gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 08:36:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix data usage cache and scanner (#2074)

Browse Source
This commit is contained in:
weisd
2026-03-04 19:55:01 +08:00
committed by GitHub
parent 05032cf887
commit ed18b3da75
4 changed files with 31 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/data_usage_define.rs
+6 -2
View File
@@ -1109,10 +1109,13 @@ impl DataUsageCache {
let mut buf = Vec::new();
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
let store_clone = store.clone();
let buf_clone = buf.clone();
let path_clone = path.clone();
let res = timeout(Duration::from_secs(5), async move {
save_config(store_clone, name, buf_clone).await?;
save_config(store_clone, &path_clone, buf_clone).await?;
Ok::<(), StorageError>(())
})
.await
@@ -1125,8 +1128,9 @@ impl DataUsageCache {
let store_clone = store.clone();
let backup_name = format!("{name}.bkp");
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
let res = timeout(Duration::from_secs(5), async move {
save_config(store_clone, backup_name.as_str(), buf).await?;
save_config(store_clone, &backup_path, buf).await?;
Ok::<(), StorageError>(())
})
.await
crates/scanner/src/scanner.rs
+3 -5
View File
@@ -259,8 +259,6 @@ pub async fn store_data_usage_in_backend(
break;
}
debug!("store_data_usage_in_backend: received data usage info: {:?}", &data_usage_info);
// Serialize to JSON
let data = match serde_json::to_vec(&data_usage_info) {
Ok(data) => data,
@@ -272,7 +270,7 @@ pub async fn store_data_usage_in_backend(
// Save a backup every 10th update
if attempts > 10 {
let backup_path = format!("{:?}.bkp", &DATA_USAGE_OBJ_NAME_PATH);
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await {
warn!("Failed to save data usage backup to {}: {}", backup_path, e);
}
@@ -280,8 +278,8 @@ pub async fn store_data_usage_in_backend(
}
// Save main configuration
if let Err(e) = save_config(storeapi.clone(), &DATA_USAGE_OBJ_NAME_PATH, data).await {
error!("Failed to save data usage info to {:?}: {e}", &DATA_USAGE_OBJ_NAME_PATH);
if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await {
error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str());
}
attempts += 1;
crates/scanner/src/scanner_io.rs
+9 -10
View File
@@ -177,17 +177,17 @@ impl ScannerIO for ECStore {
let mut all_merged = DataUsageCache::default();
for result in results.iter() {
if result.info.last_update.is_none() {
return;
continue;
}
all_merged.merge(result);
}
if all_merged.root().is_some() && all_merged.info.last_update.unwrap() > last_update
&& let Err(e) = updates
.send(all_merged.dui(&all_merged.info.name, &all_buckets_clone))
.await {
if all_merged.root().is_some() && all_merged.info.last_update.unwrap() > last_update {
let dui = all_merged.dui(&all_merged.info.name, &all_buckets_clone);
if let Err(e) = updates.send(dui).await {
error!("Failed to send data usage info: {}", e);
}
}
break;
}
_ = ticker.tick() => {
@@ -195,15 +195,14 @@ impl ScannerIO for ECStore {
let mut all_merged = DataUsageCache::default();
for result in results.iter() {
if result.info.last_update.is_none() {
return;
continue;
}
all_merged.merge(result);
}
if all_merged.root().is_some() && all_merged.info.last_update.unwrap() > last_update {
if let Err(e) = updates
.send(all_merged.dui(&all_merged.info.name, &all_buckets_clone))
.await {
let dui = all_merged.dui(&all_merged.info.name, &all_buckets_clone);
if let Err(e) = updates.send(dui).await {
error!("Failed to send data usage info: {}", e);
}
last_update = all_merged.info.last_update.unwrap();
@@ -299,7 +298,7 @@ impl ScannerIOCache for SetDisks {
let cache = cache_mutex_clone.lock().await;
if cache.info.last_update == last_update {
continue;
continue;
}
if let Err(e) = cache.save(store_clone.clone(), DATA_USAGE_CACHE_NAME).await {