mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 05:06:28 +00:00
style: format usage freshness changes
This commit is contained in:
@@ -960,7 +960,10 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
|
|||||||
match parse_usage_snapshot(&data) {
|
match parse_usage_snapshot(&data) {
|
||||||
Ok(info)
|
Ok(info)
|
||||||
if info.usage_snapshot_converged == Some(false)
|
if info.usage_snapshot_converged == Some(false)
|
||||||
&& (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) => Some(info),
|
&& (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
|
||||||
|
{
|
||||||
|
Some(info)
|
||||||
|
}
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
error!(
|
error!(
|
||||||
event = "data_usage_snapshot_load_failed",
|
event = "data_usage_snapshot_load_failed",
|
||||||
@@ -4749,12 +4752,7 @@ mod tests {
|
|||||||
async fn negative_delta_waits_for_set_reconciliation() {
|
async fn negative_delta_waits_for_set_reconciliation() {
|
||||||
clear_usage_memory_cache_for_test().await;
|
clear_usage_memory_cache_for_test().await;
|
||||||
|
|
||||||
let baseline = data_usage_info_for_test(
|
let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
|
||||||
"bucket-a",
|
|
||||||
1,
|
|
||||||
100,
|
|
||||||
SystemTime::UNIX_EPOCH + Duration::from_secs(100),
|
|
||||||
);
|
|
||||||
replace_bucket_usage_memory_from_info(&baseline).await;
|
replace_bucket_usage_memory_from_info(&baseline).await;
|
||||||
record_bucket_object_delete_memory("bucket-a", 25, true).await;
|
record_bucket_object_delete_memory("bucket-a", 25, true).await;
|
||||||
|
|
||||||
|
|||||||
@@ -28,8 +28,8 @@ use rustfs_common::heal_channel::HealScanMode;
|
|||||||
use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
|
use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
|
||||||
pub use rustfs_data_usage::{
|
pub use rustfs_data_usage::{
|
||||||
AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
|
AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
|
||||||
DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry,
|
DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, DataUsageSnapshotSetState, LEGACY_DATA_USAGE_OBJECT_NAME,
|
||||||
PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, DataUsageSnapshotSetState, hash_path,
|
PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path,
|
||||||
prefix_usage_in_cache,
|
prefix_usage_in_cache,
|
||||||
};
|
};
|
||||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||||
|
|||||||
@@ -18,9 +18,8 @@ use crate::scanner_folder::{ScannerItem, scan_data_folder};
|
|||||||
use crate::sleeper::SCANNER_SLEEPER;
|
use crate::sleeper::SCANNER_SLEEPER;
|
||||||
use crate::{
|
use crate::{
|
||||||
DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageCachePrepareOutcome,
|
DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageCachePrepareOutcome,
|
||||||
DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageSnapshotSetState, DataUsageScanPlanDigest,
|
DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState,
|
||||||
ScannerError, SizeSummary,
|
ScannerError, SizeSummary, TierStats,
|
||||||
TierStats,
|
|
||||||
};
|
};
|
||||||
use futures::future::join_all;
|
use futures::future::join_all;
|
||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
|
|||||||
@@ -353,7 +353,10 @@ pub(super) fn observational_data_usage_info(
|
|||||||
objects_total_size: u64::try_from(total.size).ok()?,
|
objects_total_size: u64::try_from(total.size).ok()?,
|
||||||
tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()),
|
tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()),
|
||||||
buckets_count: u64::try_from(buckets_usage.len()).ok()?,
|
buckets_count: u64::try_from(buckets_usage.len()).ok()?,
|
||||||
bucket_sizes: buckets_usage.iter().map(|(bucket, usage)| (bucket.clone(), usage.size)).collect(),
|
bucket_sizes: buckets_usage
|
||||||
|
.iter()
|
||||||
|
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||||
|
.collect(),
|
||||||
buckets_usage,
|
buckets_usage,
|
||||||
usage_snapshot_complete: false,
|
usage_snapshot_complete: false,
|
||||||
usage_snapshot_partial: true,
|
usage_snapshot_partial: true,
|
||||||
|
|||||||
@@ -112,9 +112,7 @@ impl ScannerIOCache for SetDisks {
|
|||||||
incomplete_scope.info.lkg_leader_epoch = Some(lkg.info.leader_epoch);
|
incomplete_scope.info.lkg_leader_epoch = Some(lkg.info.leader_epoch);
|
||||||
incomplete_scope.info.lkg_scan_plan_digest = lkg.info.scan_plan_digest;
|
incomplete_scope.info.lkg_scan_plan_digest = lkg.info.scan_plan_digest;
|
||||||
}
|
}
|
||||||
let _ = updates
|
let _ = updates.send(incomplete_scope).await;
|
||||||
.send(incomplete_scope)
|
|
||||||
.await;
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
// Preserve the original set topology across capability filtering. During
|
// Preserve the original set topology across capability filtering. During
|
||||||
@@ -214,9 +212,7 @@ impl ScannerIOCache for SetDisks {
|
|||||||
incomplete_scope.info.lkg_leader_epoch = Some(lkg.info.leader_epoch);
|
incomplete_scope.info.lkg_leader_epoch = Some(lkg.info.leader_epoch);
|
||||||
incomplete_scope.info.lkg_scan_plan_digest = lkg.info.scan_plan_digest;
|
incomplete_scope.info.lkg_scan_plan_digest = lkg.info.scan_plan_digest;
|
||||||
}
|
}
|
||||||
let _ = updates
|
let _ = updates.send(incomplete_scope).await;
|
||||||
.send(incomplete_scope)
|
|
||||||
.await;
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
|
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
|
||||||
|
|||||||
@@ -386,16 +386,19 @@ impl ScannerIOCycle for ECStore {
|
|||||||
budget_elapsed,
|
budget_elapsed,
|
||||||
ctx.is_cancelled(),
|
ctx.is_cancelled(),
|
||||||
);
|
);
|
||||||
let observational_usage = completed_usage.is_none().then(|| {
|
let observational_usage = completed_usage
|
||||||
observational_data_usage_info(
|
.is_none()
|
||||||
&results,
|
.then(|| {
|
||||||
&expected_sources,
|
observational_data_usage_info(
|
||||||
&all_buckets,
|
&results,
|
||||||
scan_plan_digest,
|
&expected_sources,
|
||||||
want_cycle,
|
&all_buckets,
|
||||||
leader_epoch,
|
scan_plan_digest,
|
||||||
)
|
want_cycle,
|
||||||
}).flatten();
|
leader_epoch,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.flatten();
|
||||||
let structurally_complete_snapshot = result.is_ok() && completed_all_sets && completed_usage.is_some();
|
let structurally_complete_snapshot = result.is_ok() && completed_all_sets && completed_usage.is_some();
|
||||||
let cycle_status = classify_nsscanner_cycle(
|
let cycle_status = classify_nsscanner_cycle(
|
||||||
structurally_complete_snapshot,
|
structurally_complete_snapshot,
|
||||||
|
|||||||
@@ -129,16 +129,11 @@ fn partial_usage_is_observational_not_authoritative_for_quota() {
|
|||||||
let stalled = lkg_root_cache("bucket", 1, stalled_source);
|
let stalled = lkg_root_cache("bucket", 1, stalled_source);
|
||||||
let expected = HashSet::from([current_source, stalled_source]);
|
let expected = HashSet::from([current_source, stalled_source]);
|
||||||
|
|
||||||
assert!(completed_data_usage_info(&[current.clone(), stalled.clone()], &expected, &all_buckets, true, false, false).is_none());
|
assert!(
|
||||||
let (observed, _) = observational_data_usage_info(
|
completed_data_usage_info(&[current.clone(), stalled.clone()], &expected, &all_buckets, true, false, false).is_none()
|
||||||
&[current, stalled],
|
);
|
||||||
&expected,
|
let (observed, _) = observational_data_usage_info(&[current, stalled], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3)
|
||||||
&all_buckets,
|
.expect("a completed set should produce an observational view");
|
||||||
TEST_PLAN_DIGEST,
|
|
||||||
8,
|
|
||||||
3,
|
|
||||||
)
|
|
||||||
.expect("a completed set should produce an observational view");
|
|
||||||
assert!(observed.usage_snapshot_partial);
|
assert!(observed.usage_snapshot_partial);
|
||||||
assert!(!observed.usage_snapshot_complete);
|
assert!(!observed.usage_snapshot_complete);
|
||||||
assert_eq!(observed.usage_snapshot_converged, Some(false));
|
assert_eq!(observed.usage_snapshot_converged, Some(false));
|
||||||
|
|||||||
Reference in New Issue
Block a user