gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-07 05:43:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): publish bounded observational usage (#5742)

Browse Source 
* fix(scanner): publish bounded observational usage

* test(ci): serialize embedded integration ports

* test(cache): isolate generation-change timeout

* fix(scanner): address observational usage review

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: Anthony Martin <949506+anthonymartin@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
anthonymartin
2026-08-06 17:52:34 -07:00
committed by GitHub
parent 83cdea1f18
commit 706a8b6061
21 changed files with 1396 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/nextest.toml
+14
View File
@@ -29,6 +29,7 @@
[test-groups]
ecstore-serial-flaky = { max-threads = 1 }
embedded-test-ports = { max-threads = 1 }
# Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS
# server and manipulate its disk directories at runtime (crates/e2e_test:
@@ -54,6 +55,13 @@ test-group = 'ecstore-serial-flaky'
filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
test-group = 'ecstore-serial-flaky'
# Embedded integration-test binaries discover an ephemeral port and release
# the probe listener before RustFS binds it. Serialize that cross-process
# TOCTOU window; retries would only hide real startup failures.
[[profile.default.overrides]]
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
test-group = 'embedded-test-ports'
# Serialize the durable manual-transition checkpoint test across nextest's
# process boundary; it mutates bucket lifecycle metadata and is not quarantined.
[[profile.default.overrides]]
@@ -143,6 +151,12 @@ test-group = 'e2e-reliability'
filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
test-group = 'ecstore-serial-flaky'
# Match the default-profile embedded test isolation without quarantining or
# retrying failures in CI.
[[profile.ci.overrides]]
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
test-group = 'embedded-test-ports'
# Serialize the durable manual-transition checkpoint test under the ci profile
# too. No retries: failures stay visible.
[[profile.ci.overrides]]
crates/data-usage/src/data_usage.rs
+143
View File
@@ -37,6 +37,10 @@ pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 *
/// Keeping the existing object name preserves rolling-upgrade and rollback
/// compatibility without allowing an ambiguous snapshot to become authoritative.
pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json";
/// Latest structurally complete scanner observation. Unlike
/// [`DATA_USAGE_OBJECT_NAME`], this object is never authoritative for quota
/// admission because namespace activity may have raced the scan.
pub const DATA_USAGE_OBSERVED_OBJECT_NAME: &str = ".usage.observed.json";
/// Usage snapshot written by scanner implementations predating distributed
/// leadership fencing. It is read only when neither authoritative snapshot
@@ -218,6 +222,20 @@ pub struct DataUsageInfo {
/// explicit entry for every bucket, including confirmed-empty buckets.
#[serde(default)]
pub usage_snapshot_complete: bool,
/// Whether no namespace activity or dirty-usage generation changed while
/// the coordinated snapshot was being produced.
///
/// `false` still describes a structurally complete, useful point-in-time
/// usage view, but follow-up scanner work remains pending. `None` is kept
/// for snapshots written before this status became observable.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage_snapshot_converged: Option<bool>,
/// Identity of the authoritative snapshot from which a nonconverged
/// observation started. Admin readers require an exact match before using
/// the observation, so bucket namespace mutations fence old observations
/// without relying on synchronized clocks.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage_snapshot_authoritative_baseline: Option<DataUsageSnapshotIdentity>,
/// Deprecated kept here for backward compatibility reasons
pub bucket_sizes: HashMap<String, u64>,
/// Per-disk snapshot information when available
@@ -225,6 +243,59 @@ pub struct DataUsageInfo {
pub disk_usage_status: Vec<DiskUsageStatus>,
}
/// Stable identity fields changed by both coordinated scanner publication and
/// backward-compatible bucket namespace cleanup.
#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
pub struct DataUsageSnapshotIdentity {
pub last_update: Option<SystemTime>,
pub scanner_cycle: Option<u64>,
pub scanner_epoch: Option<u64>,
}
impl DataUsageInfo {
pub fn snapshot_identity(&self) -> DataUsageSnapshotIdentity {
DataUsageSnapshotIdentity {
last_update: self.last_update,
scanner_cycle: self.scanner_cycle,
scanner_epoch: self.scanner_epoch,
}
}
}
/// Return whether `candidate` was produced after `baseline`.
///
/// New coordinated snapshots are ordered by leadership epoch and scanner
/// cycle. The timestamp fallback preserves ordering for legacy snapshots that
/// predate those fields.
pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUsageInfo) -> bool {
match (
candidate.scanner_epoch.zip(candidate.scanner_cycle),
baseline.scanner_epoch.zip(baseline.scanner_cycle),
) {
(Some(candidate), Some(baseline)) => candidate > baseline,
(Some(_), None) => true,
(None, Some(_)) => false,
(None, None) => match (candidate.last_update, baseline.last_update) {
(Some(candidate), Some(baseline)) => candidate > baseline,
(Some(_), None) => true,
(None, Some(_) | None) => false,
},
}
}
/// Return whether a nonconverged observation may safely supersede the admin
/// view of `authoritative`.
///
/// The exact baseline identity is independent of clock ordering. Older binaries
/// already advance the authoritative timestamp when deleting a bucket, so a
/// rollback delete/recreate fences the previous bucket incarnation too.
pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
observed.usage_snapshot_converged == Some(false)
&& observed.is_complete_bucket_usage_snapshot()
&& observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
&& data_usage_snapshot_is_newer(observed, authoritative)
}
/// Metadata describing the status of a disk-level data usage snapshot.
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DiskUsageStatus {
@@ -1783,6 +1854,8 @@ mod tests {
let current = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(false),
usage_snapshot_authoritative_baseline: Some(DataUsageSnapshotIdentity::default()),
..Default::default()
};
let encoded = rmp_serde::to_vec_named(&current).expect("encode current data usage snapshot");
@@ -1790,6 +1863,76 @@ mod tests {
assert_eq!(legacy.buckets_count, 0);
assert!(current.is_complete_bucket_usage_snapshot());
assert_eq!(current.usage_snapshot_converged, Some(false));
}
#[test]
fn convergence_marker_defaults_to_unknown_for_older_snapshots() {
let encoded = rmp_serde::to_vec_named(&DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
usage_snapshot_complete: true,
..Default::default()
})
.expect("encode pre-convergence data usage snapshot");
let decoded: DataUsageInfo = rmp_serde::from_slice(&encoded).expect("decode older data usage snapshot");
assert!(decoded.is_complete_bucket_usage_snapshot());
assert_eq!(decoded.usage_snapshot_converged, None);
}
#[test]
fn observation_selection_is_clock_independent_and_baseline_fenced() {
let mut authoritative = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(600)),
scanner_epoch: Some(7),
scanner_cycle: Some(10),
usage_snapshot_complete: true,
..Default::default()
};
let observed = DataUsageInfo {
// A newer leader may have a slower wall clock.
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
scanner_epoch: Some(8),
scanner_cycle: Some(1),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(false),
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
..Default::default()
};
assert!(observed_data_usage_is_newer(&observed, &authoritative));
authoritative.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(601));
assert!(
!observed_data_usage_is_newer(&observed, &authoritative),
"an old-binary namespace mutation must fence the prior bucket incarnation regardless of clock skew"
);
}
#[test]
fn observation_selection_requires_nonconverged_complete_newer_data() {
let authoritative = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
scanner_epoch: Some(2),
scanner_cycle: Some(10),
usage_snapshot_complete: true,
..Default::default()
};
let baseline = Some(authoritative.snapshot_identity());
let candidate = |epoch, cycle, converged, complete| DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
scanner_epoch: Some(epoch),
scanner_cycle: Some(cycle),
usage_snapshot_complete: complete,
usage_snapshot_converged: converged,
usage_snapshot_authoritative_baseline: baseline,
..Default::default()
};
assert!(observed_data_usage_is_newer(&candidate(2, 11, Some(false), true), &authoritative));
assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
}
#[test]
crates/ecstore/src/api/mod.rs
+2 -1
View File
@@ -310,7 +310,8 @@ pub mod config {
pub mod data_usage {
pub use crate::data_usage::{
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
init_compression_total_memory_from_backend, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations,
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
crates/ecstore/src/data_usage/mod.rs
+339 -11
View File
@@ -20,7 +20,7 @@ pub mod local_snapshot;
use crate::storage_api_contracts::{
bucket::{BucketOperations as _, BucketOptions},
list::{ListOperations as _, StorageListObjectVersionsInfo},
object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _},
object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
};
use crate::{
bucket::{metadata_sys::get_replication_config, versioning::VersioningApi as _, versioning_sys::BucketVersioningSys},
@@ -33,8 +33,9 @@ use crate::{
};
pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path};
use rustfs_data_usage::{
BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DataUsageCache, DataUsageEntry,
DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary, VersionsHistogram,
BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary,
VersionsHistogram, observed_data_usage_is_newer,
};
use rustfs_io_metrics::record_system_path_failure;
use rustfs_utils::path::SLASH_SEPARATOR;
@@ -162,8 +163,9 @@ fn cache_data_usage_snapshot_result(
result: Result<(DataUsageInfo, HashMap<String, u64>), Error>,
loaded_at: tokio::time::Instant,
refresh_generation: u64,
current_generation: u64,
) -> Option<Result<DataUsageInfo, Error>> {
if data_usage_snapshot_generation() != refresh_generation {
if current_generation != refresh_generation {
return None;
}
@@ -196,6 +198,9 @@ type DataUsageSnapshotCache = Arc<RwLock<Option<CachedDataUsageSnapshot>>>;
static DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
static DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
static DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
static ADMIN_DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
static ADMIN_DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
static ADMIN_DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
// Always-on revert detector for rustfs/backlog#1306: one relaxed increment per
// full-bucket version listing is negligible and lets tests prove that admin
@@ -254,11 +259,24 @@ fn data_usage_snapshot_generation() -> u64 {
DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
}
fn admin_data_usage_snapshot_cache() -> &'static DataUsageSnapshotCache {
ADMIN_DATA_USAGE_SNAPSHOT_CACHE.get_or_init(|| Arc::new(RwLock::new(None)))
}
fn admin_data_usage_snapshot_generation() -> u64 {
ADMIN_DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
}
fn clear_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
*cache = None;
}
fn clear_admin_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
ADMIN_DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
*cache = None;
}
fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
LIVE_BUCKET_USAGE_CACHE.get_or_init(|| {
moka::future::Cache::builder()
@@ -283,6 +301,11 @@ lazy_static::lazy_static! {
SLASH_SEPARATOR,
DATA_USAGE_OBJECT_NAME
);
pub static ref DATA_USAGE_OBSERVED_OBJ_NAME_PATH: String = format!("{}{}{}",
crate::disk::BUCKET_META_PREFIX,
SLASH_SEPARATOR,
DATA_USAGE_OBSERVED_OBJECT_NAME
);
static ref DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
static ref LEGACY_DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
crate::disk::BUCKET_META_PREFIX,
@@ -357,6 +380,11 @@ fn stale_data_usage_persist_reason_for_source(
/// Store data usage info to backend storage
#[instrument(skip(store))]
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
if data_usage_info.usage_snapshot_converged == Some(false) {
return Err(Error::other(
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
));
}
// Prevent older data from overwriting newer persisted stats
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
&& source.is_authoritative()
@@ -377,10 +405,12 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
// Save to backend using the same mechanism as original code
crate::config::com::save_config(store, &DATA_USAGE_OBJ_NAME_PATH, data)
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
.await
.map_err(Error::other)?;
cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
// Invalidate the cached snapshot so readers observe the new save on their
// next request instead of waiting out the remaining TTL. The next cached
// read reloads through `load_data_usage_from_backend`, keeping its
@@ -390,6 +420,64 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
Ok(())
}
#[async_trait::async_trait]
trait ObservedDataUsageSnapshotCleanup {
async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error>;
}
#[async_trait::async_trait]
impl ObservedDataUsageSnapshotCleanup for ECStore {
async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error> {
self.delete_object(
RUSTFS_META_BUCKET,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
ObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(revision.to_string()),
..Default::default()
}),
..Default::default()
},
)
.await
.map(|_| ())
}
}
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
Ok(Some(snapshot)) => snapshot,
Ok(None) => return,
Err(err) => {
record_usage_snapshot_failure(
"read_observed_before_authoritative_cleanup",
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
&err,
);
return;
}
};
if observed_data_usage_is_newer(&observed, authoritative) {
return;
}
match store.delete_observed_data_usage_snapshot(&revision).await {
Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
Err(err) => {
record_usage_snapshot_failure(
"delete_observed_after_authoritative_save",
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
&err,
);
}
}
}
fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
data_usage_info.buckets_count = u64::try_from(data_usage_info.buckets_usage.len()).unwrap_or(u64::MAX);
}
@@ -443,6 +531,11 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache cleanup")?;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
drop(snapshot_cache);
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache cleanup")?;
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
Ok(())
}
@@ -458,6 +551,10 @@ where
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
drop(snapshot_cache);
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
result
}
@@ -551,6 +648,23 @@ where
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
if let Err(err) = remove_bucket_usage_from_object_with_retries(
store,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
)
.await
{
// The authoritative timestamp was already advanced above, so admin
// selection rejects this observation even if optional cleanup fails.
// Never make an admin-only freshness artifact block DeleteBucket.
record_usage_snapshot_failure("remove_bucket_from_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
}
for object in [
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
@@ -826,6 +940,61 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
}
async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUsageInfo> {
let data = match read_config_preserve_empty(store, &DATA_USAGE_OBSERVED_OBJ_NAME_PATH).await {
Ok(data) => data,
Err(Error::ConfigNotFound) => return None,
Err(err) => {
record_usage_snapshot_failure("read_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
return None;
}
};
match parse_usage_snapshot(&data) {
Ok(info) if info.usage_snapshot_converged == Some(false) && info.is_complete_bucket_usage_snapshot() => Some(info),
Ok(_) => {
error!(
event = "data_usage_snapshot_load_failed",
component = "ecstore",
subsystem = "data_usage",
state = "invalid_observed_snapshot",
object = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
"observed data usage snapshot was not a structurally complete nonconverged view"
);
None
}
Err(err) => {
record_usage_snapshot_decode_failure("parse_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
None
}
}
}
fn select_admin_data_usage_snapshot(
mut authoritative: DataUsageInfo,
authoritative_format: bool,
observed: Option<DataUsageInfo>,
) -> (DataUsageInfo, bool) {
if authoritative_format
&& authoritative.is_complete_bucket_usage_snapshot()
&& authoritative.usage_snapshot_converged.is_none()
{
authoritative.usage_snapshot_converged = Some(true);
}
match observed {
Some(observed) if observed_data_usage_is_newer(&observed, &authoritative) => (observed, true),
_ => (authoritative, authoritative_format),
}
}
async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
let observed = load_observed_data_usage_snapshot(store).await;
let (selected, selected_is_current_format) =
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
}
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
if !data_usage_info.is_complete_bucket_usage_snapshot() {
data_usage_info.usage_snapshot_complete = false;
@@ -944,7 +1113,55 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
let result = load_data_usage_from_backend_with_baseline(store.clone()).await;
let loaded_at = tokio::time::Instant::now();
let mut cache = data_usage_snapshot_cache().write().await;
if let Some(result) = cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation) {
if let Some(result) =
cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation, data_usage_snapshot_generation())
{
return result;
}
drop(cache);
drop(refresh_guard);
}
}
/// Load the freshest structurally complete snapshot for authenticated admin
/// observability. A scan raced by namespace activity may be selected here, but
/// never by [`load_data_usage_from_backend_cached`], which remains the
/// converged source for quota admission.
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
loop {
{
let cache = admin_data_usage_snapshot_cache().read().await;
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
return result;
}
}
let refresh_guard = ADMIN_DATA_USAGE_SNAPSHOT_REFRESH
.get_or_init(|| Arc::new(TokioMutex::new(())))
.lock()
.await;
{
let cache = admin_data_usage_snapshot_cache().read().await;
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
return result;
}
}
let refresh_generation = admin_data_usage_snapshot_generation();
let result = load_admin_data_usage_from_backend(store.clone())
.await
.map(|info| (info, HashMap::new()));
let loaded_at = tokio::time::Instant::now();
let mut cache = admin_data_usage_snapshot_cache().write().await;
if let Some(result) = cache_data_usage_snapshot_result(
&mut cache,
result,
loaded_at,
refresh_generation,
admin_data_usage_snapshot_generation(),
) {
return result;
}
drop(cache);
@@ -957,6 +1174,16 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
pub async fn invalidate_data_usage_snapshot_cache() {
let mut cache = data_usage_snapshot_cache().write().await;
clear_data_usage_snapshot_cache(&mut cache);
let mut admin_cache = admin_data_usage_snapshot_cache().write().await;
clear_admin_data_usage_snapshot_cache(&mut admin_cache);
}
/// Invalidate only the admin/console view after an observational save. Quota
/// admission continues to use the independently cached converged snapshot.
pub async fn invalidate_admin_data_usage_snapshot_cache() {
let mut cache = admin_data_usage_snapshot_cache().write().await;
clear_admin_data_usage_snapshot_cache(&mut cache);
}
/// Aggregate usage information from local disk snapshots.
@@ -2080,6 +2307,7 @@ mod tests {
struct UsageCasState {
object: Option<(Vec<u8>, u64)>,
backup_object: Option<(Vec<u8>, u64)>,
observed_object: Option<(Vec<u8>, u64)>,
legacy_object: Option<(Vec<u8>, u64)>,
legacy_backup_object: Option<(Vec<u8>, u64)>,
interleaving_snapshot: Option<Vec<u8>>,
@@ -2099,10 +2327,24 @@ mod tests {
state: Mutex<UsageCasState>,
}
#[async_trait::async_trait]
impl ObservedDataUsageSnapshotCleanup for UsageCasStore {
async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error> {
let mut state = self.state.lock().await;
let current = state.observed_object.as_ref().ok_or(Error::FileNotFound)?.1;
if revision != format!("usage-{current}") {
return Err(Error::PreconditionFailed);
}
state.observed_object = None;
Ok(())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum UsageObjectSlot {
Primary,
Backup,
Observed,
LegacyPrimary,
LegacyBackup,
}
@@ -2131,6 +2373,7 @@ mod tests {
let slot = match object {
object if object == DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Primary,
object if object == DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::Backup,
object if object == DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Observed,
object if object == LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::LegacyPrimary,
object if object == LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::LegacyBackup,
_ => return Err(Error::FileNotFound),
@@ -2139,6 +2382,7 @@ mod tests {
let stored = match slot {
UsageObjectSlot::Primary => &state.object,
UsageObjectSlot::Backup => &state.backup_object,
UsageObjectSlot::Observed => &state.observed_object,
UsageObjectSlot::LegacyPrimary => &state.legacy_object,
UsageObjectSlot::LegacyBackup => &state.legacy_backup_object,
};
@@ -2178,6 +2422,7 @@ mod tests {
let slot = match object {
object if object == DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Primary,
object if object == DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::Backup,
object if object == DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Observed,
object if object == LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::LegacyPrimary,
object if object == LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::LegacyBackup,
_ => return Err(Error::FileNotFound),
@@ -2213,6 +2458,9 @@ mod tests {
let revision = state.backup_object.as_ref().map_or(1, |(_, revision)| revision + 1);
state.backup_object = Some((interleaving, revision));
}
if slot == UsageObjectSlot::Observed {
return Err(Error::other("observed test fixture writes are injected directly"));
}
if slot == UsageObjectSlot::LegacyPrimary
&& let Some(interleaving) = state.interleaving_legacy_snapshot.take()
{
@@ -2228,6 +2476,7 @@ mod tests {
let current_revision = match slot {
UsageObjectSlot::Primary => state.object.as_ref(),
UsageObjectSlot::Backup => state.backup_object.as_ref(),
UsageObjectSlot::Observed => state.observed_object.as_ref(),
UsageObjectSlot::LegacyPrimary => state.legacy_object.as_ref(),
UsageObjectSlot::LegacyBackup => state.legacy_backup_object.as_ref(),
}
@@ -2258,6 +2507,7 @@ mod tests {
match slot {
UsageObjectSlot::Primary => state.object = Some((buf, revision)),
UsageObjectSlot::Backup => state.backup_object = Some((buf, revision)),
UsageObjectSlot::Observed => state.observed_object = Some((buf, revision)),
UsageObjectSlot::LegacyPrimary => state.legacy_object = Some((buf, revision)),
UsageObjectSlot::LegacyBackup => state.legacy_backup_object = Some((buf, revision)),
}
@@ -2550,6 +2800,71 @@ mod tests {
assert!(normalized.buckets_usage.is_empty());
}
#[test]
fn admin_snapshot_selection_requires_the_current_authoritative_baseline() {
let authoritative = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH),
scanner_epoch: Some(4),
scanner_cycle: Some(10),
usage_snapshot_complete: true,
..Default::default()
};
let observed = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
scanner_epoch: Some(4),
scanner_cycle: Some(11),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(false),
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
..Default::default()
};
let (selected, _) = select_admin_data_usage_snapshot(authoritative.clone(), true, Some(observed.clone()));
assert_eq!(selected.usage_snapshot_converged, Some(false));
let mut namespace_changed = authoritative;
namespace_changed.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(2));
let (selected, _) = select_admin_data_usage_snapshot(namespace_changed, true, Some(observed));
assert_eq!(selected.usage_snapshot_converged, Some(true));
}
#[tokio::test]
async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
let store = UsageCasStore::default();
let authoritative = data_usage_info_for_test("bucket", 1, 10, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
let stale_observed = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
scanner_epoch: Some(4),
scanner_cycle: Some(10),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(false),
..Default::default()
};
store.state.lock().await.observed_object =
Some((serde_json::to_vec(&stale_observed).expect("observed snapshot should encode"), 1));
cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
assert!(store.state.lock().await.observed_object.is_none());
cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
assert!(store.state.lock().await.observed_object.is_none());
let newer_observed = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(3)),
scanner_epoch: Some(4),
scanner_cycle: Some(11),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(false),
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
..Default::default()
};
store.state.lock().await.observed_object =
Some((serde_json::to_vec(&newer_observed).expect("observed snapshot should encode"), 2));
cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
assert!(store.state.lock().await.observed_object.is_some());
}
#[test]
#[serial]
fn cached_snapshot_failure_is_reused_until_ttl_expires() {
@@ -2557,8 +2872,14 @@ mod tests {
let mut cache = None;
let refresh_generation = data_usage_snapshot_generation();
let first = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, refresh_generation)
.expect("an uninterrupted refresh should populate the cache");
let first = cache_data_usage_snapshot_result(
&mut cache,
Err(Error::ErasureReadQuorum),
loaded_at,
refresh_generation,
data_usage_snapshot_generation(),
)
.expect("an uninterrupted refresh should populate the cache");
assert!(matches!(first, Err(Error::ErasureReadQuorum)));
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
@@ -2576,9 +2897,15 @@ mod tests {
let mut cache = None;
let refresh_generation = data_usage_snapshot_generation();
let first = cache_data_usage_snapshot_result(&mut cache, Ok((expected, HashMap::new())), loaded_at, refresh_generation)
.expect("an uninterrupted refresh should populate the cache")
.expect("successful load must be returned");
let first = cache_data_usage_snapshot_result(
&mut cache,
Ok((expected, HashMap::new())),
loaded_at,
refresh_generation,
data_usage_snapshot_generation(),
)
.expect("an uninterrupted refresh should populate the cache")
.expect("successful load must be returned");
assert_snapshot_bucket(&first, "bucket");
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
@@ -2604,6 +2931,7 @@ mod tests {
Ok((data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH), HashMap::new())),
loaded_at,
refresh_generation,
data_usage_snapshot_generation(),
);
assert!(stale_result.is_none());
crates/madmin/src/user.rs
+19
View File
@@ -401,6 +401,11 @@ pub struct AccountInfo {
pub server: BackendInfo,
pub policy: serde_json::Value, // Use iam/policy::parse to parse the result, to be done by the caller.
pub buckets: Vec<BucketAccessInfo>,
/// Whether the usage values came from a fully converged scanner cycle.
/// `false` means the values are a newer structurally complete observation
/// while another convergence pass remains pending.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage_snapshot_converged: Option<bool>,
}
#[derive(Debug, Serialize, Deserialize, Default)]
@@ -798,6 +803,20 @@ mod tests {
assert_eq!(value["object_versions_histogram"], serde_json::json!({}));
}
#[test]
fn account_info_exposes_observational_usage_state_additively() {
let info = AccountInfo {
usage_snapshot_converged: Some(false),
..Default::default()
};
let value = serde_json::to_value(&info).unwrap();
assert_eq!(value["usage_snapshot_converged"], false);
let legacy_shape = serde_json::to_value(AccountInfo::default()).unwrap();
assert!(!legacy_shape.as_object().unwrap().contains_key("usage_snapshot_converged"));
}
#[test]
fn test_account_status_try_from_invalid() {
let result = AccountStatus::try_from("invalid");
crates/obs/src/metrics/collectors/cluster_usage.rs
+14 -3
View File
@@ -38,6 +38,9 @@ pub struct ClusterUsageStats {
pub delete_markers_count: u64,
/// Total number of buckets in the usage snapshot
pub buckets_count: u64,
/// Whether the selected admin usage snapshot completed without concurrent
/// namespace activity.
pub snapshot_converged: bool,
/// Object size distribution by range
pub object_size_distribution: Vec<(String, u64)>,
/// Version count distribution by range
@@ -69,7 +72,7 @@ pub struct BucketUsageStats {
///
/// Returns a vector of Prometheus metrics for cluster usage.
pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(6 + stats.object_size_distribution.len() + stats.versions_distribution.len());
let mut metrics = Vec::with_capacity(7 + stats.object_size_distribution.len() + stats.versions_distribution.len());
metrics.push(PrometheusMetric::from_descriptor(
&USAGE_SINCE_LAST_UPDATE_SECONDS_MD,
@@ -83,6 +86,10 @@ pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<Prometheu
stats.delete_markers_count as f64,
));
metrics.push(PrometheusMetric::from_descriptor(&USAGE_BUCKETS_COUNT_MD, stats.buckets_count as f64));
metrics.push(PrometheusMetric::from_descriptor(
&USAGE_SNAPSHOT_CONVERGED_MD,
if stats.snapshot_converged { 1.0 } else { 0.0 },
));
// Object size distribution
for (range, count) in &stats.object_size_distribution {
@@ -176,6 +183,7 @@ mod tests {
versions_count: 15000,
delete_markers_count: 500,
buckets_count: 8,
snapshot_converged: false,
object_size_distribution: vec![
("0-1KB".to_string(), 5000),
("1KB-1MB".to_string(), 3000),
@@ -188,8 +196,8 @@ mod tests {
let metrics = collect_cluster_usage_metrics(&stats);
report_metrics(&metrics);
// 6 base metrics + 4 size distribution + 3 version distribution = 13
assert_eq!(metrics.len(), 13);
// 7 base metrics + 4 size distribution + 3 version distribution = 14
assert_eq!(metrics.len(), 14);
let total_bytes_name = USAGE_TOTAL_BYTES_MD.get_full_metric_name();
let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
@@ -198,6 +206,9 @@ mod tests {
let stale_name = USAGE_SINCE_LAST_UPDATE_SECONDS_MD.get_full_metric_name();
let stale = metrics.iter().find(|m| m.name == stale_name && m.value == 45.0);
assert!(stale.is_some());
let converged_name = USAGE_SNAPSHOT_CONVERGED_MD.get_full_metric_name();
assert!(metrics.iter().any(|m| m.name == converged_name && m.value == 0.0));
}
#[test]
crates/obs/src/metrics/schema/cluster_usage.rs
+9
View File
@@ -76,6 +76,15 @@ pub static USAGE_BUCKETS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(||
)
});
pub static USAGE_SNAPSHOT_CONVERGED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::UsageSnapshotConverged,
"Whether the selected admin usage snapshot is converged (1) or observational (0)",
&[],
subsystems::CLUSTER_USAGE_OBJECTS,
)
});
pub static USAGE_OBJECTS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::UsageSizeDistribution,
crates/obs/src/metrics/schema/entry/metric_name.rs
+2
View File
@@ -233,6 +233,7 @@ pub enum MetricName {
UsageVersionsCount,
UsageDeleteMarkersCount,
UsageBucketsCount,
UsageSnapshotConverged,
UsageSizeDistribution,
UsageVersionCountDistribution,
@@ -642,6 +643,7 @@ impl MetricName {
Self::UsageVersionsCount => "versions_count".to_string(),
Self::UsageDeleteMarkersCount => "delete_markers_count".to_string(),
Self::UsageBucketsCount => "buckets_count".to_string(),
Self::UsageSnapshotConverged => "snapshot_converged".to_string(),
Self::UsageSizeDistribution => "size_distribution".to_string(),
Self::UsageVersionCountDistribution => "version_count_distribution".to_string(),
crates/obs/src/metrics/stats_collector.rs
+21
View File
@@ -65,6 +65,7 @@ type ObsBackendInfo = <ObsStore as StorageAdminApi>::BackendInfo;
struct ObsDataUsageInfo {
last_update: Option<SystemTime>,
usage_snapshot_complete: bool,
usage_snapshot_converged: bool,
buckets_count: u64,
objects_total_count: u64,
versions_total_count: u64,
@@ -102,6 +103,7 @@ async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreRes
Ok(ObsDataUsageInfo {
last_update: data_usage.last_update,
usage_snapshot_complete,
usage_snapshot_converged: data_usage.usage_snapshot_converged == Some(true),
buckets_count: data_usage.buckets_count,
objects_total_count: data_usage.objects_total_count,
versions_total_count: data_usage.versions_total_count,
@@ -1204,6 +1206,7 @@ async fn collect_cluster_usage_metric_stats_from_data_usage(
versions_count: data_usage.versions_total_count,
delete_markers_count: data_usage.delete_markers_total_count,
buckets_count: data_usage.buckets_count,
snapshot_converged: data_usage.usage_snapshot_converged,
object_size_distribution: data_usage
.buckets_usage
.values()
@@ -1579,6 +1582,7 @@ mod tests {
let (cluster, buckets) = collect_cluster_usage_metric_stats_from_data_usage(
ObsDataUsageInfo {
usage_snapshot_complete: true,
usage_snapshot_converged: true,
..Default::default()
},
&HashSet::new(),
@@ -1589,9 +1593,26 @@ mod tests {
assert_eq!(cluster.buckets_count, 0);
assert_eq!(cluster.objects_count, 0);
assert_eq!(cluster.total_bytes, 0);
assert!(cluster.snapshot_converged);
assert!(buckets.is_empty());
}
#[tokio::test]
async fn cluster_usage_metrics_publish_unknown_convergence_as_unconverged() {
let (cluster, _) = collect_cluster_usage_metric_stats_from_data_usage(
ObsDataUsageInfo {
usage_snapshot_complete: true,
usage_snapshot_converged: false,
..Default::default()
},
&HashSet::new(),
)
.await
.expect("complete usage with unknown convergence should remain publishable");
assert!(!cluster.snapshot_converged);
}
#[tokio::test]
async fn cluster_usage_metrics_skip_snapshot_for_a_different_bucket_namespace() {
let data_usage = ObsDataUsageInfo {
crates/obs/src/metrics/storage_api.rs
+1 -1
View File
@@ -27,8 +27,8 @@ pub(crate) use rustfs_ecstore::api::capacity::{
get_total_usable_capacity_free as obs_get_total_usable_capacity_free,
};
pub(crate) use rustfs_ecstore::api::compression::is_disk_compression_enabled as obs_is_disk_compression_enabled;
pub(crate) use rustfs_ecstore::api::data_usage::load_admin_data_usage_from_backend_cached as obs_load_data_usage_from_backend;
pub(crate) use rustfs_ecstore::api::data_usage::load_compression_total_from_memory as obs_load_compression_total_from_memory;
pub(crate) use rustfs_ecstore::api::data_usage::load_data_usage_from_backend as obs_load_data_usage_from_backend;
pub(crate) use rustfs_ecstore::api::error::Result as ObsEcstoreResult;
pub(crate) use rustfs_ecstore::api::runtime::{
bucket_monitor as obs_get_global_bucket_monitor, expiry_state_handle as obs_expiry_state_handle,
crates/scanner/src/data_usage_define.rs
+5 -2
View File
@@ -27,8 +27,8 @@ use rustfs_common::heal_channel::HealScanMode;
#[cfg(test)]
use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
pub use rustfs_data_usage::{
AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageEntry, DataUsageHash,
DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, TierStats, hash_path,
AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, TierStats, hash_path,
};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use tokio::time::{Duration, Instant, sleep, timeout};
@@ -173,6 +173,9 @@ pub static DATA_USAGE_BUCKET: LazyLock<String> =
pub static DATA_USAGE_OBJ_NAME_PATH: LazyLock<String> =
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_OBJECT_NAME}"));
pub static DATA_USAGE_OBSERVED_OBJ_NAME_PATH: LazyLock<String> =
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_OBSERVED_OBJECT_NAME}"));
pub static LEGACY_DATA_USAGE_OBJ_NAME_PATH: LazyLock<String> =
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{LEGACY_DATA_USAGE_OBJECT_NAME}"));
crates/scanner/src/lib.rs
+31 -4
View File
@@ -35,10 +35,11 @@ use storage_api::owner::{
EcstoreTierConfig, EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete,
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_data_usage_snapshot_cache,
ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
@@ -475,6 +476,10 @@ pub(crate) async fn invalidate_data_usage_snapshot_cache() {
ecstore_invalidate_data_usage_snapshot_cache().await;
}
pub(crate) async fn invalidate_admin_data_usage_snapshot_cache() {
ecstore_invalidate_admin_data_usage_snapshot_cache().await;
}
pub trait ScannerObjectIO:
ObjectIO<
Error = EcstoreError,
@@ -501,6 +506,28 @@ impl<T> ScannerObjectIO for T where
{
}
#[async_trait::async_trait]
pub trait ScannerConfigObjectDelete: Send + Sync + std::fmt::Debug + 'static {
async fn delete_config_object(
&self,
bucket: &str,
object: &str,
opts: ScannerObjectOptions,
) -> EcstoreResult<ScannerObjectInfo>;
}
#[async_trait::async_trait]
impl ScannerConfigObjectDelete for ECStore {
async fn delete_config_object(
&self,
bucket: &str,
object: &str,
opts: ScannerObjectOptions,
) -> EcstoreResult<ScannerObjectInfo> {
ObjectOperations::delete_object(self, bucket, object, opts).await
}
}
#[cfg(test)]
mod tests {
use super::*;
crates/scanner/src/scanner.rs
+640 -72
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/scanner_io.rs
+88 -17
View File
@@ -776,6 +776,34 @@ fn classify_nsscanner_cycle(
}
}
fn should_publish_usage_snapshot(status: ScannerCycleStatus) -> bool {
matches!(status, ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded)
}
fn prepare_usage_snapshot_for_publication(
status: ScannerCycleStatus,
mut data_usage_info: DataUsageInfo,
) -> Option<DataUsageInfo> {
if !should_publish_usage_snapshot(status) {
return None;
}
data_usage_info.usage_snapshot_converged = Some(status == ScannerCycleStatus::Complete);
Some(data_usage_info)
}
async fn publish_usage_snapshot(
updates: &mpsc::Sender<DataUsageInfo>,
status: ScannerCycleStatus,
data_usage_info: DataUsageInfo,
) -> Result<bool> {
let Some(data_usage_info) = prepare_usage_snapshot_for_publication(status, data_usage_info) else {
return Ok(false);
};
send_data_usage_update(updates, data_usage_info).await?;
Ok(true)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ScannerCycleActivityStatus {
Unchanged,
@@ -2347,22 +2375,29 @@ impl ScannerIOCycle for ECStore {
dirty_usage_status,
activity_status,
);
if status != ScannerCycleStatus::Complete {
if !publish_usage_snapshot(
&updates,
status,
DataUsageInfo {
last_update: Some(SystemTime::now()),
scanner_cycle: Some(want_cycle),
usage_snapshot_complete: true,
..Default::default()
},
)
.await?
{
return Ok(ScannerCycleResult::new(status, None));
}
let empty_usage = DataUsageInfo {
last_update: Some(SystemTime::now()),
scanner_cycle: Some(want_cycle),
usage_snapshot_complete: true,
..Default::default()
let dirty_usage_clear =
(status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone());
let remote_dirty_usage_acknowledgements = if status == ScannerCycleStatus::Complete {
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
} else {
Vec::new()
};
send_data_usage_update(&updates, empty_usage).await?;
let dirty_usage_clear = Some(dirty_usage_snapshot.buckets.as_ref().clone());
return Ok(
ScannerCycleResult::new(status, dirty_usage_clear).with_remote_dirty_usage_acknowledgements(
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before),
),
);
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
}
let total_results = expected_sources.len();
@@ -2576,10 +2611,8 @@ impl ScannerIOCycle for ECStore {
dirty_usage_status,
activity_status,
);
if cycle_status == ScannerCycleStatus::Complete
&& let Some((data_usage_info, _)) = completed_usage
{
send_data_usage_update(&updates, data_usage_info).await?;
if let Some((data_usage_info, _)) = completed_usage {
publish_usage_snapshot(&updates, cycle_status, data_usage_info).await?;
}
let dirty_usage_clear = should_clear_dirty_usage_snapshot(
result.is_ok(),
@@ -4597,6 +4630,44 @@ mod tests {
}
}
#[tokio::test]
async fn structurally_complete_superseded_cycles_publish_without_claiming_convergence() {
let (updates, mut receiver) = mpsc::channel(2);
assert!(
publish_usage_snapshot(&updates, ScannerCycleStatus::Complete, DataUsageInfo::default())
.await
.expect("complete snapshot publication should succeed")
);
assert!(
publish_usage_snapshot(&updates, ScannerCycleStatus::Superseded, DataUsageInfo::default())
.await
.expect("superseded snapshot publication should succeed")
);
assert!(
!publish_usage_snapshot(&updates, ScannerCycleStatus::Incomplete, DataUsageInfo::default())
.await
.expect("incomplete snapshot suppression should succeed")
);
assert_eq!(
receiver
.recv()
.await
.expect("complete update should be sent")
.usage_snapshot_converged,
Some(true)
);
assert_eq!(
receiver
.recv()
.await
.expect("superseded update should be sent")
.usage_snapshot_converged,
Some(false)
);
}
#[test]
fn scanner_cycle_fails_closed_for_namespace_disappearance() {
for activity_status in [
crates/scanner/src/storage_api.rs
+6 -5
View File
@@ -58,6 +58,7 @@ pub(crate) use rustfs_ecstore::api::config::storageclass::{
RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD,
};
pub(crate) use rustfs_ecstore::api::data_usage::{
invalidate_admin_data_usage_snapshot_cache as ecstore_invalidate_admin_data_usage_snapshot_cache,
invalidate_data_usage_snapshot_cache as ecstore_invalidate_data_usage_snapshot_cache,
replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info,
};
@@ -111,11 +112,11 @@ pub(crate) mod owner {
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config,
ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
scanner_replication_config_for_lifecycle_eval,
ecstore_invalidate_admin_data_usage_snapshot_cache, ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure,
ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle,
ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
rustfs/src/admin/handlers/account_info.rs
+3 -2
View File
@@ -18,7 +18,7 @@ use crate::admin::runtime_sources::{current_action_credentials, object_store_fro
use crate::admin::storage_api::bucket::versioning_sys::BucketVersioningSys;
use crate::admin::storage_api::contract::admin::StorageAdminApi;
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
use crate::admin::storage_api::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend_cached};
use crate::admin::storage_api::data_usage::{apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached};
use crate::admin::storage_api::metadata_sys;
use crate::auth::get_condition_values;
use crate::server::{ADMIN_PREFIX, RemoteAddr};
@@ -264,8 +264,9 @@ impl Operation for AccountInfoHandler {
// process-local absolute counters to cluster-wide bucket totals.
// This path never triggers a live full-version listing
// (rustfs/backlog#1306); freshness is owned by the scanner.
let mut data_usage_info = map_data_usage_result(load_data_usage_from_backend_cached(store.clone()).await)?;
let mut data_usage_info = map_data_usage_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
apply_bucket_usage_memory_overlay(&mut data_usage_info).await;
account_info.usage_snapshot_converged = data_usage_info.usage_snapshot_converged;
for bucket in buckets.iter() {
let (rd, wr) = is_allow(bucket.name.clone()).await;
rustfs/src/admin/storage_api.rs
+2 -2
View File
@@ -784,10 +784,10 @@ pub(crate) mod data_usage {
crate::storage::storage_api::ecstore_data_usage::load_data_usage_from_backend(store).await
}
pub(crate) async fn load_data_usage_from_backend_cached(
pub(crate) async fn load_admin_data_usage_from_backend_cached(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_data_usage_from_backend_cached(store).await
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_from_backend_cached(store).await
}
}
rustfs/src/app/admin_usecase.rs
+4 -2
View File
@@ -20,7 +20,9 @@ use super::storage_api::admin_usecase::capacity::{
};
use super::storage_api::admin_usecase::contract::StorageAdminApi;
use super::storage_api::admin_usecase::contract::bucket::{BucketOperations as _, BucketOptions};
use super::storage_api::admin_usecase::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend_cached};
use super::storage_api::admin_usecase::data_usage::{
apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached,
};
use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
use crate::app::runtime_sources::{
AppContext, current_app_context, current_endpoints_handle, current_object_store_handle_for_context,
@@ -265,7 +267,7 @@ impl DefaultAdminUsecase {
/// This path never triggers a live full-version listing
/// (rustfs/backlog#1306); freshness is owned by the scanner.
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
let mut info = Self::map_data_usage_load_result(load_data_usage_from_backend_cached(store.clone()).await)?;
let mut info = Self::map_data_usage_load_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
apply_bucket_usage_memory_overlay(&mut info).await;
let buckets = store
.list_bucket(&BucketOptions {
rustfs/src/app/object_usecase.rs
+43 -5
View File
@@ -3417,6 +3417,8 @@ static IO_QUEUE_CONGESTION_WARN_THROTTLE: IoQueueCongestionWarnThrottle = IoQueu
#[derive(Clone, Default)]
pub struct DefaultObjectUsecase {
context: Option<Arc<AppContext>>,
#[cfg(test)]
get_object_timeout_policy: Option<GetObjectTimeoutPolicy>,
}
impl DefaultObjectUsecase {
@@ -3426,12 +3428,17 @@ impl DefaultObjectUsecase {
#[cfg(test)]
pub fn without_context() -> Self {
Self { context: None }
Self {
context: None,
get_object_timeout_policy: None,
}
}
pub fn from_global() -> Self {
Self {
context: current_app_context(),
#[cfg(test)]
get_object_timeout_policy: None,
}
}
@@ -3440,7 +3447,22 @@ impl DefaultObjectUsecase {
/// so the use-case resolves that server's store; `None` falls back to the
/// ambient default.
pub fn with_context(context: Option<std::sync::Arc<crate::runtime_sources::AppContext>>) -> Self {
Self { context }
Self {
context,
#[cfg(test)]
get_object_timeout_policy: None,
}
}
#[cfg(test)]
fn with_context_and_get_object_timeout_policy(
context: Option<std::sync::Arc<crate::runtime_sources::AppContext>>,
get_object_timeout_policy: GetObjectTimeoutPolicy,
) -> Self {
Self {
context,
get_object_timeout_policy: Some(get_object_timeout_policy),
}
}
fn bucket_metadata_sys(&self) -> Option<Arc<RwLock<metadata_sys::BucketMetadataSys>>> {
@@ -3581,7 +3603,13 @@ impl DefaultObjectUsecase {
blob
}
fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> {
fn init_get_object_bootstrap(&self, bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> {
#[cfg(test)]
let timeout_config = self
.get_object_timeout_policy
.clone()
.unwrap_or_else(GetObjectTimeoutPolicy::cached_from_env);
#[cfg(not(test))]
let timeout_config = GetObjectTimeoutPolicy::cached_from_env();
let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.to_string());
let request_start = std::time::Instant::now();
@@ -5763,7 +5791,7 @@ impl DefaultObjectUsecase {
context.start_time.elapsed().as_secs_f64(),
);
}
let bootstrap = Self::init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?;
let bootstrap = self.init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?;
let timeout_config = bootstrap.timeout_config;
let wrapper = bootstrap.wrapper;
let request_start = bootstrap.request_start;
@@ -10728,7 +10756,17 @@ mod tests {
.key(object.to_string())
.build()
.expect("real cold-fill GET input must build");
let usecase = DefaultObjectUsecase::with_context(Some(context));
// The request is intentionally held behind the first producer while a
// 1.3 MiB replacement write changes its generation. Disable dynamic
// sizing for this test so runner I/O load cannot consume the five-second
// production minimum before the behavior under test is released.
let usecase = DefaultObjectUsecase::with_context_and_get_object_timeout_policy(
Some(context),
GetObjectTimeoutPolicy {
enable_dynamic_timeout: false,
..GetObjectTimeoutPolicy::default()
},
);
let request = tokio::spawn(async move { usecase.execute_get_object(build_request(input, Method::GET)).await });
tokio::time::timeout(Duration::from_secs(2), async {
while coordinator.global_waiter_count_for_test() != 1 {
rustfs/src/app/storage_api.rs
+7
View File
@@ -51,6 +51,13 @@ pub(crate) mod data_usage {
crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
}
pub(crate) async fn load_admin_data_usage_from_backend_cached(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_from_backend_cached(store).await
}
#[cfg(test)]
pub(crate) async fn load_data_usage_from_backend_cached(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
rustfs/src/storage/storage_api.rs
+3 -3
View File
@@ -415,15 +415,15 @@ pub(crate) mod ecstore_config {
pub(crate) mod ecstore_data_usage {
pub(crate) use rustfs_ecstore::api::data_usage::{
apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_data_usage_from_backend,
load_data_usage_from_backend_cached, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached,
load_data_usage_from_backend, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend,
};
// Test-only observables for the rustfs/backlog#1306 revert detector.
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::data_usage::{
compute_bucket_usage, live_bucket_usage_computations, store_data_usage_in_backend,
compute_bucket_usage, live_bucket_usage_computations, load_data_usage_from_backend_cached, store_data_usage_in_backend,
};
}