Compare commits

..

3 Commits

Author SHA1 Message Date
cxymds 46542baeb7 test(e2e): register the cluster scanner regression in distributed selection
The controlled follower test adds one unconditional case to the 4-node profile. Pin the resulting 111 test IDs so the distributed workflow reaches the test runner on both supported platforms.
2026-11-20 19:30:28 +08:00
cxymds 6cfacc42f9 fix(admin): verify scanner attempts and preserve inventory lookup budgets
Start identity validation after snapshot refresh. Record terminal scanner outcomes at the bounded wait boundary and trust the durable fence for successful completion. Assert that the real leader and followers agree on an active cycle and its completion.
2026-11-20 18:51:43 +08:00
cxymds 3cbfe81962 fix(admin): report inventory coverage and cluster scanner activity
Join the current bucket namespace with measured usage for matching bucket identities. Report the elected scanner coordinator and previous attempt to clients connected to any node, with explicit coverage when measurements are unavailable.
2026-11-20 18:14:51 +08:00
26 changed files with 1582 additions and 115 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
sha256-linux=983efba6508e684db29bbdb923485fcc1c4fb312fc47fe101f7f27620906978b
sha256-darwin=983efba6508e684db29bbdb923485fcc1c4fb312fc47fe101f7f27620906978b
+1
View File
@@ -36,6 +36,7 @@ mod replication_delete_marker_test;
mod replication_quota_test;
mod s3_basic_test;
mod s3_during_data_movement_test;
mod scanner_cluster_observation_test;
mod scanner_heal_topology_fixture;
mod scanner_heal_topology_test;
mod scanner_heal_topology_tier_test;
@@ -159,6 +159,7 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
let info: Value = serde_json::from_str(&info_body)?;
assert!(info.get("inventory").is_none(), "inventory requires explicit opt-in");
let servers = info["info"]["servers"]
.as_array()
.ok_or_else(|| format!("admin info omitted servers: {info}"))?;
@@ -200,6 +201,24 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
assert!(sample.scanner_node.is_none(), "default metrics must not expose the internal peer sample");
wait_until(
Duration::from_secs(30),
|| async {
let frame: Value = serde_json::from_str(
&node_admin_body(&dist, node_idx, "/rustfs/admin/v3/realtime?scope=cluster&types=1&n=1").await?,
)?;
assert_eq!(frame["final"], true);
let summary = &frame["cluster_scanner"];
assert_eq!(summary["nodes_total"], 4, "known cluster membership");
assert!(summary["sampled_at"].as_str().is_some());
assert!(matches!(summary["state"].as_str(), Some("scanning" | "idle" | "unknown")));
assert_eq!(summary["current"].is_object(), summary["state"] == "scanning");
Ok(summary["nodes_responded"] == 4)
},
"every request node reaches the full scanner peer set",
)
.await?;
}
let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
@@ -234,6 +253,38 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
"audit entry leaked the root secret key"
);
let pending = unique_bucket("inventory");
dist.create_bucket(&pending).await?;
let listed = dist.client(1)?.list_buckets().send().await?;
assert_eq!(listed.buckets().len(), 2, "expected the two test buckets");
wait_until(
Duration::from_secs(30),
|| async {
let body = node_admin_body(&dist, 1, "/rustfs/admin/v3/info?include=inventory").await?;
let response: Value = serde_json::from_str(&body)?;
let inventory: rustfs_madmin::ClusterInventory = serde_json::from_value(response["inventory"].clone())?;
if inventory.current_buckets != Some(2) {
return Ok(false);
}
match inventory.measured_buckets {
None | Some(0) => {
assert_eq!(inventory.measured_objects, None);
assert_eq!(inventory.measured_bytes, None);
}
Some(1 | 2) => {
assert!(inventory.measured_objects.is_some_and(|objects| objects <= 1));
}
Some(count) => panic!("measured {count} buckets across a two-bucket namespace"),
}
if inventory.measured_buckets != Some(2) {
assert_ne!(inventory.converged, Some(true));
}
Ok(true)
},
"inventory reflects the current namespace and reports scanner coverage independently",
)
.await?;
let result = verify_write_observations_during_peer_failure(&dist, &bucket).await;
collector.abort();
result
@@ -0,0 +1,111 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
use super::harness::{NODE_COUNT, TestResult, wait_until};
use super::scanner_peer_bucket_fixture::PeerBucketRun;
use crate::common::init_logging;
use futures::future::try_join_all;
use serde_json::Value;
use std::time::Duration;
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
const CLUSTER_SCANNER: &str = "/rustfs/admin/v3/realtime?n=1&types=1&scope=cluster";
const OBSERVATION_TIMEOUT: Duration = Duration::from_secs(35);
async fn scanner_frames(run: &PeerBucketRun) -> TestResult<Vec<Value>> {
try_join_all((0..NODE_COUNT).map(|node| run.admin_json(node, CLUSTER_SCANNER))).await
}
fn full_sample(frame: &Value) -> bool {
frame["final"] == true
&& frame["cluster_scanner"]["nodes_total"] == NODE_COUNT
&& frame["cluster_scanner"]["nodes_responded"] == NODE_COUNT
}
#[tokio::test]
async fn followers_report_the_coordinators_active_progress_and_committed_completion() -> TestResult {
init_logging();
let mut run = PeerBucketRun::start("cluster-scanner-observation").await?;
let baseline = run.quiet_baseline().await?;
let leader = run.leader().await?;
let pid = run.pid(leader)?;
let hot = run.buckets[2].clone();
run.barrier.arm("before_start_snapshot", pid, None)?;
run.barrier.start_session(&run.buckets)?;
// All four child processes and the harness use the same host clock.
let written_after = OffsetDateTime::now_utc();
run.put(leader, &hot, "wake-cluster-observation", 149).await?;
let start = run.barrier.wait_stage("before_start_snapshot", pid, None, true).await?;
let epoch = start["detail"]["epoch"].as_u64().ok_or("start gate omitted epoch")?;
let cycle = start["detail"]["cycle"].as_u64().ok_or("start gate omitted cycle")?;
if baseline.scanner_epoch != Some(epoch) || baseline.scanner_cycle.is_none_or(|previous| previous >= cycle) {
return Err("controlled cycle did not follow the quiet baseline under the same coordinator".into());
}
run.barrier.arm("root_durable", pid, Some((epoch, cycle)))?;
run.barrier.release("before_start_snapshot", pid)?;
run.barrier
.wait_stage("root_durable", pid, Some((epoch, cycle)), true)
.await?;
// The leader is held after the real walk and before committing its cycle completion.
// Each ingress node maintains its own short-lived cluster sample cache.
let expected_leader = &run.dist.cluster.nodes[leader].url;
let active = wait_until(
OBSERVATION_TIMEOUT,
|| async {
let frames = scanner_frames(&run).await?;
let started = frames[leader]["cluster_scanner"]["current"]["started_at"].as_str();
let consistent = started.is_some()
&& frames.iter().all(|frame| {
let summary = &frame["cluster_scanner"];
let current = &summary["current"];
full_sample(frame)
&& summary["state"] == "scanning"
&& current["cycle"] == cycle
&& current["leader"].as_str() == Some(expected_leader.as_str())
&& current["started_at"].as_str() == started
&& current["objects_scanned"].as_u64().is_some_and(|objects| objects > 0)
});
if !consistent {
tracing::info!(
?frames,
cycle,
expected_leader,
"waiting for active scanner progress on every ingress node"
);
}
Ok(consistent)
},
"leader and all three followers report the identified active cycle with nonzero work",
)
.await;
run.barrier.disarm()?;
run.barrier.release("root_durable", pid)?;
active?;
wait_until(
OBSERVATION_TIMEOUT,
|| async {
let frames = scanner_frames(&run).await?;
let settled = frames.iter().all(|frame| {
let summary = &frame["cluster_scanner"];
full_sample(frame)
&& summary["state"] == "idle"
&& summary.get("current").is_none()
&& summary["last_attempt"]["cycle"] == cycle
&& summary["last_attempt"]["result"] == "success"
&& summary["last_success_at"].as_str().is_some_and(|value| {
OffsetDateTime::parse(value, &Rfc3339).is_ok_and(|completed| completed >= written_after)
})
});
if !settled {
tracing::info!(?frames, cycle, "waiting for the controlled scanner cycle to finish");
}
Ok(settled)
},
"every ingress node reports the completed cycle and an idle cluster scanner",
)
.await
}
+2 -2
View File
@@ -28,7 +28,7 @@ pub mod diagnostic {
}
pub mod admin {
pub use crate::diagnostics::admin_server_info::{get_local_server_property, get_server_info};
pub use crate::diagnostics::admin_server_info::{get_local_server_property, get_server_info, get_server_info_with_inventory};
}
pub mod bitrot {
@@ -460,7 +460,7 @@ pub mod data_usage {
record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
store_data_usage_in_backend,
store_data_usage_in_backend, verified_account_bucket_usage,
};
pub use crate::data_usage::{get_bucket_usage_memory, lookup_degraded_bucket_usage_baseline};
}
@@ -0,0 +1,94 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::{ADMIN_USAGE_INCARNATION_CONCURRENCY, ADMIN_USAGE_INCARNATION_TIMEOUT, BucketUsageInfo, ECStore};
use futures::StreamExt;
use std::{collections::HashMap, sync::Arc};
use tokio::time::{Instant, timeout_at};
/// Read only the caller-selected bucket identities. One deadline bounds the
/// entire batch, including concurrent reads waiting for metadata locks.
pub(crate) async fn current_admin_bucket_incarnations(
store: Arc<ECStore>,
buckets: Vec<String>,
deadline: Instant,
) -> HashMap<String, String> {
let mut current = HashMap::new();
let mut reads = futures::stream::iter(buckets)
.map(move |name| {
let store = store.clone();
async move {
let id = store.bucket_incarnation_id_from_disk(&name).await;
(name, id)
}
})
.buffer_unordered(ADMIN_USAGE_INCARNATION_CONCURRENCY);
while Instant::now() < deadline {
let Ok(Some((name, id))) = timeout_at(deadline, reads.next()).await else {
break;
};
if let Ok(id) = id
&& !id.is_nil()
{
current.insert(name, id.to_string());
}
}
current
}
/// Verify both persisted scanner and single-node overlay values at the time of
/// an account-info read. Only names already selected by the IAM policy enter
/// metadata I/O; missing legacy identities and metadata timeouts remain unknown.
pub async fn verified_account_bucket_usage(
store: Arc<ECStore>,
mut measured: HashMap<String, BucketUsageInfo>,
allowed_names: Vec<String>,
) -> HashMap<String, BucketUsageInfo> {
let names = allowed_names
.into_iter()
.filter(|name| measured.get(name).is_some_and(|value| value.bucket_incarnation.is_some()))
.collect();
let current = current_admin_bucket_incarnations(store, names, Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT).await;
measured.retain(|name, value| {
current
.get(name)
.is_some_and(|id| value.bucket_incarnation.as_ref() == Some(id))
});
measured
}
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
#[tokio::test]
#[serial_test::serial]
async fn account_inventory_preserves_only_permitted_current_incarnations_including_measured_zero() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let mut usage = HashMap::new();
for (name, objects) in [("permitted", 0), ("private", 300), ("recreated", 900), ("legacy", 90)] {
store.make_bucket(name, &MakeBucketOptions::default()).await.unwrap();
let current = store.bucket_incarnation_id_from_disk(name).await.unwrap();
let incarnation = match name {
"recreated" => Some("11111111-1111-4111-8111-111111111111".to_string()),
"legacy" => None,
_ => Some(current.to_string()),
};
usage.insert(
name.to_owned(),
BucketUsageInfo {
objects_count: objects,
bucket_incarnation: incarnation,
..Default::default()
},
);
}
let allowed = ["permitted", "recreated", "legacy"].map(String::from).to_vec();
let values = verified_account_bucket_usage(store, usage, allowed).await;
assert_eq!(values.len(), 1);
assert_eq!(values["permitted"].objects_count, 0);
}
}
+6 -36
View File
@@ -14,6 +14,9 @@
// #730: scanner/data-usage state is partially migrated and still owns staged cache helpers.
mod admin_bucket_identity;
pub(crate) use admin_bucket_identity::current_admin_bucket_incarnations;
pub use admin_bucket_identity::verified_account_bucket_usage;
pub mod local_snapshot;
use crate::storage_api_contracts::{
@@ -57,7 +60,7 @@ const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json";
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
const DATA_USAGE_CACHE_TTL_SECS: u64 = 30;
const ADMIN_USAGE_INCARNATION_CONCURRENCY: usize = 4;
const ADMIN_USAGE_INCARNATION_TIMEOUT: Duration = Duration::from_secs(2);
pub(crate) const ADMIN_USAGE_INCARNATION_TIMEOUT: Duration = Duration::from_secs(2);
// A scanner save invalidates only the saving process's snapshot cache; every
// other node sees it only after reloading. A fail-closed usage lookup for a
// bucket the cached snapshot does not mention at all may reload a copy this
@@ -1464,42 +1467,9 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
let (selected, selected_is_current_format) =
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
let mut response = normalize_loaded_data_usage(selected, selected_is_current_format).await.0;
use futures::StreamExt as _;
let buckets = response.usage_snapshot_bucket_observations.keys().cloned().collect();
let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT;
let mut incarnations = HashMap::new();
{
// Display-only identity reads must not serialize every bucket behind a
// slow metadata lock while holding the shared admin refresh permit.
// The common deadline bounds their total added latency; dropping this
// stream cancels all unfinished reads without spawning detached work.
// Own the names and store across awaits so this future remains Send
// when composed into spawned metrics collectors.
let buckets = response
.usage_snapshot_bucket_observations
.keys()
.cloned()
.collect::<Vec<_>>();
let mut identities = futures::stream::iter(buckets)
.map(move |bucket| {
let store = store.clone();
async move {
let identity = store.bucket_incarnation_id_from_disk(&bucket).await;
(bucket, identity)
}
})
.buffer_unordered(ADMIN_USAGE_INCARNATION_CONCURRENCY);
while tokio::time::Instant::now() < deadline {
let Ok(Some((bucket, identity))) = tokio::time::timeout_at(deadline, identities.next()).await else {
break;
};
// Unavailable, timed-out or recreated buckets retain canonical values.
if let Ok(incarnation) = identity
&& !incarnation.is_nil()
{
incarnations.insert(bucket, incarnation.to_string());
}
}
}
let incarnations = current_admin_bucket_incarnations(store, buckets, deadline).await;
response.project_bucket_observations_for_admin(&incarnations);
Ok(response)
}
@@ -50,6 +50,8 @@ const EVENT_SERVER_INFO: &str = "server_info";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_DIAGNOSTICS: &str = "diagnostics";
mod inventory;
use shadow_rs::shadow;
shadow!(build);
@@ -276,6 +278,14 @@ fn collect_runtime_server_stats() -> (MemStats, u64, u64) {
}
pub async fn get_server_info(get_pools: bool) -> InfoMessage {
get_server_info_with_inventory(get_pools, false).await.0
}
pub async fn get_server_info_with_inventory(
get_pools: bool,
include_inventory: bool,
) -> (InfoMessage, Option<rustfs_madmin::ClusterInventory>) {
let mut inventory = include_inventory.then(rustfs_madmin::ClusterInventory::default);
let started = tracing::enabled!(tracing::Level::TRACE).then(Instant::now);
let local = get_local_server_property().await;
@@ -301,25 +311,24 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
if let Some(store) = runtime_sources::object_store_handle() {
mode = ITEM_ONLINE;
apply_data_usage_result(
load_server_info_usage_snapshot(store.clone()).await,
&mut buckets,
&mut objects,
&mut versions,
&mut delete_markers,
&mut usage,
);
let namespace_options = BucketOptions {
cached: true,
no_metadata: true,
..Default::default()
};
let load_snapshot = load_server_info_usage_snapshot(store.clone());
let (snapshot, namespace) = if include_inventory {
let (snapshot, namespace, measured) =
inventory::collect(&store, load_snapshot, store.list_bucket_for_scanner(&namespace_options)).await;
inventory = Some(measured);
(snapshot, Some(namespace))
} else {
(load_snapshot.await, None)
};
apply_data_usage_result(snapshot, &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage);
if buckets.error.is_some() {
apply_bucket_namespace_count(
store
.list_bucket_for_scanner(&BucketOptions {
cached: true,
no_metadata: true,
..Default::default()
})
.await,
&mut buckets,
);
let listing = inventory::namespace_or_fallback(namespace, store.list_bucket_for_scanner(&namespace_options)).await;
apply_bucket_namespace_count(listing, &mut buckets);
}
let backend_info = StorageAdminApi::backend_info(store.as_ref()).await;
@@ -375,7 +384,7 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
);
}
InfoMessage {
let info = InfoMessage {
mode: Some(mode.to_string()),
domain: None,
region: None,
@@ -390,7 +399,8 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
services: Some(services),
servers: Some(servers),
pools: Some(pools),
}
};
(info, inventory)
}
fn is_online_drive_state(state: &str) -> bool {
@@ -0,0 +1,345 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::ScannerBucketListing;
use rustfs_data_usage::DataUsageInfo;
use rustfs_madmin::ClusterInventory;
use std::collections::HashSet;
use crate::data_usage::{ADMIN_USAGE_INCARNATION_TIMEOUT, current_admin_bucket_incarnations};
use crate::error::Result;
use crate::store::ECStore;
use std::collections::HashMap;
use std::sync::Arc;
pub(super) async fn collect(
store: &Arc<ECStore>,
load_snapshot: impl Future<Output = Result<DataUsageInfo>>,
list_namespace: impl Future<Output = Result<ScannerBucketListing>>,
) -> (Result<DataUsageInfo>, Result<ScannerBucketListing>, ClusterInventory) {
let snapshot = load_snapshot.await;
let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT;
let namespace = bounded_namespace(deadline, list_namespace).await;
let mut current_ids = HashMap::new();
if let (Ok(namespace), Ok(snapshot)) = (&namespace, &snapshot)
&& namespace.topology_complete
{
let names = namespace
.buckets
.iter()
.filter(|bucket| {
!bucket.name.starts_with('.')
&& snapshot
.buckets_usage
.get(&bucket.name)
.is_some_and(|usage| usage.bucket_incarnation.is_some())
})
.map(|bucket| bucket.name.clone())
.collect();
current_ids = current_admin_bucket_incarnations(store.clone(), names, deadline).await;
}
let measured = summarize(namespace.as_ref().ok(), snapshot.as_ref().ok(), &current_ids);
(snapshot, namespace, measured)
}
pub(super) async fn bounded_namespace(
deadline: tokio::time::Instant,
read: impl Future<Output = crate::error::Result<ScannerBucketListing>>,
) -> crate::error::Result<ScannerBucketListing> {
tokio::time::timeout_at(deadline, read)
.await
.map_err(|_| crate::error::Error::other("inventory namespace read timed out"))?
}
pub(super) async fn namespace_or_fallback(
previous: Option<crate::error::Result<ScannerBucketListing>>,
read: impl Future<Output = crate::error::Result<ScannerBucketListing>>,
) -> crate::error::Result<ScannerBucketListing> {
// A failed or timed-out optional listing is already an observation.
// Retrying it without the deadline would stall the independent health response.
match previous {
Some(result) => result,
None => read.await,
}
}
pub(super) fn summarize(
namespace: Option<&ScannerBucketListing>,
snapshot: Option<&DataUsageInfo>,
current_ids: &HashMap<String, String>,
) -> ClusterInventory {
// An admin projection is a validated display-only view produced by the loader.
let snapshot = snapshot.filter(|info| {
info.usage_snapshot_admin_projection || info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()
});
let mut inventory = ClusterInventory {
snapshot_at: snapshot
.and_then(|info| info.last_update)
.and_then(|time| time.try_into().ok()),
..Default::default()
};
let Some(namespace) = namespace.filter(|listing| listing.topology_complete) else {
return inventory;
};
let names: HashSet<_> = namespace
.buckets
.iter()
.map(|bucket| bucket.name.as_str())
.filter(|name| !name.starts_with('.'))
.collect();
inventory.current_buckets = u64::try_from(names.len()).ok();
let Some(snapshot) = snapshot else {
return inventory;
};
let mut measured = 0_u64;
let mut objects = Some(0_u64);
let mut bytes = Some(0_u64);
for &name in &names {
let Some(usage) = snapshot.buckets_usage.get(name) else { continue };
if usage
.bucket_incarnation
.as_ref()
.is_none_or(|id| current_ids.get(name) != Some(id))
{
continue;
}
measured += 1;
objects = objects.and_then(|total| total.checked_add(usage.objects_count));
bytes = bytes.and_then(|total| total.checked_add(usage.size));
}
inventory.measured_buckets = Some(measured);
if measured != 0 || names.is_empty() {
inventory.measured_objects = objects;
inventory.measured_bytes = bytes;
}
inventory.converged = if objects.is_none() || bytes.is_none() {
None
} else if inventory.current_buckets != Some(measured)
|| snapshot.buckets_usage.len() != names.len()
|| snapshot.usage_snapshot_pending_bucket_count > 0
|| !snapshot.usage_snapshot_pending_buckets.is_empty()
|| !snapshot.is_complete_bucket_usage_snapshot()
{
Some(false)
} else {
snapshot.usage_snapshot_converged
};
inventory
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_data_usage::BucketUsageInfo;
use std::time::{Duration, SystemTime};
fn namespace(names: &[&str]) -> ScannerBucketListing {
ScannerBucketListing {
buckets: names
.iter()
.map(|name| crate::storage_api_contracts::bucket::BucketInfo {
name: name.to_string(),
..Default::default()
})
.collect(),
set_buckets: vec![],
topology_complete: true,
}
}
fn snapshot(names: &[(&str, u64)]) -> DataUsageInfo {
DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(true),
buckets_count: u64::try_from(names.len()).unwrap(),
buckets_usage: names
.iter()
.map(|&(name, count)| {
(
name.to_owned(),
BucketUsageInfo {
objects_count: count,
size: count,
bucket_incarnation: Some("11111111-1111-4111-8111-111111111111".into()),
..Default::default()
},
)
})
.collect(),
..Default::default()
}
}
fn view(namespace: Option<&ScannerBucketListing>, snapshot: Option<&DataUsageInfo>) -> ClusterInventory {
let ids = namespace
.into_iter()
.flat_map(|ns| &ns.buckets)
.map(|bucket| (bucket.name.clone(), "11111111-1111-4111-8111-111111111111".to_string()))
.collect();
summarize(namespace, snapshot, &ids)
}
#[tokio::test(start_paused = true)]
async fn inventory_namespace_timeout_is_reused_by_the_legacy_fallback() {
use tokio::time::{Duration, Instant};
let deadline = Instant::now() + Duration::from_secs(2);
let result = bounded_namespace(deadline, std::future::pending()).await;
assert!(result.is_err());
assert_eq!(Instant::now(), deadline);
let result = namespace_or_fallback(Some(result), async { panic!("must not retry a failed namespace read") }).await;
assert!(result.is_err());
let inventory = view(result.as_ref().ok(), None);
assert_eq!(inventory.current_buckets, None);
assert_eq!(inventory.measured_objects, None);
}
#[tokio::test]
#[serial_test::serial]
async fn inventory_excludes_stale_bucket_incarnations_after_a_slow_cold_snapshot_refresh() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::data_usage::{DATA_USAGE_OBJ_NAME_PATH, invalidate_data_usage_snapshot_cache};
use crate::storage_api_contracts::bucket::{BucketOperations as _, BucketOptions, MakeBucketOptions};
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
for bucket in ["unchanged", "recreated", "new"] {
store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
}
let mut scanned = snapshot(&[("unchanged", 4), ("recreated", 99), ("deleted", 6)]);
let current = store.bucket_incarnation_id_from_disk("unchanged").await.unwrap();
scanned.buckets_usage.get_mut("unchanged").unwrap().bucket_incarnation = Some(current.to_string());
scanned.usage_snapshot_converged = None;
scanned.calculate_totals();
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&scanned).unwrap())
.await
.unwrap();
invalidate_data_usage_snapshot_cache().await;
let options = BucketOptions {
no_metadata: true,
..Default::default()
};
let cold_refresh = async {
// Model a refresh that outlasts the inventory budget; keep disk I/O
// on the real clock so its independent timeouts behave normally.
tokio::time::pause();
tokio::time::advance(ADMIN_USAGE_INCARNATION_TIMEOUT + Duration::from_secs(1)).await;
tokio::time::resume();
super::super::load_server_info_usage_snapshot(store.clone()).await
};
let (loaded, listing, result) = collect(&store, cold_refresh, store.list_bucket_for_scanner(&options)).await;
assert_eq!(loaded.unwrap().usage_snapshot_converged, Some(true));
assert!(listing.unwrap().topology_complete);
assert_eq!(result.current_buckets, Some(3));
assert_eq!(result.measured_buckets, Some(1));
assert_eq!(result.measured_objects, Some(4));
assert_eq!(result.converged, Some(false));
// A complete canonical snapshot from before the convergence field was
// added inherits the admin loader's existing convergence interpretation.
let mut legacy = snapshot(&[("unchanged", 4), ("recreated", 0), ("new", 0)]);
legacy.usage_snapshot_converged = None;
for (name, usage) in &mut legacy.buckets_usage {
usage.bucket_incarnation = Some(store.bucket_incarnation_id_from_disk(name).await.unwrap().to_string());
}
legacy.calculate_totals();
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&legacy).unwrap())
.await
.unwrap();
invalidate_data_usage_snapshot_cache().await;
let (_, _, result) = collect(
&store,
super::super::load_server_info_usage_snapshot(store.clone()),
store.list_bucket_for_scanner(&options),
)
.await;
assert_eq!(result.measured_objects, Some(4));
assert_eq!(result.converged, Some(true));
invalidate_data_usage_snapshot_cache().await;
}
#[test]
fn inventory_reports_only_the_intersection_of_current_and_measured_buckets() {
let scope = namespace(&["existing", "new", ".rustfs.sys"]);
let before = snapshot(&[("existing", 300_000), ("deleted", 7)]);
let result = view(Some(&scope), Some(&before));
assert_eq!(result.current_buckets, Some(2));
assert_eq!(result.measured_buckets, Some(1));
assert_eq!(result.measured_objects, Some(300_000));
assert_eq!(result.measured_bytes, Some(300_000));
assert_eq!(result.converged, Some(false));
assert_eq!(result.snapshot_at.unwrap().as_second(), 1);
}
#[test]
fn inventory_distinguishes_unavailable_and_confirmed_empty_measurements() {
let empty = namespace(&[]);
let missing = view(Some(&empty), Some(&DataUsageInfo::default()));
assert_eq!(missing.current_buckets, Some(0));
assert_eq!(missing.measured_objects, None);
assert_eq!(missing.measured_buckets, None);
let available = view(Some(&empty), Some(&snapshot(&[])));
assert_eq!(available.measured_objects, Some(0));
assert_eq!(available.converged, Some(true));
let scope = namespace(&["new"]);
let no_overlap = view(Some(&scope), Some(&snapshot(&[("deleted", 7)])));
assert_eq!(no_overlap.measured_buckets, Some(0));
assert_eq!(no_overlap.measured_objects, None);
let known_empty = view(Some(&scope), Some(&snapshot(&[("new", 0)])));
assert_eq!(known_empty.measured_objects, Some(0));
assert_eq!(known_empty.measured_buckets, Some(1));
}
#[test]
fn inventory_keeps_the_recorded_snapshot_convergence_and_age() {
let scope = namespace(&["a"]);
let mut scanned = snapshot(&[("a", 12)]);
scanned.usage_snapshot_converged = None;
assert_eq!(view(Some(&scope), Some(&scanned)).converged, None);
scanned.usage_snapshot_converged = Some(true);
scanned.usage_snapshot_pending_buckets = vec!["a".into()];
assert_eq!(view(Some(&scope), Some(&scanned)).converged, Some(false));
scanned.usage_snapshot_pending_buckets.clear();
scanned.usage_snapshot_converged = Some(false);
scanned.usage_snapshot_complete = false;
scanned.usage_snapshot_admin_projection = true;
let result = view(Some(&scope), Some(&scanned));
assert_eq!(result.measured_objects, Some(12));
assert_eq!(result.converged, Some(false));
assert_eq!(result.snapshot_at.unwrap().as_second(), 1);
}
#[test]
fn inventory_requires_complete_namespace_and_checks_each_sum() {
let mut scope = namespace(&["a", "b"]);
let mut scanned = snapshot(&[("a", u64::MAX), ("b", 1)]);
scanned.buckets_usage.get_mut("a").unwrap().size = 1;
scope.topology_complete = false;
let missing = view(Some(&scope), Some(&scanned));
assert_eq!(missing.current_buckets, None);
assert_eq!(missing.measured_buckets, None);
assert_eq!(missing.converged, None);
scope.topology_complete = true;
let overflow = view(Some(&scope), Some(&scanned));
assert_eq!(overflow.measured_buckets, Some(2));
assert_eq!(overflow.measured_objects, None);
assert_eq!(overflow.measured_bytes, Some(2));
assert_eq!(overflow.converged, None);
}
#[test]
fn inventory_checks_even_cached_canonical_incarnations_on_main() {
let scope = namespace(&["recreated", "unreadable", "legacy"]);
let mut scanned = snapshot(&[("recreated", 5), ("unreadable", 6), ("legacy", 7)]);
scanned.buckets_usage.get_mut("legacy").unwrap().bucket_incarnation = None;
let ids = HashMap::from([
("recreated".to_string(), "22222222-2222-4222-8222-222222222222".to_string()),
("legacy".to_string(), "33333333-3333-4333-8333-333333333333".to_string()),
]);
let result = summarize(Some(&scope), Some(&scanned), &ids);
assert_eq!(result.current_buckets, Some(3));
assert_eq!(result.measured_buckets, Some(0));
assert_eq!(result.measured_objects, None);
}
}
@@ -62,6 +62,10 @@ impl MetricType {
pub const RPC: MetricType = MetricType(1 << 8);
pub const HTTP: MetricType = MetricType(1 << 9);
/// Internal GetMetrics selector for a compact, generation-bound node sample.
/// Excluded from ALL so existing HTTP and rolling-upgrade peers keep their wire shape.
pub const SCANNER_SUMMARY: MetricType = MetricType(1 << 10);
// MetricsAll must be last.
pub const ALL: MetricType = MetricType((1 << 10) - 1);
@@ -405,6 +409,27 @@ impl MetricType {
pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) -> RealtimeMetrics {
debug!("collect_local_metrics");
let mut real_time_metrics = RealtimeMetrics::default();
if types.0 == MetricType::SCANNER_SUMMARY.0 {
let sample = global_metrics().scanner_node_observation().await;
real_time_metrics.scanner_node = Some(rustfs_madmin::metrics::ScannerNodeSample {
active: sample.active,
leader: sample.leader,
cycle: sample.cycle,
epoch: sample.epoch,
started_at: sample.started_at,
objects_scanned: sample.objects_scanned,
last_attempt: sample
.last_attempt
.map(|attempt| rustfs_madmin::metrics::ScannerAttemptSample {
epoch: attempt.epoch,
cycle: attempt.cycle,
result: attempt.result.to_string(),
partial_reason: attempt.partial_reason.map(str::to_string),
duration_seconds: attempt.duration.as_secs_f64(),
}),
});
return real_time_metrics;
}
if types.0 == MetricType::NONE.0 {
info!("types is None, return");
return real_time_metrics;
+11
View File
@@ -322,6 +322,17 @@ pub struct Services {
pub notifications: Option<Vec<HashMap<String, Vec<TargetIDStatus>>>>,
}
/// Namespace coverage and scanner measurements for the optional RustFS admin inventory.
#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ClusterInventory {
pub current_buckets: Option<u64>,
pub measured_buckets: Option<u64>,
pub measured_objects: Option<u64>,
pub measured_bytes: Option<u64>,
pub snapshot_at: Option<jiff::Timestamp>,
pub converged: Option<bool>,
}
#[derive(Serialize, Deserialize, Debug, Default)]
pub struct Buckets {
pub count: u64,
+53
View File
@@ -1506,6 +1506,27 @@ pub struct KeyRotationInfo {
pub objects_failed: i64,
}
/// A single configured node's scanner observation. The epoch fences concurrent coordinator samples.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ScannerNodeSample {
pub active: bool,
pub leader: bool,
pub cycle: u64,
pub epoch: Option<u64>,
pub started_at: Option<Timestamp>,
pub objects_scanned: u64,
pub last_attempt: Option<ScannerAttemptSample>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ScannerAttemptSample {
pub epoch: u64,
pub cycle: u64,
pub result: String,
pub partial_reason: Option<String>,
pub duration_seconds: f64,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct RealtimeMetrics {
#[serde(rename = "errors")]
@@ -1520,6 +1541,9 @@ pub struct RealtimeMetrics {
pub by_disk: HashMap<String, DiskMetric>,
#[serde(rename = "final")]
pub finally: bool,
/// Internal GetMetrics reply to a scanner summary probe. Absent on older peers.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scanner_node: Option<ScannerNodeSample>,
}
impl RealtimeMetrics {
@@ -1583,6 +1607,35 @@ mod tests {
Timestamp::constant(1_700_000_000, 123_456_000)
}
#[test]
fn cluster_scanner_node_sample_is_additive_on_the_peer_metrics_wire() {
let ordinary = serde_json::to_value(RealtimeMetrics::default()).expect("ordinary response");
assert!(ordinary.get("scanner_node").is_none());
let old = rmp_serde::to_vec_named(&ordinary).expect("previous map reply");
assert!(
rmp_serde::from_slice::<RealtimeMetrics>(&old)
.expect("old peers remain readable")
.scanner_node
.is_none()
);
let reply = RealtimeMetrics {
scanner_node: Some(ScannerNodeSample {
active: true,
leader: true,
epoch: Some(8),
cycle: 40,
..Default::default()
}),
..Default::default()
};
let encoded = rmp_serde::to_vec_named(&reply).expect("scoped peer sample");
let sample = rmp_serde::from_slice::<RealtimeMetrics>(&encoded)
.expect("round trip")
.scanner_node
.expect("new capability field");
assert_eq!((sample.active, sample.leader, sample.epoch, sample.cycle), (true, true, Some(8), 40));
}
#[test]
fn http_metrics_merge_preserves_outcomes_and_missing_node_support() {
#[derive(Serialize, Deserialize, Default)]
+111
View File
@@ -983,6 +983,7 @@ pub struct Metrics {
actions_latency: Vec<LockedLastMinuteLatency>,
current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
last_cluster_attempt: Mutex<Option<ScannerAttemptRecord>>,
// Lock order: cycle_info, usage_publication, then the observation's status. No awaits under either mutex.
usage_publication: Mutex<Arc<ScannerUsagePublicationObservation>>,
current_scan_mode: AtomicU8,
@@ -1214,6 +1215,27 @@ impl ScannerUsageSaveResult {
}
}
/// Completed coordinator attempt, retained with the generation of its originating worker.
#[derive(Clone, Copy, Debug)]
pub struct ScannerAttemptRecord {
pub epoch: u64,
pub cycle: u64,
pub result: &'static str,
pub partial_reason: Option<&'static str>,
pub duration: Duration,
}
/// Compact process-local state for a cluster status probe.
pub struct ScannerNodeObservation {
pub active: bool,
pub leader: bool,
pub epoch: Option<u64>,
pub cycle: u64,
pub started_at: Option<Timestamp>,
pub objects_scanned: u64,
pub last_attempt: Option<ScannerAttemptRecord>,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct CurrentCycle {
pub current: u64,
@@ -1713,6 +1735,11 @@ pub struct ScannerRuntimeDetailsReport {
}
impl CurrentCycle {
/// Last successful completion recorded in the persisted cycle history.
pub fn last_success_at(&self) -> Option<Timestamp> {
self.cycle_completed.last().copied()
}
pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
*self = rmp_serde::from_slice(buf)?;
Ok(())
@@ -2140,6 +2167,7 @@ impl Metrics {
.collect(),
current_paths: Arc::new(RwLock::new(HashMap::new())),
cycle_info: Arc::new(RwLock::new(None)),
last_cluster_attempt: Mutex::new(None),
usage_publication: Mutex::new(Arc::new(ScannerUsagePublicationObservation::default())),
current_scan_mode: AtomicU8::new(HealScanMode::Unknown as u8),
current_scan_cycle_work_active: AtomicBool::new(false),
@@ -3106,6 +3134,45 @@ impl Metrics {
*current_cycle = Some(cycle);
}
/// Keep delayed results from an older coordinator from replacing a newer attempt.
pub fn record_cluster_attempt(&self, attempt: ScannerAttemptRecord) {
let mut latest = self.last_cluster_attempt.lock().unwrap_or_else(|err| err.into_inner());
if latest.is_none_or(|old| (old.epoch, old.cycle) <= (attempt.epoch, attempt.cycle)) {
*latest = Some(attempt);
}
}
/// Sample only the fields required by the admin cluster status API. The cycle read
/// lock also protects installation of the publication and per-cycle work baseline.
pub async fn scanner_node_observation(&self) -> ScannerNodeObservation {
let cycle = self.cycle_info.read().await;
let active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
let generation = self
.usage_publication
.lock()
.unwrap_or_else(|err| err.into_inner())
.snapshot();
let cycle_id = cycle.as_ref().map_or(0, |c| c.current);
let epoch = generation
.scanner_epoch
.filter(|_| !active || generation.cycle == Some(cycle_id));
let last_attempt = *self.last_cluster_attempt.lock().unwrap_or_else(|err| err.into_inner());
ScannerNodeObservation {
active,
leader: self.scanner_leader_lock_held.load(Ordering::Acquire),
cycle: cycle_id,
epoch,
started_at: cycle.as_ref().filter(|_| active).map(|c| c.started),
objects_scanned: if active {
self.scan_cycle_work_since(self.current_scan_cycle_work_start())
.objects_scanned
} else {
0
},
last_attempt,
}
}
/// Read the current cycle record.
pub async fn get_cycle(&self) -> Option<CurrentCycle> {
self.cycle_info.read().await.clone()
@@ -5693,6 +5760,50 @@ mod tests {
assert_eq!(report.last_cycle_usage_saves, 2);
}
#[tokio::test]
async fn cluster_scanner_keeps_last_attempt_when_leadership_moves() {
let metrics = Metrics::new();
let previous = ScannerAttemptRecord {
epoch: 7,
cycle: 40,
result: "partial",
partial_reason: Some("runtime"),
duration: Duration::from_secs(1801),
};
metrics.record_cluster_attempt(previous);
metrics.record_cluster_attempt(ScannerAttemptRecord {
epoch: 6,
cycle: 100,
..previous
});
metrics.set_cycle(None).await;
let follower = metrics.scanner_node_observation().await;
assert!(!follower.active);
let history = follower.last_attempt.expect("previous leader's terminal record");
assert_eq!(
(history.epoch, history.cycle, history.result, history.partial_reason),
(7, 40, "partial", Some("runtime"))
);
let cycle = CurrentCycle {
current: 41,
..Default::default()
};
let remote = Arc::new(RemoteScanProgress::default());
metrics
.start_scan_cycle_work_with_publication(
cycle,
remote.clone(),
Arc::new(ScannerUsagePublicationObservation::new(8, 41)),
)
.await;
metrics.record_scanner_leader_liveness("acquired", true, "").await;
remote.record(814, 1);
let active = metrics.scanner_node_observation().await;
assert!(active.active && active.leader);
assert_eq!((active.epoch, active.cycle, active.objects_scanned), (Some(8), 41, 814));
assert_eq!(active.last_attempt.expect("prior cycle").cycle, 40);
}
#[tokio::test]
async fn scan_cycle_activity_and_cycle_state_publish_together() {
let metrics = Metrics::new();
+51 -2
View File
@@ -1883,6 +1883,48 @@ where
.await
}
/// Observe the outer wait so that an expired drain records the attempt even when
/// the worker is dropped before its first poll. Only one terminal record is emitted.
async fn await_observed_scanner_cycle(
metrics: &rustfs_scanner_metrics::metrics::Metrics,
epoch: u64,
cycle: u64,
ctx: &CancellationToken,
budget: &ScannerCycleBudget,
worker: impl Future<Output = ScannerCycleOutcome>,
lock_lost: impl Future<Output = ()>,
) -> ScannerCycleWaitOutcome<ScannerCycleOutcome> {
let started = tokio::time::Instant::now();
let mut completed = None;
let worker = async {
let outcome = worker.await;
completed = Some(outcome);
outcome
};
let wait = await_scanner_cycle_with_budget_fence(ctx, budget, worker, lock_lost).await;
let partial = || ("partial", budget.reason().map(|reason| scan_cycle_partial_reason(Some(reason)).as_str()));
let (result, partial_reason) = match (&wait, completed.as_ref()) {
(ScannerCycleWaitOutcome::Completed(outcome), _) | (_, Some(outcome)) => match outcome {
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => ("success", None),
ScannerCycleOutcome::Partial => partial(),
ScannerCycleOutcome::Superseded => ("superseded", None),
ScannerCycleOutcome::Deferred(_) => ("deferred", None),
ScannerCycleOutcome::Failed | ScannerCycleOutcome::StatePersistenceFailed => ("error", None),
},
(ScannerCycleWaitOutcome::LockLost, None) => ("lock_lost", None),
(ScannerCycleWaitOutcome::Cancelled, None) => ("cancelled", None),
(ScannerCycleWaitOutcome::Deadline { .. }, None) => partial(),
};
metrics.record_cluster_attempt(rustfs_scanner_metrics::metrics::ScannerAttemptRecord {
epoch,
cycle,
result,
partial_reason,
duration: started.elapsed(),
});
wait
}
#[instrument(skip_all)]
#[hotpath::measure]
async fn run_data_scanner_cycle_with_budget<S>(
@@ -3369,7 +3411,10 @@ where
let cycle_ctx = ctx.child_token();
let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
record_scanner_cycle_attempt(None);
let initial_outcome = match await_scanner_cycle_with_budget_fence(
let initial_outcome = match await_observed_scanner_cycle(
global_metrics(),
leader_epoch,
cycle_info.next,
&cycle_ctx,
&cycle_budget,
run_data_scanner_cycle_with_budget(
@@ -3707,7 +3752,10 @@ where
let cycle_ctx = ctx.child_token();
let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
record_scanner_cycle_attempt(Some(wake_reason));
let outcome = match await_scanner_cycle_with_budget_fence(
let outcome = match await_observed_scanner_cycle(
global_metrics(),
leader_epoch,
cycle_info.next,
&cycle_ctx,
&cycle_budget,
run_data_scanner_cycle_with_budget(
@@ -4462,6 +4510,7 @@ pub use backlog::{
};
#[cfg(test)]
pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
pub use cycle_state::read_scanner_status_fence;
pub use cycle_state::{
SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD, ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus,
ScannerRecoveryIntentAcceptResult, ScannerRecoveryIntentConflict, ScannerRecoveryIntentRecord, ScannerRecoveryIntentRequest,
+17 -9
View File
@@ -3881,16 +3881,24 @@ pub(crate) fn encode_scanner_cycle_fence_for_test(next_cycle: u64, leader_epoch:
pub(crate) async fn current_scanner_leader_epoch() -> Result<u64, ScannerError> {
let store = crate::resolve_scanner_object_store_handle()
.ok_or_else(|| ScannerError::Other("scanner object layer is unavailable".to_string()))?;
match read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await {
Ok(buf) => {
let (_, leader_epoch) = decode_persisted_scanner_cycle_fence(&buf)?;
if leader_epoch == 0 {
return Err(ScannerError::Other("persisted scanner cycle state has no leader epoch".to_string()));
}
Ok(leader_epoch)
}
Err(err) => Err(ScannerError::Other(format!("failed to read persisted scanner leader epoch: {err}"))),
Ok(read_scanner_status_fence(store).await?.0)
}
/// Read the committed leader generation, next cycle and successful completion history
/// for a point-in-time scanner observation. This does not initialize or repair state.
///
/// # Errors
/// Returns an error if the requested store has no readable, valid, fenced cycle state.
pub async fn read_scanner_status_fence<S: ScannerObjectIO>(store: Arc<S>) -> Result<(u64, CurrentCycle), ScannerError> {
let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
.await
.map_err(|err| ScannerError::Other(format!("failed to read persisted scanner leader epoch: {err}")))?;
let (cycle, epoch) = decode_scanner_cycle_state(&buf)
.map_err(|err| ScannerError::Other(format!("persisted scanner cycle state is invalid: {err}")))?;
if epoch == 0 {
return Err(ScannerError::Other("persisted scanner cycle state has no leader epoch".to_string()));
}
Ok((epoch, cycle))
}
pub(super) fn decode_scanner_cycle_state_for_startup(buf: &[u8]) -> Result<(CurrentCycle, u64), ScannerCycleStateError> {
+66
View File
@@ -566,6 +566,72 @@ async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() {
assert!(cycle_ctx.is_cancelled());
}
#[tokio::test(start_paused = true)]
async fn cluster_scanner_records_a_budget_deadline_before_the_first_worker_poll() {
let metrics = rustfs_scanner_metrics::metrics::Metrics::new();
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&ctx,
ScannerCycleBudgetConfig {
max_duration: Some(Duration::from_secs(1)),
..Default::default()
},
);
budget.wait_for_stop().await;
advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT + Duration::from_secs(1)).await;
let worker = std::future::poll_fn(|_| panic!("expired drain must win before polling the worker"));
let wait = await_observed_scanner_cycle(&metrics, 9, 40, &ctx, &budget, worker, std::future::pending()).await;
assert_eq!(wait, ScannerCycleWaitOutcome::Deadline { worker_stopped: false });
let attempt = metrics
.scanner_node_observation()
.await
.last_attempt
.expect("dropped attempt");
assert_eq!(
(attempt.epoch, attempt.cycle, attempt.result, attempt.partial_reason),
(9, 40, "partial", Some("runtime"))
);
}
#[tokio::test(start_paused = true)]
async fn cluster_scanner_records_lock_loss_cancellation_and_cooperative_completion() {
let metrics = rustfs_scanner_metrics::metrics::Metrics::new();
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, Default::default());
let wait = await_observed_scanner_cycle(&metrics, 9, 40, &ctx, &budget, std::future::pending(), std::future::ready(())).await;
assert_eq!(wait, ScannerCycleWaitOutcome::LockLost);
let attempt = metrics.scanner_node_observation().await.last_attempt.expect("lost lock");
assert_eq!(attempt.result, "lock_lost");
assert!(attempt.duration >= SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT);
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, Default::default());
ctx.cancel();
let wait = await_observed_scanner_cycle(&metrics, 9, 41, &ctx, &budget, std::future::pending(), std::future::pending()).await;
assert_eq!(wait, ScannerCycleWaitOutcome::Cancelled);
assert_eq!(
metrics
.scanner_node_observation()
.await
.last_attempt
.expect("cancelled attempt")
.result,
"cancelled"
);
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, Default::default());
let worker = std::future::ready(ScannerCycleOutcome::Completed);
let wait = await_observed_scanner_cycle(&metrics, 9, 42, &ctx, &budget, worker, std::future::ready(())).await;
assert_eq!(wait, ScannerCycleWaitOutcome::LockLost);
let attempt = metrics
.scanner_node_observation()
.await
.last_attempt
.expect("cooperative result");
assert_eq!((attempt.cycle, attempt.result), (42, "success"));
}
#[tokio::test(start_paused = true)]
async fn cycle_budget_fences_late_writer_after_timeout() {
let cycle_ctx = CancellationToken::new();
@@ -38,3 +38,11 @@ See [scanner runtime controls](scanner-runtime-controls.md) for configuration, e
Record the RustFS source revision and binary identity, effective scanner controls, exact S3 object/version inventory, per-bucket value and provenance, original certificate or observation times, pending work, source coverage and relevant request errors. Compare the same workload and configuration before and after a change. Preserve HTTP 503 counts and latency evidence separately from accounting convergence so an improved snapshot timestamp cannot conceal foreground regressions.
The companion [auto-testing scanner usage suite](https://github.com/rustfs/auto-testing) covers public observations and candidate-native race assertions. Its deterministic cases require evidence from the exact candidate test executable. Missing internal proof, an unavailable hook, a failed cleanup or an incomplete source matrix must remain a reported limitation instead of a passing case. Multi-node load and rolling-version acceptance still require execution in the intended test environment.
## Console inventory and scanner status
`GET /rustfs/admin/v3/info?include=inventory` adds an `inventory` object to the usual server information response. `current_buckets` counts the current namespace. `measured_buckets`, `measured_objects`, `measured_bytes`, `snapshot_at` and `converged` describe the available scanner snapshot. An unavailable measurement is unknown. A measured bucket count can lag a new bucket even while that bucket appears in S3 listings. The request uses the existing server information permission.
`GET /rustfs/admin/v3/realtime?scope=cluster&types=1&n=1` returns one NDJSON frame containing `cluster_scanner` and `final: true` under the existing realtime metrics permission. `nodes_responded` counts configured members that supplied a supported sample. Compare it with `nodes_total`. `sampled_at` identifies the observation time, including when concurrent refreshes share the result. Scanning requires an active coordinator whose epoch and cycle match the durable scanner record. Idle requires complete coverage and an identified coordinator. Missing or contradictory evidence gives `unknown`. The optional `current`, `last_attempt` and `last_success_at` fields distinguish work in progress, a completed attempt (which may be partial), and a successful cycle. An interrupted worker records its budget reason, lost leadership or cancellation even when the drain deadline prevents completion. Successful completion time comes only from persisted cycle history. The current cycle's object count measures work performed during that cycle.
Older servers can ignore the query and return their ordinary response. Check for `inventory` or `cluster_scanner` before interpreting the corresponding response as the new contract.
+10 -2
View File
@@ -285,7 +285,7 @@ impl Operation for AccountInfoHandler {
for bucket in buckets.iter() {
let (rd, wr) = is_allow(bucket.name.clone()).await;
if rd || wr {
let mut bucket_info = rustfs_madmin::BucketAccessInfo {
let bucket_info = rustfs_madmin::BucketAccessInfo {
name: bucket.name.clone(),
details: Some(rustfs_madmin::BucketDetails {
versioning: BucketVersioningSys::enabled(bucket.name.as_str()).await,
@@ -298,11 +298,19 @@ impl Operation for AccountInfoHandler {
access: rustfs_madmin::AccountAccess { read: rd, write: wr },
..Default::default()
};
apply_usage_to_bucket_access_info(&mut bucket_info, data_usage_info.buckets_usage.get(&bucket.name));
account_info.buckets.push(bucket_info);
}
}
let allowed = account_info.buckets.iter().map(|bucket| bucket.name.clone()).collect();
let measured =
crate::admin::storage_api::data_usage::verified_account_bucket_usage(store, data_usage_info.buckets_usage, allowed)
.await;
for bucket in &mut account_info.buckets {
let usage = measured.get(&bucket.name);
apply_usage_to_bucket_access_info(bucket, usage);
}
let data = serde_json::to_vec(&AccountInfoResponse {
account: account_info,
username,
+87 -10
View File
@@ -18,6 +18,8 @@
//! keeping the response format explicitly NDJSON. It is not a Prometheus text
//! exposition endpoint.
mod cluster_scanner;
use crate::admin::auth::authorize_admin_envelope;
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
use crate::admin::storage_api::access::spawn_traced;
@@ -62,6 +64,8 @@ struct MetricsParams {
by_dep_id: String,
#[serde(skip)]
n_set: bool,
scope: String,
valid_cluster_count: bool,
}
impl Default for MetricsParams {
@@ -77,10 +81,25 @@ impl Default for MetricsParams {
by_job_id: Default::default(),
by_dep_id: Default::default(),
n_set: false,
scope: String::new(),
valid_cluster_count: true,
}
}
}
impl MetricsParams {
fn is_single_cluster_scanner_sample(&self) -> bool {
self.valid_cluster_count
&& self.types == 1
&& self.hosts.is_empty()
&& self.disks.is_empty()
&& self.by_job_id.is_empty()
&& self.by_dep_id.is_empty()
&& self.by_host != "true"
&& self.by_disk != "true"
}
}
fn extract_metrics_init_params(uri: &Uri) -> MetricsParams {
let mut mp = MetricsParams::default();
if let Some(query) = uri.query() {
@@ -88,6 +107,11 @@ fn extract_metrics_init_params(uri: &Uri) -> MetricsParams {
for param in params {
let mut parts = param.split('=');
if let Some(key) = parts.next() {
if key == "scope"
&& let Some(value) = parts.next()
{
mp.scope = value.to_owned();
}
if key == "disks"
&& let Some(value) = parts.next()
{
@@ -107,7 +131,9 @@ fn extract_metrics_init_params(uri: &Uri) -> MetricsParams {
&& let Some(value) = parts.next()
{
mp.n_set = true;
mp.n = value.parse::<u64>().unwrap_or(DEFAULT_METRICS_SAMPLES);
let count = value.parse::<u64>();
mp.valid_cluster_count = count == Ok(1);
mp.n = count.unwrap_or(DEFAULT_METRICS_SAMPLES);
}
if key == "types"
&& let Some(value) = parts.next()
@@ -182,6 +208,32 @@ impl Operation for MetricsHandler {
let mp = extract_metrics_init_params(req.uri());
debug!("mp: {:?}", mp);
if mp.scope == "cluster" {
if !mp.is_single_cluster_scanner_sample() {
return Err(s3_error!(
InvalidRequest,
"cluster scanner summaries require n=1, types=1 and no node filters"
));
}
let cluster_scanner =
cluster_scanner::read(crate::admin::runtime_sources::app_context_from_envelope(req.envelope())).await;
#[derive(Serialize)]
struct ClusterFrame {
cluster_scanner: cluster_scanner::Summary,
#[serde(rename = "final")]
finished: bool,
}
let mut encoded = serde_json::to_vec(&ClusterFrame {
cluster_scanner,
finished: true,
})
.map_err(|_| s3_error!(InternalError, "could not encode cluster scanner status"))?;
encoded.push(b'\n');
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, HeaderValue::from_static(CONSOLE_METRICS_CONTENT_TYPE));
return Ok(AdminResponse::with_headers(StatusCode::OK, Body::from(encoded), header));
}
let tick = parse_duration(&mp.tick).unwrap_or_else(|_| StdDuration::from_secs(3));
let mut n = resolve_sample_count(&mp);
@@ -274,6 +326,28 @@ mod tests {
.build()
}
#[test]
fn cluster_scanner_scope_only_accepts_one_unfiltered_scanner_sample() {
fn params(query: &str) -> super::MetricsParams {
let uri = format!("/rustfs/admin/v3/realtime?scope=cluster&{query}")
.parse()
.expect("test URI");
extract_metrics_init_params(&uri)
}
let valid = params("n=1&types=1");
assert_eq!(valid.scope, "cluster");
assert!(valid.is_single_cluster_scanner_sample());
for query in [
"types=1&n=5",
"types=1&n=bad",
"types=3&n=1",
"types=1&n=1&hosts=only-one",
"types=1&by-host=true",
] {
assert!(!params(query).is_single_cluster_scanner_sample(), "{query}");
}
}
#[test]
fn metrics_params_default_to_single_sample() {
let uri: Uri = "/rustfs/admin/v3/realtime".parse().unwrap();
@@ -305,15 +379,18 @@ mod tests {
#[tokio::test]
async fn metrics_handler_rejects_missing_credentials() {
let result = MetricsHandler {}
.call(build_metrics_request("/rustfs/admin/v3/realtime"), Params::new())
.await;
let err = match result {
Ok(_) => panic!("metrics handler must reject unauthenticated requests"),
Err(err) => err,
};
for uri in [
"/rustfs/admin/v3/realtime",
"/rustfs/admin/v3/realtime?scope=cluster&types=1&n=1",
] {
let result = MetricsHandler {}.call(build_metrics_request(uri), Params::new()).await;
let err = match result {
Ok(_) => panic!("metrics handler must reject unauthenticated requests"),
Err(err) => err,
};
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
assert_eq!(err.message(), Some("Signature is required"));
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
assert_eq!(err.message(), Some("Signature is required"));
}
}
}
@@ -0,0 +1,386 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Read-only coordinator observations from the configured members of this deployment.
use crate::admin::runtime_sources::AppContext;
use crate::admin::storage_api::metrics::{CollectMetricsOpts, MetricType, collect_local_metrics};
use futures::{StreamExt, stream};
use jiff::Timestamp;
use rustfs_madmin::metrics::{ScannerAttemptSample, ScannerNodeSample};
use serde::Serialize;
use std::sync::{Arc, Mutex, Weak};
use std::time::Duration;
use tokio::sync::OnceCell;
use tokio::time::{Instant, timeout_at};
const PROBE_WINDOW: Duration = Duration::from_secs(2);
const FENCE_WINDOW: Duration = Duration::from_secs(1);
const REUSE_WINDOW: Duration = Duration::from_secs(2);
const PROBE_CONCURRENCY: usize = 8;
#[derive(Clone, Debug, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
enum State {
Scanning,
Idle,
Unknown,
}
#[derive(Clone, Debug, Serialize)]
pub(super) struct Summary {
state: State,
nodes_total: usize,
nodes_responded: usize,
sampled_at: Timestamp,
#[serde(skip_serializing_if = "Option::is_none")]
current: Option<Current>,
#[serde(skip_serializing_if = "Option::is_none")]
last_attempt: Option<LastAttempt>,
#[serde(skip_serializing_if = "Option::is_none")]
last_success_at: Option<Timestamp>,
}
#[derive(Clone, Debug, Serialize)]
struct Current {
cycle: u64,
started_at: Timestamp,
objects_scanned: u64,
leader: String,
}
#[derive(Clone, Debug, Serialize)]
struct LastAttempt {
cycle: u64,
result: String,
#[serde(skip_serializing_if = "Option::is_none")]
partial_reason: Option<String>,
duration_seconds: f64,
}
struct DurableCycle {
epoch: u64,
cycle: u64,
last_success_at: Option<Timestamp>,
}
fn summarize(total: usize, samples: &[(String, ScannerNodeSample)], durable: Option<DurableCycle>) -> Summary {
let fence = durable.as_ref().map(|record| (record.epoch, record.cycle));
let covered = total > 0 && total == samples.len();
let active_count = samples.iter().filter(|(_, node)| node.active).count();
let mut holders = samples.iter().filter(|(_, node)| node.leader);
let holder = holders.next();
let conflict = holders.next().is_some();
let mut attempts = samples
.iter()
.filter_map(|(_, node)| node.last_attempt.as_ref())
.collect::<Vec<_>>();
attempts.sort_unstable_by_key(|a| (a.epoch, a.cycle));
let mut attempts = attempts.into_iter().rev();
let latest = attempts.next();
let tied = latest
.zip(attempts.next())
.is_some_and(|(a, b)| (a.epoch, a.cycle) == (b.epoch, b.cycle));
let fenced_leader = holder.filter(|(_, node)| node.epoch.is_some() && node.epoch == fence.map(|(epoch, _)| epoch));
let current = fenced_leader.and_then(|(host, node)| {
let epoch = node.epoch?;
let started_at = node.started_at?;
if fence != Some((epoch, node.cycle))
|| conflict
|| active_count != 1
|| !node.active
|| latest.is_some_and(|a| (a.epoch, a.cycle) > (epoch, node.cycle))
{
return None;
}
Some(Current {
cycle: node.cycle,
started_at,
objects_scanned: node.objects_scanned,
leader: host.clone(),
})
});
let state = if current.is_some() {
State::Scanning
} else if covered && active_count == 0 && !conflict && fenced_leader.is_some() {
State::Idle
} else {
State::Unknown
};
let last_attempt = latest
.filter(|a| covered && !tied && fence.is_some_and(|head| (a.epoch, a.cycle) <= head))
.map(|a: &ScannerAttemptSample| LastAttempt {
cycle: a.cycle,
result: a.result.clone(),
partial_reason: a.partial_reason.clone(),
duration_seconds: a.duration_seconds,
});
Summary {
state,
nodes_total: total,
nodes_responded: samples.len(),
sampled_at: Timestamp::now(),
current,
last_attempt,
last_success_at: durable.and_then(|record| record.last_success_at),
}
}
async fn probe(context: &AppContext) -> Summary {
let Some(endpoints) = context.endpoints().handle() else { return summarize(0, &[], None) };
let nodes = endpoints.get_nodes();
let total = nodes.len();
let notification = context.notification_system().handle();
let deadline = Instant::now() + PROBE_WINDOW;
let mut samples = stream::iter(nodes)
.map(|node| {
let notification = notification.as_deref();
async move {
let opts = CollectMetricsOpts::default();
let host = if node.grid_host.is_empty() && node.is_local {
"local".to_owned()
} else {
node.grid_host
};
let fetch = async {
if node.is_local {
collect_local_metrics(MetricType::SCANNER_SUMMARY, &opts).await.scanner_node
} else {
notification?
.peer_client_for_grid_host(&host)?
.get_metrics(MetricType::SCANNER_SUMMARY, &opts)
.await
.ok()?
.scanner_node
}
};
let sample = timeout_at(deadline, fetch).await.ok().flatten()?;
Some((host, sample))
}
})
.buffer_unordered(PROBE_CONCURRENCY)
.filter_map(|sample| async { sample })
.collect::<Vec<_>>()
.await;
// Require distinct configured identities, even if a malformed endpoint table
// contains overlapping aliases. Missing coverage cannot establish idleness.
samples.sort_unstable_by(|a, b| a.0.cmp(&b.0));
let responses = samples.len();
samples.dedup_by(|a, b| a.0 == b.0);
if responses != samples.len() {
return summarize(total, &samples, None);
}
let fence = if let Some(store) = crate::admin::runtime_sources::current_object_store_handle_for_context(Some(context)) {
timeout_at(deadline + FENCE_WINDOW, rustfs_scanner::scanner::read_scanner_status_fence(store))
.await
.ok()
.and_then(Result::ok)
} else {
None
};
let durable = fence.map(|(epoch, record)| DurableCycle {
epoch,
cycle: record.next,
last_success_at: record.last_success_at(),
});
summarize(total, &samples, durable)
}
struct Cache {
owner: Weak<AppContext>,
observation: Arc<OnceCell<(Instant, Summary)>>,
}
/// Concurrent Console refreshes share a cancellable, bounded fan-out. Never reuse a
/// sample from another server context or refresh its observation timestamp on a hit.
pub(super) async fn read(context: Option<Arc<AppContext>>) -> Summary {
static CACHE: Mutex<Option<Cache>> = Mutex::new(None);
let Some(context) = context else { return summarize(0, &[], None) };
let owner = Arc::downgrade(&context);
let observation = {
let mut cache = CACHE.lock().unwrap_or_else(|e| e.into_inner());
let cached = cache.as_ref().filter(|entry| {
entry.owner.ptr_eq(&owner) && entry.observation.get().is_none_or(|(time, _)| time.elapsed() < REUSE_WINDOW)
});
if let Some(entry) = cached {
Arc::clone(&entry.observation)
} else {
let observation = Arc::new(OnceCell::new());
*cache = Some(Cache {
owner,
observation: Arc::clone(&observation),
});
observation
}
};
observation
.get_or_init(|| async {
let summary = probe(&context).await;
(Instant::now(), summary)
})
.await
.1
.clone()
}
#[cfg(test)]
mod tests {
use super::*;
fn durable(epoch: u64, cycle: u64) -> Option<DurableCycle> {
Some(DurableCycle {
epoch,
cycle,
last_success_at: None,
})
}
fn with_fence(total: usize, samples: &[(String, ScannerNodeSample)]) -> Summary {
let cycle = samples.iter().find(|(_, node)| node.leader).map_or(0, |(_, node)| node.cycle);
summarize(total, samples, durable(9, cycle))
}
fn node(name: &str, sample: ScannerNodeSample) -> (String, ScannerNodeSample) {
(name.to_owned(), sample)
}
fn leader(cycle: u64) -> ScannerNodeSample {
ScannerNodeSample {
active: true,
leader: true,
epoch: Some(9),
cycle,
started_at: Some(Timestamp::UNIX_EPOCH),
objects_scanned: 301_363,
..Default::default()
}
}
fn partial(epoch: u64, cycle: u64) -> ScannerAttemptSample {
ScannerAttemptSample {
epoch,
cycle,
result: "partial".into(),
partial_reason: Some("runtime".into()),
duration_seconds: 1801.225,
}
}
#[test]
fn cluster_scanner_only_reports_success_from_durable_history() {
let when = Timestamp::constant(1_700_000_000, 0);
// An older peer may send its speculative timestamp even if persistence failed.
let mut peer = serde_json::to_value(leader(40)).expect("peer sample");
peer["last_success_at"] = serde_json::json!(when);
let samples = [node("owner", serde_json::from_value(peer).expect("compatible peer"))];
assert!(summarize(1, &samples, None).last_success_at.is_none());
assert!(summarize(1, &samples, durable(9, 40)).last_success_at.is_none());
let confirmed = Timestamp::constant(1_699_000_000, 0);
let record = DurableCycle {
epoch: 9,
cycle: 40,
last_success_at: Some(confirmed),
};
assert_eq!(summarize(1, &samples, Some(record)).last_success_at, Some(confirmed));
}
#[test]
fn cluster_scanner_requires_a_current_persisted_fence_to_trust_a_leader() {
let samples = [node("former", leader(40))];
assert_eq!(summarize(4, &samples, None).state, State::Unknown);
assert_eq!(summarize(4, &samples, durable(10, 41)).state, State::Unknown);
assert_eq!(summarize(4, &samples, durable(9, 41)).state, State::Unknown);
assert_eq!(summarize(4, &samples, durable(9, 40)).state, State::Scanning);
let idle = [node(
"owner",
ScannerNodeSample {
leader: true,
epoch: Some(9),
..Default::default()
},
)];
assert_eq!(summarize(1, &idle, durable(9, 41)).state, State::Idle);
assert_eq!(summarize(2, &idle, durable(9, 41)).state, State::Unknown);
assert_eq!(summarize(1, &idle, None).state, State::Unknown);
}
#[test]
fn cluster_scanner_returns_leader_progress_and_previous_attempt_on_a_follower_request() {
let mut old_leader = ScannerNodeSample {
last_attempt: Some(partial(8, 39)),
..Default::default()
};
old_leader.objects_scanned = 999;
let status = with_fence(2, &[node("receiving-node", old_leader), node("scan-leader", leader(40))]);
assert_eq!(status.state, State::Scanning);
assert_eq!((status.nodes_responded, status.nodes_total), (2, 2));
let current = status.current.expect("leader is known");
assert_eq!(
(current.leader.as_str(), current.cycle, current.objects_scanned),
("scan-leader", 40, 301_363)
);
let attempt = status.last_attempt.expect("completed partial");
assert_eq!(
(attempt.cycle, attempt.result.as_str(), attempt.partial_reason.as_deref()),
(39, "partial", Some("runtime"))
);
assert!(status.last_success_at.is_none());
}
#[test]
fn cluster_scanner_missing_peers_allow_positive_activity_but_cannot_establish_idle() {
let active = with_fence(4, &[node("leader", leader(0))]);
assert_eq!(active.state, State::Scanning);
assert_eq!(active.current.expect("first cycle").cycle, 0);
assert_eq!(with_fence(4, &[node("follower", Default::default())]).state, State::Unknown);
assert_eq!(with_fence(1, &[node("local", Default::default())]).state, State::Unknown);
assert_eq!(with_fence(0, &[]).state, State::Unknown);
}
#[test]
fn cluster_scanner_withholds_conflicting_or_unidentified_coordinators() {
let ambiguous = with_fence(2, &[node("former", leader(39)), node("new", leader(40))]);
assert_eq!(ambiguous.state, State::Unknown);
assert!(ambiguous.current.is_none());
let unverified = ScannerNodeSample {
epoch: None,
..leader(40)
};
assert_eq!(with_fence(1, &[node("unknown", unverified)]).state, State::Unknown);
let later = ScannerNodeSample {
last_attempt: Some(partial(10, 41)),
..Default::default()
};
assert_eq!(with_fence(2, &[node("stale", leader(40)), node("new", later)]).state, State::Unknown);
}
#[test]
fn cluster_scanner_selects_latest_attempt_by_generation_without_fabricating_success() {
let a = ScannerNodeSample {
last_attempt: Some(partial(9, 500)),
..Default::default()
};
let b = ScannerNodeSample {
last_attempt: Some(partial(10, 2)),
..Default::default()
};
let members = [node("new", b), node("old", a)];
assert!(summarize(2, &members, durable(9, 501)).last_attempt.is_none());
let status = summarize(2, &members, durable(10, 3));
assert_eq!(status.last_attempt.expect("latest epoch").cycle, 2);
let json = serde_json::to_value(status.state).expect("state");
assert_eq!(json, "unknown");
let json = serde_json::to_value(with_fence(2, &[node("one", Default::default())])).expect("frame");
assert!(json.get("last_success_at").is_none() && json.get("last_attempt").is_none() && json.get("current").is_none());
}
}
+77 -26
View File
@@ -413,6 +413,8 @@ struct SystemAdminDiscovery {
#[derive(Serialize)]
struct ServerInfoResponse {
info: InfoMessage,
#[serde(skip_serializing_if = "Option::is_none")]
inventory: Option<rustfs_madmin::ClusterInventory>,
admin_discovery: SystemAdminDiscovery,
/// Startup bitrot algorithm self-test outcome (rustfs/backlog#1873):
/// `passed` (algorithms verified at boot), `failed` (a drifted hash
@@ -440,6 +442,13 @@ fn encode_server_info_response(path: &str, response: &ServerInfoResponse) -> ser
}
}
fn server_info_inventory_requested(uri: &http::Uri) -> bool {
!is_compat_admin_request(uri.path())
&& uri.query().is_some_and(|query| {
url::form_urlencoded::parse(query.as_bytes()).any(|(key, value)| key == "include" && value == "inventory")
})
}
/// Same split as [`encode_server_info_response`]: madmin decodes `StorageInfo`
/// from the bare body.
fn encode_storage_info_response(path: &str, response: &StorageInfoResponse) -> serde_json::Result<Vec<u8>> {
@@ -477,13 +486,16 @@ impl Operation for ServerInfoHandler {
authorize_admin_envelope(req.envelope(), vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]).await?;
let usecase = default_admin_usecase();
let info = usecase
.execute_query_server_info(QueryServerInfoRequest { include_pools: true })
let result = usecase
.execute_query_server_info(QueryServerInfoRequest {
include_pools: true,
include_inventory: server_info_inventory_requested(req.uri()),
})
.await
.map_err(S3Error::from)?
.info;
.map_err(S3Error::from)?;
let response = ServerInfoResponse {
info,
info: result.info,
inventory: result.inventory,
admin_discovery: system_admin_discovery(&usecase),
bitrot_selftest: bitrot_selftest_status_str(),
};
@@ -1722,29 +1734,38 @@ mod tests {
Arc::new(rustfs_kms::KmsServiceManager::new()),
)));
let request = crate::admin::operation::test_support::TestRequest::new(Method::GET, "/rustfs/admin/v3/info")
.headers(HeaderMap::new())
.extensions(Extensions::new())
.body(Body::empty())
.credentials(ROOT_ACCESS_KEY, ROOT_SECRET_KEY, None)
.build();
for (path, expect_inventory) in [
("/rustfs/admin/v3/info", false),
("/rustfs/admin/v3/info?include=inventory", true),
] {
let request = crate::admin::operation::test_support::TestRequest::new(Method::GET, path)
.headers(HeaderMap::new())
.extensions(Extensions::new())
.body(Body::empty())
.credentials(ROOT_ACCESS_KEY, ROOT_SECRET_KEY, None)
.build();
let (status, _headers, body) = super::ServerInfoHandler {}
.call(request, Params::new())
.await
.expect("root admin credentials must be served server info")
.into_parts();
assert_eq!(status, hyper::StatusCode::OK);
let (status, _headers, body) = super::ServerInfoHandler {}
.call(request, Params::new())
.await
.expect("root admin credentials must be served server info")
.into_parts();
assert_eq!(status, hyper::StatusCode::OK);
let bytes = body.collect().await.expect("server info body should read").to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&bytes).expect("server info must be json");
let pools = payload["info"]["pools"]
.as_object()
.expect("server info must carry a pools object");
assert!(
pools.contains_key("0"),
"server info must report the erasure-set topology of pool 0, got {pools:?}"
);
let bytes = body.collect().await.expect("server info body should read").to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&bytes).expect("server info must be json");
let pools = payload["info"]["pools"]
.as_object()
.expect("server info must carry a pools object");
assert!(
pools.contains_key("0"),
"server info must report the erasure-set topology of pool 0, got {pools:?}"
);
assert_eq!(payload.get("inventory").is_some(), expect_inventory);
if expect_inventory {
assert!(payload["inventory"].as_object().unwrap().contains_key("current_buckets"));
}
}
}
/// Authorization denial for this exact action is pinned to AccessDenied by
@@ -1773,6 +1794,7 @@ mod tests {
fn server_info_response_exposes_admin_discovery_paths() {
let usecase = DefaultAdminUsecase::without_context();
let response = ServerInfoResponse {
inventory: None,
info: InfoMessage {
mode: None,
domain: None,
@@ -1815,6 +1837,7 @@ mod tests {
fn server_info_response_with_deployment_id(deployment_id: &str) -> ServerInfoResponse {
ServerInfoResponse {
inventory: None,
info: InfoMessage {
mode: Some("online".to_string()),
domain: None,
@@ -1870,6 +1893,33 @@ mod tests {
assert!(value.get("deploymentID").is_none(), "envelope must not duplicate the message: {value}");
}
#[test]
fn server_info_inventory_requires_native_opt_in_and_preserves_legacy_envelopes() {
let native = "/rustfs/admin/v3/info";
let compat = "/minio/admin/v3/info";
for (path, expected) in [
(native.to_owned(), false),
(format!("{native}?include=other"), false),
(format!("{native}?include=inventory"), true),
(format!("{compat}?include=inventory"), false),
] {
assert_eq!(super::server_info_inventory_requested(&path.parse().unwrap()), expected);
}
let mut response = server_info_response_with_deployment_id("inventory-test");
let encode = |path, response: &ServerInfoResponse| -> serde_json::Value {
serde_json::from_slice(&super::encode_server_info_response(path, response).unwrap()).unwrap()
};
assert!(encode(native, &response).get("inventory").is_none());
response.inventory = Some(rustfs_madmin::ClusterInventory {
current_buckets: Some(2),
..Default::default()
});
let value = encode(native, &response);
assert_eq!(value["inventory"]["current_buckets"], 2);
assert!(value["inventory"]["measured_objects"].is_null());
assert!(encode(compat, &response).get("inventory").is_none());
}
#[test]
fn storage_info_wire_shape_depends_on_admin_prefix() {
let response = super::StorageInfoResponse {
@@ -2034,6 +2084,7 @@ mod tests {
assert_missing_credentials(&ServiceHandle {}, Method::POST, "/rustfs/admin/v3/service").await;
assert_missing_credentials(&UpdateHandler {}, Method::POST, "/rustfs/admin/v3/update").await;
assert_missing_credentials(&ServerInfoHandler {}, Method::GET, "/rustfs/admin/v3/info").await;
assert_missing_credentials(&ServerInfoHandler {}, Method::GET, "/rustfs/admin/v3/info?include=inventory").await;
assert_missing_credentials(&InspectDataHandler {}, Method::GET, "/rustfs/admin/v3/inspect-data").await;
assert_missing_credentials(&StorageInfoHandler {}, Method::GET, "/rustfs/admin/v3/storageinfo").await;
assert_missing_credentials(&RuntimeCapabilitiesHandler {}, Method::GET, "/rustfs/admin/v4/runtime/capabilities").await;
+1
View File
@@ -943,6 +943,7 @@ pub(crate) static ERR_TIER_NOT_FOUND: AdminErrorRef = AdminErrorRef(|| &ecstore_
pub(crate) static ERR_TIER_RESERVED_NAME: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_RESERVED_NAME);
pub(crate) mod data_usage {
pub(crate) use crate::storage::storage_api::ecstore_data_usage::verified_account_bucket_usage;
use std::sync::Arc;
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
+4 -2
View File
@@ -55,10 +55,12 @@ pub const ADMIN_RUNTIME_CAPABILITIES_ROUTE: &str = "/rustfs/admin/v4/runtime/cap
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct QueryServerInfoRequest {
pub include_pools: bool,
pub include_inventory: bool,
}
pub struct QueryServerInfoResponse {
pub info: InfoMessage,
pub inventory: Option<rustfs_madmin::ClusterInventory>,
}
impl std::fmt::Debug for QueryServerInfoResponse {
@@ -319,8 +321,8 @@ impl DefaultAdminUsecase {
}
pub async fn execute_query_server_info(&self, req: QueryServerInfoRequest) -> AdminUsecaseResult<QueryServerInfoResponse> {
let info = get_server_info(req.include_pools).await;
Ok(QueryServerInfoResponse { info })
let (info, inventory) = get_server_info(req.include_pools, req.include_inventory).await;
Ok(QueryServerInfoResponse { info, inventory })
}
pub async fn execute_query_storage_info(&self) -> AdminUsecaseResult<StorageInfo> {
+5 -2
View File
@@ -91,8 +91,11 @@ pub(crate) mod s3 {
}
pub(crate) mod admin {
pub(crate) async fn get_server_info(get_pools: bool) -> rustfs_madmin::InfoMessage {
crate::storage::storage_api::ecstore_admin::get_server_info(get_pools).await
pub(crate) async fn get_server_info(
get_pools: bool,
include_inventory: bool,
) -> (rustfs_madmin::InfoMessage, Option<rustfs_madmin::ClusterInventory>) {
crate::storage::storage_api::ecstore_admin::get_server_info_with_inventory(get_pools, include_inventory).await
}
}
+26
View File
@@ -11441,6 +11441,32 @@ mod tests {
serde::Deserialize::deserialize(&mut de).expect("ServerProperties round-trip failed");
}
#[tokio::test]
async fn cluster_scanner_rpc_returns_compact_node_sample() {
let service = create_test_node_service();
let request = GetMetricsRequest {
metric_type: rmp_serde::to_vec(&MetricType::SCANNER_SUMMARY)
.expect("encode scanner selector")
.into(),
opts: rmp_serde::to_vec(&CollectMetricsOpts::default())
.expect("encode empty filters")
.into(),
};
let response = service
.get_metrics(Request::new(request))
.await
.expect("GetMetrics")
.into_inner();
assert!(response.success);
let sample: rustfs_madmin::metrics::RealtimeMetrics =
rmp_serde::from_slice(&response.realtime_metrics).expect("named reply");
assert!(sample.scanner_node.is_some());
assert!(
sample.aggregated.scanner.is_none(),
"peer probe excludes the full scanner report and paths"
);
}
#[tokio::test]
async fn test_get_metrics_round_trip() {
let service = create_test_node_service();
+2 -2
View File
@@ -497,7 +497,7 @@ pub(crate) mod test_consumer {
}
pub(crate) mod ecstore_admin {
pub(crate) use rustfs_ecstore::api::admin::{get_local_server_property, get_server_info};
pub(crate) use rustfs_ecstore::api::admin::{get_local_server_property, get_server_info_with_inventory};
}
pub(crate) mod ecstore_bucket {
@@ -551,7 +551,7 @@ pub(crate) mod ecstore_data_usage {
load_admin_data_usage_from_backend_cached, load_bucket_usage_from_backend_cached, load_data_usage_from_backend,
lookup_degraded_bucket_usage_baseline, quota_object_size, record_bucket_delete_marker_memory,
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend,
record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend, verified_account_bucket_usage,
};
// Test-only observables for the rustfs/backlog#1306 revert detector.
#[cfg(test)]