gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): make distributed usage convergence authoritative (#5151)

Browse Source 
* fix(scanner): make distributed usage cycles authoritative

* fix(scanner): close distributed refresh races

* fix(config): align scanner reload integration

* fix(admin): scope config test helpers

* fix(scanner): harden distributed usage convergence

* fix(scanner): preserve rolling activity compatibility

* fix(admin): expose non-secret optional config values

* fix(scanner): acknowledge distributed dirty usage

* fix(ecstore): make bucket mutations cancellation safe

* fix(scanner): preserve pending dirty acknowledgements

* test(obs): account for superseded scanner metric

* fix(api): reject excess detached bucket mutations

* test: close scanner convergence coverage gaps

* fix(scanner): make path tracking cleanup one-shot

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-07-25 18:45:16 +08:00
committed by GitHub
parent 0364523dad
commit a63b79004c
69 changed files with 18073 additions and 1585 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+6
View File
@@ -9899,14 +9899,18 @@ name = "rustfs-scanner"
version = "1.0.0-beta.11"
dependencies = [
"async-trait",
"bytes",
"chrono",
"futures",
"hex-simd",
"hmac 0.13.0",
"http 1.4.2",
"metrics",
"rand 0.10.2",
"rmp-serde",
"rustfs-common",
"rustfs-config",
"rustfs-credentials",
"rustfs-data-usage",
"rustfs-ecstore",
"rustfs-filemeta",
@@ -9916,7 +9920,9 @@ dependencies = [
"serde",
"serde_json",
"serial_test",
"sha2 0.11.0",
"temp-env",
"tempfile",
"thiserror 2.0.19",
"time",
"tokio",
crates/common/src/metrics.rs
+113 -17
View File
@@ -768,6 +768,7 @@ pub struct Metrics {
last_scan_cycle_replication_checks: AtomicU64,
last_scan_cycle_usage_saves: AtomicU64,
failed_scan_cycles: AtomicU64,
superseded_scan_cycles: AtomicU64,
partial_scan_cycles_unknown: AtomicU64,
partial_scan_cycles_runtime: AtomicU64,
partial_scan_cycles_objects: AtomicU64,
@@ -833,10 +834,12 @@ const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4;
const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial";
const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded";
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum ScanCyclePartialReason {
@@ -1208,6 +1211,8 @@ pub struct ScannerMetricsReport {
pub last_cycle_usage_saves: u64,
pub failed_cycles: u64,
#[serde(default)]
pub superseded_cycles: u64,
#[serde(default)]
pub partial_cycles_unknown: u64,
#[serde(default)]
pub partial_cycles_runtime: u64,
@@ -1310,6 +1315,7 @@ fn scan_cycle_result_label(result: u8) -> &'static str {
SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL,
_ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
}
}
@@ -1633,6 +1639,11 @@ pub fn emit_scan_cycle_partial_with_source(
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1);
}
pub fn emit_scan_cycle_superseded(duration: Duration) {
global_metrics().record_scan_cycle_superseded(duration);
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1);
}
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
let result = if success { "success" } else { "error" };
metrics::counter!(
@@ -1726,6 +1737,7 @@ impl Metrics {
last_scan_cycle_replication_checks: AtomicU64::new(0),
last_scan_cycle_usage_saves: AtomicU64::new(0),
failed_scan_cycles: AtomicU64::new(0),
superseded_scan_cycles: AtomicU64::new(0),
partial_scan_cycles_unknown: AtomicU64::new(0),
partial_scan_cycles_runtime: AtomicU64::new(0),
partial_scan_cycles_objects: AtomicU64::new(0),
@@ -2349,6 +2361,18 @@ impl Metrics {
.store(duration_millis_saturated(duration), Ordering::Relaxed);
}
pub fn record_scan_cycle_superseded(&self, duration: Duration) {
self.record_scanner_cycle_end_time();
self.superseded_scan_cycles.fetch_add(1, Ordering::Relaxed);
self.last_scan_cycle_result
.store(SCAN_CYCLE_RESULT_SUPERSEDED, Ordering::Relaxed);
self.last_scan_cycle_partial_reason
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
self.last_scan_cycle_duration_millis
.store(duration_millis_saturated(duration), Ordering::Relaxed);
}
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
self.record_scan_cycle_partial_with_source(duration, reason, None);
}
@@ -2802,6 +2826,7 @@ impl Metrics {
m.last_cycle_replication_repair =
self.scanner_replication_repair_work_counter_snapshots(&self.last_scan_cycle_replication_repair_work);
m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
m.superseded_cycles = self.superseded_scan_cycles.load(Ordering::Relaxed);
m.partial_cycles_unknown = self.partial_scan_cycles_unknown.load(Ordering::Relaxed);
m.partial_cycles_runtime = self.partial_scan_cycles_runtime.load(Ordering::Relaxed);
m.partial_cycles_objects = self.partial_scan_cycles_objects.load(Ordering::Relaxed);
@@ -2950,19 +2975,15 @@ pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>>
/// Register a new disk in the global path tracker and return two callbacks:
/// one to update the current path and one to deregister the disk when done.
pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
pub async fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
let disk_name = disk.to_string();
let tracker_clone = Arc::clone(&tracker);
let disk_insert = disk_name.clone();
tokio::spawn(async move {
global_metrics()
.current_paths
.write()
.await
.insert(disk_insert, tracker_clone);
});
global_metrics()
.current_paths
.write()
.await
.insert(disk_name.clone(), Arc::clone(&tracker));
let update_fn: UpdateCurrentPathFn = {
let tracker = Arc::clone(&tracker);
@@ -2990,23 +3011,28 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn,
// CloseDiskGuard
// ---------------------------------------------------------------------------
pub struct CloseDiskGuard(CloseDiskFn);
pub struct CloseDiskGuard(Option<CloseDiskFn>);
impl CloseDiskGuard {
pub fn new(close_disk: CloseDiskFn) -> Self {
Self(close_disk)
Self(Some(close_disk))
}
pub async fn close(&self) {
self.0().await;
pub async fn close(&mut self) {
let Some(close_disk) = self.0.clone() else {
return;
};
close_disk().await;
self.0 = None;
}
}
impl Drop for CloseDiskGuard {
fn drop(&mut self) {
if let Ok(handle) = tokio::runtime::Handle::try_current() {
let close_fn = self.0.clone();
handle.spawn(async move { close_fn().await });
if let Some(close_disk) = self.0.take()
&& let Ok(handle) = tokio::runtime::Handle::try_current()
{
handle.spawn(close_disk());
}
// If there is no runtime we are in a test or shutdown path; skip cleanup.
}
@@ -3016,6 +3042,61 @@ impl Drop for CloseDiskGuard {
mod tests {
use super::*;
#[tokio::test]
async fn close_disk_guard_runs_cleanup_when_an_early_return_drops_it() {
let (closed_tx, closed_rx) = tokio::sync::oneshot::channel();
let closed_tx = Arc::new(std::sync::Mutex::new(Some(closed_tx)));
let close_disk: CloseDiskFn = {
let closed_tx = Arc::clone(&closed_tx);
Arc::new(move || {
let closed_tx = closed_tx.lock().expect("close callback lock").take();
Box::pin(async move {
if let Some(closed_tx) = closed_tx {
let _ = closed_tx.send(());
}
})
})
};
let guard = CloseDiskGuard::new(close_disk);
drop(guard);
tokio::time::timeout(std::time::Duration::from_secs(1), closed_rx)
.await
.expect("drop cleanup should run")
.expect("drop cleanup should signal");
}
#[tokio::test]
async fn close_disk_guard_runs_explicit_cleanup_once() {
let close_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let close_disk: CloseDiskFn = {
let close_count = Arc::clone(&close_count);
Arc::new(move || {
close_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
Box::pin(std::future::ready(()))
})
};
let mut guard = CloseDiskGuard::new(close_disk);
guard.close().await;
drop(guard);
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
assert_eq!(close_count.load(std::sync::atomic::Ordering::Relaxed), 1);
}
#[tokio::test]
async fn current_path_updater_registers_before_return() {
let disk = format!("test-disk-{}", uuid::Uuid::new_v4());
let (_update_path, close_disk) = current_path_updater(&disk, "bucket-a").await;
assert!(global_metrics().current_paths.read().await.contains_key(&disk));
close_disk().await;
assert!(!global_metrics().current_paths.read().await.contains_key(&disk));
}
#[tokio::test]
async fn report_counts_active_scan_paths() {
let metrics = Metrics::new();
@@ -3850,6 +3931,21 @@ mod tests {
assert_eq!(report.failed_cycles, 1);
}
#[tokio::test]
async fn report_tracks_superseded_cycle_without_failed_increment() {
let metrics = Metrics::new();
metrics.record_scan_cycle_superseded(Duration::from_millis(750));
let report = metrics.report().await;
assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_SUPERSEDED_LABEL);
assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_SUPERSEDED));
assert_eq!(report.last_cycle_duration_seconds, 0.75);
assert_eq!(report.failed_cycles, 0);
assert_eq!(report.superseded_cycles, 1);
assert_eq!(report.partial_cycles, 0);
}
#[tokio::test]
async fn report_tracks_successful_scan_cycle_without_failed_increment() {
let metrics = Metrics::new();
crates/config/src/constants/scanner.rs
+2 -4
View File
@@ -220,12 +220,10 @@ pub const ENV_SCANNER_YIELD_EVERY_N_OBJECTS: &str = "RUSTFS_SCANNER_YIELD_EVERY_
pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
/// Default set scan concurrency budget.
/// `0` means no additional limit beyond deployment topology.
pub const DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS: usize = 0;
pub const DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS: usize = 4;
/// Default disk scan concurrency budget.
/// `0` means no additional limit beyond available disks in the set.
pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 0;
pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 4;
/// Default object interval for cooperative scanner yields.
pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128;
crates/data-usage/src/data_usage.rs
+234 -14
View File
@@ -32,6 +32,20 @@ use std::{
/// save forever and freeze admin usage stats; callers must bypass the skip instead.
pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 * 60);
/// Authoritative cluster-wide usage snapshot written by coordinated scanners.
///
/// This object name is intentionally distinct from the legacy unfenced
/// snapshot. Older binaries can continue writing the legacy object during a
/// rolling upgrade without overwriting a snapshot produced by the current
/// scanner protocol.
pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json";
/// Usage snapshot written by scanner implementations predating distributed
/// leadership fencing. It is read only when neither authoritative snapshot
/// copy exists.
// RUSTFS_COMPAT_TODO(scanner-usage-v2): keep .usage.json readable and removable during rolling upgrades from pre-v2 scanners. Remove after supported direct-upgrade sources all write .usage.v2.json.
pub const LEGACY_DATA_USAGE_OBJECT_NAME: &str = ".usage.json";
/// Returns true when `existing_last_update` is ahead of `now` by more than
/// [`USAGE_LAST_UPDATE_FUTURE_TOLERANCE`], i.e. the persisted timestamp cannot be
/// trusted for staleness comparisons and a fresh snapshot save must be allowed.
@@ -95,7 +109,7 @@ impl AllTierStats {
}
/// Bucket target usage info provides replication statistics
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct BucketTargetUsageInfo {
pub replication_pending_size: u64,
pub replication_failed_size: u64,
@@ -107,7 +121,7 @@ pub struct BucketTargetUsageInfo {
}
/// Bucket usage info provides bucket-level statistics
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct BucketUsageInfo {
pub size: u64,
// Following five fields suffixed with V1 are here for backward compatibility
@@ -133,7 +147,7 @@ pub struct BucketUsageInfo {
}
/// DataUsageInfo represents data usage stats of the underlying storage
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DataUsageInfo {
/// Total capacity
pub total_capacity: u64,
@@ -145,6 +159,22 @@ pub struct DataUsageInfo {
/// LastUpdate is the timestamp of when the data usage info was last updated
pub last_update: Option<SystemTime>,
/// Monotonic scanner cycle that produced this complete snapshot.
///
/// Older snapshots omit this field and continue to use `last_update` for
/// compatibility. New scanner snapshots use the cycle to fence stale
/// leaders independently of wall-clock skew.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scanner_cycle: Option<u64>,
/// Persisted scanner leadership epoch that produced this snapshot.
///
/// The epoch is claimed through the cycle-state CAS before scanning. It
/// orders snapshots from different leaders even when their wall clocks or
/// cycle counters coincide.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scanner_epoch: Option<u64>,
/// Objects total count across all buckets
pub objects_total_count: u64,
/// Versions total count across all buckets
@@ -168,7 +198,7 @@ pub struct DataUsageInfo {
}
/// Metadata describing the status of a disk-level data usage snapshot.
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DiskUsageStatus {
pub disk_id: String,
pub pool_index: Option<usize>,
@@ -268,12 +298,30 @@ impl DataUsageHash {
pub type DataUsageHashMap = HashSet<String>;
/// Size histogram for object size distribution
#[derive(Clone, Debug, Serialize, Deserialize)]
const SIZE_HISTOGRAM_LEN: usize = 11;
#[derive(Clone, Debug, Serialize)]
pub struct SizeHistogram(Vec<u64>);
impl Default for SizeHistogram {
fn default() -> Self {
Self(vec![0; 11]) // DATA_USAGE_BUCKET_LEN = 11
Self(vec![0; SIZE_HISTOGRAM_LEN])
}
}
impl<'de> Deserialize<'de> for SizeHistogram {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let values = Vec::<u64>::deserialize(deserializer)?;
if values.len() != SIZE_HISTOGRAM_LEN {
return Err(serde::de::Error::invalid_length(
values.len(),
&"exactly 11 object-size histogram buckets",
));
}
Ok(Self(values))
}
}
@@ -343,7 +391,7 @@ impl SizeHistogram {
.zip(names.iter())
.filter(|((_, (start, end)), name)| name != &&"BETWEEN_1024B_AND_1_MB" && *start >= 1024 && *end < ONE_MIB)
.map(|((count, _), _)| *count)
.sum();
.fold(0, u64::saturating_add);
let mut res = HashMap::new();
for (count, name) in self.0.iter().zip(names.iter()) {
@@ -364,12 +412,30 @@ impl SizeHistogram {
}
/// Versions histogram for version count distribution
#[derive(Clone, Debug, Serialize, Deserialize)]
const VERSIONS_HISTOGRAM_LEN: usize = 7;
#[derive(Clone, Debug, Serialize)]
pub struct VersionsHistogram(Vec<u64>);
impl Default for VersionsHistogram {
fn default() -> Self {
Self(vec![0; 7]) // DATA_USAGE_VERSION_LEN = 7
Self(vec![0; VERSIONS_HISTOGRAM_LEN])
}
}
impl<'de> Deserialize<'de> for VersionsHistogram {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let values = Vec::<u64>::deserialize(deserializer)?;
if values.len() != VERSIONS_HISTOGRAM_LEN {
return Err(serde::de::Error::invalid_length(
values.len(),
&"exactly 7 object-version histogram buckets",
));
}
Ok(Self(values))
}
}
@@ -535,13 +601,74 @@ impl DataUsageEntry {
}
}
for (i, v) in other.obj_sizes.0.iter().enumerate() {
self.obj_sizes.0[i] += v;
}
self.obj_sizes.merge_from(&other.obj_sizes);
self.obj_versions.merge_from(&other.obj_versions);
}
for (i, v) in other.obj_versions.0.iter().enumerate() {
self.obj_versions.0[i] += v;
pub fn checked_merge(&mut self, other: &DataUsageEntry) -> bool {
let scalar_counts_fit = self.objects.checked_add(other.objects).is_some()
&& self.versions.checked_add(other.versions).is_some()
&& self.delete_markers.checked_add(other.delete_markers).is_some()
&& self.size.checked_add(other.size).is_some()
&& self.failed_objects.checked_add(other.failed_objects).is_some();
let histograms_fit = self.obj_sizes.0.len() == SIZE_HISTOGRAM_LEN
&& other.obj_sizes.0.len() == SIZE_HISTOGRAM_LEN
&& self.obj_versions.0.len() == VERSIONS_HISTOGRAM_LEN
&& other.obj_versions.0.len() == VERSIONS_HISTOGRAM_LEN
&& self
.obj_sizes
.0
.iter()
.zip(other.obj_sizes.0.iter())
.all(|(left, right)| left.checked_add(*right).is_some())
&& self
.obj_versions
.0
.iter()
.zip(other.obj_versions.0.iter())
.all(|(left, right)| left.checked_add(*right).is_some());
let replication_fits = match (&self.replication_stats, &other.replication_stats) {
(_, None) | (None, Some(_)) => true,
(Some(left), Some(right)) => {
left.replica_size.checked_add(right.replica_size).is_some()
&& left.replica_count.checked_add(right.replica_count).is_some()
&& right.targets.iter().all(|(target, right_stats)| {
left.targets.get(target).is_none_or(|left_stats| {
left_stats.pending_size.checked_add(right_stats.pending_size).is_some()
&& left_stats.replicated_size.checked_add(right_stats.replicated_size).is_some()
&& left_stats.failed_size.checked_add(right_stats.failed_size).is_some()
&& left_stats.failed_count.checked_add(right_stats.failed_count).is_some()
&& left_stats.pending_count.checked_add(right_stats.pending_count).is_some()
&& left_stats
.missed_threshold_size
.checked_add(right_stats.missed_threshold_size)
.is_some()
&& left_stats
.after_threshold_size
.checked_add(right_stats.after_threshold_size)
.is_some()
&& left_stats
.missed_threshold_count
.checked_add(right_stats.missed_threshold_count)
.is_some()
&& left_stats
.after_threshold_count
.checked_add(right_stats.after_threshold_count)
.is_some()
&& left_stats
.replicated_count
.checked_add(right_stats.replicated_count)
.is_some()
})
})
}
};
if !scalar_counts_fit || !histograms_fit || !replication_fits {
return false;
}
self.merge(other);
true
}
}
@@ -1395,6 +1522,17 @@ mod tests {
assert_eq!(map["BETWEEN_512_KB_AND_1_MB"], 1);
}
#[test]
fn test_size_histogram_compat_rollup_saturates_on_corrupt_counts() {
let mut hist = SizeHistogram::default();
hist.0[1] = u64::MAX;
hist.0[2] = 1;
let map = hist.to_map();
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], u64::MAX);
}
#[test]
fn test_data_usage_cache_merge_adds_missing_child() {
let mut base = DataUsageCache::default();
@@ -1708,4 +1846,86 @@ mod tests {
assert!(cache.find("bucket/large/a").is_some());
assert!(cache.find("bucket/large/b").is_some());
}
#[test]
fn checked_merge_rejects_scalar_and_replication_overflow_without_mutation() {
let mut entry = DataUsageEntry {
objects: usize::MAX,
replication_stats: Some(ReplicationAllStats {
replica_size: 7,
..Default::default()
}),
..Default::default()
};
let other = DataUsageEntry {
objects: 1,
replication_stats: Some(ReplicationAllStats {
replica_size: u64::MAX,
..Default::default()
}),
..Default::default()
};
assert!(!entry.checked_merge(&other));
assert_eq!(entry.objects, usize::MAX);
assert_eq!(entry.replication_stats.as_ref().map(|stats| stats.replica_size), Some(7));
}
#[test]
fn checked_merge_accepts_valid_usage() {
let mut entry = DataUsageEntry {
objects: 2,
size: 20,
..Default::default()
};
let other = DataUsageEntry {
objects: 3,
size: 30,
..Default::default()
};
assert!(entry.checked_merge(&other));
assert_eq!(entry.objects, 5);
assert_eq!(entry.size, 50);
}
#[test]
fn histogram_deserialization_rejects_noncanonical_lengths() {
let invalid_sizes =
rmp_serde::to_vec(&vec![0_u64; SIZE_HISTOGRAM_LEN + 1]).expect("encode invalid object-size histogram fixture");
let invalid_versions =
rmp_serde::to_vec(&vec![0_u64; VERSIONS_HISTOGRAM_LEN - 1]).expect("encode invalid object-version histogram fixture");
assert!(rmp_serde::from_slice::<SizeHistogram>(&invalid_sizes).is_err());
assert!(rmp_serde::from_slice::<VersionsHistogram>(&invalid_versions).is_err());
}
#[test]
fn replication_target_deserialization_preserves_large_historical_maps() {
let mut stats = ReplicationAllStats::default();
for index in 0..=1024 {
stats.targets.insert(format!("target-{index}"), ReplicationStats::default());
}
let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode");
let decoded = rmp_serde::from_slice::<ReplicationAllStats>(&encoded)
.expect("historical replication target maps must remain readable");
assert_eq!(decoded.targets.len(), stats.targets.len());
}
#[test]
fn checked_merge_rejects_noncanonical_histograms_without_mutation() {
let mut entry = DataUsageEntry {
objects: 2,
..Default::default()
};
let other = DataUsageEntry {
objects: 3,
obj_sizes: SizeHistogram(vec![0; SIZE_HISTOGRAM_LEN + 1]),
..Default::default()
};
assert!(!entry.checked_merge(&other));
assert_eq!(entry.objects, 2);
}
}
crates/e2e_test/src/content_encoding_test.rs
+19
View File
@@ -80,6 +80,25 @@ mod tests {
assert_eq!(head_resp.content_encoding(), Some("zstd"), "HEAD should return Content-Encoding: zstd");
assert_eq!(head_resp.content_type(), Some("text/plain"), "HEAD should return correct Content-Type");
client
.delete_object()
.bucket(bucket)
.key(key)
.send()
.await
.expect("DELETE object failed");
client
.delete_bucket()
.bucket(bucket)
.send()
.await
.expect("DELETE bucket failed");
client
.list_buckets()
.send()
.await
.expect("RustFS must remain available after deleting a bucket");
env.stop_server();
}
crates/ecstore/src/api/mod.rs
+15 -14
View File
@@ -244,11 +244,12 @@ pub mod config {
pub mod com {
pub use crate::config::com::{
COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
ServerConfigCorruptError, delete_config, is_server_config_corrupt_error, lookup_configs, read_config,
read_config_no_lock, read_config_with_metadata, read_config_without_migrate, read_config_without_migrate_no_lock,
read_existing_server_config_no_lock, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
save_server_config_no_lock, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
with_server_config_read_lock, with_server_config_write_lock,
ServerConfigCorruptError, ServerConfigSnapshot, delete_config, is_server_config_corrupt_error, lookup_configs,
read_config, read_config_no_lock, read_config_with_metadata, read_config_without_migrate,
read_config_without_migrate_no_lock, read_existing_server_config_no_lock, read_server_config_snapshot, save_config,
save_config_no_lock, save_config_with_opts, save_server_config, save_server_config_no_lock,
save_server_config_snapshot, server_config_path, try_migrate_server_config, with_config_object_read_lock,
with_config_object_write_lock, with_server_config_read_lock, with_server_config_write_lock,
};
}
@@ -287,9 +288,9 @@ pub mod disk {
pub use crate::disk::{
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, OldCurrentSize, RUSTFS_META_BUCKET, ReadMultipleReq,
ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
new_disk, validate_batch_read_version_item_count,
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, RUSTFS_META_BUCKET,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions, new_disk, validate_batch_read_version_item_count,
};
pub use bytes::Bytes;
pub use endpoint::Endpoint;
@@ -390,12 +391,12 @@ pub mod rio {
pub mod rpc {
pub use crate::cluster::rpc::{
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor,
gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature,
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
};
}
crates/ecstore/src/cluster/rpc/http_auth.rs
+52
View File
@@ -27,6 +27,7 @@
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
use crate::cluster::rpc::context_propagation::{inject_request_id_into_http_headers, inject_trace_context_into_http_headers};
use crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION;
use base64::Engine as _;
use base64::engine::general_purpose;
use hmac::{Hmac, KeyInit, Mac};
@@ -61,6 +62,7 @@ const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536;
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
get_env_bool(
@@ -209,6 +211,42 @@ fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, si
mac.verify_slice(&signature).is_ok()
}
fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
mac.update(challenge.as_bytes());
mac.update(server_epoch.as_bytes());
}
fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
Ok(mac.finalize().into_bytes().to_vec())
}
fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
mac.verify_slice(proof)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
}
pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch)
}
pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof)
}
#[derive(Clone, Copy)]
struct SignatureV2Scope<'a> {
audience: &'a str,
@@ -629,6 +667,20 @@ mod tests {
runtime_sources::ensure_test_rpc_secret();
}
#[test]
fn namespace_scanner_capability_proof_binds_challenge_and_server_epoch() {
let secret = "test-scanner-capability-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof =
generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err());
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err());
}
/// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
/// fixed in rustfs/rustfs#4402): secret resolution must never silently fall
/// back to a default/empty shared secret. Missing and default secrets both
crates/ecstore/src/cluster/rpc/internode_data_transport.rs
+220 -2
View File
@@ -12,11 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::build_auth_headers;
use crate::cluster::rpc::{build_auth_headers, verify_ns_scanner_capability};
use crate::disk::error::{Error, Result};
use crate::disk::{FileReader, FileWriter};
use crate::storage_api_contracts::internode::{
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY,
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
};
use async_trait::async_trait;
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
@@ -28,13 +31,18 @@ use rustfs_rio::{HttpReader, HttpWriter};
use sha2::{Digest, Sha256};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use tokio::io::AsyncReadExt;
use uuid::Uuid;
static INTERNODE_DATA_TRANSPORT: OnceLock<std::result::Result<Arc<dyn InternodeDataTransport>, String>> = OnceLock::new();
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner";
const NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE: usize = 1024;
const CONTENT_TYPE_JSON: &str = "application/json";
const CONTENT_TYPE_MSGPACK: &str = "application/msgpack";
fn unsupported_transport_message(transport: &str) -> String {
format!(
@@ -101,6 +109,25 @@ pub struct WalkDirStreamRequest {
pub stall_timeout: Option<Duration>,
}
#[derive(Debug, Clone)]
pub struct NsScannerStreamRequest {
pub endpoint: String,
pub disk: String,
pub request_id: Uuid,
pub server_epoch: Uuid,
pub session_id: Uuid,
pub session_sequence: u64,
pub next_cycle: u64,
pub leader_epoch: u64,
pub body: Vec<u8>,
pub stall_timeout: Option<Duration>,
}
#[derive(Debug, Clone)]
pub struct NsScannerCapabilityRequest {
pub endpoint: String,
}
/// Data-plane stream opener used by `RemoteDisk`.
///
/// This boundary is limited to remote disk streams that can move large payloads.
@@ -114,6 +141,12 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader>;
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter>;
async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader>;
async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result<FileReader> {
Err(Error::MethodNotAllowed)
}
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
Err(Error::MethodNotAllowed)
}
fn name(&self) -> &'static str;
fn capabilities(&self) -> InternodeDataTransportCapabilities;
}
@@ -148,6 +181,39 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
))
}
async fn open_ns_scanner(&self, request: NsScannerStreamRequest) -> Result<FileReader> {
let url = build_ns_scanner_url(&request);
let mut headers = msgpack_headers();
build_auth_headers(&url, &Method::POST, &mut headers)?;
Ok(Box::new(
HttpReader::new_with_stall_timeout(url, Method::POST, headers, Some(request.body), request.stall_timeout).await?,
))
}
async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(&request, challenge);
let mut headers = msgpack_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
let reader = HttpReader::new(url, Method::GET, headers, None).await?;
let mut body = Vec::new();
reader
.take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
.read_to_end(&mut body)
.await?;
if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
return Err(Error::other("invalid remote namespace scanner capability response size"));
}
let response: NsScannerCapabilityResponse =
rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
return Err(Error::other("incompatible remote namespace scanner capability response"));
}
verify_ns_scanner_capability(challenge, response.server_epoch, &response.proof)
.map_err(|err| Error::other(format!("remote namespace scanner capability authentication failed: {err}")))?;
Ok(response.server_epoch)
}
fn name(&self) -> &'static str {
DEFAULT_INTERNODE_DATA_TRANSPORT
}
@@ -197,12 +263,54 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
)
}
fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
let body_sha256 = hex_simd::encode_to_string(Sha256::digest(&request.body), hex_simd::AsciiCase::Lower);
format!(
"{}{}?disk={}&{}={}&{}={}&{}={}&{}={}&{}={}&{}={}&{}={}",
request.endpoint,
NS_SCANNER_PATH,
urlencoding::encode(&request.disk),
NS_SCANNER_REQUEST_ID_QUERY,
request.request_id,
NS_SCANNER_SERVER_EPOCH_QUERY,
request.server_epoch,
NS_SCANNER_SESSION_ID_QUERY,
request.session_id,
NS_SCANNER_SESSION_SEQUENCE_QUERY,
request.session_sequence,
NS_SCANNER_CYCLE_QUERY,
request.next_cycle,
NS_SCANNER_LEADER_EPOCH_QUERY,
request.leader_epoch,
NS_SCANNER_BODY_SHA256_QUERY,
body_sha256
)
}
fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
format!(
"{}{}?{}={}&{}={}",
request.endpoint,
NS_SCANNER_PATH,
NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_PROTOCOL_VERSION,
NS_SCANNER_CAPABILITY_CHALLENGE_QUERY,
challenge
)
}
fn json_headers() -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static(CONTENT_TYPE_JSON));
headers
}
fn msgpack_headers() -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static(CONTENT_TYPE_MSGPACK));
headers
}
fn build_internode_data_transport_result(
configured_transport: Option<&str>,
) -> std::result::Result<Arc<dyn InternodeDataTransport>, String> {
@@ -241,6 +349,65 @@ pub fn build_internode_data_transport_from_env() -> Result<Arc<dyn InternodeData
mod tests {
use super::*;
#[derive(Debug)]
struct LegacyTestTransport;
#[async_trait::async_trait]
impl InternodeDataTransport for LegacyTestTransport {
async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
Ok(Box::new(tokio::io::empty()))
}
async fn open_write(&self, _request: WriteStreamRequest) -> Result<FileWriter> {
Ok(Box::new(tokio::io::sink()))
}
async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result<FileReader> {
Ok(Box::new(tokio::io::empty()))
}
fn name(&self) -> &'static str {
"legacy-test"
}
fn capabilities(&self) -> InternodeDataTransportCapabilities {
InternodeDataTransportCapabilities::tcp_http()
}
}
#[tokio::test]
async fn legacy_transport_defaults_namespace_scanner_to_unsupported() {
let transport = LegacyTestTransport;
let probe_err = transport
.probe_ns_scanner(NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
})
.await
.expect_err("legacy transport should report namespace scanner as unsupported");
assert!(matches!(probe_err, Error::MethodNotAllowed));
let open_result = transport
.open_ns_scanner(NsScannerStreamRequest {
endpoint: "http://node1:9000".to_string(),
disk: "http://node1:9000/data/rustfs0".to_string(),
request_id: Uuid::new_v4(),
server_epoch: Uuid::new_v4(),
session_id: Uuid::new_v4(),
session_sequence: 0,
next_cycle: 7,
leader_epoch: 9,
body: Vec::new(),
stall_timeout: None,
})
.await;
let open_err = match open_result {
Ok(_) => panic!("legacy transport should not open namespace scanner streams"),
Err(err) => err,
};
assert!(matches!(open_err, Error::MethodNotAllowed));
}
#[test]
fn tcp_http_capabilities_are_behavior_preserving() {
let transport = TcpHttpInternodeDataTransport;
@@ -322,6 +489,57 @@ mod tests {
);
}
#[test]
fn ns_scanner_url_binds_body_and_encodes_disk_ref() {
let request_id = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("request ID");
let server_epoch = Uuid::parse_str("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee").expect("server epoch");
let session_id = Uuid::parse_str("99999999-8888-4777-8666-555555555555").expect("session ID");
let url = build_ns_scanner_url(&NsScannerStreamRequest {
endpoint: "http://node1:9000".to_string(),
disk: "http://node1:9000/data/rustfs0".to_string(),
request_id,
server_epoch,
session_id,
session_sequence: 3,
next_cycle: 7,
leader_epoch: 9,
body: b"scanner-request".to_vec(),
stall_timeout: None,
});
assert_eq!(
url,
concat!(
"http://node1:9000/rustfs/rpc/ns_scanner?disk=http%3A%2F%2Fnode1%3A9000%2Fdata%2Frustfs0",
"&ns_scanner_request_id=11111111-2222-4333-8444-555555555555",
"&ns_scanner_server_epoch=aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee",
"&ns_scanner_session_id=99999999-8888-4777-8666-555555555555",
"&ns_scanner_session_sequence=3",
"&ns_scanner_cycle=7",
"&ns_scanner_leader_epoch=9",
"&ns_scanner_body_sha256=c958f15ca28422275c1245399f4c44eaba628ca453fcd77d6b3d4484573e4387"
)
);
}
#[test]
fn ns_scanner_capability_url_binds_version_and_challenge() {
let challenge = Uuid::parse_str("12345678-1234-4234-8234-123456789abc").expect("challenge");
let url = build_ns_scanner_capability_url(
&NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
},
challenge,
);
assert_eq!(
url,
format!(
"http://node1:9000/rustfs/rpc/ns_scanner?ns_scanner_protocol={NS_SCANNER_PROTOCOL_VERSION}&ns_scanner_challenge={challenge}"
)
);
}
#[test]
fn transport_config_defaults_to_tcp_http() {
let transport = build_internode_data_transport(None).unwrap();
crates/ecstore/src/cluster/rpc/mod.rs
+3 -3
View File
@@ -30,8 +30,8 @@ pub use client::{
};
pub use http_auth::{
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability,
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
};
#[cfg(test)]
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
@@ -41,6 +41,6 @@ pub use peer_rest_client::{
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
};
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys};
pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys, ScannerBucketListing, ScannerSetBucketListing};
pub use remote_disk::RemoteDisk;
pub use remote_locker::RemoteClient;
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+337 -12
View File
@@ -18,6 +18,9 @@ use crate::cluster::rpc::client::{
};
use crate::cluster::rpc::{set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof};
use crate::error::{Error, Result};
use crate::storage_api_contracts::internode::{
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
};
use crate::{
bucket::replication::BucketStats,
disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout},
@@ -27,6 +30,7 @@ use crate::{
};
use bytes::Bytes;
use rmp_serde::{Deserializer, Serializer};
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
use rustfs_madmin::{
ServerProperties,
health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysServices},
@@ -97,15 +101,34 @@ fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<
Ok(stats)
}
fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u32) -> Result<()> {
if sig == SERVICE_SIGNAL_RELOAD_DYNAMIC
&& matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS)
&& protocol_version < rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION
{
return Err(Error::other(format!("peer does not support dynamic {sub_sys} config convergence")));
}
Ok(())
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ScannerPeerActivity {
pub instance_id: String,
pub namespace_generation: u64,
pub maintenance_generation: u64,
pub protocol_version: u32,
pub topology_digest: Option<[u8; 32]>,
pub data_movement_active: Option<bool>,
pub dirty_usage_generation: Option<u64>,
pub dirty_usage_pending: Option<bool>,
}
fn decode_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerPeerActivity> {
let instance_id = response.instance_id;
fn decode_scanner_activity_with_verifier(
response: ScannerActivityResponse,
challenge: &[u8; 16],
verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>,
) -> Result<ScannerPeerActivity> {
let instance_id = &response.instance_id;
if instance_id.len() != 32
|| !instance_id
.as_bytes()
@@ -114,10 +137,80 @@ fn decode_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerP
{
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
}
let (topology_digest, data_movement_active, dirty_usage_generation, dirty_usage_pending) = match response.protocol_version {
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy response fields are unauthenticated. Remove after protocol v0 peers are unsupported.
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION
if response.topology_digest.is_empty()
&& response.response_proof.is_empty()
&& !response.data_movement_active
&& response.dirty_usage_generation == 0
&& !response.dirty_usage_pending =>
{
(None, None, None, None)
}
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(Error::other("legacy scanner activity peer returned unexpected extended fields"));
}
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
if response.dirty_usage_generation != 0 || response.dirty_usage_pending {
return Err(Error::other("scanner activity protocol v4 peer returned unauthenticated v5 fields"));
}
let canonical = rustfs_protos::canonical_scanner_activity_v4_response_body(challenge, &response)
.map_err(|_| Error::other("peer scanner activity response is too large to authenticate"))?;
verify_proof(&canonical, &response.response_proof)?;
(
Some(
response
.topology_digest
.as_ref()
.try_into()
.map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?,
),
Some(response.data_movement_active),
None,
None,
)
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
}
let canonical = rustfs_protos::canonical_scanner_activity_response_body(challenge, &response)
.map_err(|_| Error::other("peer scanner activity response is too large to authenticate"))?;
verify_proof(&canonical, &response.response_proof)?;
(
Some(
response
.topology_digest
.as_ref()
.try_into()
.map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?,
),
Some(response.data_movement_active),
Some(response.dirty_usage_generation),
Some(response.dirty_usage_pending),
)
}
version => {
return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}")));
}
};
Ok(ScannerPeerActivity {
instance_id,
instance_id: response.instance_id,
namespace_generation: response.namespace_generation,
maintenance_generation: response.maintenance_generation,
protocol_version: response.protocol_version,
topology_digest,
data_movement_active,
dirty_usage_generation,
dirty_usage_pending,
})
}
fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 16]) -> Result<ScannerPeerActivity> {
decode_scanner_activity_with_verifier(response, challenge, |canonical, proof| {
verify_tonic_rpc_response_proof(canonical, proof)
.map_err(|_| Error::other("peer returned an invalid scanner activity response proof"))
})
}
@@ -1331,6 +1424,7 @@ impl PeerRestClient {
}
return Err(Error::other(""));
}
validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
Ok(())
}
.await,
@@ -1338,25 +1432,44 @@ impl PeerRestClient {
.await
}
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
async fn scanner_activity_request(
&self,
acknowledge_instance_id: String,
acknowledge_dirty_usage_generation: u64,
) -> Result<ScannerPeerActivity> {
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self
.get_client()
.await?
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let response = client
.scanner_activity(Request::new(ScannerActivityRequest {}))
.await?
.into_inner();
decode_scanner_activity(response)
let mut request = Request::new(ScannerActivityRequest {
challenge: challenge.as_bytes().to_vec().into(),
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id,
acknowledge_dirty_usage_generation,
});
let canonical = rustfs_protos::canonical_scanner_activity_request_body(request.get_ref())
.map_err(|_| Error::other("scanner activity request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.scanner_activity(request).await?.into_inner();
decode_scanner_activity(response, challenge.as_bytes())
}
.await,
)
.await
}
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
self.scanner_activity_request(String::new(), 0).await
}
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
self.scanner_activity_request(instance_id, generation).await
}
pub async fn get_metacache_listing(&self) -> Result<()> {
warn!("get_metacache_listing is not implemented in PeerRestClient");
Err(Error::NotImplemented)
@@ -1542,6 +1655,7 @@ impl PeerRestClient {
#[cfg(test)]
mod tests {
use super::*;
use crate::config::com::STORAGE_CLASS_SUB_SYS;
use serde_json::Value;
use std::io::{self, Write};
use std::sync::{Arc, Mutex};
@@ -1650,6 +1764,14 @@ mod tests {
)
}
fn decode_test_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerPeerActivity> {
decode_scanner_activity_with_verifier(response, &[9; 16], |_canonical, proof| {
(proof == b"proof")
.then_some(())
.ok_or_else(|| Error::other("peer returned an invalid scanner activity response proof"))
})
}
#[test]
fn build_clients_from_slots_preserves_missing_remote_topology_slots() {
let slots = vec![
@@ -1682,13 +1804,89 @@ mod tests {
#[test]
fn scanner_activity_requires_restart_safe_peer_identity() {
let legacy = decode_test_scanner_activity(ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
topology_digest: Vec::new().into(),
data_movement_active: false,
response_proof: Vec::new().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
})
.expect("legacy peers should retain their activity generations during a rolling upgrade");
assert_eq!(
legacy,
ScannerPeerActivity {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
topology_digest: None,
data_movement_active: None,
dirty_usage_generation: None,
dirty_usage_pending: None,
}
);
let previous = decode_test_scanner_activity(ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: true,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
})
.expect("protocol v4 peers should remain observable during a rolling upgrade");
assert_eq!(
previous,
ScannerPeerActivity {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
topology_digest: Some([7; 32]),
data_movement_active: Some(true),
dirty_usage_generation: None,
dirty_usage_pending: None,
}
);
let malformed_topology = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 31].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_test_scanner_activity(malformed_topology)
.expect_err("activity topology digests must have the protocol-defined length")
.to_string()
.contains("topology digest")
);
let missing_instance = ScannerActivityResponse {
instance_id: String::new(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_scanner_activity(missing_instance)
decode_test_scanner_activity(missing_instance)
.expect_err("an empty instance ID is not restart safe")
.to_string()
.contains("instance ID")
@@ -1698,18 +1896,30 @@ mod tests {
instance_id: "ABCDEF0123456789ABCDEF0123456789".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_scanner_activity(malformed_instance)
decode_test_scanner_activity(malformed_instance)
.expect_err("activity instance IDs must use the canonical lowercase hex form")
.to_string()
.contains("instance ID")
);
let activity = decode_scanner_activity(ScannerActivityResponse {
let activity = decode_test_scanner_activity(ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: true,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
})
.expect("complete activity responses should be accepted");
assert_eq!(
@@ -1718,8 +1928,123 @@ mod tests {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: Some([7; 32]),
data_movement_active: Some(true),
dirty_usage_generation: Some(11),
dirty_usage_pending: Some(true),
}
);
let pending_without_generation = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: true,
};
assert!(
decode_test_scanner_activity(pending_without_generation)
.expect_err("pending dirty usage must carry a nonzero generation")
.to_string()
.contains("without a generation")
);
let previous_with_dirty_usage = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_test_scanner_activity(previous_with_dirty_usage)
.expect_err("protocol v4 responses must not claim unauthenticated dirty usage fields")
.to_string()
.contains("unauthenticated v5 fields")
);
let legacy_with_topology = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
};
assert!(
decode_test_scanner_activity(legacy_with_topology)
.expect_err("legacy protocol responses must not claim extended fields")
.to_string()
.contains("unexpected extended fields")
);
let unsupported_protocol = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION + 1,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_test_scanner_activity(unsupported_protocol)
.expect_err("unknown activity protocols must fail closed")
.to_string()
.contains("unsupported scanner activity protocol")
);
let missing_proof = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: Vec::new().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_test_scanner_activity(missing_proof)
.expect_err("unsigned scanner activity responses must fail closed")
.to_string()
.contains("response proof")
);
}
#[test]
fn dynamic_scanner_config_requires_versioned_peer_acknowledgement() {
for sub_system in [SCANNER_SUB_SYS, HEAL_SUB_SYS] {
let err = validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, sub_system, 0)
.expect_err("an unversioned peer must not claim scanner config convergence");
assert!(err.to_string().contains("does not support dynamic"));
validate_signal_service_protocol(
SERVICE_SIGNAL_RELOAD_DYNAMIC,
sub_system,
rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
)
.expect("a current peer should support dynamic scanner config");
}
validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, 0)
.expect("unrelated dynamic config keeps its existing compatibility contract");
validate_signal_service_protocol(SERVICE_SIGNAL_REFRESH_CONFIG, SCANNER_SUB_SYS, 0)
.expect("full refresh compatibility is guarded by its scanner preflight");
}
#[test]
crates/ecstore/src/cluster/rpc/peer_s3_client.rs
+108 -6
View File
@@ -49,6 +49,20 @@ use tracing::{debug, info, warn};
type Client = Arc<Box<dyn PeerS3Client>>;
#[derive(Clone, Debug)]
pub struct ScannerBucketListing {
pub buckets: Vec<BucketInfo>,
pub set_buckets: Vec<ScannerSetBucketListing>,
pub topology_complete: bool,
}
#[derive(Clone, Debug)]
pub struct ScannerSetBucketListing {
pub pool_index: usize,
pub set_index: usize,
pub buckets: Vec<BucketInfo>,
}
fn pool_participant_errors(clients: &[Client], errors: &[Option<Error>], pool_idx: usize) -> Vec<Option<Error>> {
clients
.iter()
@@ -216,6 +230,10 @@ impl S3PeerSys {
Ok(())
}
pub async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
Ok(self.list_bucket_for_scanner(opts).await?.buckets)
}
pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result<ScannerBucketListing> {
let mut futures = Vec::with_capacity(self.clients.len());
for cli in self.clients.iter() {
futures.push(cli.list_bucket(opts));
@@ -239,9 +257,12 @@ impl S3PeerSys {
}
let mut result_map: HashMap<&String, BucketInfo> = HashMap::new();
let mut topology_complete = true;
for i in 0..self.pools_count {
let per_pool_errs = pool_participant_errors(&self.clients, &errors, i);
let quorum = pool_write_quorum(per_pool_errs.len());
topology_complete &=
!per_pool_errs.is_empty() && per_pool_errs.iter().all(|participant_error| participant_error.is_none());
if let Some(pool_err) = reduce_pool_write_quorum_errs(&per_pool_errs) {
tracing::error!("list_bucket per_pool_errs: {per_pool_errs:?}");
@@ -261,20 +282,17 @@ impl S3PeerSys {
}
for bucket in buckets.iter() {
if result_map.contains_key(&bucket.name) {
continue;
}
// incr bucket_map count create if not exists
let count = bucket_map.entry(&bucket.name).or_insert(0usize);
*count += 1;
if *count >= quorum {
result_map.insert(&bucket.name, bucket.clone());
result_map.entry(&bucket.name).or_insert_with(|| bucket.clone());
}
}
}
}
topology_complete &= bucket_map.values().all(|count| *count >= quorum);
// TODO: MRF
}
@@ -282,7 +300,11 @@ impl S3PeerSys {
buckets.sort_by_key(|b| b.name.clone());
Ok(buckets)
Ok(ScannerBucketListing {
buckets,
set_buckets: Vec::new(),
topology_complete,
})
}
pub async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
let mut futures = Vec::with_capacity(self.clients.len());
@@ -1645,6 +1667,86 @@ mod tests {
assert_eq!(buckets[0].name, bucket.name);
}
#[tokio::test]
async fn scanner_bucket_listing_marks_quorum_result_incomplete_when_a_peer_is_missing() {
let bucket = BucketInfo {
name: "bucket-hidden-by-quorum".to_string(),
..Default::default()
};
let peer_sys = S3PeerSys {
clients: vec![
test_peer_with_list_bucket(&[0], Ok(vec![bucket])),
test_peer_with_list_bucket(&[0], Ok(Vec::new())),
test_peer_with_list_bucket(&[0], Ok(Vec::new())),
test_peer_with_list_bucket(&[0], Err(Error::DiskAccessDenied)),
],
pools_count: 1,
};
let listing = peer_sys
.list_bucket_for_scanner(&BucketOptions::default())
.await
.expect("peer quorum should still produce a scanner candidate listing");
assert!(listing.buckets.is_empty());
assert!(!listing.topology_complete);
}
#[tokio::test]
async fn scanner_bucket_listing_marks_divergent_successful_peers_incomplete() {
let bucket = BucketInfo {
name: "bucket-below-quorum".to_string(),
..Default::default()
};
let peer_sys = S3PeerSys {
clients: vec![
test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[0], Ok(vec![bucket])),
test_peer_with_list_bucket(&[0], Ok(Vec::new())),
test_peer_with_list_bucket(&[0], Ok(Vec::new())),
],
pools_count: 1,
};
let listing = peer_sys
.list_bucket_for_scanner(&BucketOptions::default())
.await
.expect("successful peer responses should still produce a scanner candidate listing");
assert!(listing.buckets.is_empty());
assert!(!listing.topology_complete);
}
#[tokio::test]
async fn scanner_bucket_listing_checks_same_bucket_in_every_pool() {
let bucket = BucketInfo {
name: "shared-bucket".to_string(),
..Default::default()
};
let peer_sys = S3PeerSys {
clients: vec![
test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[1], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[1], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[1], Ok(Vec::new())),
test_peer_with_list_bucket(&[1], Ok(Vec::new())),
],
pools_count: 2,
};
let listing = peer_sys
.list_bucket_for_scanner(&BucketOptions::default())
.await
.expect("a bucket visible in one pool should remain a scan candidate");
assert_eq!(listing.buckets.len(), 1);
assert_eq!(listing.buckets[0].name, bucket.name);
assert!(!listing.topology_complete);
}
#[tokio::test]
async fn test_delete_bucket_fails_when_any_pool_misses_write_quorum() {
let peer_sys = S3PeerSys {
crates/ecstore/src/cluster/rpc/remote_disk.rs
+279 -6
View File
@@ -17,7 +17,8 @@ use crate::cluster::rpc::client::{
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
};
use crate::cluster::rpc::internode_data_transport::{
InternodeDataTransport, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest,
InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
WriteStreamRequest,
};
use crate::disk::error::{Error, Result};
use crate::disk::{
@@ -81,6 +82,7 @@ const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2;
const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20);
const REMOTE_DISK_OPEN_READ_MAX_ATTEMPTS: usize = 2;
const REMOTE_DISK_OPEN_READ_RETRY_BACKOFF: Duration = Duration::from_millis(20);
const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
/// Base backoff for idempotent read-only RPC retries (grpc-optimization P3-3); doubles per attempt.
const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
@@ -97,6 +99,17 @@ const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
volume_infos
.into_iter()
.enumerate()
.map(|(index, json)| {
serde_json::from_str::<VolumeInfo>(&json)
.map_err(|err| Error::other(format!("decode list volumes entry {index} failed: {err}")))
})
.collect()
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum BatchMetadataRpcMode {
Off,
@@ -264,6 +277,71 @@ fn spawn_control_channel_prewarm(addr: String) {
}
impl RemoteDisk {
pub(crate) async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let probe = self.data_transport.probe_ns_scanner(NsScannerCapabilityRequest {
endpoint: self.endpoint.grid_host(),
});
let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
.await
.map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?;
match result {
Ok(server_epoch) => Ok(Some(server_epoch)),
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
Err(DiskError::MethodNotAllowed) => Ok(None),
Err(err)
if matches!(
err.internode_http_error_kind(),
Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status))
if matches!(status.as_u16(), 404 | 405)
) =>
{
Ok(None)
}
Err(err)
if matches!(
err.internode_http_error_kind(),
Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status)) if status.as_u16() == 426
) =>
{
Ok(None)
}
Err(err) => Err(err),
}
}
pub(crate) async fn open_ns_scanner_stream(&self, request: crate::disk::NsScannerOpenRequest) -> Result<FileReader> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let crate::disk::NsScannerOpenRequest {
request_id,
server_epoch,
session_id,
session_sequence,
next_cycle,
leader_epoch,
body,
stall_timeout,
} = request;
self.data_transport
.open_ns_scanner(NsScannerStreamRequest {
endpoint: self.endpoint.grid_host(),
disk: self.disk_ref().await,
request_id,
server_epoch,
session_id,
session_sequence,
next_cycle,
leader_epoch,
body,
stall_timeout,
})
.await
}
fn recovery_monitor_span(addr: &str, endpoint: &Endpoint) -> tracing::Span {
tracing::info_span!(
"recovery-monitor",
@@ -1335,11 +1413,7 @@ impl DiskAPI for RemoteDisk {
return Err(response.error.unwrap_or_default().into());
}
let infos = response
.volume_infos
.into_iter()
.filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
.collect();
let infos = decode_volume_infos(response.volume_infos)?;
Ok(infos)
},
@@ -2722,6 +2796,19 @@ mod tests {
static INIT: Once = Once::new();
#[test]
fn list_volumes_decode_rejects_a_malformed_entry() {
let valid = serde_json::to_string(&VolumeInfo {
name: "bucket".to_string(),
created: None,
})
.expect("volume info should serialize");
let err = decode_volume_infos(vec![valid, "{".to_string()])
.expect_err("a malformed volume entry must fail the complete response");
assert!(err.to_string().contains("entry 1"));
}
#[test]
fn decoded_remote_metadata_rejects_default_like_delete_marker() {
let forged = FileInfo {
@@ -2794,14 +2881,31 @@ mod tests {
Read(ReadStreamRequest),
Write(WriteStreamRequest),
WalkDir(WalkDirStreamRequest),
NsScanner(NsScannerStreamRequest),
NsScannerProbe(NsScannerCapabilityRequest),
}
#[derive(Debug, Clone, Default)]
struct RecordingInternodeDataTransport {
calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
ns_scanner_probe_status: Arc<StdMutex<Option<u16>>>,
}
impl RecordingInternodeDataTransport {
fn with_ns_scanner_probe_status(status: u16) -> Self {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))),
}
}
fn set_ns_scanner_probe_status(&self, status: Option<u16>) {
*self
.ns_scanner_probe_status
.lock()
.expect("namespace scanner probe status lock poisoned") = status;
}
fn calls(&self) -> Vec<RecordedTransportCall> {
self.calls.lock().expect("recorded transport calls lock poisoned").clone()
}
@@ -3369,6 +3473,26 @@ mod tests {
Ok(Box::new(EmptyTestReader))
}
async fn open_ns_scanner(&self, request: NsScannerStreamRequest) -> Result<FileReader> {
self.record(RecordedTransportCall::NsScanner(request));
Ok(Box::new(EmptyTestReader))
}
async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
self.record(RecordedTransportCall::NsScannerProbe(request));
if let Some(status) = *self
.ns_scanner_probe_status
.lock()
.expect("namespace scanner probe status lock poisoned")
{
let status = reqwest::StatusCode::from_u16(status).expect("test status code should be valid");
return Err(
rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status)).into(),
);
}
Ok(Uuid::from_u128(1))
}
fn name(&self) -> &'static str {
"recording"
}
@@ -3401,6 +3525,14 @@ mod tests {
}
}
async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result<FileReader> {
panic!("open_ns_scanner should not be used in walk_dir retry test");
}
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
Ok(Uuid::from_u128(1))
}
fn name(&self) -> &'static str {
"retrying-walk-dir"
}
@@ -3429,6 +3561,14 @@ mod tests {
panic!("open_walk_dir should not be used in open_write retry test");
}
async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result<FileReader> {
panic!("open_ns_scanner should not be used in open_write retry test");
}
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
Ok(Uuid::from_u128(1))
}
fn name(&self) -> &'static str {
"retrying-open-write"
}
@@ -4092,6 +4232,139 @@ mod tests {
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_uses_configured_data_transport() {
let transport = RecordingInternodeDataTransport::default();
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let expected_disk = remote_disk.disk_ref().await;
let expected_body = b"namespace-scanner-request".to_vec();
let expected_request_id = Uuid::new_v4();
let expected_server_epoch = Uuid::new_v4();
let expected_session_id = Uuid::new_v4();
let _reader = remote_disk
.open_ns_scanner_stream(crate::disk::NsScannerOpenRequest {
request_id: expected_request_id,
server_epoch: expected_server_epoch,
session_id: expected_session_id,
session_sequence: 3,
next_cycle: 7,
leader_epoch: 9,
body: expected_body.clone(),
stall_timeout: Some(Duration::from_secs(15)),
})
.await
.expect("namespace scanner stream should open");
let calls = transport.calls();
assert_eq!(calls.len(), 1);
match &calls[0] {
RecordedTransportCall::NsScanner(request) => {
assert_eq!(request.endpoint, "http://remote-node:9000");
assert_eq!(request.disk, expected_disk);
assert_eq!(request.request_id, expected_request_id);
assert_eq!(request.server_epoch, expected_server_epoch);
assert_eq!(request.session_id, expected_session_id);
assert_eq!(request.session_sequence, 3);
assert_eq!(request.next_cycle, 7);
assert_eq!(request.leader_epoch, 9);
assert_eq!(request.body, expected_body);
assert_eq!(request.stall_timeout, Some(Duration::from_secs(15)));
}
other => panic!("expected namespace scanner transport call, got {other:?}"),
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_uses_configured_data_transport() {
let transport = RecordingInternodeDataTransport::default();
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("namespace scanner capability probe should succeed"),
Some(Uuid::from_u128(1))
);
let calls = transport.calls();
assert_eq!(calls.len(), 1);
match &calls[0] {
RecordedTransportCall::NsScannerProbe(request) => {
assert_eq!(request.endpoint, "http://remote-node:9000");
}
other => panic!("expected namespace scanner capability probe, got {other:?}"),
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_rejects_old_and_incompatible_peers() {
for status in [404, 405, 426] {
let transport = RecordingInternodeDataTransport::with_ns_scanner_probe_status(status);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("unsupported namespace scanner response should be classified"),
None
);
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_rejects_legacy_transport() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RetryingOpenReadInternodeDataTransport::default())).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("legacy transport should be classified as unsupported"),
None
);
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_reprobes_after_peer_upgrade() {
let transport = RecordingInternodeDataTransport::with_ns_scanner_probe_status(404);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("old peer should be classified as unsupported"),
None
);
transport.set_ns_scanner_probe_status(None);
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("upgraded peer should be re-probed"),
Some(Uuid::from_u128(1))
);
assert_eq!(transport.calls().len(), 2);
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_propagates_transient_failure() {
let transport = RecordingInternodeDataTransport::with_ns_scanner_probe_status(503);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await;
let err = remote_disk
.ns_scanner_server_epoch()
.await
.expect_err("transient capability failure must not be reported as unsupported");
assert!(matches!(
err.internode_http_error_kind(),
Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status)) if status.as_u16() == 503
));
}
#[tokio::test]
async fn test_remote_disk_walk_dir_preserves_skip_total_timeout_option() {
let transport = RecordingInternodeDataTransport::default();
crates/ecstore/src/config/com.rs
+1108 -49
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/data_movement/mod.rs
+16 -11
View File
@@ -560,17 +560,22 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
set.delete_object(
bucket,
cleanup_key.as_str(),
ObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
no_lock: true,
..Default::default()
},
)
.await
let result = set
.delete_object(
bucket,
cleanup_key.as_str(),
ObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
no_lock: true,
..Default::default()
},
)
.await;
if result.is_ok() {
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
result
}
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
crates/ecstore/src/data_usage/mod.rs
+1304 -62
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/disk/error.rs
+18
View File
@@ -231,6 +231,13 @@ impl DiskError {
}
}
pub fn is_internode_http_status(&self, status: u16) -> bool {
matches!(
self.internode_http_error_kind(),
Some(InternodeHttpErrorKind::HttpStatus(actual)) if actual.as_u16() == status
)
}
// /// If all errors are of the same fatal disk error type, returns the corresponding error.
// /// Otherwise, returns Ok.
// pub fn check_disk_fatal_errs(errs: &[Option<Error>]) -> Result<()> {
@@ -981,6 +988,17 @@ mod tests {
assert!(!disk_error.contains_io_error_kind(std::io::ErrorKind::TimedOut));
}
#[test]
fn test_internode_http_status_classification() {
let too_many_requests = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::TOO_MANY_REQUESTS),
));
assert!(too_many_requests.is_internode_http_status(429));
assert!(!too_many_requests.is_internode_http_status(500));
assert!(!DiskError::FileNotFound.is_internode_http_status(429));
}
#[test]
fn test_metacache_output_stream_closed_classification_survives_clone() {
let disk_error = DiskError::metacache_output_stream_closed();
crates/ecstore/src/disk/mod.rs
+27 -1
View File
@@ -52,7 +52,7 @@ use local::LocalDisk;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_madmin::info_commands::DiskMetrics;
use serde::{Deserialize, Serialize};
use std::{fmt::Debug, path::PathBuf, sync::Arc};
use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncWrite};
use uuid::Uuid;
@@ -453,6 +453,20 @@ impl DiskAPI for Disk {
}
impl Disk {
pub async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
match self {
Disk::Local(_) => Ok(None),
Disk::Remote(remote_disk) => remote_disk.ns_scanner_server_epoch().await,
}
}
pub async fn open_ns_scanner_stream(&self, request: NsScannerOpenRequest) -> Result<FileReader> {
match self {
Disk::Remote(remote_disk) => remote_disk.open_ns_scanner_stream(request).await,
Disk::Local(_) => Err(Error::other("namespace scanner stream requires a remote disk")),
}
}
pub fn runtime_state(&self) -> RuntimeDriveHealthState {
match self {
Disk::Local(local_disk) => local_disk.runtime_state(),
@@ -498,6 +512,18 @@ impl Disk {
}
}
#[derive(Debug)]
pub struct NsScannerOpenRequest {
pub request_id: Uuid,
pub server_epoch: Uuid,
pub session_id: Uuid,
pub session_sequence: u64,
pub next_cycle: u64,
pub leader_epoch: u64,
pub body: Vec<u8>,
pub stall_timeout: Option<Duration>,
}
impl Disk {
/// Reset drive health so `connect_load_init_formats` retries are not blocked by a prior
/// transient mark-faulty (same disk handles are reused across retries).
crates/ecstore/src/object_api/readers.rs
+8
View File
@@ -471,6 +471,14 @@ impl PutObjReader {
}
}
pub fn from_prehashed_bytes(data: Bytes, sha256hex: Option<String>) -> std::io::Result<Self> {
let content_length =
i64::try_from(data.len()).map_err(|_| std::io::Error::other("prehashed object payload exceeds i64 length"))?;
Ok(PutObjReader {
stream: HashReader::from_stream(Cursor::new(data), content_length, content_length, None, sha256hex, false)?,
})
}
pub fn size(&self) -> i64 {
self.stream.size()
}
crates/ecstore/src/services/metrics_realtime.rs
+3
View File
@@ -154,6 +154,7 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
failed_cycles: metrics.failed_cycles,
superseded_cycles: metrics.superseded_cycles,
partial_cycles_unknown: metrics.partial_cycles_unknown,
partial_cycles_runtime: metrics.partial_cycles_runtime,
partial_cycles_objects: metrics.partial_cycles_objects,
@@ -796,6 +797,7 @@ mod test {
last_cycle_replication_checks: 27,
last_cycle_usage_saves: 28,
failed_cycles: 29,
superseded_cycles: 30,
partial_cycles_unknown: 30,
partial_cycles_runtime: 31,
partial_cycles_objects: 32,
@@ -927,6 +929,7 @@ mod test {
assert_eq!(scanner.last_cycle_replication_checks, 27);
assert_eq!(scanner.last_cycle_usage_saves, 28);
assert_eq!(scanner.failed_cycles, 29);
assert_eq!(scanner.superseded_cycles, 30);
assert_eq!(scanner.partial_cycles_unknown, 30);
assert_eq!(scanner.partial_cycles_runtime, 31);
assert_eq!(scanner.partial_cycles_objects, 32);
crates/ecstore/src/services/notification_sys.rs
+124 -1
View File
@@ -29,7 +29,7 @@ use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_madmin::net::NetInfo;
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
use rustfs_utils::XHost;
use std::collections::hash_map::DefaultHasher;
use std::collections::{HashMap, hash_map::DefaultHasher};
use std::future::Future;
use std::hash::{Hash, Hasher};
use std::sync::{Arc, Mutex, OnceLock};
@@ -1041,6 +1041,40 @@ impl NotificationSys {
Ok(generations)
}
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<(String, String, u64)>) -> Result<bool> {
let mut by_host = HashMap::with_capacity(acknowledgements.len());
for (host, instance_id, generation) in acknowledgements {
if by_host.insert(host.clone(), (instance_id, generation)).is_some() {
return Err(Error::other(format!("duplicate scanner dirty usage acknowledgement target: {host}")));
}
}
let clients = self
.peer_clients
.iter()
.flatten()
.map(|client| (client.grid_host.clone(), client.clone()))
.collect::<HashMap<_, _>>();
let mut failures = Vec::new();
let mut futures = Vec::with_capacity(by_host.len());
for (host, (instance_id, generation)) in by_host {
let Some(client) = clients.get(&host).cloned() else {
failures.push(format!("peer {host} scanner dirty usage acknowledgement failed: peer is not reachable"));
continue;
};
futures.push(async move {
let result = scanner_activity_with_timeout(
SCANNER_ACTIVITY_PROBE_TIMEOUT,
&host,
client.acknowledge_scanner_dirty_usage(instance_id, generation),
)
.await;
(host, result)
});
}
aggregate_scanner_dirty_usage_acknowledgement_results(join_all(futures).await, failures)
}
pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
let mut futures = Vec::with_capacity(self.peer_clients.len());
for client in self.peer_clients.iter() {
@@ -1591,6 +1625,23 @@ fn aggregate_notification_failures(operation: &str, failures: Vec<String>) -> Re
)))
}
fn aggregate_scanner_dirty_usage_acknowledgement_results(
results: Vec<(String, Result<ScannerPeerActivity>)>,
mut failures: Vec<String>,
) -> Result<bool> {
let mut dirty_usage_pending = false;
for (host, result) in results {
match result {
Ok(activity) => {
dirty_usage_pending |= activity.dirty_usage_pending != Some(false);
}
Err(err) => failures.push(format!("peer {host} scanner dirty usage acknowledgement failed: {err}")),
}
}
aggregate_notification_failures("acknowledge_scanner_dirty_usage", failures)?;
Ok(dirty_usage_pending)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1842,6 +1893,78 @@ mod tests {
assert!(err.to_string().contains("peer-1"));
}
#[tokio::test]
async fn scanner_dirty_usage_acknowledgement_rejects_missing_and_duplicate_targets() {
let sys = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: Vec::new(),
peer_admin_caches: Vec::new(),
peer_topology_hosts: Vec::new(),
};
let missing = sys
.acknowledge_scanner_dirty_usage(vec![("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7)])
.await
.expect_err("a missing acknowledgement target must remain pending");
assert!(missing.to_string().contains("peer is not reachable"));
let duplicate = sys
.acknowledge_scanner_dirty_usage(vec![
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
])
.await
.expect_err("duplicate acknowledgement targets must be rejected");
assert!(
duplicate
.to_string()
.contains("duplicate scanner dirty usage acknowledgement target")
);
}
#[test]
fn scanner_dirty_usage_acknowledgement_preserves_newer_pending_work() {
let activity = |dirty_usage_pending| ScannerPeerActivity {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 1,
maintenance_generation: 1,
protocol_version: crate::storage_api_contracts::internode::SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: Some([0; 32]),
data_movement_active: Some(false),
dirty_usage_generation: Some(2),
dirty_usage_pending,
};
let pending = aggregate_scanner_dirty_usage_acknowledgement_results(
vec![
("peer-1".to_string(), Ok(activity(Some(false)))),
("peer-2".to_string(), Ok(activity(Some(true)))),
],
Vec::new(),
)
.expect("successful acknowledgements should return their pending state");
assert!(pending, "new dirty usage reported by an acknowledged peer must remain pending");
let cleared = aggregate_scanner_dirty_usage_acknowledgement_results(
vec![("peer-1".to_string(), Ok(activity(Some(false))))],
Vec::new(),
)
.expect("a cleared acknowledgement should succeed");
assert!(!cleared, "an explicitly cleared peer must not remain pending");
let unknown =
aggregate_scanner_dirty_usage_acknowledgement_results(vec![("peer-1".to_string(), Ok(activity(None)))], Vec::new())
.expect("an acknowledgement without a pending field should remain retryable");
assert!(unknown, "a peer that cannot prove its dirty state is clear must remain pending");
let err = aggregate_scanner_dirty_usage_acknowledgement_results(
vec![("peer-1".to_string(), Err(Error::other("injected acknowledgement failure")))],
Vec::new(),
)
.expect_err("a reachable peer acknowledgement failure must be reported");
assert!(err.to_string().contains("peer-1"));
assert!(err.to_string().contains("injected acknowledgement failure"));
}
#[tokio::test]
async fn load_bucket_metadata_reports_unreachable_peers() {
let sys = NotificationSys {
crates/ecstore/src/set_disk/core/io_primitives.rs
+55 -13
View File
@@ -3862,6 +3862,24 @@ impl SetDisks {
Ok(removed)
}
fn write_precondition_lookup_error(
error: StorageError,
http_preconditions: &HTTPPreconditions,
bucket: &str,
object: &str,
) -> Option<StorageError> {
match error {
StorageError::VersionNotFound(_, _, _) | StorageError::ObjectNotFound(_, _) => {
if http_preconditions.if_match_value().is_some() {
Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()))
} else {
None
}
}
error => Some(error),
}
}
pub(in crate::set_disk) async fn check_write_precondition(
&self,
bucket: &str,
@@ -3892,19 +3910,8 @@ impl SetDisks {
}
}
Err(StorageError::VersionNotFound(_, _, _))
| Err(StorageError::ObjectNotFound(_, _))
| Err(StorageError::ErasureReadQuorum) => {
// When the object is not found,
// - if If-Match is set, we should return 404 NotFound
// - if If-None-Match is set, we should be able to proceed with the request
if http_preconditions.if_match_value().is_some() {
return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()));
}
}
Err(e) => {
return Some(e);
Err(error) => {
return Self::write_precondition_lookup_error(error, &http_preconditions, bucket, object);
}
}
@@ -4407,6 +4414,41 @@ mod tests {
use tempfile::TempDir;
use tokio::io::AsyncReadExt;
#[test]
fn write_precondition_lookup_errors_fail_closed_unless_absence_is_known() {
let create_only = HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
};
let replace_only = HTTPPreconditions {
if_match: Some("etag".to_string()),
..Default::default()
};
assert!(matches!(
SetDisks::write_precondition_lookup_error(StorageError::ErasureReadQuorum, &create_only, "bucket", "object",),
Some(StorageError::ErasureReadQuorum)
));
assert!(
SetDisks::write_precondition_lookup_error(
StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
&create_only,
"bucket",
"object",
)
.is_none()
);
assert!(matches!(
SetDisks::write_precondition_lookup_error(
StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
&replace_only,
"bucket",
"object",
),
Some(StorageError::ObjectNotFound(_, _))
));
}
fn metadata_test_fileinfo(object: &str) -> FileInfo {
let mut fi = FileInfo::new(object, 2, 2);
fi.volume = "bucket".to_string();
crates/ecstore/src/set_disk/mod.rs
+42
View File
@@ -9275,6 +9275,48 @@ mod tests {
));
}
#[tokio::test]
async fn set_level_if_none_match_fails_closed_without_read_quorum() {
let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
let bucket = "bucket-write-precondition-quorum";
let object = "existing-object.txt";
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before disk loss");
let mut reader = PutObjReader::from_vec(b"existing object body".to_vec());
set_disks
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("object should be written before disk loss");
{
let mut disks = set_disks.disks.write().await;
disks[1..].fill(None);
}
let create_only = ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
}),
..Default::default()
};
let result = set_disks.check_write_precondition(bucket, object, &create_only).await;
assert!(
matches!(result, Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))),
"expected read-quorum failure, got {result:?}"
);
}
#[tokio::test]
async fn set_level_versioned_delete_marker_hides_object_without_corrupting_version_metadata() {
let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
crates/ecstore/src/set_disk/ops/bucket.rs
+67 -59
View File
@@ -21,6 +21,71 @@
use super::super::*;
impl SetDisks {
pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec<BucketInfo>, bool)> {
let disks = self.disk_inventory().await;
let write_quorum = (disks.len() / 2) + 1;
let mut futures = Vec::with_capacity(disks.len());
for disk in disks {
futures.push(async move {
match disk {
Some(disk) => disk.list_volumes().await,
None => Err(DiskError::DiskNotFound),
}
});
}
let results = join_all(futures).await;
let mut topology_complete = results.iter().all(|result| result.is_ok());
let mut infos = Vec::with_capacity(results.len());
let mut errs = Vec::with_capacity(results.len());
for result in results {
match result {
Ok(volumes) => {
infos.push(Some(volumes));
errs.push(None);
}
Err(err) => {
infos.push(None);
errs.push(Some(err));
}
}
}
if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
return Err(err.into());
}
let mut counts: HashMap<String, (usize, BucketInfo)> = HashMap::new();
for volumes in infos.into_iter().flatten() {
for volume in volumes {
if is_reserved_or_invalid_bucket(&volume.name, false) {
continue;
}
let entry = counts.entry(volume.name.clone()).or_insert((
0,
BucketInfo {
name: volume.name.clone(),
created: volume.created,
..Default::default()
},
));
entry.0 += 1;
}
}
topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum);
let mut buckets = counts
.into_values()
.filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket))
.collect::<Vec<_>>();
buckets.sort_by(|left, right| left.name.cmp(&right.name));
Ok((buckets, topology_complete))
}
}
#[async_trait::async_trait]
impl BucketOperations for SetDisks {
type Error = Error;
@@ -117,65 +182,8 @@ impl BucketOperations for SetDisks {
}
#[tracing::instrument(skip(self))]
async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
let disks = self.disk_inventory().await;
let write_quorum = (disks.len() / 2) + 1;
let mut futures = Vec::with_capacity(disks.len());
for disk in disks {
futures.push(async move {
match disk {
Some(disk) => disk.list_volumes().await,
None => Err(DiskError::DiskNotFound),
}
});
}
let results = join_all(futures).await;
let mut infos = Vec::with_capacity(results.len());
let mut errs = Vec::with_capacity(results.len());
for result in results {
match result {
Ok(volumes) => {
infos.push(Some(volumes));
errs.push(None);
}
Err(err) => {
infos.push(None);
errs.push(Some(err));
}
}
}
if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
return Err(err.into());
}
let mut counts: HashMap<String, (usize, BucketInfo)> = HashMap::new();
for volumes in infos.into_iter().flatten() {
for volume in volumes {
if is_reserved_or_invalid_bucket(&volume.name, false) {
continue;
}
let entry = counts.entry(volume.name.clone()).or_insert((
0,
BucketInfo {
name: volume.name.clone(),
created: volume.created,
..Default::default()
},
));
entry.0 += 1;
}
}
let mut buckets = counts
.into_values()
.filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket))
.collect::<Vec<_>>();
buckets.sort_by(|left, right| left.name.cmp(&right.name));
Ok(buckets)
async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
Ok(self.list_bucket_for_scanner(opts).await?.0)
}
#[tracing::instrument(skip(self))]
crates/ecstore/src/storage_api_contracts/mod.rs
+6 -1
View File
@@ -23,7 +23,12 @@ pub(crate) mod heal {
pub(crate) mod internode {
pub(crate) use rustfs_storage_api::{
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY,
NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY,
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
};
}
crates/ecstore/src/store/bucket.rs
+746 -30
View File
@@ -17,12 +17,21 @@ use crate::bucket::{
metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix},
utils::is_meta_bucketname,
};
use crate::error::is_err_bucket_not_found;
use crate::runtime::sources as runtime_sources;
use crate::set_disk::get_lock_acquire_timeout;
use crate::storage_api_contracts::bucket::SRBucketDeleteOp;
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use futures::stream::{self, StreamExt};
use std::collections::BTreeMap;
use std::future::Future;
const DELETED_BUCKETS_PREFIX: &str = ".deleted";
const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
fn scanner_bucket_list_set_concurrency(set_count: usize) -> usize {
set_count.clamp(1, SCANNER_BUCKET_LIST_SET_CONCURRENCY)
}
fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
created != OffsetDateTime::UNIX_EPOCH
@@ -78,6 +87,49 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
}
async fn await_bucket_namespace_operation<T, F>(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
operation: &'static str,
future: F,
) -> Result<T>
where
F: Future<Output = Result<T>>,
{
let Some(guard) = guard else {
return future.await;
};
if guard.is_lock_lost() {
return Err(StorageError::other(format!(
"bucket namespace lock was lost before {operation}: {bucket}"
)));
}
tokio::select! {
biased;
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
"bucket namespace lock was lost during {operation}: {bucket}"
))),
result = future => result,
}
}
async fn run_bucket_usage_cleanup<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
where
F: Future<Output = Result<()>>,
{
await_bucket_namespace_operation(guard, bucket, "bucket usage cleanup", future).await
}
async fn run_physical_bucket_deletion<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
where
F: Future<Output = Result<()>>,
{
// Fence before polling deletion: the physical namespace may become
// invisible at any await point inside the storage operation.
list_objects::observe_scanner_namespace_mutations(bucket, 1);
await_bucket_namespace_operation(guard, bucket, "physical bucket deletion", future).await
}
impl ECStore {
async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> {
let marker_volume = bucket_deleted_marker_volume(bucket);
@@ -96,21 +148,84 @@ impl ECStore {
Ok(())
}
async fn cleanup_deleted_bucket_metadata(&self, bucket: &str, include_deleted_marker: bool) -> Result<()> {
async fn cleanup_deleted_bucket_metadata(
&self,
bucket: &str,
include_deleted_marker: bool,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<()> {
for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()).await?;
await_bucket_namespace_operation(
guard,
bucket,
"deleted bucket metadata cleanup",
self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()),
)
.await?;
}
if include_deleted_marker {
let marker_prefix = bucket_deleted_marker_prefix(bucket);
self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()).await?;
await_bucket_namespace_operation(
guard,
bucket,
"deleted bucket marker cleanup",
self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()),
)
.await?;
}
metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket).await?;
await_bucket_namespace_operation(
guard,
bucket,
"deleted bucket metadata cache cleanup",
metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket),
)
.await?;
runtime_sources::delete_bucket_monitor_entry(bucket);
Ok(())
}
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
run_bucket_usage_cleanup(guard, bucket, async {
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await
})
.await
}
async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
if let Err(err) = self.cleanup_bucket_usage(bucket, guard).await {
warn!(
bucket = %bucket,
error = ?err,
"bucket data usage cleanup deferred to scanner reconciliation"
);
}
}
async fn rollback_failed_bucket_creation(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
let rollback_opts = DeleteBucketOptions {
no_lock: true,
no_recreate: true,
..Default::default()
};
if let Err(err) = await_bucket_namespace_operation(guard, bucket, "failed bucket creation rollback", async {
self.peer_sys
.delete_bucket(bucket, &rollback_opts)
.await
.map_err(|rollback_err| to_object_err(rollback_err.into(), vec![bucket]))
})
.await
{
warn!(
bucket = %bucket,
error = ?err,
"failed bucket creation rollback did not remove every physical bucket volume"
);
}
}
#[instrument(skip(self))]
pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
if !is_meta_bucketname(bucket)
@@ -119,7 +234,7 @@ impl ECStore {
return Err(StorageError::BucketNameInvalid(err.to_string()));
}
let _ns_guard = if !opts.no_lock {
let ns_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
Some(
ns_lock
@@ -142,8 +257,34 @@ impl ECStore {
None
};
if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await {
let err = to_object_err(err.into(), vec![bucket]);
let confirmed_missing = match self.peer_sys.get_bucket_info(bucket, &BucketOptions::default()).await {
Ok(_) => false,
Err(err) => {
let err: StorageError = err.into();
if is_err_bucket_not_found(&err) {
true
} else {
return Err(to_object_err(err, vec![bucket]));
}
}
};
if confirmed_missing && !is_meta_bucketname(bucket) {
// Fence every scanner cycle that could have observed the namespace
// before physical creation. Creation may become visible even when a
// later metadata write or namespace-lock check fails.
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?;
}
if let Err(err) = await_bucket_namespace_operation(ns_guard.as_ref(), bucket, "physical bucket creation", async {
self.peer_sys
.make_bucket(bucket, opts)
.await
.map_err(|err| to_object_err(err.into(), vec![bucket]))
})
.await
{
if is_err_bucket_exists(&err)
&& let Err(heal_err) = self
.handle_heal_bucket(
@@ -157,18 +298,9 @@ impl ECStore {
{
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
}
if !is_err_bucket_exists(&err) {
if !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
error!("make bucket failed: {err}");
let _ = self
.delete_bucket(
bucket,
&DeleteBucketOptions {
no_lock: true,
no_recreate: true,
..Default::default()
},
)
.await;
self.rollback_failed_bucket_creation(bucket, ns_guard.as_ref()).await;
}
return Err(err);
};
@@ -186,7 +318,13 @@ impl ECStore {
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
}
metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await?;
await_bucket_namespace_operation(
ns_guard.as_ref(),
bucket,
"bucket metadata initialization",
metadata_sys::set_bucket_metadata_in(&self.ctx, meta),
)
.await?;
Ok(())
}
@@ -224,6 +362,80 @@ impl ECStore {
Ok(buckets)
}
pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result<crate::cluster::rpc::ScannerBucketListing> {
let sets = self
.pools
.iter()
.flat_map(|pool| {
pool.disk_set
.iter()
.map(|set| (set.pool_index, set.set_index, Arc::clone(set)))
})
.collect::<Vec<_>>();
let set_count = sets.len();
let deleted = opts.deleted;
let cached = opts.cached;
let no_metadata = opts.no_metadata;
let mut set_listings = stream::iter(sets.into_iter().map(move |(pool_index, set_index, set)| {
let opts = BucketOptions {
deleted,
cached,
no_metadata,
};
async move {
set.list_bucket_for_scanner(&opts)
.await
.map(|(buckets, complete)| (pool_index, set_index, buckets, complete))
}
}))
.buffer_unordered(scanner_bucket_list_set_concurrency(set_count));
let mut topology_complete = set_count != 0;
let mut bucket_map = BTreeMap::<String, BucketInfo>::new();
let mut scoped_buckets = Vec::with_capacity(set_count);
while let Some(set_listing) = set_listings.next().await {
let (pool_index, set_index, buckets, set_complete) = set_listing?;
topology_complete &= set_complete;
for bucket in &buckets {
bucket_map.entry(bucket.name.clone()).or_insert_with(|| bucket.clone());
}
scoped_buckets.push(crate::cluster::rpc::ScannerSetBucketListing {
pool_index,
set_index,
buckets,
});
}
scoped_buckets.sort_unstable_by_key(|scope| (scope.pool_index, scope.set_index));
let mut listing = crate::cluster::rpc::ScannerBucketListing {
buckets: bucket_map.into_values().collect(),
set_buckets: scoped_buckets,
topology_complete,
};
if !opts.no_metadata {
for bucket in &mut listing.buckets {
if let Ok(created) = metadata_sys::created_at_in(&self.ctx, &bucket.name).await
&& should_override_created_from_metadata(created)
{
bucket.created = Some(created);
}
}
let created_by_bucket = listing
.buckets
.iter()
.map(|bucket| (bucket.name.as_str(), bucket.created))
.collect::<BTreeMap<_, _>>();
for scope in &mut listing.set_buckets {
for bucket in &mut scope.buckets {
if let Some(created) = created_by_bucket.get(bucket.name.as_str()) {
bucket.created = *created;
}
}
}
}
Ok(listing)
}
#[instrument(skip(self))]
pub(super) async fn handle_delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
if is_meta_bucketname(bucket) {
@@ -234,7 +446,7 @@ impl ECStore {
return Err(StorageError::BucketNameInvalid(err.to_string()));
}
let _ns_guard = if !opts.no_lock {
let ns_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
Some(
ns_lock
@@ -299,17 +511,40 @@ impl ECStore {
}
if sr_mark_delete {
self.mark_bucket_deleted(bucket).await?;
await_bucket_namespace_operation(
ns_guard.as_ref(),
bucket,
"bucket delete marker creation",
self.mark_bucket_deleted(bucket),
)
.await?;
}
if let Err(err) = self.peer_sys.delete_bucket(bucket, &delete_opts).await {
let storage_err = to_object_err(err.into(), vec![bucket]);
if !sr_delete || !is_err_strict_volume_not_found(&storage_err) {
return Err(storage_err);
}
let delete_result = run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
self.peer_sys
.delete_bucket(bucket, &delete_opts)
.await
.map_err(|err| to_object_err(err.into(), vec![bucket]))
})
.await;
if let Err(err) = delete_result
&& (!sr_delete || !is_err_strict_volume_not_found(&err))
{
return Err(err);
}
self.cleanup_deleted_bucket_metadata(bucket, sr_purge).await?;
self.cleanup_bucket_usage_best_effort(bucket, ns_guard.as_ref()).await;
if let Err(err) = self
.cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_ref())
.await
{
warn!(
bucket = %bucket,
error = ?err,
"physical bucket deletion succeeded but metadata cleanup remains pending"
);
}
Ok(())
}
}
@@ -317,8 +552,9 @@ impl ECStore {
#[cfg(test)]
mod tests {
use super::{
bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix, bucket_deleted_marker_volume,
should_override_created_from_metadata, validate_table_bucket_delete_allowed,
SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes,
bucket_deleted_marker_prefix, bucket_deleted_marker_volume, run_bucket_usage_cleanup, run_physical_bucket_deletion,
scanner_bucket_list_set_concurrency, should_override_created_from_metadata, validate_table_bucket_delete_allowed,
};
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
use crate::bucket::metadata_sys;
@@ -335,16 +571,146 @@ mod tests {
disk::endpoint::Endpoint,
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
};
use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
use serial_test::serial;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, SystemTime};
use time::OffsetDateTime;
use tokio::sync::OnceCell;
use tokio::sync::{Notify, OnceCell};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
#[tokio::test(start_paused = true)]
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
let ttl = Duration::from_millis(20);
let lock = NamespaceLock::new("bucket-operation-test".to_string(), Arc::new(LocalClient::new()));
let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
.with_acquire_timeout(Duration::from_secs(1))
.with_ttl(ttl)
.with_refresh_interval(ttl);
let guard = lock
.acquire_guard(&request)
.await
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired");
tokio::time::advance(ttl + Duration::from_millis(1)).await;
let operation_ran = Arc::new(AtomicBool::new(false));
let operation_ran_for_future = operation_ran.clone();
let result = await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
operation_ran_for_future.store(true, Ordering::SeqCst);
Ok(())
})
.await;
assert!(result.is_err(), "an expired namespace lease must fence the operation");
assert!(
!operation_ran.load(Ordering::SeqCst),
"a fenced namespace operation must not poll its mutation future"
);
}
#[tokio::test]
#[serial]
async fn physical_bucket_delete_fences_scanner_before_polling_storage() {
let generation_before = crate::store::list_objects::scanner_namespace_mutation_generation();
let storage_polled = Arc::new(AtomicBool::new(false));
let storage_polled_for_future = storage_polled.clone();
run_physical_bucket_deletion(None, "generation-order-bucket", async move {
assert!(
crate::store::list_objects::scanner_namespace_mutation_generation() > generation_before,
"scanner generation must advance before physical deletion is polled"
);
storage_polled_for_future.store(true, Ordering::SeqCst);
Ok(())
})
.await
.expect("synthetic physical deletion should succeed");
assert!(storage_polled.load(Ordering::SeqCst));
}
#[tokio::test]
async fn bucket_usage_cleanup_stops_after_parent_cancellation() {
let started = Arc::new(Notify::new());
let started_wait = started.notified();
let release = Arc::new(Notify::new());
let completed = Arc::new(AtomicBool::new(false));
let started_for_cleanup = started.clone();
let release_for_cleanup = release.clone();
let completed_for_cleanup = completed.clone();
let parent = tokio::spawn(run_bucket_usage_cleanup(None, "bucket", async move {
started_for_cleanup.notify_one();
release_for_cleanup.notified().await;
completed_for_cleanup.store(true, Ordering::SeqCst);
Ok(())
}));
started_wait.await;
parent.abort();
let _ = parent.await;
tokio::task::yield_now().await;
release.notify_waiters();
tokio::task::yield_now().await;
assert!(
!completed.load(Ordering::SeqCst),
"a cancelled cleanup future must not continue in a detached task"
);
}
#[tokio::test(start_paused = true)]
#[serial]
async fn bucket_namespace_operation_stops_in_flight_work_after_lock_loss() {
let ttl = Duration::from_millis(20);
let lock = NamespaceLock::new("bucket-operation-in-flight-loss-test".to_string(), Arc::new(LocalClient::new()));
let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
.with_acquire_timeout(Duration::from_secs(1))
.with_ttl(ttl)
.with_refresh_interval(ttl);
let guard = lock
.acquire_guard(&request)
.await
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired");
let operation_started = Arc::new(Notify::new());
let started_wait = operation_started.notified();
let operation_release = Arc::new(Notify::new());
let operation_completed = Arc::new(AtomicBool::new(false));
let started_for_operation = operation_started.clone();
let release_for_operation = operation_release.clone();
let completed_for_operation = operation_completed.clone();
let task = tokio::spawn(async move {
await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
started_for_operation.notify_one();
release_for_operation.notified().await;
completed_for_operation.store(true, Ordering::SeqCst);
Ok(())
})
.await
});
started_wait.await;
tokio::time::advance(ttl + Duration::from_millis(1)).await;
let err = task
.await
.expect("operation task should join")
.expect_err("an operation still running after lease loss must be fenced");
assert!(err.to_string().contains("namespace lock was lost during test operation"));
operation_release.notify_waiters();
tokio::task::yield_now().await;
assert!(
!operation_completed.load(Ordering::SeqCst),
"lock loss must stop the old owner before a successor can acquire the namespace"
);
}
async fn setup_bucket_delete_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
BUCKET_DELETE_TEST_ENV
.get_or_init(|| async {
@@ -413,6 +779,58 @@ mod tests {
.clone()
}
async fn setup_multi_pool_scanner_listing_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
let temp_dir = tempfile::tempdir().expect("multi-pool scanner test directory should be created");
let mut pools = Vec::new();
for pool_index in 0..2 {
let mut endpoints = Vec::new();
for disk_index in 0..4 {
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
tokio::fs::create_dir_all(&disk_path)
.await
.expect("multi-pool scanner test disk should be created");
let mut endpoint =
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
endpoints.push(endpoint);
}
pools.push(PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("scanner-listing-pool-{pool_index}"),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
});
}
let endpoint_pools = EndpointServerPools(pools);
let instance_ctx = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("multi-pool local disks should initialize");
let ecstore = ECStore::new_with_instance_ctx(
"127.0.0.1:0".parse().expect("test address"),
endpoint_pools,
CancellationToken::new(),
instance_ctx,
)
.await
.expect("multi-pool ECStore should initialize");
let storage_class =
crate::config::storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4, 4])
.expect("multi-pool storage class should match both four-disk pools");
for pool in &ecstore.pools {
for set in &pool.disk_set {
set.set_test_storage_class_config(storage_class.clone());
}
}
(temp_dir, ecstore)
}
async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
let generation_before_make = ecstore.scanner_namespace_mutation_generation();
ecstore
@@ -515,6 +933,98 @@ mod tests {
);
}
#[test]
fn scanner_bucket_listing_bounds_set_fanout() {
assert_eq!(scanner_bucket_list_set_concurrency(0), 1);
assert_eq!(scanner_bucket_list_set_concurrency(2), 2);
assert_eq!(scanner_bucket_list_set_concurrency(100), SCANNER_BUCKET_LIST_SET_CONCURRENCY);
}
#[tokio::test]
#[serial]
async fn scanner_bucket_listing_unions_every_erasure_set() {
let (_temp_dir, ecstore) = setup_multi_pool_scanner_listing_test_env().await;
let bucket = format!("second-pool-only-{}", Uuid::new_v4().simple());
ecstore.pools[1].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created in the second pool only");
let listing = ecstore
.list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("scanner should enumerate every pool and set");
assert!(listing.topology_complete);
assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
assert_eq!(listing.set_buckets.len(), 2);
assert!(
listing
.set_buckets
.iter()
.find(|scope| scope.pool_index == 0 && scope.set_index == 0)
.is_some_and(|scope| scope.buckets.is_empty())
);
assert!(
listing
.set_buckets
.iter()
.find(|scope| scope.pool_index == 1 && scope.set_index == 0)
.is_some_and(|scope| scope.buckets.iter().any(|entry| entry.name == bucket))
);
}
#[tokio::test]
async fn scanner_bucket_listing_marks_degraded_set_incomplete() {
let (_temp_dir, ecstore) = setup_multi_pool_scanner_listing_test_env().await;
let bucket = format!("degraded-set-{}", Uuid::new_v4().simple());
let set = &ecstore.pools[0].disk_set[0];
set.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before a disk is removed");
set.disks.write().await[0] = None;
let listing = ecstore
.list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("a degraded set with quorum should still return candidate buckets");
assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
assert!(!listing.topology_complete);
}
#[tokio::test]
async fn scanner_bucket_listing_marks_divergent_disk_views_incomplete() {
let (temp_dir, ecstore) = setup_multi_pool_scanner_listing_test_env().await;
let bucket = format!("divergent-set-{}", Uuid::new_v4().simple());
ecstore.pools[0].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before disk views diverge");
for disk_index in 0..2 {
tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool0-disk{disk_index}")).join(&bucket))
.await
.expect("test bucket directory should be removed from a minority disk view");
}
let listing = ecstore
.list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("responsive disks should still produce a scanner candidate listing");
assert!(listing.buckets.iter().all(|entry| entry.name != bucket));
assert!(!listing.topology_complete);
}
// These tests share one isolated instance and mutate its bucket metadata;
// serialize them so their assertions cannot observe each other's operations.
#[tokio::test]
@@ -736,4 +1246,210 @@ mod tests {
"failed default S3 DeleteBucket must keep metadata cache"
);
}
#[tokio::test]
#[serial]
async fn bucket_delete_finishes_usage_cleanup_before_same_name_recreation() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut snapshot = DataUsageInfo {
last_update: Some(SystemTime::now()),
buckets_count: 1,
..Default::default()
};
snapshot.buckets_usage.insert(
bucket.clone(),
BucketUsageInfo {
size: 42,
objects_count: 1,
versions_count: 1,
..Default::default()
},
);
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
.await
.expect("usage snapshot should be stored");
ecstore
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("empty bucket should be deleted");
let deleted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("usage snapshot should remain readable");
assert!(!deleted.buckets_usage.contains_key(&bucket));
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("same bucket name should be recreated after delete returns");
crate::data_usage::record_bucket_object_write_memory(&bucket, None, 84).await;
let mut recreated = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("recreated bucket usage base should load");
crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
assert_eq!(
recreated
.buckets_usage
.get(&bucket)
.map(|usage| (usage.objects_count, usage.versions_count, usage.size)),
Some((1, 1, 84))
);
}
#[tokio::test]
#[serial]
async fn bucket_create_removes_stale_usage_before_physical_creation() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-create-stale-usage-{}", Uuid::new_v4().simple());
let mut snapshot = DataUsageInfo {
last_update: Some(SystemTime::now()),
buckets_count: 1,
..Default::default()
};
snapshot.buckets_usage.insert(
bucket.clone(),
BucketUsageInfo {
size: 42,
objects_count: 1,
versions_count: 1,
..Default::default()
},
);
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
.await
.expect("stale usage fixture should be stored");
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("CreateBucket should succeed");
let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("usage snapshot should remain readable");
assert!(
!persisted.buckets_usage.contains_key(&bucket),
"a newly created bucket must not inherit the predecessor generation's usage"
);
assert!(
ecstore.get_bucket_info(&bucket, &BucketOptions::default()).await.is_ok(),
"physical creation should happen after the usage fence succeeds"
);
}
#[tokio::test]
#[serial]
async fn failed_create_rollback_does_not_run_unfenced_usage_cleanup() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-create-rollback-{}", Uuid::new_v4().simple());
ecstore
.peer_sys
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("the partial-create fixture should expose a physical bucket");
let mut snapshot = DataUsageInfo {
last_update: Some(SystemTime::now()),
buckets_count: 1,
..Default::default()
};
snapshot.buckets_usage.insert(
bucket.clone(),
BucketUsageInfo {
size: 42,
objects_count: 1,
versions_count: 1,
..Default::default()
},
);
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
.await
.expect("the usage fixture should be stored");
ecstore.rollback_failed_bucket_creation(&bucket, None).await;
assert!(
ecstore
.peer_sys
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.is_err(),
"failed-create rollback should remove the partial physical bucket"
);
let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("the usage snapshot should remain readable");
assert!(
persisted.buckets_usage.contains_key(&bucket),
"failed-create rollback must not start an unfenced usage cleanup"
);
crate::data_usage::store_data_usage_in_backend(DataUsageInfo::default(), ecstore.clone())
.await
.expect("the rollback usage fixture should be cleared");
}
#[tokio::test]
#[serial]
async fn bucket_create_fails_closed_when_usage_snapshot_cannot_be_fenced() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let deleted_bucket = format!("bucket-delete-corrupt-usage-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&deleted_bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before corrupting usage");
let usage_path = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}");
crate::config::com::save_config(ecstore.clone(), &usage_path, b"{".to_vec())
.await
.expect("corrupt usage fixture should be stored");
ecstore
.delete_bucket(&deleted_bucket, &DeleteBucketOptions::default())
.await
.expect("usage snapshot corruption must not block DeleteBucket");
assert!(
ecstore
.get_bucket_info(&deleted_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
.await
.is_err(),
"successful DeleteBucket must remove the physical bucket"
);
let new_bucket = format!("bucket-create-corrupt-usage-{}", Uuid::new_v4().simple());
let create_err = ecstore
.make_bucket(&new_bucket, &MakeBucketOptions::default())
.await
.expect_err("MakeBucket must fail when stale usage cannot be fenced");
assert!(!create_err.to_string().is_empty(), "the usage snapshot failure should be surfaced");
assert!(
ecstore
.get_bucket_info(&new_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
.await
.is_err(),
"failed MakeBucket must not expose a new physical bucket"
);
let restored = serde_json::to_vec(&DataUsageInfo {
last_update: Some(SystemTime::now()),
..Default::default()
})
.expect("restored usage fixture should encode");
crate::config::com::save_config(ecstore, &usage_path, restored)
.await
.expect("usage fixture should be restored after the failure-path test");
}
}
crates/ecstore/src/store/list_objects.rs
+1 -1
View File
@@ -619,7 +619,7 @@ pub(super) fn scanner_namespace_mutation_generation() -> u64 {
SCANNER_NAMESPACE_MUTATION_GENERATION.load(Ordering::Acquire)
}
pub(super) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) {
pub(crate) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) {
if bucket == RUSTFS_META_BUCKET {
return;
}
crates/ecstore/src/store/mod.rs
+10 -13
View File
@@ -322,6 +322,11 @@ impl ECStore {
pub fn scanner_namespace_mutation_generation(&self) -> u64 {
list_objects::scanner_namespace_mutation_generation()
}
pub async fn scanner_data_movement_active(&self) -> bool {
let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
decommission || rebalance
}
}
// impl Clone for ECStore {
@@ -440,11 +445,7 @@ impl BucketOperations for ECStore {
#[instrument(skip(self))]
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
let result = self.handle_make_bucket(bucket, opts).await;
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
result
Box::pin(self.handle_make_bucket(bucket, opts)).await
}
#[instrument(skip(self))]
@@ -457,11 +458,7 @@ impl BucketOperations for ECStore {
}
#[instrument(skip(self))]
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
let result = self.handle_delete_bucket(bucket, opts).await;
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
result
Box::pin(self.handle_delete_bucket(bucket, opts)).await
}
}
@@ -871,9 +868,9 @@ mod tests {
// Build a minimal ECStore carrying an explicit instance context. Empty
// pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> ECStore {
fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
let endpoint_pools = EndpointServerPools::default();
ECStore {
Arc::new(ECStore {
id: uuid::Uuid::new_v4(),
disk_map: std::collections::HashMap::new(),
pools: Vec::new(),
@@ -884,7 +881,7 @@ mod tests {
start_gate: Mutex::new(()),
pool_meta_save_gate: Mutex::new(()),
ctx,
}
})
}
// The object graph is the isolation carrier: two ECStore instances holding
crates/io-metrics/src/internode_metrics.rs
+1
View File
@@ -23,6 +23,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
pub const INTERNODE_OPERATION_READ_FILE_STREAM: &str = "read_file_stream";
pub const INTERNODE_OPERATION_PUT_FILE_STREAM: &str = "put_file_stream";
pub const INTERNODE_OPERATION_WALK_DIR: &str = "walk_dir";
pub const INTERNODE_OPERATION_NS_SCANNER: &str = "ns_scanner";
pub const INTERNODE_OPERATION_GRPC_READ_ALL: &str = "grpc_read_all";
pub const INTERNODE_OPERATION_GRPC_WRITE_ALL: &str = "grpc_write_all";
pub const INTERNODE_OPERATION_GRPC_READ_MULTIPLE: &str = "grpc_read_multiple";
crates/lock/src/distributed_lock.rs
+399 -118
View File
@@ -18,16 +18,20 @@ use crate::{
error::{LockError, Result},
types::{LockId, LockInfo, LockRequest, LockResponse, LockStatus, LockType},
};
use futures::future::join_all;
use futures::{
future::join_all,
stream::{FuturesUnordered, StreamExt},
};
use rustfs_io_metrics::{
record_lock_refresh_quorum_lost, record_read_lock_held_acquire, record_read_lock_held_release,
record_write_lock_held_acquire, record_write_lock_held_release,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::sync::Notify;
use tokio::task::{JoinHandle, JoinSet};
use tokio::time::Instant;
use tracing::{debug, warn};
use uuid::Uuid;
@@ -80,10 +84,10 @@ fn is_unrecoverable_quorum_error(error: &str) -> bool {
/// Deliberately avoids `Duration::clamp` (which asserts `min <= max` and would panic for
/// sub-second ttls where `ttl - 1s` underflows to zero). Returns `None` for degenerate cases so
/// no heartbeat is spawned:
/// - `entries_len <= 1`: single/degenerate path (matches a local, non-distributed lock),
/// - `entries_len == 0`: no acquired backend lease to renew,
/// - `interval.is_zero()` or `interval >= ttl`: too small a ttl / too large an interval to renew.
fn derive_refresh_interval(entries_len: usize, ttl: Duration, injected: Option<Duration>) -> Option<Duration> {
if entries_len <= 1 {
if entries_len == 0 {
return None;
}
let interval = injected.unwrap_or(ttl / 3);
@@ -130,10 +134,29 @@ fn should_warn_lock_failure(error: &str) -> bool {
#[derive(Debug, Default)]
pub struct LockLostSignal {
lost: AtomicBool,
valid_until: Mutex<Option<Instant>>,
notify: Notify,
}
impl LockLostSignal {
fn set_valid_until(&self, valid_until: Instant) {
if self.lost.load(Ordering::SeqCst) {
return;
}
match self.valid_until.lock() {
Ok(mut deadline) => {
if deadline.is_some_and(|current| Instant::now() >= current) {
drop(deadline);
self.mark_lost();
return;
}
*deadline = Some(valid_until);
self.notify.notify_waiters();
}
Err(_) => self.mark_lost(),
}
}
fn mark_lost(&self) {
self.lost.store(true, Ordering::SeqCst);
self.notify.notify_waiters();
@@ -141,16 +164,81 @@ impl LockLostSignal {
/// Whether refresh quorum has been lost for the associated guard.
pub fn is_lost(&self) -> bool {
self.lost.load(Ordering::SeqCst)
if self.lost.load(Ordering::SeqCst) {
return true;
}
let expired = match self.valid_until.lock() {
Ok(deadline) => deadline.is_some_and(|valid_until| Instant::now() >= valid_until),
Err(_) => true,
};
if expired {
self.mark_lost();
}
expired
}
/// Resolves once the lock is declared lost (immediately if already lost).
pub async fn notified(&self) {
if self.is_lost() {
return;
}
self.notify.notified().await;
self.notified_after_registration(|| {}).await;
}
async fn notified_after_registration(&self, after_registration: impl FnOnce()) {
let mut after_registration = Some(after_registration);
loop {
let notified = self.notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if let Some(after_registration) = after_registration.take() {
after_registration();
}
if self.is_lost() {
return;
}
let valid_until = match self.valid_until.lock() {
Ok(deadline) => *deadline,
Err(_) => return,
};
match valid_until {
Some(valid_until) => {
tokio::select! {
_ = tokio::time::sleep_until(valid_until) => {}
_ = &mut notified => {}
}
}
None => notified.await,
}
}
}
}
#[derive(Clone, Debug)]
struct HeldLockEntry {
lock_id: LockId,
client: Arc<dyn LockClient>,
valid_until: Instant,
}
impl HeldLockEntry {
fn release_entry(&self) -> (LockId, Arc<dyn LockClient>) {
(self.lock_id.clone(), self.client.clone())
}
}
#[derive(Clone, Debug)]
struct LockHeartbeatConfig {
ttl: Duration,
interval: Option<Duration>,
refresh_quorum: usize,
owner: String,
resource: ObjectKey,
}
#[derive(Clone, Copy, Debug, Default)]
struct LockRefreshStats {
refreshed: usize,
not_found: usize,
refresh_errors: usize,
}
/// A RAII guard for distributed locks that releases the lock asynchronously when dropped.
@@ -159,14 +247,12 @@ pub struct DistributedLockGuard {
/// The public-facing lock id. For multi-client scenarios this is typically
/// an aggregate id; for single-client it is the only id.
lock_id: LockId,
/// All underlying (LockId, client) entries that should be released when the
/// guard is dropped.
entries: Vec<(LockId, Arc<dyn LockClient>)>,
/// All underlying leases that should be refreshed and released with the guard.
entries: Vec<HeldLockEntry>,
lock_type: LockType,
/// If true, Drop will not try to release (used if user manually released).
disarmed: bool,
/// Background heartbeat task that periodically refreshes the per-client leases.
/// `None` for degenerate/single-client paths that do not need renewal.
/// Background lease task for renewable leases.
refresh_task: Option<JoinHandle<()>>,
/// Lock-loss signal, shared with the heartbeat task.
lock_lost: Arc<LockLostSignal>,
@@ -176,31 +262,22 @@ impl DistributedLockGuard {
/// Create a new guard.
///
/// - `lock_id` is the id returned to the caller (`lock_id()`).
/// - `entries` is the full list of underlying (LockId, client) pairs
/// that should be released when this guard is dropped.
/// - `refresh_interval`: `Some` spawns a heartbeat that refreshes every entry on that
/// cadence; `None` (degenerate/single-client, or interval derived away) spawns nothing.
/// - `refresh_quorum`: minimum refreshes that must keep succeeding; if `not_found` exceeds
/// `entries.len() - refresh_quorum` the guard is declared lost.
/// - `owner`/`resource`: diagnostics only.
pub(crate) fn new(
lock_id: LockId,
entries: Vec<(LockId, Arc<dyn LockClient>)>,
lock_type: LockType,
refresh_interval: Option<Duration>,
refresh_quorum: usize,
owner: String,
resource: ObjectKey,
) -> Self {
/// - `entries` is the full list of underlying leases that should be refreshed and released
/// with this guard.
/// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics.
fn new(lock_id: LockId, entries: Vec<HeldLockEntry>, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self {
record_lock_held_acquire(lock_type);
let lock_lost = Arc::new(LockLostSignal::default());
let refresh_task = refresh_interval.and_then(|interval| {
// Only spawn when a tokio runtime is available; guard construction off-runtime
// (e.g. some tests) simply skips the heartbeat.
match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) {
Some(valid_until) => lock_lost.set_valid_until(valid_until),
None => lock_lost.mark_lost(),
}
// Non-renewable guards are fenced directly by LockLostSignal's deadline checks.
let refresh_task = heartbeat.interval.and_then(|_| {
tokio::runtime::Handle::try_current().ok().map(|handle| {
let entries = entries.clone();
let lock_lost = lock_lost.clone();
handle.spawn(Self::run_heartbeat(entries, interval, refresh_quorum, lock_lost, owner, resource))
handle.spawn(Self::run_heartbeat(entries, heartbeat, lock_lost))
})
});
Self {
@@ -213,59 +290,107 @@ impl DistributedLockGuard {
}
}
/// Heartbeat loop: every `interval`, refresh all entries and classify the outcomes.
/// `Ok(true)` = refreshed, `Ok(false)` = not_found, `Err` = RPC jitter (ignored, absorbed by
/// the ttl > interval margin and retried next tick). Declares the lock lost when
/// `not_found > entries.len() - refresh_quorum`. Phase 1 does not release on loss: the guarded
/// operation is still running, and tearing the lock down here would only widen the window.
async fn run_heartbeat(
entries: Vec<(LockId, Arc<dyn LockClient>)>,
interval: Duration,
refresh_quorum: usize,
lock_lost: Arc<LockLostSignal>,
owner: String,
resource: ObjectKey,
fn quorum_valid_until(entries: &[HeldLockEntry], refresh_quorum: usize) -> Option<Instant> {
if refresh_quorum == 0 || entries.len() < refresh_quorum {
return None;
}
let mut deadlines: Vec<_> = entries.iter().map(|entry| entry.valid_until).collect();
deadlines.sort_unstable();
deadlines.get(deadlines.len() - refresh_quorum).copied()
}
fn signal_refresh_quorum_lost(
lock_lost: &LockLostSignal,
config: &LockHeartbeatConfig,
entries: usize,
stats: LockRefreshStats,
) {
let tolerable_not_found = entries.len().saturating_sub(refresh_quorum);
let mut ticker = tokio::time::interval(interval);
warn!(
resource = %config.resource,
owner = config.owner,
refreshed = stats.refreshed,
not_found = stats.not_found,
refresh_errors = stats.refresh_errors,
entries,
refresh_quorum = config.refresh_quorum,
"lock refresh lost quorum"
);
record_lock_refresh_quorum_lost();
lock_lost.mark_lost();
}
/// Refresh every tracked lease while a quorum is still known to be valid.
///
/// A successful refresh extends only that entry's conservative local deadline. RPC errors
/// retain the previous deadline, so transient jitter is tolerated but an unconfirmed lease
/// can never remain valid beyond its backend TTL. Waiting for a slow refresh is also fenced
/// by the current quorum deadline.
async fn run_heartbeat(mut entries: Vec<HeldLockEntry>, config: LockHeartbeatConfig, lock_lost: Arc<LockLostSignal>) {
let Some(interval) = config.interval else {
return;
};
let now = Instant::now();
let first_refresh = now.checked_add(interval).unwrap_or(now);
let mut ticker = tokio::time::interval_at(first_refresh, interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
// The first tick fires immediately; skip it so the first refresh lands one interval
// after acquisition rather than right away.
ticker.tick().await;
loop {
ticker.tick().await;
let Some(valid_until) = Self::quorum_valid_until(&entries, config.refresh_quorum) else {
Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), LockRefreshStats::default());
return;
};
lock_lost.set_valid_until(valid_until);
let results = join_all(entries.iter().map(|(lock_id, client)| {
let lock_id = lock_id.clone();
let client = client.clone();
async move { client.refresh(&lock_id).await }
}))
.await;
let mut refreshed = 0usize;
let mut not_found = 0usize;
for result in &results {
match result {
Ok(true) => refreshed += 1,
Ok(false) => not_found += 1,
// RPC jitter: count as neither; the ttl > interval margin covers a transient
// dip and the next tick retries.
Err(_) => {}
tokio::select! {
biased;
_ = tokio::time::sleep_until(valid_until) => {
Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), LockRefreshStats::default());
return;
}
_ = ticker.tick() => {}
}
if not_found > tolerable_not_found {
warn!(
resource = %resource,
owner = %owner,
refreshed,
not_found,
entries = entries.len(),
refresh_quorum,
"lock refresh lost quorum"
);
record_lock_refresh_quorum_lost();
lock_lost.mark_lost();
let mut stats = LockRefreshStats::default();
let mut pending = FuturesUnordered::new();
for (idx, entry) in entries.iter().enumerate() {
let refresh_started = Instant::now();
let lock_id = entry.lock_id.clone();
let client = entry.client.clone();
pending.push(async move { (idx, refresh_started, client.refresh(&lock_id).await) });
}
while !pending.is_empty() {
let Some(valid_until) = Self::quorum_valid_until(&entries, config.refresh_quorum) else {
Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), stats);
return;
};
lock_lost.set_valid_until(valid_until);
let next = tokio::select! {
biased;
_ = tokio::time::sleep_until(valid_until) => {
Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), stats);
return;
}
next = pending.next() => next
};
let Some((idx, refresh_started, result)) = next else {
break;
};
match result {
Ok(true) => {
stats.refreshed += 1;
entries[idx].valid_until = refresh_started.checked_add(config.ttl).unwrap_or(refresh_started);
}
Ok(false) => {
stats.not_found += 1;
entries[idx].valid_until = Instant::now();
}
Err(_) => stats.refresh_errors += 1,
}
}
}
}
@@ -326,7 +451,7 @@ impl DistributedLockGuard {
return true;
}
let entries = self.entries.clone();
let entries = self.entries.iter().map(HeldLockEntry::release_entry).collect();
DistributedLock::spawn_release_cleanup(entries, "distributed_lock_guard_release");
// Disarm to prevent double-release on drop
@@ -361,7 +486,7 @@ type LockAcquireTaskResult = (usize, Result<LockResponse>);
struct LockAcquireQuorumResult {
response: LockResponse,
individual_locks: Vec<(LockId, Arc<dyn LockClient>)>,
individual_locks: Vec<HeldLockEntry>,
failure_kind: Option<LockAcquireFailureKind>,
quorum_impossible: bool,
}
@@ -438,15 +563,18 @@ impl DistributedLock {
// Heartbeat operates on the per-client individual locks (their per-client ids), never
// the aggregate id, so refreshes round-trip to the exact backend entries.
let refresh_interval = derive_refresh_interval(individual_locks.len(), request.ttl, request.refresh_interval);
let heartbeat = LockHeartbeatConfig {
ttl: request.ttl,
interval: derive_refresh_interval(individual_locks.len(), request.ttl, request.refresh_interval),
refresh_quorum: required_quorum,
owner: request.owner.clone(),
resource: request.resource.clone(),
};
Ok(Some(DistributedLockGuard::new(
aggregate_lock_id,
individual_locks,
request.lock_type,
refresh_interval,
required_quorum,
request.owner.clone(),
request.resource.clone(),
heartbeat,
)))
} else {
// Check if it's a timeout or quorum failure
@@ -738,7 +866,7 @@ impl DistributedLock {
fn lock_acquire_attempt_timeout_result(
timeout: Duration,
individual_locks: Vec<(LockId, Arc<dyn LockClient>)>,
individual_locks: Vec<HeldLockEntry>,
last_failure: Option<String>,
last_failure_kind: Option<LockAcquireFailureKind>,
) -> LockAcquireQuorumResult {
@@ -770,19 +898,23 @@ impl DistributedLock {
/// Returns the LockResponse with aggregate lock_id and individual lock mappings.
async fn acquire_lock_quorum_once(&self, request: &LockRequest) -> Result<LockAcquireQuorumResult> {
let required_quorum = self.required_quorum(request.lock_type);
let attempt_started = Instant::now();
let mut pending = self.spawn_lock_requests(request);
let mut individual_locks: Vec<(LockId, Arc<dyn LockClient>)> = Vec::new();
let mut individual_locks: Vec<HeldLockEntry> = Vec::new();
let fallback_lock_id = request.lock_id.clone();
let mut last_failure = None;
let mut last_failure_kind = None;
let mut last_hard_failure_kind = None;
let mut hard_failures = 0usize;
let start = tokio::time::Instant::now();
let start = Instant::now();
while !pending.is_empty() {
let remaining = request.acquire_timeout.saturating_sub(start.elapsed());
if remaining.is_zero() {
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_attempt_timeout");
Self::spawn_release_cleanup(
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
"distributed_lock_attempt_timeout",
);
Self::spawn_pending_cleanup(
pending,
self.clients.clone(),
@@ -801,7 +933,10 @@ impl DistributedLock {
Ok(Some(join_result)) => join_result,
Ok(None) => break,
Err(_) => {
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_attempt_timeout");
Self::spawn_release_cleanup(
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
"distributed_lock_attempt_timeout",
);
Self::spawn_pending_cleanup(
pending,
self.clients.clone(),
@@ -827,7 +962,11 @@ impl DistributedLock {
.unwrap_or_else(|| fallback_lock_id.clone());
if let Some(client) = self.clients.get(idx) {
individual_locks.push((lock_id, client.clone()));
individual_locks.push(HeldLockEntry {
lock_id,
client: client.clone(),
valid_until: attempt_started.checked_add(request.ttl).unwrap_or(attempt_started),
});
} else {
tracing::warn!("Missing lock client at index {} while recording success", idx);
}
@@ -862,7 +1001,10 @@ impl DistributedLock {
if self.clients.len().saturating_sub(hard_failures) < required_quorum {
let rollback_count = individual_locks.len();
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
Self::spawn_release_cleanup(
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
"distributed_lock_quorum_rollback",
);
if !pending.is_empty() {
Self::spawn_pending_cleanup(
pending,
@@ -933,7 +1075,10 @@ impl DistributedLock {
if individual_locks.len() + pending.len() < required_quorum {
let rollback_count = individual_locks.len();
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
Self::spawn_release_cleanup(
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
"distributed_lock_quorum_rollback",
);
if !pending.is_empty() {
Self::spawn_pending_cleanup(
pending,
@@ -967,7 +1112,10 @@ impl DistributedLock {
}
let rollback_count = individual_locks.len();
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
Self::spawn_release_cleanup(
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
"distributed_lock_quorum_rollback",
);
let mut error = format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required");
if let Some(last_failure) = &last_failure {
error.push_str("; last failure: ");
@@ -1006,7 +1154,7 @@ fn record_lock_held_release(lock_type: LockType) {
#[cfg(test)]
mod tests {
use super::{
DistributedLock, LOCK_ACQUIRE_ATTEMPT_TIMEOUT, LockAcquireFailureKind, is_remote_lock_rpc_failure,
DistributedLock, LOCK_ACQUIRE_ATTEMPT_TIMEOUT, LockAcquireFailureKind, LockLostSignal, is_remote_lock_rpc_failure,
should_warn_lock_failure,
};
use crate::{LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockType, ObjectKey, client::LockClient};
@@ -1020,9 +1168,10 @@ mod tests {
#[derive(Clone, Copy, Debug)]
enum RefreshOutcome {
Alive, // Ok(true) refresh succeeded
NotFound, // Ok(false) remote already lost the lock (reclaimed / never held)
RpcError, // Err RPC jitter
Alive, // Ok(true) refresh succeeded
SlowAlive(Duration), // Ok(true) after a delayed response
NotFound, // Ok(false) remote already lost the lock (reclaimed / never held)
RpcError, // Err RPC jitter
}
/// Counting test client: acquires successfully and echoes back request.lock_id as
@@ -1071,6 +1220,10 @@ mod tests {
self.refresh_calls.fetch_add(1, Ordering::SeqCst);
match self.outcome {
RefreshOutcome::Alive => Ok(true),
RefreshOutcome::SlowAlive(delay) => {
tokio::time::sleep(delay).await;
Ok(true)
}
RefreshOutcome::NotFound => Ok(false),
RefreshOutcome::RpcError => Err(LockError::internal("refresh rpc jitter")),
}
@@ -1140,19 +1293,37 @@ mod tests {
drop(guard);
}
// A2 -- single client (degenerate path) spawns no heartbeat (matches localLockInstance).
// A2 -- a quorum-one distributed lease still requires renewal.
#[tokio::test]
async fn single_client_guard_spawns_no_heartbeat() {
async fn single_client_guard_keeps_backend_lease_refreshed() {
let (clients, counters) = counting_clients(&[RefreshOutcome::Alive]);
let lock = DistributedLock::new("test".to_string(), clients, 1);
let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner")
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock.acquire_guard(&request).await.unwrap().unwrap();
tokio::time::sleep(Duration::from_millis(90)).await;
let guard = lock
.acquire_guard(&request)
.await
.expect("single-client lock acquisition should not error")
.expect("single-client lock quorum should be reached");
assert!(
guard.refresh_task.is_some(),
"single-client distributed guard must spawn a background task"
);
tokio::time::timeout(Duration::from_secs(2), async {
while counters[0].load(Ordering::SeqCst) < 2 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("single-client distributed guard should renew before the test deadline");
assert_eq!(counters[0].load(Ordering::SeqCst), 0, "single-client guard must not run a heartbeat");
assert!(
counters[0].load(Ordering::SeqCst) >= 2,
"single-client distributed guard must renew beyond the original backend TTL"
);
assert!(!guard.is_lock_lost(), "successful quorum-one refresh must preserve the lease");
drop(guard);
}
@@ -1170,7 +1341,11 @@ mod tests {
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock.acquire_guard(&request).await.unwrap().unwrap();
let guard = lock
.acquire_guard(&request)
.await
.expect("distributed lock acquisition should not error")
.expect("distributed lock quorum should be reached");
tokio::time::sleep(Duration::from_millis(50)).await;
drop(guard); // should abort the heartbeat first
let after_drop: Vec<usize> = counters.iter().map(|c| c.load(Ordering::SeqCst)).collect();
@@ -1200,7 +1375,11 @@ mod tests {
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock.acquire_guard(&request).await.unwrap().unwrap();
let guard = lock
.acquire_guard(&request)
.await
.expect("distributed lock acquisition should not error")
.expect("distributed lock quorum should be reached");
tokio::time::sleep(Duration::from_millis(60)).await; // at least one tick
assert!(
@@ -1214,6 +1393,52 @@ mod tests {
drop(guard);
}
#[tokio::test]
async fn lock_lost_notification_cannot_be_missed_after_waiter_registration() {
let signal = LockLostSignal::default();
tokio::time::timeout(Duration::from_secs(2), signal.notified_after_registration(|| signal.mark_lost()))
.await
.expect("loss between waiter registration and state recheck must still resolve");
}
#[tokio::test]
async fn expired_lease_is_fenced_without_waiting_for_monitor_scheduling() {
let signal = LockLostSignal::default();
signal.set_valid_until(tokio::time::Instant::now());
assert!(signal.is_lost(), "an elapsed backend deadline must fence the owner immediately");
tokio::time::timeout(Duration::from_secs(2), signal.notified())
.await
.expect("an elapsed backend deadline must resolve lock-loss waiters immediately");
}
#[tokio::test]
async fn deadline_notification_permanently_fences_the_signal() {
let signal = LockLostSignal::default();
signal.set_valid_until(tokio::time::Instant::now() + Duration::from_millis(10));
tokio::time::timeout(Duration::from_secs(2), signal.notified())
.await
.expect("the backend deadline must resolve lock-loss waiters");
assert!(
signal.lost.load(Ordering::SeqCst),
"deadline notification must persist the lock-loss fence"
);
}
#[test]
fn expired_lease_cannot_be_revived_by_a_later_deadline() {
let signal = LockLostSignal::default();
signal.set_valid_until(tokio::time::Instant::now());
assert!(signal.is_lost(), "an elapsed lease must become permanently fenced");
signal.set_valid_until(tokio::time::Instant::now() + Duration::from_secs(60));
assert!(signal.is_lost(), "a later heartbeat must not revive an expired lease");
}
// Phase 2 passthrough: NamespaceLockGuard::Standard must forward the distributed
// guard's lock-loss signal so the write path can fence its commit on it.
#[tokio::test]
@@ -1229,7 +1454,11 @@ mod tests {
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock.acquire_guard(&request).await.unwrap().unwrap();
let guard = lock
.acquire_guard(&request)
.await
.expect("namespace lock acquisition should not error")
.expect("namespace lock quorum should be reached");
tokio::time::sleep(Duration::from_millis(60)).await; // at least one tick -> lost
let ns_guard = crate::namespace::NamespaceLockGuard::Standard(guard);
@@ -1239,9 +1468,10 @@ mod tests {
);
}
// A5 -- refresh RPC jitter (Err) is not counted as not_found; the lock is not declared lost.
// A5 -- refresh RPC jitter is tolerated within the lease, but an unconfirmed lease cannot
// remain valid after its backend TTL.
#[tokio::test]
async fn heartbeat_rpc_error_not_counted_as_lock_lost() {
async fn heartbeat_rpc_error_expires_at_backend_ttl() {
let (clients, _counters) = counting_clients(&[
RefreshOutcome::RpcError,
RefreshOutcome::RpcError,
@@ -1253,17 +1483,53 @@ mod tests {
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock.acquire_guard(&request).await.unwrap().unwrap();
let guard = lock
.acquire_guard(&request)
.await
.expect("distributed lock acquisition should not error")
.expect("distributed lock quorum should be reached");
tokio::time::sleep(Duration::from_millis(60)).await;
assert!(
!guard.is_lock_lost(),
"RPC jitter (Err) must NOT be counted as not_found; lock must not be declared lost"
"transient RPC jitter before the backend TTL must not immediately signal lock loss"
);
let signal = guard.lock_lost();
tokio::time::timeout(Duration::from_secs(2), signal.notified())
.await
.expect("unconfirmed leases must signal lock loss when their backend TTL expires");
assert!(guard.is_lock_lost(), "the guard must fail closed once refresh quorum expires");
drop(guard);
}
// A6 -- a refresh call that remains in flight cannot keep an expired quorum alive.
#[tokio::test]
async fn heartbeat_slow_refresh_is_fenced_by_backend_ttl() {
let (clients, _counters) =
counting_clients(&[RefreshOutcome::SlowAlive(Duration::from_millis(200)), RefreshOutcome::Alive]);
let lock = DistributedLock::new("test".to_string(), clients, 2);
let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner")
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock
.acquire_guard(&request)
.await
.expect("distributed lock acquisition should not error")
.expect("distributed lock quorum should be reached");
let signal = guard.lock_lost();
tokio::time::timeout(Duration::from_secs(2), signal.notified())
.await
.expect("an in-flight refresh must not extend an unconfirmed lease past its TTL");
assert!(
guard.is_lock_lost(),
"slow refresh must fail closed at the last confirmed quorum deadline"
);
drop(guard);
}
// A6 -- sub-second ttl boundary: no panic, and skip heartbeat when interval>=ttl
// A7 -- sub-second ttl boundary: no panic, and skip heartbeat when interval>=ttl
// (fixes clamp panic; aligns with the 50ms-ttl distributed path in namespace tests).
#[tokio::test]
async fn subsecond_ttl_guard_does_not_panic_and_skips_heartbeat() {
@@ -1286,15 +1552,26 @@ mod tests {
let request2 = LockRequest::new(ObjectKey::new("bucket", "object2"), LockType::Exclusive, "owner")
.with_ttl(Duration::from_millis(50))
.with_refresh_interval(Duration::from_millis(50)); // interval >= ttl -> None
let guard2 = lock2.acquire_guard(&request2).await.unwrap().unwrap();
tokio::time::sleep(Duration::from_millis(60)).await;
let guard2 = lock2
.acquire_guard(&request2)
.await
.expect("subsecond lock acquisition should not error")
.expect("subsecond lock quorum should be reached");
assert!(
guard2.refresh_task.is_none(),
"non-renewable subsecond guard must not spawn a background task"
);
tokio::time::timeout(Duration::from_secs(2), guard2.lock_lost().notified())
.await
.expect("non-renewable subsecond guard must signal loss at its backend TTL");
for c in &counters2 {
assert_eq!(c.load(Ordering::SeqCst), 0, "interval>=ttl must not spawn heartbeat");
assert_eq!(c.load(Ordering::SeqCst), 0, "interval>=ttl must not refresh the backend lease");
}
assert!(guard2.is_lock_lost(), "interval>=ttl guard must fail closed after backend TTL");
drop(guard2);
}
// A7 -- disarm() must also stop the heartbeat (public API gap).
// A8 -- disarm() must also stop the heartbeat (public API gap).
#[tokio::test]
async fn disarm_stops_heartbeat() {
let (clients, counters) = counting_clients(&[
@@ -1308,7 +1585,11 @@ mod tests {
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let mut guard = lock.acquire_guard(&request).await.unwrap().unwrap();
let mut guard = lock
.acquire_guard(&request)
.await
.expect("distributed lock acquisition should not error")
.expect("distributed lock quorum should be reached");
tokio::time::sleep(Duration::from_millis(50)).await;
guard.disarm(); // should abort the heartbeat
let after_disarm: Vec<usize> = counters.iter().map(|c| c.load(Ordering::SeqCst)).collect();
crates/lock/src/namespace/mod.rs
+13
View File
@@ -128,6 +128,19 @@ impl NamespaceLockGuard {
Self::Fast(_) => false,
}
}
/// Resolves when a distributed guard loses refresh quorum.
///
/// Local fast locks cannot lose distributed quorum, so their future remains pending.
pub async fn lock_lost_notified(&self) {
match self {
Self::Standard(guard) => {
let signal = guard.lock_lost();
signal.notified().await;
}
Self::Fast(_) => std::future::pending().await,
}
}
}
/// Namespace lock for managing locks by resource namespaces
crates/madmin/src/metrics.rs
+6
View File
@@ -640,6 +640,8 @@ pub struct ScannerMetrics {
pub last_cycle_usage_saves: u64,
#[serde(rename = "failed_cycles", default)]
pub failed_cycles: u64,
#[serde(rename = "superseded_cycles", default)]
pub superseded_cycles: u64,
#[serde(rename = "partial_cycles_unknown", default)]
pub partial_cycles_unknown: u64,
#[serde(rename = "partial_cycles_runtime", default)]
@@ -820,6 +822,7 @@ impl ScannerMetrics {
.saturating_add(other.last_cycle_replication_checks);
self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves);
self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles);
self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles);
self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown);
self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime);
self.partial_cycles_objects = self.partial_cycles_objects.saturating_add(other.partial_cycles_objects);
@@ -1862,6 +1865,7 @@ mod tests {
last_cycle_replication_checks: 7,
last_cycle_usage_saves: 8,
failed_cycles: 1,
superseded_cycles: 2,
partial_cycles_unknown: 2,
partial_cycles_runtime: 3,
partial_cycles_objects: 4,
@@ -1933,6 +1937,7 @@ mod tests {
last_cycle_replication_checks: 70,
last_cycle_usage_saves: 80,
failed_cycles: 10,
superseded_cycles: 20,
partial_cycles_unknown: 20,
partial_cycles_runtime: 30,
partial_cycles_objects: 40,
@@ -2014,6 +2019,7 @@ mod tests {
assert_eq!(scanner.last_cycle_replication_checks, 77);
assert_eq!(scanner.last_cycle_usage_saves, 88);
assert_eq!(scanner.failed_cycles, 11);
assert_eq!(scanner.superseded_cycles, 22);
assert_eq!(scanner.partial_cycles_unknown, 22);
assert_eq!(scanner.partial_cycles_runtime, 33);
assert_eq!(scanner.partial_cycles_objects, 44);
crates/obs/src/metrics/collectors/scanner.rs
+15 -5
View File
@@ -44,8 +44,9 @@ use crate::metrics::schema::scanner::{
SCANNER_LAST_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_THROTTLE_SLEEP_EVENTS_MD,
SCANNER_LAST_CYCLE_USAGE_SAVES_MD, SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD,
SCANNER_OBJECTS_SCANNED_MD, SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD, SCANNER_PARTIAL_CYCLES_BY_REASON_MD,
SCANNER_PARTIAL_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD,
SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD,
SCANNER_PARTIAL_CYCLES_MD, SCANNER_SUPERSEDED_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD,
SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD,
SCANNER_YIELD_EVERY_N_OBJECTS_MD,
};
/// Scanner statistics.
@@ -139,7 +140,7 @@ pub struct ScannerStats {
pub current_cycle_usage_saves: u64,
/// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
pub current_scan_mode: u64,
/// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial
/// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded
pub last_cycle_result: u64,
/// Last scanner partial cycle reason: 0 unknown, 1 runtime, 2 objects, 3 directories
pub last_cycle_partial_reason: u64,
@@ -177,6 +178,8 @@ pub struct ScannerStats {
pub last_cycle_usage_saves: u64,
/// Number of scanner cycles that failed since server start
pub failed_cycles: u64,
/// Number of scanner cycles superseded by concurrent namespace activity
pub superseded_cycles: u64,
/// Number of scanner cycles stopped by runtime budget since server start
pub partial_cycles: u64,
/// Number of scanner cycles stopped by an unknown budget reason
@@ -318,6 +321,7 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, stats.last_cycle_replication_checks as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_USAGE_SAVES_MD, stats.last_cycle_usage_saves as f64),
PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
PrometheusMetric::from_descriptor(&SCANNER_SUPERSEDED_CYCLES_MD, stats.superseded_cycles as f64),
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_MD, stats.partial_cycles as f64),
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_unknown as f64)
.with_label("reason", "unknown"),
@@ -405,6 +409,7 @@ mod tests {
last_cycle_replication_checks: 12,
last_cycle_usage_saves: 9,
failed_cycles: 3,
superseded_cycles: 5,
partial_cycles: 10,
partial_cycles_unknown: 1,
partial_cycles_runtime: 2,
@@ -415,7 +420,7 @@ mod tests {
let metrics = collect_scanner_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 68);
assert_eq!(metrics.len(), 69);
let objects = metrics.iter().find(|m| m.value == 1000000.0);
assert!(objects.is_some());
@@ -708,6 +713,11 @@ mod tests {
.find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
assert_eq!(failed_cycles.map(|m| m.value), Some(3.0));
let superseded_cycles = metrics
.iter()
.find(|m| m.name == SCANNER_SUPERSEDED_CYCLES_MD.get_full_metric_name());
assert_eq!(superseded_cycles.map(|m| m.value), Some(5.0));
let partial_cycles = metrics
.iter()
.find(|m| m.name == SCANNER_PARTIAL_CYCLES_MD.get_full_metric_name());
@@ -743,7 +753,7 @@ mod tests {
let stats = ScannerStats::default();
let metrics = collect_scanner_metrics(&stats);
assert_eq!(metrics.len(), 68);
assert_eq!(metrics.len(), 69);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
if metric.name == SCANNER_PARTIAL_CYCLES_BY_REASON_MD.get_full_metric_name() {
crates/obs/src/metrics/schema/entry/metric_name.rs
+2
View File
@@ -337,6 +337,7 @@ pub enum MetricName {
ScannerLastCycleReplicationChecks,
ScannerLastCycleUsageSaves,
ScannerFailedCycles,
ScannerSupersededCycles,
ScannerPartialCycles,
ScannerPartialCyclesByReason,
@@ -745,6 +746,7 @@ impl MetricName {
Self::ScannerLastCycleReplicationChecks => "last_cycle_replication_checks".to_string(),
Self::ScannerLastCycleUsageSaves => "last_cycle_usage_saves".to_string(),
Self::ScannerFailedCycles => "failed_cycles".to_string(),
Self::ScannerSupersededCycles => "superseded_cycles".to_string(),
Self::ScannerPartialCycles => "partial_cycles".to_string(),
Self::ScannerPartialCyclesByReason => "partial_cycles_by_reason".to_string(),
crates/obs/src/metrics/schema/scanner.rs
+10 -1
View File
@@ -416,7 +416,7 @@ pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::
pub static SCANNER_LAST_CYCLE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleResult,
"Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial.",
"Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded.",
&[],
subsystems::SCANNER,
)
@@ -584,6 +584,15 @@ pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(
)
});
pub static SCANNER_SUPERSEDED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::ScannerSupersededCycles,
"Total number of clean scanner cycles superseded by concurrent cluster, configuration, topology, or namespace activity since server start.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_PARTIAL_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::ScannerPartialCycles,
crates/obs/src/metrics/stats_collector.rs
+1
View File
@@ -1135,6 +1135,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
failed_cycles: metrics.failed_cycles,
superseded_cycles: metrics.superseded_cycles,
partial_cycles: metrics.partial_cycles,
partial_cycles_unknown: metrics.partial_cycles_unknown,
partial_cycles_runtime: metrics.partial_cycles_runtime,
crates/protos/src/generated/proto_gen/node_service.rs
+25 -2
View File
@@ -1076,9 +1076,20 @@ pub struct SignalServiceResponse {
pub success: bool,
#[prost(string, optional, tag = "2")]
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
#[prost(uint32, tag = "3")]
pub protocol_version: u32,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerActivityRequest {
#[prost(bytes = "bytes", tag = "1")]
pub challenge: ::prost::bytes::Bytes,
#[prost(uint32, tag = "2")]
pub protocol_version: u32,
#[prost(string, tag = "3")]
pub acknowledge_instance_id: ::prost::alloc::string::String,
#[prost(uint64, tag = "4")]
pub acknowledge_dirty_usage_generation: u64,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerActivityRequest {}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerActivityResponse {
#[prost(string, tag = "1")]
@@ -1087,6 +1098,18 @@ pub struct ScannerActivityResponse {
pub namespace_generation: u64,
#[prost(uint64, tag = "3")]
pub maintenance_generation: u64,
#[prost(uint32, tag = "4")]
pub protocol_version: u32,
#[prost(bytes = "bytes", tag = "5")]
pub topology_digest: ::prost::bytes::Bytes,
#[prost(bool, tag = "6")]
pub data_movement_active: bool,
#[prost(bytes = "bytes", tag = "7")]
pub response_proof: ::prost::bytes::Bytes,
#[prost(uint64, tag = "8")]
pub dirty_usage_generation: u64,
#[prost(bool, tag = "9")]
pub dirty_usage_pending: bool,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BackgroundHealStatusRequest {}
crates/protos/src/lib.rs
+225
View File
@@ -169,6 +169,7 @@ pub fn internode_rpc_max_message_size() -> usize {
pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SIZE + 1024;
pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
@@ -353,6 +354,230 @@ pub fn canonical_tier_mutation_rpc_response_body(
Ok(body)
}
/// Builds the stable byte representation authenticated for a scanner activity request.
pub fn canonical_scanner_activity_request_body(
request: &proto_gen::node_service::ScannerActivityRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-activity-request-v1\0";
let challenge = request.challenge.as_ref();
let acknowledge_instance_id = request.acknowledge_instance_id.as_bytes();
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + acknowledge_instance_id.len() + 4 + 8 * 3);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&request.protocol_version.to_be_bytes());
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.extend_from_slice(&u64::try_from(acknowledge_instance_id.len())?.to_be_bytes());
body.extend_from_slice(acknowledge_instance_id);
body.extend_from_slice(&request.acknowledge_dirty_usage_generation.to_be_bytes());
Ok(body)
}
/// Builds the protocol-v4 byte representation authenticated for a scanner activity response.
pub fn canonical_scanner_activity_v4_response_body(
challenge: &[u8],
response: &proto_gen::node_service::ScannerActivityResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-activity-response-v1\0";
let instance_id = response.instance_id.as_bytes();
let topology_digest = response.topology_digest.as_ref();
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + instance_id.len() + topology_digest.len() + 4 + 8 * 5 + 1);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.extend_from_slice(&u64::try_from(instance_id.len())?.to_be_bytes());
body.extend_from_slice(instance_id);
body.extend_from_slice(&response.namespace_generation.to_be_bytes());
body.extend_from_slice(&response.maintenance_generation.to_be_bytes());
body.extend_from_slice(&response.protocol_version.to_be_bytes());
body.extend_from_slice(&u64::try_from(topology_digest.len())?.to_be_bytes());
body.extend_from_slice(topology_digest);
body.push(u8::from(response.data_movement_active));
Ok(body)
}
/// Builds the stable byte representation authenticated for a scanner activity response.
pub fn canonical_scanner_activity_response_body(
challenge: &[u8],
response: &proto_gen::node_service::ScannerActivityResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-activity-response-v2\0";
let instance_id = response.instance_id.as_bytes();
let topology_digest = response.topology_digest.as_ref();
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + instance_id.len() + topology_digest.len() + 4 + 8 * 6 + 2);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.extend_from_slice(&u64::try_from(instance_id.len())?.to_be_bytes());
body.extend_from_slice(instance_id);
body.extend_from_slice(&response.namespace_generation.to_be_bytes());
body.extend_from_slice(&response.maintenance_generation.to_be_bytes());
body.extend_from_slice(&response.protocol_version.to_be_bytes());
body.extend_from_slice(&u64::try_from(topology_digest.len())?.to_be_bytes());
body.extend_from_slice(topology_digest);
body.push(u8::from(response.data_movement_active));
body.extend_from_slice(&response.dirty_usage_generation.to_be_bytes());
body.push(u8::from(response.dirty_usage_pending));
Ok(body)
}
#[cfg(test)]
mod scanner_activity_tests {
use super::{
canonical_scanner_activity_request_body, canonical_scanner_activity_response_body,
canonical_scanner_activity_v4_response_body,
proto_gen::node_service::{ScannerActivityRequest, ScannerActivityResponse},
};
#[test]
fn canonical_scanner_activity_request_binds_every_field() {
let request = ScannerActivityRequest {
challenge: vec![1; 16].into(),
protocol_version: 5,
acknowledge_instance_id: "0123456789abcdef0123456789abcdef".to_string(),
acknowledge_dirty_usage_generation: 11,
};
let baseline = canonical_scanner_activity_request_body(&request).expect("scanner activity request should encode");
let variants = [
ScannerActivityRequest {
challenge: vec![2; 16].into(),
..request.clone()
},
ScannerActivityRequest {
protocol_version: 4,
..request.clone()
},
ScannerActivityRequest {
acknowledge_instance_id: "1123456789abcdef0123456789abcdef".to_string(),
..request.clone()
},
ScannerActivityRequest {
acknowledge_dirty_usage_generation: 12,
..request
},
];
for variant in variants {
assert_ne!(
baseline,
canonical_scanner_activity_request_body(&variant).expect("scanner activity request variant should encode")
);
}
}
#[test]
fn canonical_scanner_activity_response_binds_challenge_and_every_status_field() {
let response = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: 4,
topology_digest: vec![9; 32].into(),
data_movement_active: true,
response_proof: Vec::new().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
let baseline =
canonical_scanner_activity_response_body(&[1; 16], &response).expect("scanner activity response should encode");
let variants = [
ScannerActivityResponse {
instance_id: "1123456789abcdef0123456789abcdef".to_string(),
..response.clone()
},
ScannerActivityResponse {
namespace_generation: 8,
..response.clone()
},
ScannerActivityResponse {
maintenance_generation: 4,
..response.clone()
},
ScannerActivityResponse {
protocol_version: 5,
..response.clone()
},
ScannerActivityResponse {
topology_digest: vec![8; 32].into(),
..response.clone()
},
ScannerActivityResponse {
data_movement_active: false,
..response.clone()
},
ScannerActivityResponse {
dirty_usage_generation: 12,
..response.clone()
},
ScannerActivityResponse {
dirty_usage_pending: false,
..response.clone()
},
];
for variant in variants {
assert_ne!(
baseline,
canonical_scanner_activity_response_body(&[1; 16], &variant)
.expect("scanner activity response variant should encode")
);
}
assert_ne!(
baseline,
canonical_scanner_activity_response_body(&[2; 16], &response)
.expect("scanner activity response with a different challenge should encode")
);
}
#[test]
fn scanner_activity_v4_response_canonicalization_ignores_v5_fields() {
let response = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: 4,
topology_digest: vec![9; 32].into(),
data_movement_active: true,
response_proof: Vec::new().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
};
let baseline =
canonical_scanner_activity_v4_response_body(&[1; 16], &response).expect("scanner activity v4 response should encode");
let expected = [
b"rustfs-scanner-activity-response-v1\0".as_slice(),
16u64.to_be_bytes().as_slice(),
[1u8; 16].as_slice(),
32u64.to_be_bytes().as_slice(),
b"0123456789abcdef0123456789abcdef".as_slice(),
7u64.to_be_bytes().as_slice(),
3u64.to_be_bytes().as_slice(),
4u32.to_be_bytes().as_slice(),
32u64.to_be_bytes().as_slice(),
[9u8; 32].as_slice(),
[1u8].as_slice(),
]
.concat();
assert_eq!(
baseline, expected,
"protocol v4 response bytes must remain stable during rolling upgrades"
);
let extended = ScannerActivityResponse {
dirty_usage_generation: 11,
dirty_usage_pending: true,
..response
};
assert_eq!(
baseline,
canonical_scanner_activity_v4_response_body(&[1; 16], &extended)
.expect("scanner activity v4 response should ignore v5 fields")
);
}
}
#[cfg(test)]
mod heal_control_tests {
use super::{
crates/protos/src/node.proto
+13 -1
View File
@@ -756,14 +756,26 @@ message SignalServiceRequest {
message SignalServiceResponse {
bool success = 1;
optional string error_info = 2;
uint32 protocol_version = 3;
}
message ScannerActivityRequest {}
message ScannerActivityRequest {
bytes challenge = 1;
uint32 protocol_version = 2;
string acknowledge_instance_id = 3;
uint64 acknowledge_dirty_usage_generation = 4;
}
message ScannerActivityResponse {
string instance_id = 1;
uint64 namespace_generation = 2;
uint64 maintenance_generation = 3;
uint32 protocol_version = 4;
bytes topology_digest = 5;
bool data_movement_active = 6;
bytes response_proof = 7;
uint64 dirty_usage_generation = 8;
bool dirty_usage_pending = 9;
}
message BackgroundHealStatusRequest {}
crates/rio/src/http_reader.rs
+8 -1
View File
@@ -19,7 +19,8 @@ use http::{HeaderMap, Version};
use pin_project_lite::pin_project;
use reqwest::{Certificate, Client, Identity, Method, RequestBuilder};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_OPERATION_WALK_DIR,
};
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_opt_u64, get_env_opt_usize};
@@ -43,6 +44,7 @@ use tracing::{error, warn};
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner";
const HTTP_VERSION_09_LABEL: &str = "http/0.9";
const HTTP_VERSION_10_LABEL: &str = "http/1.0";
const HTTP_VERSION_11_LABEL: &str = "http/1.1";
@@ -1120,6 +1122,7 @@ fn internode_rpc_operation(url: &str) -> Option<&'static str> {
READ_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_READ_FILE_STREAM),
PUT_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_PUT_FILE_STREAM),
WALK_DIR_PATH => Some(INTERNODE_OPERATION_WALK_DIR),
NS_SCANNER_PATH => Some(INTERNODE_OPERATION_NS_SCANNER),
_ => None,
}
}
@@ -1770,6 +1773,10 @@ mod tests {
internode_rpc_operation(&format!("http://node:9000{WALK_DIR_PATH}?disk=d")),
Some(INTERNODE_OPERATION_WALK_DIR)
);
assert_eq!(
internode_rpc_operation(&format!("http://node:9000{NS_SCANNER_PATH}?disk=d")),
Some(INTERNODE_OPERATION_NS_SCANNER)
);
assert_eq!(internode_rpc_operation("http://node:9000/rustfs/rpc/unknown"), None);
assert_eq!(
internode_rpc_operation("http://node:9000/rustfs/rpc/unknown?next=/rustfs/rpc/read_file_stream"),
crates/scanner/Cargo.toml
+7
View File
@@ -32,6 +32,7 @@ workspace = true
[dependencies]
rustfs-config = { workspace = true, features = ["server-config-model"] }
rustfs-common = { workspace = true }
rustfs-credentials = { workspace = true }
rustfs-utils = { workspace = true }
tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] }
tracing = { workspace = true }
@@ -43,6 +44,8 @@ futures = { workspace = true }
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
chrono = { workspace = true, features = ["serde"] }
rmp-serde = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
rustfs-filemeta = { workspace = true }
tokio-util = { workspace = true, features = ["io", "compat"] }
rustfs-ecstore = { workspace = true }
@@ -52,11 +55,15 @@ rand = { workspace = true, features = ["serde"] }
s3s = { workspace = true, features = ["minio"] }
metrics = { workspace = true }
rustfs-data-usage = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde", "fast-rng"] }
bytes.workspace = true
hex-simd.workspace = true
[dev-dependencies]
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
serial_test = { workspace = true }
temp-env = { workspace = true }
tempfile = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
# Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via
crates/scanner/src/data_usage_define.rs
+1485 -128
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/error.rs
+16
View File
@@ -36,7 +36,23 @@ pub enum ScannerError {
#[error("Scanner error: {0}")]
Other(String),
/// A remote namespace scanner request ID was already accepted.
#[error("Remote namespace scanner request replay detected")]
RemoteRequestReplay,
/// The bounded remote namespace scanner replay cache cannot accept more IDs yet.
#[error("Remote namespace scanner replay cache capacity exceeded")]
RemoteReplayCapacity,
/// A remote namespace scanner request is already active for the target disk.
#[error("Remote namespace scanner disk is already active")]
RemoteDiskBusy,
/// Partial data usage cache produced before the scanner stopped.
#[error("Scanner stopped with partial data usage cache")]
PartialCache(Box<DataUsageCache>),
/// Partial cache retained because the bucket or scan root disappeared.
#[error("Scanner namespace path disappeared during the scan")]
NamespaceNotFoundCache(Box<DataUsageCache>),
}
crates/scanner/src/lib.rs
+104 -8
View File
@@ -20,6 +20,7 @@
rust_2018_idioms
)]
use bytes::Bytes;
use http::HeaderMap;
use rustfs_config::server_config::{Config as ServerConfig, get_global_server_config as config_get_global_server_config};
use std::path::PathBuf;
@@ -29,11 +30,11 @@ use storage_api::owner::{
ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk,
EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, EcstoreDiskLocation,
EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc, EcstoreLifecycle,
EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge,
EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig,
EcstoreVersioningApi, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, ScannerReplicationHealObject,
ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule,
ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, EcstoreReplicationConfigurationExt,
EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore,
EcstoreTierConfig, EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete,
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
@@ -41,11 +42,16 @@ use storage_api::owner::{
scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
use storage_api::owner::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
use storage_api::owner::{
EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints, EcstoreInstanceContext,
EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
ecstore_new_disk,
};
use tokio_util::sync::CancellationToken;
pub mod data_usage_define;
pub mod error;
mod remote_scanner;
pub mod runtime_config;
pub mod scanner;
pub mod scanner_budget;
@@ -56,16 +62,23 @@ pub(crate) mod storage_api;
pub use data_usage_define::*;
pub use error::ScannerError;
pub use remote_scanner::{
NS_SCANNER_MAX_REQUEST_BODY_SIZE, RemoteScannerAdmission, RemoteScannerRequest, admit_remote_scanner_request,
claim_remote_scanner_request, decode_remote_scanner_request, preflight_remote_scanner_request,
remote_scanner_request_matches_envelope, serve_remote_scanner_request, validate_remote_scanner_request_fence,
};
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
pub use rustfs_common::last_minute;
pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status};
pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status, scanner_topology_digest};
pub use scanner_io::{
clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch,
ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,
scanner_maintenance_generation,
};
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
use std::sync::atomic::{AtomicU64, Ordering};
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
pub use storage_api::scan::SCANNER_ACTIVITY_PROTOCOL_VERSION;
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
static SCANNER_FOREGROUND_READ_ACTIVITY: AtomicU64 = AtomicU64::new(0);
@@ -150,6 +163,7 @@ pub(crate) type BucketTargetSys = EcstoreBucketTargetSys;
pub(crate) type BucketVersioningSys = EcstoreBucketVersioningSys;
pub(crate) type DiskInfo = EcstoreDiskInfo;
pub(crate) type DiskInfoOptions = EcstoreDiskInfoOptions;
pub(crate) type NsScannerOpenRequest = EcstoreNsScannerOpenRequest;
pub(crate) type DiskBytes = EcstoreDiskBytes;
pub(crate) type Evaluator = EcstoreEvaluator;
pub(crate) type Event = EcstoreEvent;
@@ -185,6 +199,27 @@ pub(crate) fn init_ecstore_config_for_scanner_tests() {
pub(crate) type DiskOption = EcstoreDiskOption;
#[cfg(test)]
pub(crate) type Endpoint = EcstoreEndpoint;
#[cfg(test)]
pub(crate) type EndpointServerPools = EcstoreEndpointServerPools;
#[cfg(test)]
pub(crate) type Endpoints = EcstoreEndpoints;
#[cfg(test)]
pub(crate) type InstanceContext = EcstoreInstanceContext;
#[cfg(test)]
pub(crate) type PoolEndpoints = EcstorePoolEndpoints;
#[cfg(test)]
pub(crate) async fn init_local_disks_with_instance_ctx(
ctx: &Arc<InstanceContext>,
pools: EndpointServerPools,
) -> EcstoreResult<()> {
ecstore_init_local_disks_with_instance_ctx(ctx, pools).await
}
#[cfg(test)]
pub(crate) async fn init_bucket_metadata_sys_for_scanner_tests(store: Arc<ECStore>) {
ecstore_init_bucket_metadata_sys(store, Vec::new()).await;
}
#[cfg(test)]
pub(crate) async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> DiskResult<DiskStore> {
@@ -247,6 +282,8 @@ impl ScannerVersioningConfigExt for s3s::dto::VersioningConfiguration {
pub(crate) trait ScannerDiskExt {
async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult<DiskInfo>;
async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult<DiskBytes>;
fn is_local(&self) -> bool;
fn host_name(&self) -> String;
fn path(&self) -> PathBuf;
fn get_disk_location(&self) -> DiskLocation;
fn start_scan(&self) -> ScanGuard;
@@ -264,6 +301,14 @@ where
EcstoreDiskAPI::read_metadata(self, volume, path).await
}
fn is_local(&self) -> bool {
EcstoreDiskAPI::is_local(self)
}
fn host_name(&self) -> String {
EcstoreDiskAPI::host_name(self)
}
fn path(&self) -> PathBuf {
EcstoreDiskAPI::path(self)
}
@@ -344,6 +389,10 @@ pub(crate) async fn scanner_is_erasure_sd() -> bool {
ecstore_is_erasure_sd().await
}
pub(crate) async fn scanner_disk_is_online(disk: &Disk) -> bool {
EcstoreDiskAPI::is_online(disk).await
}
pub(crate) async fn read_config<S>(api: Arc<S>, file: &str) -> EcstoreResult<Vec<u8>>
where
S: ScannerObjectIO,
@@ -358,6 +407,53 @@ where
ecstore_save_config(api, file, data).await
}
pub(crate) async fn save_config_with_preconditions<S>(
api: Arc<S>,
file: &str,
data: Vec<u8>,
preconditions: HTTPPreconditions,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO,
{
let mut reader = ScannerPutObjReader::from_vec(data);
api.put_object(
RUSTFS_META_BUCKET,
file,
&mut reader,
&ScannerObjectOptions {
max_parity: true,
http_preconditions: Some(preconditions),
..Default::default()
},
)
.await
}
pub(crate) async fn save_config_shared_with_preconditions<S>(
api: Arc<S>,
file: &str,
data: Bytes,
sha256hex: Option<String>,
preconditions: HTTPPreconditions,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO,
{
let mut reader = ScannerPutObjReader::from_prehashed_bytes(data, sha256hex)?;
api.put_object(
RUSTFS_META_BUCKET,
file,
&mut reader,
&ScannerObjectOptions {
max_parity: true,
http_preconditions: Some(preconditions),
..Default::default()
},
)
.await
}
pub(crate) async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> std::result::Result<(), DiskError> {
ecstore_list_path_raw(rx, opts).await
}
crates/scanner/src/remote_scanner.rs
+3011
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/scanner.rs
+3089 -167
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/scanner_budget.rs
+223 -12
View File
@@ -16,6 +16,7 @@ use std::sync::{
Arc,
atomic::{AtomicU8, AtomicU64, Ordering},
};
use std::time::Instant;
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
@@ -61,15 +62,26 @@ impl ScannerCycleBudgetReason {
pub struct ScannerCycleBudget {
token: CancellationToken,
reason: Arc<AtomicU8>,
started_at: Instant,
max_duration: Option<Duration>,
max_objects: Option<u64>,
max_directories: Option<u64>,
track_progress: bool,
objects_scanned: AtomicU64,
directories_started: AtomicU64,
entries_visited: AtomicU64,
}
impl ScannerCycleBudget {
pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
Self::new_inner(parent, config, false)
}
pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
Self::new_inner(parent, config, true)
}
fn new_inner(parent: &CancellationToken, config: ScannerCycleBudgetConfig, track_progress: bool) -> Arc<Self> {
let token = parent.child_token();
let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE));
@@ -92,11 +104,14 @@ impl ScannerCycleBudget {
Arc::new(Self {
token,
reason,
started_at: Instant::now(),
max_duration: config.max_duration,
max_objects: config.max_objects,
max_directories: config.max_directories,
track_progress,
objects_scanned: AtomicU64::new(0),
directories_started: AtomicU64::new(0),
entries_visited: AtomicU64::new(0),
})
}
@@ -124,27 +139,97 @@ impl ScannerCycleBudget {
self.max_directories
}
pub(crate) fn try_start_directory(&self) -> bool {
let Some(max_directories) = self.max_directories else {
return true;
};
pub(crate) fn requires_serial_progress_accounting(&self) -> bool {
self.max_objects.is_some() || self.max_directories.is_some()
}
let directories = self.directories_started.fetch_add(1, Ordering::Relaxed) + 1;
if directories <= max_directories {
pub(crate) fn remaining_config(&self) -> ScannerCycleBudgetConfig {
let max_duration = self
.max_duration
.map(|duration| duration.saturating_sub(self.started_at.elapsed()));
if max_duration.is_some_and(|duration| duration.is_zero()) {
self.cancel_for(ScannerCycleBudgetReason::Runtime);
}
ScannerCycleBudgetConfig {
max_duration,
max_objects: self
.max_objects
.map(|max| max.saturating_sub(self.objects_scanned.load(Ordering::Relaxed))),
max_directories: self
.max_directories
.map(|max| max.saturating_sub(self.directories_started.load(Ordering::Relaxed))),
}
}
pub(crate) fn progress(&self) -> (u64, u64) {
(
self.objects_scanned.load(Ordering::Relaxed),
self.directories_started.load(Ordering::Relaxed),
)
}
pub(crate) fn entries_visited(&self) -> u64 {
self.entries_visited.load(Ordering::Relaxed)
}
pub(crate) fn record_entries_visited(&self, entries_visited: u64) {
if self.track_progress {
saturating_fetch_add(&self.entries_visited, entries_visited);
}
}
pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) {
if self.track_progress || self.max_objects.is_some() {
let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned);
if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
self.cancel_for(ScannerCycleBudgetReason::Objects);
}
}
if self.track_progress || self.max_directories.is_some() {
let directories = saturating_fetch_add(&self.directories_started, directories_started);
if self
.max_directories
.is_some_and(|max_directories| directories > max_directories)
{
self.cancel_for(ScannerCycleBudgetReason::Directories);
}
}
}
pub(crate) fn cancel_after_unreported_remote_progress(&self) {
if self.max_objects.is_some() {
self.cancel_for(ScannerCycleBudgetReason::Objects);
} else if self.max_directories.is_some() {
self.cancel_for(ScannerCycleBudgetReason::Directories);
}
}
pub(crate) fn try_start_directory(&self) -> bool {
if !self.track_progress && self.max_directories.is_none() {
return true;
}
self.cancel_for(ScannerCycleBudgetReason::Directories);
false
let directories = saturating_fetch_add(&self.directories_started, 1);
if self
.max_directories
.is_some_and(|max_directories| directories > max_directories)
{
self.cancel_for(ScannerCycleBudgetReason::Directories);
return false;
}
true
}
pub(crate) fn record_object_scanned(&self) {
let Some(max_objects) = self.max_objects else {
if !self.track_progress && self.max_objects.is_none() {
return;
};
}
let objects = self.objects_scanned.fetch_add(1, Ordering::Relaxed) + 1;
if objects >= max_objects {
let objects = saturating_fetch_add(&self.objects_scanned, 1);
if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
self.cancel_for(ScannerCycleBudgetReason::Objects);
}
}
@@ -163,6 +248,17 @@ impl ScannerCycleBudget {
}
}
fn saturating_fetch_add(value: &AtomicU64, delta: u64) -> u64 {
let mut current = value.load(Ordering::Relaxed);
loop {
let next = current.saturating_add(delta);
match value.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return next,
Err(observed) => current = observed,
}
}
}
impl Drop for ScannerCycleBudget {
fn drop(&mut self) {
self.token.cancel();
@@ -250,4 +346,119 @@ mod tests {
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
assert!(budget.token().is_cancelled());
}
#[tokio::test]
async fn remaining_config_accounts_for_local_and_remote_progress() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_duration: Some(Duration::from_secs(60)),
max_objects: Some(10),
max_directories: Some(5),
},
);
budget.record_object_scanned();
assert!(budget.try_start_directory());
budget.record_remote_progress(3, 2);
let remaining = budget.remaining_config();
assert!(
remaining
.max_duration
.is_some_and(|duration| duration <= Duration::from_secs(60))
);
assert_eq!(remaining.max_objects, Some(6));
assert_eq!(remaining.max_directories, Some(2));
assert_eq!(budget.progress(), (4, 3));
assert!(!budget.budget_elapsed());
}
#[test]
fn remote_progress_preserves_object_and_directory_limit_semantics() {
let parent = CancellationToken::new();
let object_budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(2),
..Default::default()
},
);
object_budget.record_remote_progress(2, 0);
assert_eq!(object_budget.reason(), Some(ScannerCycleBudgetReason::Objects));
let directory_budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_directories: Some(2),
..Default::default()
},
);
directory_budget.record_remote_progress(0, 2);
assert!(!directory_budget.budget_elapsed());
directory_budget.record_remote_progress(0, 1);
assert_eq!(directory_budget.reason(), Some(ScannerCycleBudgetReason::Directories));
}
#[test]
fn explicit_progress_tracking_counts_unbounded_remote_work_without_cancelling() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default());
assert!(budget.try_start_directory());
budget.record_object_scanned();
budget.record_remote_progress(2, 3);
assert_eq!(budget.progress(), (3, 4));
assert!(!budget.budget_elapsed());
assert!(!budget.token().is_cancelled());
}
#[test]
fn unreported_remote_progress_cancels_count_budget() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(10),
..Default::default()
},
);
budget.cancel_after_unreported_remote_progress();
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
assert!(budget.token().is_cancelled());
}
#[tokio::test]
async fn count_budgets_require_serial_progress_accounting() {
let parent = CancellationToken::new();
let runtime_only = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_duration: Some(Duration::from_secs(1)),
..Default::default()
},
);
let object_limited = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(1),
..Default::default()
},
);
let directory_limited = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
assert!(!runtime_only.requires_serial_progress_accounting());
assert!(object_limited.requires_serial_progress_accounting());
assert!(directory_limited.requires_serial_progress_accounting());
}
}
crates/scanner/src/scanner_folder.rs
+341 -49
View File
@@ -38,8 +38,8 @@ use rustfs_common::heal_channel::{
HealRequestSource, HealScanMode, send_heal_request_with_admission,
};
use rustfs_common::metrics::{
IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn,
current_path_updater, global_metrics,
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
UpdateCurrentPathFn, current_path_updater, global_metrics,
};
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
@@ -76,6 +76,8 @@ const DATA_SCANNER_COMPACT_AT_CHILDREN: usize = 10000;
const DATA_SCANNER_COMPACT_AT_FOLDERS: usize = DATA_SCANNER_COMPACT_AT_CHILDREN / 4;
const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000;
const SCANNER_LIST_PATH_RAW_TIMEOUT: Duration = Duration::from_secs(60);
const SCANNER_ENTRY_PROGRESS_BATCH: u64 = 32;
const SCANNER_ENTRY_PROGRESS_INTERVAL: Duration = Duration::from_secs(30);
const DEFAULT_HEAL_OBJECT_SELECT_PROB: u32 = 1024;
const ENV_DATA_USAGE_UPDATE_DIR_CYCLES: &str = "RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES";
const ENV_HEAL_OBJECT_SELECT_PROB: &str = "RUSTFS_HEAL_OBJECT_SELECT_PROB";
@@ -411,24 +413,36 @@ fn non_negative_i64_to_u64(value: i64) -> u64 {
}
fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary) {
into.size += summary.total_size;
into.versions += summary.versions;
into.delete_markers += summary.delete_markers;
into.obj_sizes.add(summary.total_size as u64);
into.obj_versions.add(summary.versions as u64);
into.size = into.size.saturating_add(summary.total_size);
into.versions = into.versions.saturating_add(summary.versions);
into.delete_markers = into.delete_markers.saturating_add(summary.delete_markers);
into.obj_sizes.add(u64::try_from(summary.total_size).unwrap_or(u64::MAX));
into.obj_versions.add(u64::try_from(summary.versions).unwrap_or(u64::MAX));
let replication_stats = into.replication_stats.get_or_insert_with(Default::default);
replication_stats.replica_size += non_negative_i64_to_u64(summary.replica_size);
replication_stats.replica_count += summary.replica_count as u64;
replication_stats.replica_size = replication_stats
.replica_size
.saturating_add(non_negative_i64_to_u64(summary.replica_size));
replication_stats.replica_count = replication_stats
.replica_count
.saturating_add(u64::try_from(summary.replica_count).unwrap_or(u64::MAX));
for (arn, st) in &summary.repl_target_stats {
let tgt_stat = replication_stats.targets.entry(arn.clone()).or_default();
tgt_stat.pending_size += non_negative_i64_to_u64(st.pending_size);
tgt_stat.failed_size += non_negative_i64_to_u64(st.failed_size);
tgt_stat.replicated_size += non_negative_i64_to_u64(st.replicated_size);
tgt_stat.replicated_count += st.replicated_count as u64;
tgt_stat.failed_count += st.failed_count as u64;
tgt_stat.pending_count += st.pending_count as u64;
tgt_stat.pending_size = tgt_stat.pending_size.saturating_add(non_negative_i64_to_u64(st.pending_size));
tgt_stat.failed_size = tgt_stat.failed_size.saturating_add(non_negative_i64_to_u64(st.failed_size));
tgt_stat.replicated_size = tgt_stat
.replicated_size
.saturating_add(non_negative_i64_to_u64(st.replicated_size));
tgt_stat.replicated_count = tgt_stat
.replicated_count
.saturating_add(u64::try_from(st.replicated_count).unwrap_or(u64::MAX));
tgt_stat.failed_count = tgt_stat
.failed_count
.saturating_add(u64::try_from(st.failed_count).unwrap_or(u64::MAX));
tgt_stat.pending_count = tgt_stat
.pending_count
.saturating_add(u64::try_from(st.pending_count).unwrap_or(u64::MAX));
}
}
@@ -1036,22 +1050,23 @@ impl ScannerItem {
if let Some(repl_target_size_summary) = size_summary.repl_target_stats.get_mut(arn.as_str()) {
match target_status {
ReplicationStatusType::Pending => {
repl_target_size_summary.pending_size += roi.size;
repl_target_size_summary.pending_count += 1;
size_summary.pending_size += roi.size;
size_summary.pending_count += 1;
repl_target_size_summary.pending_size = repl_target_size_summary.pending_size.saturating_add(roi.size);
repl_target_size_summary.pending_count = repl_target_size_summary.pending_count.saturating_add(1);
size_summary.pending_size = size_summary.pending_size.saturating_add(roi.size);
size_summary.pending_count = size_summary.pending_count.saturating_add(1);
}
ReplicationStatusType::Failed => {
repl_target_size_summary.failed_size += roi.size;
repl_target_size_summary.failed_count += 1;
size_summary.failed_size += roi.size;
size_summary.failed_count += 1;
repl_target_size_summary.failed_size = repl_target_size_summary.failed_size.saturating_add(roi.size);
repl_target_size_summary.failed_count = repl_target_size_summary.failed_count.saturating_add(1);
size_summary.failed_size = size_summary.failed_size.saturating_add(roi.size);
size_summary.failed_count = size_summary.failed_count.saturating_add(1);
}
ReplicationStatusType::Completed | ReplicationStatusType::CompletedLegacy => {
repl_target_size_summary.replicated_size += roi.size;
repl_target_size_summary.replicated_count += 1;
size_summary.replicated_size += roi.size;
size_summary.replicated_count += 1;
repl_target_size_summary.replicated_size =
repl_target_size_summary.replicated_size.saturating_add(roi.size);
repl_target_size_summary.replicated_count = repl_target_size_summary.replicated_count.saturating_add(1);
size_summary.replicated_size = size_summary.replicated_size.saturating_add(roi.size);
size_summary.replicated_count = size_summary.replicated_count.saturating_add(1);
}
_ => {}
}
@@ -1059,8 +1074,8 @@ impl ScannerItem {
}
if oi.replication_status == ReplicationStatusType::Replica {
size_summary.replica_size += roi.size;
size_summary.replica_count += 1;
size_summary.replica_size = size_summary.replica_size.saturating_add(roi.size);
size_summary.replica_count = size_summary.replica_count.saturating_add(1);
}
}
@@ -1835,21 +1850,10 @@ impl FolderScanner {
let mut dir_reader = match tokio::fs::read_dir(&dir_path).await {
Ok(dir_reader) => dir_reader,
Err(e) if e.kind() == ErrorKind::NotFound => {
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
dir_path = %dir_path,
state = "dir_missing_before_read",
error = %e,
"Scanner folder state updated"
);
return Ok(());
}
Err(e) => return Err(ScannerError::Io(e)),
};
let mut pending_entry_progress = 0_u64;
let mut last_entry_progress = Instant::now();
loop {
let entry = match dir_reader.next_entry().await {
@@ -1883,6 +1887,14 @@ impl FolderScanner {
}
Err(e) => return Err(ScannerError::Io(e)),
};
pending_entry_progress = pending_entry_progress.saturating_add(1);
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
{
self.budget.record_entries_visited(pending_entry_progress);
pending_entry_progress = 0;
last_entry_progress = Instant::now();
}
let file_name = entry.file_name().to_string_lossy().to_string();
if file_name.is_empty() || file_name == "." || file_name == ".." {
continue;
@@ -2122,6 +2134,7 @@ impl FolderScanner {
global_metrics().record_scanner_yield(yield_start.elapsed());
}
}
self.budget.record_entries_visited(pending_entry_progress);
if ctx.is_cancelled() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
@@ -2759,7 +2772,8 @@ pub async fn scan_data_folder(
// Get disk path
let base_path = local_disk.path().to_string_lossy().to_string();
let (update_current_path, close_disk) = current_path_updater(&base_path, &cache.info.name);
let (update_current_path, close_disk) = current_path_updater(&base_path, &cache.info.name).await;
let mut close_disk_guard = CloseDiskGuard::new(close_disk);
// Create skip_heal flag
let is_erasure_mode = scanner_is_erasure().await;
@@ -2829,6 +2843,8 @@ pub async fn scan_data_folder(
new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
new_cache.info.last_update = Some(SystemTime::now());
new_cache.info.next_cycle = cache.info.next_cycle;
let unresolved_objects = root.failed_objects > 0 || !new_cache.info.failed_objects.is_empty();
new_cache.info.snapshot_complete = !unresolved_objects;
let had_scan_checkpoint = cache.info.scan_checkpoint.is_some() || new_cache.info.scan_checkpoint.is_some();
new_cache.info.scan_resume_after = None;
new_cache.info.scan_checkpoint = None;
@@ -2836,8 +2852,12 @@ pub async fn scan_data_folder(
global_metrics().record_scanner_checkpoint_cleared();
}
close_disk().await;
Ok(new_cache.clone())
close_disk_guard.close().await;
if unresolved_objects {
Err(ScannerError::PartialCache(Box::new(new_cache.clone())))
} else {
Ok(new_cache.clone())
}
}
Err(e) => {
if ctx.is_cancelled() {
@@ -2857,14 +2877,23 @@ pub async fn scan_data_folder(
}
new_cache.info.last_update = Some(SystemTime::now());
new_cache.info.next_cycle = cache.info.next_cycle;
new_cache.info.snapshot_complete = false;
if root_has_progress {
set_scan_checkpoint(new_cache, checkpoint_reason_from_budget(budget.reason()));
}
close_disk().await;
close_disk_guard.close().await;
return Err(ScannerError::PartialCache(Box::new(new_cache.clone())));
}
}
close_disk().await;
if matches!(&e, ScannerError::Io(io) if io.kind() == ErrorKind::NotFound) {
let mut partial_cache = scanner.old_cache.clone();
partial_cache.info.last_update = Some(SystemTime::now());
partial_cache.info.next_cycle = cache.info.next_cycle;
partial_cache.info.snapshot_complete = false;
close_disk_guard.close().await;
return Err(ScannerError::NamespaceNotFoundCache(Box::new(partial_cache)));
}
close_disk_guard.close().await;
// No useful information, return original cache
Err(e)
}
@@ -2877,7 +2906,7 @@ mod tests {
use super::*;
use crate::storage_api::VersionPurgeStatusType;
use crate::{DiskOption, Endpoint, new_disk};
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, new_disk};
use rustfs_filemeta::{FileInfo, FileMeta};
use serial_test::serial;
#[cfg(unix)]
@@ -2886,6 +2915,76 @@ mod tests {
use temp_env::{with_var, with_var_unset};
use uuid::Uuid;
#[test]
fn scanner_size_summary_application_saturates_usage_counters() {
let target = "arn:minio:replication::target".to_string();
let mut entry = DataUsageEntry {
size: usize::MAX,
versions: usize::MAX,
delete_markers: usize::MAX,
replication_stats: Some(Default::default()),
..Default::default()
};
let replication_stats = entry
.replication_stats
.as_mut()
.expect("replication statistics should be initialized");
replication_stats.replica_size = u64::MAX;
replication_stats.replica_count = u64::MAX;
let target_stats = replication_stats.targets.entry(target.clone()).or_default();
target_stats.pending_size = u64::MAX;
target_stats.failed_size = u64::MAX;
target_stats.replicated_size = u64::MAX;
target_stats.replicated_count = u64::MAX;
target_stats.failed_count = u64::MAX;
target_stats.pending_count = u64::MAX;
let mut summary = SizeSummary {
total_size: 1,
versions: 1,
delete_markers: 1,
replica_size: 1,
replica_count: 1,
..Default::default()
};
summary.repl_target_stats.insert(
target.clone(),
ReplTargetSizeSummary {
replicated_size: 1,
replicated_count: 1,
pending_size: 1,
failed_size: 1,
pending_count: 1,
failed_count: 1,
},
);
apply_scanner_size_summary(&mut entry, &summary);
assert_eq!(entry.size, usize::MAX);
assert_eq!(entry.versions, usize::MAX);
assert_eq!(entry.delete_markers, usize::MAX);
assert_eq!(entry.obj_sizes.to_map()["LESS_THAN_1024_B"], 1);
assert_eq!(entry.obj_versions.to_map()["SINGLE_VERSION"], 1);
let replication_stats = entry
.replication_stats
.as_ref()
.expect("replication statistics should remain present");
assert_eq!(replication_stats.replica_size, u64::MAX);
assert_eq!(replication_stats.replica_count, u64::MAX);
let target_stats = replication_stats
.targets
.get(&target)
.expect("replication target statistics should remain present");
assert_eq!(target_stats.pending_size, u64::MAX);
assert_eq!(target_stats.failed_size, u64::MAX);
assert_eq!(target_stats.replicated_size, u64::MAX);
assert_eq!(target_stats.replicated_count, u64::MAX);
assert_eq!(target_stats.failed_count, u64::MAX);
assert_eq!(target_stats.pending_count, u64::MAX);
}
async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
let temp_dir = std::env::temp_dir().join(format!("rustfs-scanner-test-{}", Uuid::new_v4()));
tokio::fs::create_dir_all(&temp_dir)
@@ -4145,7 +4244,7 @@ mod tests {
scanner.update_cache.info.name = "bucket".to_string();
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(1),
@@ -4168,6 +4267,7 @@ mod tests {
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
assert!(budget.token().is_cancelled());
assert!(budget.entries_visited() >= 1);
}
#[tokio::test]
@@ -4212,6 +4312,51 @@ mod tests {
assert!(update.compacted, "partial update should preserve compacted state");
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir),
};
let parent = CancellationToken::new();
parent.cancel();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let cache = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
let disk_path = scanner.local_disk.path().to_string_lossy().to_string();
let result = scan_data_folder(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
scanner.local_disk,
cache,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
assert!(matches!(result, Err(ScannerError::Other(message)) if message == "Operation cancelled"));
tokio::time::timeout(Duration::from_secs(1), async {
loop {
let report = global_metrics().report().await;
if report.active_paths.iter().all(|path| !path.starts_with(&disk_path)) {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("cancelled scan should deregister its active path");
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
@@ -4262,6 +4407,7 @@ mod tests {
assert!(partial_cache.info.last_update.is_some());
assert_eq!(partial_cache.info.next_cycle, 7);
assert!(!partial_cache.info.snapshot_complete);
assert!(partial_cache.root().is_some(), "partial cache should keep completed scan progress");
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
@@ -4386,6 +4532,104 @@ mod tests {
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
}
#[tokio::test]
#[serial]
async fn scan_data_folder_missing_bucket_returns_partial() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir),
};
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let mut cache = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "missing-bucket".to_string(),
next_cycle: 9,
..Default::default()
},
..Default::default()
};
cache.replace(
"missing-bucket",
crate::data_usage_define::DATA_USAGE_ROOT,
DataUsageEntry {
objects: 7,
size: 70,
..Default::default()
},
);
let result = scan_data_folder(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
scanner.local_disk,
cache,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
let partial = match result {
Err(ScannerError::NamespaceNotFoundCache(partial)) => partial,
other => panic!("missing bucket should keep the scan incomplete, got {other:?}"),
};
assert!(!partial.info.snapshot_complete);
assert_eq!(partial.info.next_cycle, 9);
let root = partial
.checked_flatten("missing-bucket")
.expect("missing bucket partial must retain the last durable usage");
assert_eq!(root.objects, 7);
assert_eq!(root.size, 70);
}
#[tokio::test]
#[serial]
async fn scan_data_folder_missing_scan_root_returns_partial() {
let (scanner, temp_dir) = build_test_scanner().await;
tokio::fs::remove_dir_all(&temp_dir)
.await
.expect("failed to remove scanner root");
let _guard = TestGuard {
temp_dir: Some(temp_dir),
};
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let cache = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "missing-bucket".to_string(),
next_cycle: 9,
..Default::default()
},
..Default::default()
};
let result = scan_data_folder(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
scanner.local_disk,
cache,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
let partial = match result {
Err(ScannerError::NamespaceNotFoundCache(partial)) => partial,
other => panic!("missing scan root should keep the scan incomplete, got {other:?}"),
};
assert!(!partial.info.snapshot_complete);
assert_eq!(partial.info.next_cycle, 9);
let root = partial
.checked_flatten("missing-bucket")
.expect("missing scan root partial should retain a non-publishable root");
assert_eq!(root.objects, 0);
assert_eq!(root.size, 0);
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folders() {
@@ -4533,6 +4777,7 @@ mod tests {
.size_recursive("bucket")
.expect("completed cache should retain bucket usage");
assert_eq!(root.objects, 5);
assert!(result.info.snapshot_complete);
assert!(result.info.scan_resume_after.is_none());
assert!(result.info.scan_checkpoint.is_none());
}
@@ -4626,6 +4871,53 @@ mod tests {
assert_eq!(result.info.next_cycle, 11);
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_keeps_unresolved_objects_partial() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir.clone()),
};
write_test_object_metadata(&temp_dir, "bucket", "object").await;
let failed_path = temp_dir
.join("bucket")
.join("object")
.join(STORAGE_FORMAT_FILE)
.to_string_lossy()
.into_owned();
let mut cache = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 12,
..Default::default()
},
..Default::default()
};
cache.info.failed_objects.insert(failed_path, FolderScanner::now_secs());
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let result = scan_data_folder(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
scanner.local_disk.clone(),
cache,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
let partial = match result {
Err(ScannerError::PartialCache(partial)) => partial,
other => panic!("expected unresolved object to keep the cache partial, got {other:?}"),
};
assert!(!partial.info.snapshot_complete);
assert!(!partial.info.failed_objects.is_empty());
}
#[tokio::test]
#[serial]
#[cfg(unix)]
crates/scanner/src/scanner_io.rs
+2407 -360
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/storage_api.rs
+38 -15
View File
@@ -27,6 +27,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE, object_opts_from_object_info as ecstore_object_opts_from_object_info,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
get_replication_config as ecstore_get_replication_config,
@@ -62,8 +64,8 @@ pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError,
pub(crate) use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI,
DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation,
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE,
ScanGuard as EcstoreScanGuard,
NsScannerOpenRequest as EcstoreNsScannerOpenRequest, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE, ScanGuard as EcstoreScanGuard,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{
@@ -72,6 +74,12 @@ pub(crate) use rustfs_ecstore::api::disk::{
pub(crate) use rustfs_ecstore::api::error::{
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::layout::{
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
pub(crate) use rustfs_ecstore::api::runtime::{
expiry_state_handle as ecstore_expiry_state_handle, global_tier_config_mgr as ecstore_get_global_tier_config_mgr,
object_store_handle as ecstore_resolve_object_store_handle, setup_is_erasure as ecstore_is_erasure,
@@ -79,30 +87,39 @@ pub(crate) use rustfs_ecstore::api::runtime::{
};
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx;
pub(crate) use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
use rustfs_storage_api as storage_contracts;
pub(crate) mod owner {
pub(crate) use super::storage_contracts::{HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete};
pub(crate) use super::storage_contracts::{
HTTPPreconditions, HTTPRangeSpec, NS_SCANNER_PROTOCOL_VERSION, ObjectIO, ObjectOperations, ObjectToDelete,
};
pub(crate) use super::{
ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS,
ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys,
EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions,
EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc,
EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfigurationExt,
EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore,
EcstoreTierConfig, EcstoreVersioningApi, ScannerReplicationHealObject, ScannerReplicationHealResult,
ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle,
ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket,
ecstore_list_path_raw, ecstore_object_opts_from_object_info, ecstore_path2_bucket_object,
ecstore_path2_bucket_object_with_base_path, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info,
ecstore_resolve_object_store_handle, ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts,
EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard,
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi,
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure,
ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle,
ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
pub(crate) use super::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
pub(crate) use super::{
EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints,
EcstoreInstanceContext, EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys,
ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
};
}
#[derive(Clone, Debug, Serialize, Deserialize)]
@@ -244,11 +261,17 @@ impl From<EcstoreReplicationHealQueueResult> for ScannerReplicationHealResult {
}
pub(crate) mod scan {
pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, NamespaceLocking};
pub use super::storage_contracts::SCANNER_ACTIVITY_PROTOCOL_VERSION;
pub(crate) use super::storage_contracts::{
BucketOperations, BucketOptions, NamespaceLocking, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
};
#[cfg(test)]
pub(crate) use super::storage_contracts::{MakeBucketOptions, ObjectIO};
}
pub(crate) mod scanner_io {
pub(crate) use super::storage_contracts::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
pub(crate) use super::storage_contracts::{BucketInfo, BucketOptions};
#[cfg(test)]
pub(crate) use super::storage_contracts::{HTTPRangeSpec, ObjectIO};
}
crates/storage-api/src/lib.rs
+21
View File
@@ -17,6 +17,27 @@
pub const WALK_DIR_STREAM_COMPLETION_QUERY: &str = "walk_dir_stream_completion";
pub const WALK_DIR_STREAM_COMPLETION_V1: &str = "error-v1";
pub const WALK_DIR_BODY_SHA256_QUERY: &str = "walk_dir_body_sha256";
pub const NS_SCANNER_BODY_SHA256_QUERY: &str = "ns_scanner_body_sha256";
pub const NS_SCANNER_CAPABILITY_CHALLENGE_QUERY: &str = "ns_scanner_challenge";
pub const NS_SCANNER_CYCLE_QUERY: &str = "ns_scanner_cycle";
pub const NS_SCANNER_LEADER_EPOCH_QUERY: &str = "ns_scanner_leader_epoch";
pub const NS_SCANNER_REQUEST_ID_QUERY: &str = "ns_scanner_request_id";
pub const NS_SCANNER_SERVER_EPOCH_QUERY: &str = "ns_scanner_server_epoch";
pub const NS_SCANNER_SESSION_ID_QUERY: &str = "ns_scanner_session_id";
pub const NS_SCANNER_SESSION_SEQUENCE_QUERY: &str = "ns_scanner_session_sequence";
pub const NS_SCANNER_PROTOCOL_VERSION_QUERY: &str = "ns_scanner_protocol";
pub const NS_SCANNER_PROTOCOL_VERSION: u16 = 3;
pub const SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION: u32 = 0;
pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 4;
pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 5;
#[derive(Debug, serde::Deserialize, serde::Serialize)]
#[serde(deny_unknown_fields)]
pub struct NsScannerCapabilityResponse {
pub version: u16,
pub server_epoch: uuid::Uuid,
pub proof: Vec<u8>,
}
pub mod admin;
pub mod bucket;
crates/test-utils/src/data_usage_snapshot_tests.rs
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
use std::time::{Duration, SystemTime};
use crate::TestECStoreEnv;
@@ -48,7 +48,7 @@ async fn data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache()
.init_bucket_metadata(false)
.build()
.await;
let primary = format!("{BUCKET_META_PREFIX}/.usage.json");
let primary = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}");
let backup = format!("{primary}.bkp");
let last_update = SystemTime::UNIX_EPOCH + Duration::from_secs(100);
let expected = snapshot("bucket-a", last_update);
docs/architecture/compat-cleanup-register.md
+2
View File
@@ -12,6 +12,8 @@ for later deletion.
## Open Items
- `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while the authoritative `.usage.v2.json` pair is absent and continue removing deleted buckets from legacy copies that still exist. Remove the fallback and legacy cleanup after every supported direct-upgrade source version writes `.usage.v2.json`.
- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Current protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while the distributed scanner publishes usage only after every peer returns authenticated protocol v5 state. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, and remove the protocol-v4 activity codec after every supported peer implements protocol v5; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
- `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
- `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
- `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus.
docs/operations/scanner-runtime-controls.md
+2 -2
View File
@@ -58,8 +58,8 @@ The `/v3/scanner/status` response reports each effective runtime value with a
| `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. |
| `scanner.idle_mode` | `RUSTFS_SCANNER_IDLE_MODE` | boolean | `true` | Enables scanner sleeps and cooperative throttling. |
| `scanner.cache_save_timeout` | `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` | seconds | `30` | Timeout for saving scanner cache; runtime enforces a minimum of `1`. |
| `scanner.max_concurrent_set_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` | count | `0` | Caps concurrent set-level scanner tasks. `0` keeps topology-derived concurrency. |
| `scanner.max_concurrent_disk_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` | count | `0` | Caps concurrent disk bucket walks per set. `0` keeps disk-count-derived concurrency. |
| `scanner.max_concurrent_set_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` | count | `4` | Caps concurrent set-level scanner tasks. `0` keeps topology-derived concurrency. |
| `scanner.max_concurrent_disk_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` | count | `4` | Caps concurrent disk bucket walks per set. `0` keeps disk-count-derived concurrency. |
| `scanner.yield_every_n_objects` | `RUSTFS_SCANNER_YIELD_EVERY_N_OBJECTS` | objects | `128` | Controls how often object loops yield to the async runtime. `0` disables this extra yield. |
| `scanner.alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | versions | `100` | Version count threshold for scanner alerts. |
| `scanner.alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | bytes | `1099511627776` | Retained version byte threshold for scanner alerts. |
rustfs/src/admin/handlers/account_info.rs
+4 -2
View File
@@ -259,8 +259,10 @@ impl Operation for AccountInfoHandler {
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
// Serve the last persisted scanner snapshot plus the in-memory overlay.
// This request path must never trigger a live full-version listing
// Serve the last persisted scanner snapshot plus an authoritative
// single-process overlay. Distributed nodes must not promote their
// process-local absolute counters to cluster-wide bucket totals.
// This path never triggers a live full-version listing
// (rustfs/backlog#1306); freshness is owned by the scanner.
let mut data_usage_info = map_data_usage_result(load_data_usage_from_backend_cached(store.clone()).await)?;
apply_bucket_usage_memory_overlay(&mut data_usage_info).await;
rustfs/src/admin/handlers/audit_runtime_config.rs
+7 -9
View File
@@ -12,11 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::handlers::supervise_admin_mutation;
use crate::admin::handlers::target_descriptor::AdminTargetSpec;
use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context};
use crate::admin::storage_api::config::{
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot,
};
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
use rustfs_config::DEFAULT_DELIMITER;
@@ -96,20 +96,20 @@ where
let Some(store) = current_object_store_handle_for_context(context) else {
return Err(s3_error!(InternalError, "server storage not initialized"));
};
let specs = specs.to_vec();
let lock_store = store.clone();
with_admin_server_config_write_lock(lock_store, move || async move {
let mut config = read_admin_config_without_migrate_no_lock(store.clone())
supervise_admin_mutation("audit config update", async move {
let snapshot = read_admin_server_config_snapshot(store.clone())
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
let mut config = snapshot.config.clone();
if !modifier(&mut config) {
return Ok(());
}
save_admin_server_config_no_lock(store, &config)
save_admin_server_config_snapshot(store, &config, &snapshot)
.await
.map(|_| ())
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
// Keep persistence and runtime publication in one detached, serialized
@@ -118,8 +118,6 @@ where
apply_audit_runtime_config(&specs, config).await
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
Ok(())
}
pub(crate) async fn update_audit_config_and_reload<F>(specs: &[AdminTargetSpec], modifier: F) -> S3Result<()>
rustfs/src/admin/handlers/config_admin.rs
+303 -238
View File
@@ -21,15 +21,14 @@ use crate::admin::runtime_sources::{
};
use crate::admin::service::config::{
CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN,
LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem,
preflight_dynamic_config_reload, prepare_server_config, signal_config_snapshot_reload_checked,
signal_dynamic_config_reload_checked,
LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, is_dynamic_config_subsystem, preflight_dynamic_config_reload,
prepare_server_config, reload_dynamic_config_runtime_state, reload_runtime_config_snapshot, signal_config_snapshot_reload,
signal_config_snapshot_reload_checked, signal_dynamic_config_reload_checked,
};
use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
use crate::admin::storage_api::config::{
RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate,
read_admin_config_without_migrate_no_lock, save_admin_config, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config,
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, save_admin_server_config_snapshot,
};
use crate::admin::storage_api::contract::list::ListOperations as _;
use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
@@ -64,7 +63,7 @@ use rustfs_config::oidc::{
};
use rustfs_config::server_config::{Config as ServerConfig, DEFAULT_KVS, KV, KVS};
use rustfs_config::{
COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, ENV_SCANNER_ALERT_EXCESS_FOLDERS,
BASE_DSN_STRING, COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, ENV_SCANNER_ALERT_EXCESS_FOLDERS,
ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, ENV_SCANNER_ALERT_EXCESS_VERSIONS, ENV_SCANNER_BITROT_CYCLE_SECS,
ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DIRECTORIES,
ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_IDLE_MODE,
@@ -86,7 +85,6 @@ use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}
use serde::Serialize;
use std::collections::{BTreeSet, HashMap};
use std::env;
use std::future::Future;
use std::mem::size_of;
use time::OffsetDateTime;
use tracing::warn;
@@ -750,14 +748,6 @@ async fn load_server_config_from_store() -> S3Result<ServerConfig> {
.map_err(Into::into)
}
async fn load_server_config_from_store_locked() -> S3Result<ServerConfig> {
let store = object_store()?;
read_admin_config_without_migrate_no_lock(store)
.await
.map_err(ApiError::from)
.map_err(Into::into)
}
async fn load_active_server_config() -> S3Result<ServerConfig> {
if let Ok(config) = load_server_config_from_store().await {
return Ok(config);
@@ -768,9 +758,17 @@ async fn load_active_server_config() -> S3Result<ServerConfig> {
.ok_or_else(|| s3_error!(InternalError, "server config is not initialized"))
}
async fn save_server_config_to_store_locked(config: &ServerConfig) -> S3Result<()> {
async fn load_server_config_snapshot_from_store() -> S3Result<AdminServerConfigSnapshot> {
let store = object_store()?;
save_admin_server_config_no_lock(store, config)
read_admin_server_config_snapshot(store)
.await
.map_err(ApiError::from)
.map_err(Into::into)
}
async fn save_server_config_to_store(config: &ServerConfig, snapshot: &AdminServerConfigSnapshot) -> S3Result<bool> {
let store = object_store()?;
save_admin_server_config_snapshot(store, config, snapshot)
.await
.map_err(ApiError::from)
.map_err(Into::into)
@@ -864,7 +862,17 @@ fn encode_config_history_recovery(config: &ServerConfig) -> Vec<u8> {
fn encode_config_history_snapshot(config: &ServerConfig) -> S3Result<Vec<u8>> {
let recovery = encode_config_history_recovery(config);
let display = render_full_config(config, true);
let mut display_config = config.clone();
for targets in display_config.0.values_mut() {
for kvs in targets.values_mut() {
for entry in &mut kvs.0 {
if (entry.hidden_if_empty || is_sensitive_key_name(&entry.key)) && !entry.value.trim().is_empty() {
entry.value = REDACTED_VALUE.to_string();
}
}
}
}
let display = render_full_config(&display_config, false);
let recovery_len =
u64::try_from(recovery.len()).map_err(|_| s3_error!(InternalError, "config history recovery snapshot is too large"))?;
let mut snapshot =
@@ -1267,6 +1275,7 @@ fn is_sensitive_key_name(key: &str) -> bool {
|| normalized.ends_with("_tls_client_key")
|| normalized == "private_key"
|| normalized.ends_with("_private_key")
|| normalized == BASE_DSN_STRING
}
fn is_sensitive_env_key_name(key: &str) -> bool {
@@ -1415,7 +1424,7 @@ fn apply_delete_directives(config: &mut ServerConfig, directives: &[ConfigDirect
}
fn render_entry_value(entry: &KV, redact_secrets: bool) -> String {
if redact_secrets && (entry.hidden_if_empty || is_sensitive_key_name(&entry.key)) && !entry.value.trim().is_empty() {
if redact_secrets && is_sensitive_key_name(&entry.key) && !entry.value.trim().is_empty() {
REDACTED_VALUE.to_string()
} else {
entry.value.clone()
@@ -1824,22 +1833,22 @@ fn build_help_response(sub_system: Option<&str>, key: Option<&str>, env_only: bo
})
}
async fn reject_lost_config_transaction<T>(snapshot: AdminServerConfigSnapshot, phase: &'static str) -> S3Result<T> {
drop(snapshot);
reload_runtime_config_snapshot().await?;
signal_config_snapshot_reload().await;
Err(s3_error!(
InternalError,
"server config transaction lock was lost {phase}; runtime state was reloaded from durable config"
))
}
fn publish_prepared_config_snapshots(config: ServerConfig, prepared: PreparedRuntimeConfig) -> S3Result<()> {
prepared.publish_storage_class()?;
publish_server_config(config);
Ok(())
}
async fn commit_prepared_config(
config: ServerConfig,
prepared: PreparedRuntimeConfig,
persist: impl Future<Output = S3Result<()>>,
publish: impl FnOnce(ServerConfig, PreparedRuntimeConfig) -> S3Result<()>,
) -> S3Result<()> {
persist.await?;
publish(config, prepared)
}
/// Re-apply local mutable worker families after a full-config replacement.
/// Peers receive one full-snapshot signal after this returns; signaling each
/// family here as well would recreate audit/scanner targets twice per peer.
@@ -1851,9 +1860,26 @@ fn publish_notify_config_intent(
.then(|| rustfs_notify::ensure_live_events().publish_config(config.clone()))
}
async fn preflight_notify_config_intent(sub_system: Option<&str>) -> S3Result<()> {
if sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)) {
preflight_dynamic_config_reload(sub_system.unwrap_or(NOTIFY_WEBHOOK_SUB_SYS)).await?;
fn config_preflight_subsystems(sub_system: Option<&str>) -> Vec<&str> {
if let Some(sub_system) = sub_system {
return is_dynamic_config_subsystem(sub_system)
.then_some(sub_system)
.into_iter()
.collect();
}
let mut sub_systems = vec![NOTIFY_WEBHOOK_SUB_SYS];
for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
if !NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
sub_systems.push(sub_system);
}
}
sub_systems
}
async fn preflight_config_intent(sub_system: Option<&str>) -> S3Result<()> {
for sub_system in config_preflight_subsystems(sub_system) {
preflight_dynamic_config_reload(sub_system).await?;
}
Ok(())
}
@@ -1869,13 +1895,13 @@ async fn wait_notify_config_intent(transition: Option<rustfs_notify::Notificatio
Ok(true)
}
async fn apply_non_notify_dynamic_subsystems(config: &ServerConfig) -> Vec<String> {
async fn reload_non_notify_dynamic_subsystems() -> Vec<String> {
let mut failures = Vec::new();
for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
if NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
continue;
}
if apply_dynamic_config_for_subsystem(config, sub_system).await.is_err() {
if reload_dynamic_config_runtime_state(sub_system).await.is_err() {
failures.push(format!("local {sub_system}"));
warn!(
event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
@@ -1904,7 +1930,6 @@ fn finish_config_reconciliation(errors: Vec<String>) -> S3Result<()> {
}
async fn reconcile_targeted_config(
config: ServerConfig,
sub_system: Option<String>,
storage_class_applied: bool,
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
@@ -1933,8 +1958,8 @@ async fn reconcile_targeted_config(
} else if let Some(sub_system) = sub_system.as_deref()
&& is_dynamic_config_subsystem(sub_system)
{
let config_applied = match apply_dynamic_config_for_subsystem(&config, sub_system).await {
Ok(applied) => applied,
let config_applied = match reload_dynamic_config_runtime_state(sub_system).await {
Ok(()) => true,
Err(_) => {
warn!(config_subsystem = sub_system, reason = "apply_failed", "Local config reload failed");
errors.push(format!("local {sub_system}"));
@@ -1958,10 +1983,7 @@ async fn reconcile_targeted_config(
Ok(config_applied)
}
async fn reconcile_full_config(
config: ServerConfig,
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
) -> S3Result<()> {
async fn reconcile_full_config(notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<()> {
let mut errors = Vec::new();
if let Err(err) = wait_notify_config_intent(notify_transition).await {
warn!(error = %err, "Local notification config failed to converge");
@@ -1971,7 +1993,7 @@ async fn reconcile_full_config(
warn!(error = %err, "Peer storage-class reload failed");
errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
}
errors.extend(apply_non_notify_dynamic_subsystems(&config).await);
errors.extend(reload_non_notify_dynamic_subsystems().await);
if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await {
warn!(error = %err, "Peer notification config reload failed");
errors.push("peer notify".to_string());
@@ -1983,6 +2005,90 @@ async fn reconcile_full_config(
finish_config_reconciliation(errors)
}
struct PersistedConfigTransaction {
previous_config: ServerConfig,
history_restore_id: Option<String>,
storage_class_applied: bool,
notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
}
async fn persist_server_config_transaction(
config: ServerConfig,
prepared: PreparedRuntimeConfig,
snapshot: AdminServerConfigSnapshot,
sub_system: Option<&str>,
) -> S3Result<PersistedConfigTransaction> {
snapshot.ensure_lock_held().map_err(ApiError::from).map_err(S3Error::from)?;
let previous_config = snapshot.config.clone();
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
if snapshot.is_lock_lost() {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
return reject_lost_config_transaction(snapshot, "before persistence").await;
}
let persisted = match save_server_config_to_store(&config, &snapshot).await {
Ok(persisted) => persisted,
Err(err) => {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
return Err(err);
}
};
let history_restore_id = if persisted {
Some(history_restore_id)
} else {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
None
};
if snapshot.is_lock_lost() {
return reject_lost_config_transaction(snapshot, "after persistence").await;
}
let storage_class_applied = sub_system.is_none_or(|value| value == STORAGE_CLASS_SUB_SYS);
if storage_class_applied {
if let Err(err) = publish_prepared_config_snapshots(config.clone(), prepared) {
return Err(match history_restore_id.as_deref() {
Some(restore_id) => s3_error!(
InternalError,
"config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
restore_id,
err
),
None => err,
});
}
} else {
publish_server_config(config.clone());
}
let notify_transition = publish_notify_config_intent(&config, sub_system);
if snapshot.is_lock_lost() {
return reject_lost_config_transaction(snapshot, "while publishing runtime snapshots").await;
}
drop(snapshot);
Ok(PersistedConfigTransaction {
previous_config,
history_restore_id,
storage_class_applied,
notify_transition,
})
}
async fn commit_server_config_transaction(
config: ServerConfig,
prepared: PreparedRuntimeConfig,
snapshot: AdminServerConfigSnapshot,
sub_system: Option<String>,
) -> S3Result<bool> {
let transaction = persist_server_config_transaction(config, prepared, snapshot, sub_system.as_deref()).await?;
if sub_system.is_none() {
reconcile_full_config(transaction.notify_transition).await?;
return Ok(false);
}
reconcile_targeted_config(sub_system, transaction.storage_class_applied, transaction.notify_transition).await
}
pub struct GetConfigKVHandler {}
#[async_trait::async_trait]
@@ -2016,41 +2122,13 @@ impl Operation for SetConfigKVHandler {
validate_config_directives(&directives)?;
let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
let transaction_sub_system = sub_system.clone();
let config_store = object_store()?;
let config_applied = supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(sub_system.as_deref()).await?;
let (config, storage_class_applied, notify_transition) =
with_admin_server_config_write_lock(config_store, move || async move {
let sub_system = transaction_sub_system.as_deref();
let previous_config = load_server_config_from_store_locked().await?;
let mut config = previous_config.clone();
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, sub_system).await?;
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
if let Err(err) = save_server_config_to_store_locked(&config).await {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
return Err(err);
}
if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
publish_prepared_config_snapshots(config.clone(), prepared).map_err(|err| {
s3_error!(
InternalError,
"config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
history_restore_id,
err
)
})?;
} else {
publish_server_config(config.clone());
}
let notify_transition = publish_notify_config_intent(&config, sub_system);
Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
preflight_config_intent(sub_system.as_deref()).await?;
let snapshot = load_server_config_snapshot_from_store().await?;
let mut config = snapshot.config.clone();
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, sub_system.as_deref()).await?;
commit_server_config_transaction(config, prepared, snapshot, sub_system).await
})
.await?;
@@ -2072,41 +2150,13 @@ impl Operation for DelConfigKVHandler {
validate_config_directives(&directives)?;
let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
let transaction_sub_system = sub_system.clone();
let config_store = object_store()?;
let config_applied = supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(sub_system.as_deref()).await?;
let (config, storage_class_applied, notify_transition) =
with_admin_server_config_write_lock(config_store, move || async move {
let sub_system = transaction_sub_system.as_deref();
let previous_config = load_server_config_from_store_locked().await?;
let mut config = previous_config.clone();
apply_delete_directives(&mut config, &directives);
let prepared = prepare_server_config(&config, sub_system).await?;
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
if let Err(err) = save_server_config_to_store_locked(&config).await {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
return Err(err);
}
if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
publish_prepared_config_snapshots(config.clone(), prepared).map_err(|err| {
s3_error!(
InternalError,
"config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
history_restore_id,
err
)
})?;
} else {
publish_server_config(config.clone());
}
let notify_transition = publish_notify_config_intent(&config, sub_system);
Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
preflight_config_intent(sub_system.as_deref()).await?;
let snapshot = load_server_config_snapshot_from_store().await?;
let mut config = snapshot.config.clone();
apply_delete_directives(&mut config, &directives);
let prepared = prepare_server_config(&config, sub_system.as_deref()).await?;
commit_server_config_transaction(config, prepared, snapshot, sub_system).await
})
.await?;
@@ -2186,79 +2236,108 @@ impl Operation for RestoreConfigHistoryKVHandler {
.ok_or_else(|| s3_error!(InvalidRequest, "missing restoreId query parameter"))?;
let history = read_server_config_history(restore_id).await?;
let config = decode_config_history_snapshot(&history)?;
let prepared = prepare_server_config(&config, None).await?;
let config_store = object_store()?;
supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(None).await?;
let persisted_config = config.clone();
preflight_config_intent(None).await?;
let snapshot = load_server_config_snapshot_from_store().await?;
let prepared = prepare_server_config(&config, None).await?;
let restored_config = config.clone();
let rollback_store = config_store.clone();
let (previous_config, recovery_restore_id, notify_transition) =
with_admin_server_config_write_lock(config_store, move || async move {
let previous_config = load_server_config_from_store_locked().await?;
let recovery_restore_id = save_server_config_history_snapshot(&previous_config).await?;
if let Err(err) = save_server_config_to_store_locked(&persisted_config).await {
cleanup_failed_config_history_snapshot(&recovery_restore_id).await;
return Err(err);
}
publish_prepared_config_snapshots(persisted_config.clone(), prepared).map_err(|err| {
s3_error!(
InternalError,
"config restore persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
recovery_restore_id,
err
)
})?;
let notify_transition = publish_notify_config_intent(&persisted_config, None);
Ok::<_, S3Error>((previous_config, recovery_restore_id, notify_transition))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config restore: {}", err))??;
let Err(restore_error) = reconcile_full_config(config, notify_transition).await else {
let transaction = persist_server_config_transaction(config, prepared, snapshot, None).await?;
let Err(restore_error) = reconcile_full_config(transaction.notify_transition).await else {
return Ok(());
};
let rollback_config = previous_config.clone();
let rollback_expected_config = restored_config.clone();
let rollback_transition = with_admin_server_config_write_lock(rollback_store, move || async move {
let current_config = load_server_config_from_store_locked().await?;
validate_restore_rollback_generation(&current_config, &rollback_expected_config)?;
let rollback_prepared = prepare_server_config(&rollback_config, None).await?;
commit_prepared_config(
rollback_config.clone(),
rollback_prepared,
save_server_config_to_store_locked(&rollback_config),
publish_prepared_config_snapshots,
)
.await?;
Ok::<_, S3Error>(publish_notify_config_intent(&rollback_config, None))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config rollback: {}", err))?;
let rollback_transition = match rollback_transition {
Ok(transition) => transition,
let previous_config = transaction.previous_config;
let recovery_restore_id = transaction.history_restore_id;
let recovery_reference = recovery_restore_id.as_deref().unwrap_or("not-created");
let rollback_snapshot = match load_server_config_snapshot_from_store().await {
Ok(snapshot) => snapshot,
Err(rollback_error) => {
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_restore_id
recovery_reference
));
}
};
if let Err(rollback_error) = reconcile_full_config(previous_config, rollback_transition).await {
if let Err(rollback_error) = validate_restore_rollback_generation(&rollback_snapshot.config, &restored_config) {
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_reference
));
}
let rollback_prepared = prepare_server_config(&previous_config, None).await?;
rollback_snapshot
.ensure_lock_held()
.map_err(ApiError::from)
.map_err(S3Error::from)?;
if let Err(rollback_error) = save_server_config_to_store(&previous_config, &rollback_snapshot).await {
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_reference
));
}
if rollback_snapshot.is_lock_lost() {
if let Err(rollback_error) =
reject_lost_config_transaction::<()>(rollback_snapshot, "after restore rollback persistence").await
{
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_reference
));
}
return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly"));
}
if let Err(rollback_error) = publish_prepared_config_snapshots(previous_config.clone(), rollback_prepared) {
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback publish failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_reference
));
}
let rollback_transition = publish_notify_config_intent(&previous_config, None);
if rollback_snapshot.is_lock_lost() {
if let Err(rollback_error) =
reject_lost_config_transaction::<()>(rollback_snapshot, "while publishing restore rollback").await
{
return Err(s3_error!(
InternalError,
"config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_reference
));
}
return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly"));
}
drop(rollback_snapshot);
if let Err(rollback_error) = reconcile_full_config(rollback_transition).await {
return Err(s3_error!(
InternalError,
"config restore failed: {}; persisted rollback did not converge: {}; recovery snapshot restoreId={}",
restore_error,
rollback_error,
recovery_restore_id
recovery_reference
));
}
cleanup_failed_config_history_snapshot(&recovery_restore_id).await;
if let Some(recovery_restore_id) = recovery_restore_id.as_deref() {
cleanup_failed_config_history_snapshot(recovery_restore_id).await;
}
Err(s3_error!(InternalError, "config restore failed and was rolled back: {}", restore_error))
})
.await?;
@@ -2293,34 +2372,14 @@ impl Operation for SetConfigHandler {
}
validate_config_directives(&directives)?;
let mut config = ServerConfig::new();
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, None).await?;
let config_store = object_store()?;
supervise_admin_mutation("config mutation", async move {
preflight_notify_config_intent(None).await?;
let persisted_config = config.clone();
let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
let previous_config = load_server_config_from_store_locked().await?;
let history_restore_id = save_server_config_history_snapshot(&previous_config).await?;
if let Err(err) = save_server_config_to_store_locked(&persisted_config).await {
cleanup_failed_config_history_snapshot(&history_restore_id).await;
return Err(err);
}
publish_prepared_config_snapshots(persisted_config.clone(), prepared).map_err(|err| {
s3_error!(
InternalError,
"full config persisted but runtime publish failed; recovery snapshot restoreId={}: {}",
history_restore_id,
err
)
})?;
Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock full server config update: {}", err))??;
reconcile_full_config(config, notify_transition).await
preflight_config_intent(None).await?;
let snapshot = load_server_config_snapshot_from_store().await?;
let mut config = ServerConfig::new();
apply_set_directives(&mut config, &directives)?;
let prepared = prepare_server_config(&config, None).await?;
commit_server_config_transaction(config, prepared, snapshot, None).await?;
Ok(())
})
.await?;
@@ -2332,56 +2391,21 @@ impl Operation for SetConfigHandler {
mod tests {
use super::*;
use serial_test::serial;
use std::sync::{Arc, Mutex};
use temp_env::with_vars;
#[tokio::test]
async fn prepared_config_commit_persists_before_publish() {
let events = Arc::new(Mutex::new(Vec::new()));
let persist_events = events.clone();
let publish_events = events.clone();
commit_prepared_config(
ServerConfig::new(),
PreparedRuntimeConfig::default(),
async move {
persist_events.lock().expect("persist events lock").push("persist");
Ok(())
},
move |_, _| {
publish_events.lock().expect("publish events lock").push("publish");
Ok(())
},
)
.await
.expect("prepared config commit");
assert_eq!(*events.lock().expect("result events lock"), ["persist", "publish"]);
}
#[tokio::test]
async fn prepared_config_commit_does_not_publish_after_persist_failure() {
let events = Arc::new(Mutex::new(Vec::new()));
let persist_events = events.clone();
let publish_events = events.clone();
let err = commit_prepared_config(
ServerConfig::new(),
PreparedRuntimeConfig::default(),
async move {
persist_events.lock().expect("persist events lock").push("persist");
Err(s3_error!(InternalError, "injected persistence failure"))
},
move |_, _| {
publish_events.lock().expect("publish events lock").push("publish");
Ok(())
},
)
.await
.expect_err("persistence failure must propagate");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(*events.lock().expect("result events lock"), ["persist"]);
#[test]
fn config_preflight_covers_each_runtime_worker_family() {
assert_eq!(config_preflight_subsystems(Some(SCANNER_SUB_SYS)), [SCANNER_SUB_SYS]);
assert_eq!(config_preflight_subsystems(Some(HEAL_SUB_SYS)), [HEAL_SUB_SYS]);
assert!(config_preflight_subsystems(Some("identity_openid")).is_empty());
assert_eq!(
config_preflight_subsystems(None),
[
NOTIFY_WEBHOOK_SUB_SYS,
rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
SCANNER_SUB_SYS
]
);
}
#[test]
@@ -2499,6 +2523,17 @@ mod tests {
assert!(!rendered_after_delete.contains("client_secret="));
}
#[test]
fn config_rendering_redacts_database_dsn() {
let entry = KV {
key: BASE_DSN_STRING.to_string(),
value: "postgres://user:password@db.example/database".to_string(),
hidden_if_empty: true,
};
assert_eq!(render_entry_value(&entry, true), REDACTED_VALUE);
}
#[test]
fn full_config_export_can_be_reapplied() {
crate::admin::storage_api::config::init_admin_config_defaults();
@@ -2621,6 +2656,36 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
validate_config_directives(&directives).expect("scanner pacing keys should be supported");
}
#[test]
fn scanner_cycle_config_set_and_get_round_trip() {
let mut config = ServerConfig::new();
let directives = parse_config_directives("scanner cycle=61", false).expect("parse scanner cycle directive");
validate_config_directives(&directives).expect("scanner cycle should be supported");
apply_set_directives(&mut config, &directives).expect("apply scanner cycle directive");
assert_eq!(
config
.get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
.expect("scanner KVS should exist")
.lookup(SCANNER_CYCLE)
.as_deref(),
Some("61")
);
let rendered = String::from_utf8(
render_selected_config(
&config,
&ConfigSelector {
sub_system: SCANNER_SUB_SYS.to_string(),
target: Some(DEFAULT_DELIMITER.to_string()),
},
true,
)
.expect("render scanner cycle config"),
)
.expect("scanner config should be UTF-8");
assert!(rendered.contains("cycle=\"61\""));
}
#[test]
fn build_help_response_reports_scanner_delay() {
let response = build_help_response(Some(SCANNER_SUB_SYS), Some(SCANNER_DELAY), false).expect("scanner help response");
rustfs/src/admin/handlers/oidc.rs
+28 -34
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use crate::admin::auth::validate_admin_request;
use crate::admin::handlers::supervise_admin_mutation;
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::runtime_sources::{
current_app_context, current_federated_identity_service, current_object_store_handle_for_context,
@@ -20,8 +21,7 @@ use crate::admin::runtime_sources::{
};
use crate::admin::service::federated_identity::DefaultFederatedSessionBinding;
use crate::admin::storage_api::config::{
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot,
};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
@@ -381,17 +381,13 @@ impl Operation for PutOidcConfigHandler {
let provider_id = provider_id.to_owned();
let request: OidcConfigUpsertRequest = parse_json_body(&mut req).await?;
let store = oidc_config_store()?;
let lock_store = store.clone();
with_admin_server_config_write_lock(lock_store, move || async move {
let mut config = load_server_config_from_store_locked(store.clone()).await?;
let existing_secret = persisted_provider_secret(&config, &provider_id);
update_oidc_server_config(move |config| {
let existing_secret = persisted_provider_secret(config, &provider_id);
let provider_config = build_provider_config_from_upsert(&provider_id, request, existing_secret)?;
upsert_persisted_provider_config(&mut config, &provider_config);
save_server_config_to_store_locked(store, &config).await
upsert_persisted_provider_config(config, &provider_config);
Ok(())
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
.await?;
json_response(
StatusCode::OK,
@@ -422,15 +418,11 @@ impl Operation for DeleteOidcConfigHandler {
}
let provider_id = provider_id.to_owned();
let store = oidc_config_store()?;
let lock_store = store.clone();
with_admin_server_config_write_lock(lock_store, move || async move {
let mut config = load_server_config_from_store_locked(store.clone()).await?;
delete_persisted_provider_config(&mut config, &provider_id)?;
save_server_config_to_store_locked(store, &config).await
update_oidc_server_config(move |config| {
delete_persisted_provider_config(config, &provider_id)?;
Ok(())
})
.await
.map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
.await?;
json_response(
StatusCode::OK,
@@ -916,21 +908,23 @@ fn oidc_config_store() -> S3Result<std::sync::Arc<crate::admin::storage_api::run
.ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))
}
async fn load_server_config_from_store_locked(
store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
) -> S3Result<ServerConfig> {
read_admin_config_without_migrate_no_lock(store)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
}
async fn save_server_config_to_store_locked(
store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
config: &ServerConfig,
) -> S3Result<()> {
save_admin_server_config_no_lock(store, config)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}")))
async fn update_oidc_server_config<F>(modifier: F) -> S3Result<()>
where
F: FnOnce(&mut ServerConfig) -> S3Result<()> + Send + 'static,
{
let store = oidc_config_store()?;
supervise_admin_mutation("OIDC config update", async move {
let snapshot = read_admin_server_config_snapshot(store.clone())
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))?;
let mut config = snapshot.config.clone();
modifier(&mut config)?;
save_admin_server_config_snapshot(store, &config, &snapshot)
.await
.map(|_| ())
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}")))
})
.await
}
fn is_env_managed_provider(provider_id: &str) -> bool {
rustfs/src/admin/runtime_sources.rs
+3 -1
View File
@@ -30,7 +30,9 @@ pub(crate) use crate::runtime_sources::{
current_replication_stats_handle_for_context, current_server_config_for_context, current_token_signing_key,
};
#[cfg(test)]
pub(crate) use crate::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
pub(crate) use crate::runtime_sources::{
IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface,
};
use rustfs_config::server_config::Config;
use rustfs_kms::KmsServiceManager;
use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, TlsGeneration};
rustfs/src/admin/service/config.rs
+99 -2
View File
@@ -37,9 +37,12 @@ use rustfs_targets::config::{
};
use s3s::{S3Error, S3ErrorCode, S3Result};
use std::future::Future;
use tokio::sync::Mutex as AsyncMutex;
use tracing::warn;
use url::Url;
static RUNTIME_CONFIG_RELOAD_MUTEX: AsyncMutex<()> = AsyncMutex::const_new(());
pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool {
NOTIFY_SUB_SYSTEMS.contains(&sub_system)
|| matches!(
@@ -393,6 +396,7 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste
}
pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await;
if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM {
let store = resolve_runtime_config_store_for_context(context)?;
let notify_result = reconcile_event_notifier_from_store(store).await;
@@ -414,6 +418,14 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap
warn!("peer reload_dynamic_config: failed to load server config for {sub_system}: {err}");
internal_error(format!("failed to load server config: {err}"))
})?;
if matches!(sub_system, SCANNER_SUB_SYS | HEAL_SUB_SYS) {
validate_server_config_for_context(context, &config, Some(sub_system)).await?;
// Scanner cycles refresh from the process-wide server config before
// each pass. Publish the same validated snapshot first so that refresh
// cannot overwrite this peer's dynamic scanner update with stale data.
publish_server_config_for_context(context, config.clone());
}
apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
.await
.inspect_err(|_| {
@@ -463,6 +475,7 @@ where
}
pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppContext>) -> S3Result<()> {
let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await;
let store = resolve_runtime_config_store_for_context(context)?;
reload_runtime_config_snapshot_with(
@@ -641,13 +654,16 @@ pub async fn signal_config_snapshot_reload_checked() -> S3Result<()> {
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
use crate::admin::runtime_sources::{
IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface,
};
use crate::admin::storage_api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
use crate::admin::storage_api::config::{
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config,
save_admin_server_config_no_lock, with_admin_server_config_write_lock,
};
use crate::admin::storage_api::error::StorageError;
use crate::admin::storage_api::runtime_sources::NotificationSys;
use crate::server::{
ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot, is_event_notifier_reconciled,
refresh_persisted_module_switches_from, save_persisted_module_switches_to,
@@ -656,7 +672,7 @@ mod tests {
use crate::storage_api::startup::storage::{init_local_disks_with_instance_ctx, new_instance_ctx};
use rustfs_config::notify::{ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
use rustfs_config::{ENV_SCANNER_CYCLE, HEAL_SUB_SYS, SCANNER_CYCLE, SCANNER_SUB_SYS};
use rustfs_config::{MQTT_BROKER, MQTT_QUEUE_DIR, MQTT_TOPIC, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
@@ -675,6 +691,36 @@ mod tests {
const REAL_STORE_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
const REPLICATION_RELOAD_LABEL: &str = "replication";
struct TestNotificationSystemInterface(Arc<NotificationSys>);
impl NotificationSystemInterface for TestNotificationSystemInterface {
fn handle(&self) -> Option<Arc<NotificationSys>> {
Some(self.0.clone())
}
}
#[tokio::test]
async fn checked_scanner_reload_reports_unreachable_peer() {
let temp_dir = TempDir::new().expect("scanner reload temp dir");
let store = build_isolated_heterogeneous_store(temp_dir.path()).await;
let mut notification_system = NotificationSys::new(EndpointServerPools::default()).await;
notification_system.peer_clients.push(None);
let context = AppContext::new(
store,
Arc::new(TestIamInterface),
Arc::new(TestKmsInterface {
manager: Arc::new(KmsServiceManager::new()),
}),
)
.with_test_notification_system_interface(Arc::new(TestNotificationSystemInterface(Arc::new(notification_system))));
let err = signal_dynamic_config_reload_checked_for_context(Some(&context), SCANNER_SUB_SYS)
.await
.expect_err("scanner config must not report success when a peer did not reload it");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(err.message(), Some("1 peer(s) failed dynamic config reload for scanner"));
}
fn without_storage_class_env<R>(f: impl FnOnce() -> R) -> R {
temp_env::with_vars_unset(
[
@@ -778,6 +824,16 @@ mod tests {
config
}
fn scanner_server_config(cycle: &str) -> ServerConfig {
let mut config = ServerConfig::new();
let mut kvs = KVS::new();
kvs.insert(SCANNER_CYCLE.to_string(), cycle.to_string());
config
.0
.insert(SCANNER_SUB_SYS.to_string(), HashMap::from([(DEFAULT_DELIMITER.to_string(), kvs)]));
config
}
async fn build_isolated_heterogeneous_store(temp_dir: &Path) -> Arc<ECStore> {
let mut pools = Vec::new();
for (pool_index, drives_per_set) in [4, 2].into_iter().enumerate() {
@@ -1124,6 +1180,47 @@ mod tests {
.await;
}
#[tokio::test]
#[serial_test::serial(scanner_env)]
async fn peer_scanner_reload_publishes_server_snapshot_used_by_cycle_refresh() {
temp_env::async_with_vars([(ENV_SCANNER_CYCLE, None::<&str>)], async {
let fixture = runtime_config_reload_fixture().await;
let candidate = scanner_server_config("61");
save_admin_server_config(fixture.context.object_store(), &candidate)
.await
.expect("persist scanner config");
reload_dynamic_config_runtime_state_for_context(Some(&fixture.context), SCANNER_SUB_SYS)
.await
.expect("scanner peer reload should apply persisted config");
assert_eq!(fixture.server_set_calls.load(Ordering::SeqCst), 1);
let snapshot = fixture
.server_snapshot
.lock()
.expect("server config result lock")
.clone()
.expect("scanner peer reload should publish a server config snapshot");
assert_eq!(
snapshot
.get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
.expect("scanner defaults should be present")
.get(SCANNER_CYCLE),
"61",
"scanner peer reload must keep the process-wide config source current"
);
let runtime = rustfs_scanner::scanner_runtime_config_status();
assert_eq!(runtime.cycle_interval_seconds.value, 61);
assert_eq!(
runtime.cycle_interval_seconds.source,
rustfs_scanner::runtime_config::ScannerRuntimeConfigSource::Config
);
rustfs_scanner::apply_scanner_runtime_config(&ServerConfig::new()).expect("restore scanner runtime defaults");
})
.await;
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn peer_full_reload_rejects_later_pool_without_publishing() {
rustfs/src/admin/storage_api.rs
+25 -5
View File
@@ -639,10 +639,13 @@ pub(crate) async fn read_admin_config_without_migrate(api: Arc<ECStore>) -> Resu
ecstore_config::com::read_config_without_migrate(api).await
}
#[cfg(test)]
pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> {
ecstore_config::com::read_config_without_migrate_no_lock(api).await
}
pub(crate) type AdminServerConfigSnapshot = ecstore_config::com::ServerConfigSnapshot;
pub(crate) async fn save_admin_config(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
ecstore_config::com::save_config(api, file, data).await
}
@@ -656,6 +659,7 @@ pub(crate) async fn save_admin_server_config(api: Arc<ECStore>, cfg: &rustfs_con
ecstore_config::com::save_server_config(api, cfg).await
}
#[cfg(test)]
pub(crate) async fn save_admin_server_config_no_lock(
api: Arc<ECStore>,
cfg: &rustfs_config::server_config::Config,
@@ -663,6 +667,7 @@ pub(crate) async fn save_admin_server_config_no_lock(
ecstore_config::com::save_server_config_no_lock(api, cfg).await
}
#[cfg(test)]
pub(crate) async fn with_admin_server_config_write_lock<F, Fut, T>(api: Arc<ECStore>, operation: F) -> Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
@@ -672,6 +677,18 @@ where
ecstore_config::com::with_server_config_write_lock(api, operation).await
}
pub(crate) async fn read_admin_server_config_snapshot(api: Arc<ECStore>) -> Result<AdminServerConfigSnapshot> {
ecstore_config::com::read_server_config_snapshot(api).await
}
pub(crate) async fn save_admin_server_config_snapshot(
api: Arc<ECStore>,
cfg: &rustfs_config::server_config::Config,
snapshot: &AdminServerConfigSnapshot,
) -> Result<bool> {
ecstore_config::com::save_server_config_snapshot(api, cfg, snapshot).await
}
pub(crate) fn init_admin_config_defaults() {
ecstore_config::init();
}
@@ -769,13 +786,16 @@ pub(crate) mod cluster {
}
pub(crate) mod config {
#[cfg(test)]
pub(crate) use super::save_admin_server_config;
pub(crate) use super::storageclass;
pub(crate) use super::{
RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults, read_admin_config,
read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_config,
save_admin_server_config_no_lock, with_admin_server_config_write_lock,
AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults,
read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config,
save_admin_server_config_snapshot,
};
#[cfg(test)]
pub(crate) use super::{
read_admin_config_without_migrate_no_lock, save_admin_server_config, save_admin_server_config_no_lock,
with_admin_server_config_write_lock,
};
}
rustfs/src/app/admin_usecase.rs
+4 -2
View File
@@ -247,8 +247,10 @@ impl DefaultAdminUsecase {
Self::query_data_usage_info_with_store(store).await
}
/// Serve the last persisted scanner snapshot plus the in-memory overlay.
/// This request path must never trigger a live full-version listing
/// Serve the last persisted scanner snapshot plus an authoritative
/// single-process overlay. Distributed nodes must not promote their
/// process-local absolute counters to cluster-wide bucket totals.
/// This path never triggers a live full-version listing
/// (rustfs/backlog#1306); freshness is owned by the scanner.
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
let mut info = Self::map_data_usage_load_result(load_data_usage_from_backend_cached(store.clone()).await)?;
rustfs/src/app/bucket_usecase.rs
+224 -9
View File
@@ -47,7 +47,6 @@ use super::storage_api::bucket_usecase::contract::bucket::{
use super::storage_api::bucket_usecase::contract::list::{
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _,
};
use super::storage_api::bucket_usecase::data_usage::remove_bucket_usage_from_backend;
use super::storage_api::bucket_usecase::error::StorageError;
use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context};
use super::storage_api::bucket_usecase::object_utils::to_s3s_etag;
@@ -120,13 +119,18 @@ use std::{
borrow::Cow,
collections::{HashMap, HashSet},
fmt::Display,
future::Future,
io::Write,
sync::Arc,
sync::{Arc, LazyLock},
};
use tracing::{debug, error, info, instrument, warn};
use tokio::sync::{Semaphore, TryAcquireError};
use tracing::{Instrument as _, debug, error, info, instrument, warn};
const LOG_COMPONENT_APP: &str = "app";
const LOG_SUBSYSTEM_BUCKET: &str = "bucket";
const BUCKET_OPERATION_CONCURRENCY: usize = 8;
static BUCKET_OPERATION_ADMISSION: LazyLock<Arc<Semaphore>> =
LazyLock::new(|| Arc::new(Semaphore::new(BUCKET_OPERATION_CONCURRENCY)));
use urlencoding::encode;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
@@ -1087,6 +1091,40 @@ pub struct DefaultBucketUsecase {
context: Option<Arc<AppContext>>,
}
async fn await_bucket_usecase_on_fresh_task<T, F>(operation: &'static str, future: F) -> S3Result<T>
where
T: Send + 'static,
F: Future<Output = S3Result<T>> + Send + 'static,
{
await_bucket_usecase_on_fresh_task_with_admission(operation, Arc::clone(&BUCKET_OPERATION_ADMISSION), future).await
}
async fn await_bucket_usecase_on_fresh_task_with_admission<T, F>(
operation: &'static str,
admission: Arc<Semaphore>,
future: F,
) -> S3Result<T>
where
T: Send + 'static,
F: Future<Output = S3Result<T>> + Send + 'static,
{
let permit = admission.try_acquire_owned().map_err(|err| match err {
TryAcquireError::NoPermits => {
S3Error::with_message(S3ErrorCode::SlowDown, format!("{operation} concurrency limit reached; retry later"))
}
TryAcquireError::Closed => S3Error::with_message(S3ErrorCode::InternalError, format!("{operation} admission closed")),
})?;
tokio::spawn(
async move {
let _permit = permit;
future.await
}
.in_current_span(),
)
.await
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{operation} task failed: {err}")))?
}
impl DefaultBucketUsecase {
#[cfg(test)]
pub fn without_context() -> Self {
@@ -1121,6 +1159,11 @@ impl DefaultBucketUsecase {
fields(start_time=?time::OffsetDateTime::now_utc())
)]
pub async fn execute_create_bucket(&self, req: S3Request<CreateBucketInput>) -> S3Result<S3Response<CreateBucketOutput>> {
let usecase = self.clone();
await_bucket_usecase_on_fresh_task("bucket creation", async move { usecase.execute_create_bucket_inner(req).await }).await
}
async fn execute_create_bucket_inner(&self, req: S3Request<CreateBucketInput>) -> S3Result<S3Response<CreateBucketOutput>> {
let helper = OperationHelper::new(&req, EventName::BucketCreated, S3Operation::CreateBucket);
let requester_is_owner = match req_info_ref(&req) {
Ok(r) => r.is_owner,
@@ -1179,7 +1222,15 @@ impl DefaultBucketUsecase {
}
#[instrument(level = "debug", skip(self, req))]
pub async fn execute_delete_bucket(&self, mut req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
pub async fn execute_delete_bucket(&self, req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
let usecase = self.clone();
await_bucket_usecase_on_fresh_task("bucket deletion", async move { usecase.execute_delete_bucket_inner(req).await }).await
}
async fn execute_delete_bucket_inner(
&self,
mut req: S3Request<DeleteBucketInput>,
) -> S3Result<S3Response<DeleteBucketOutput>> {
let helper = OperationHelper::new(&req, EventName::BucketRemoved, S3Operation::DeleteBucket);
let input = req.input.clone();
@@ -1218,12 +1269,8 @@ impl DefaultBucketUsecase {
// Invalidate bucket validation cache
crate::storage::invalidate_bucket_validation_cache(&input.bucket);
// Re-evaluate lifecycle/replication after bucket removal and keep an
// absent-bucket dirty marker until a complete usage snapshot is durable.
// Re-evaluate lifecycle and replication after bucket removal.
rustfs_scanner::record_scanner_maintenance_change(&input.bucket);
if let Err(err) = remove_bucket_usage_from_backend(store.clone(), &input.bucket).await {
warn!(bucket = %input.bucket, error = ?err, "failed to remove deleted bucket from data usage");
}
if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await {
warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed");
@@ -2633,6 +2680,155 @@ mod tests {
ServerSideEncryptionConfiguration, Tag, Transition, TransitionStorageClass,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Duration;
use tokio::sync::Notify;
#[tokio::test]
async fn bucket_usecase_task_finishes_post_commit_hooks_after_parent_cancellation() {
let admission = Arc::new(Semaphore::new(1));
let committed = Arc::new(Notify::new());
let committed_wait = committed.notified();
let release_hook = Arc::new(Notify::new());
let finished = Arc::new(Notify::new());
let finished_wait = finished.notified();
let hook_ran = Arc::new(AtomicBool::new(false));
let committed_for_task = committed.clone();
let release_hook_for_task = release_hook.clone();
let finished_for_task = finished.clone();
let hook_ran_for_task = hook_ran.clone();
let parent = tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission(
"test bucket operation",
admission.clone(),
async move {
committed_for_task.notify_one();
release_hook_for_task.notified().await;
hook_ran_for_task.store(true, Ordering::SeqCst);
finished_for_task.notify_one();
Ok(())
},
));
committed_wait.await;
parent.abort();
let _ = parent.await;
release_hook.notify_waiters();
tokio::time::timeout(Duration::from_secs(1), finished_wait)
.await
.expect("the post-commit hook should finish after request cancellation");
assert!(
hook_ran.load(Ordering::SeqCst),
"the fresh task must own both the storage mutation and its post-commit hooks"
);
assert_eq!(admission.available_permits(), 1);
}
#[tokio::test]
async fn saturated_bucket_usecase_admission_does_not_start_work() {
let admission = Arc::new(Semaphore::new(1));
let held = admission
.clone()
.acquire_owned()
.await
.expect("test admission should remain open");
let started = Arc::new(AtomicBool::new(false));
let started_for_task = started.clone();
let result = tokio::time::timeout(
Duration::from_secs(1),
await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission, async move {
started_for_task.store(true, Ordering::SeqCst);
Ok(())
}),
)
.await
.expect("saturated admission must fail within the bounded timeout");
drop(held);
assert!(
!started.load(Ordering::SeqCst),
"saturated admission must not start a detached bucket operation"
);
assert_eq!(result.expect_err("saturated admission must fail fast").code(), &S3ErrorCode::SlowDown);
}
#[tokio::test]
async fn bucket_usecase_admission_rejects_excess_detached_tasks() {
let admission = Arc::new(Semaphore::new(BUCKET_OPERATION_CONCURRENCY));
let release = Arc::new(Semaphore::new(0));
let started = Arc::new(AtomicUsize::new(0));
let mut tasks = Vec::with_capacity(BUCKET_OPERATION_CONCURRENCY);
for _ in 0..BUCKET_OPERATION_CONCURRENCY {
let admission_for_task = admission.clone();
let release_for_task = release.clone();
let started_for_task = started.clone();
tasks.push(tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission(
"test bucket operation",
admission_for_task,
async move {
started_for_task.fetch_add(1, Ordering::SeqCst);
let _release = release_for_task
.acquire()
.await
.expect("test release gate should remain open");
Ok(())
},
)));
}
tokio::time::timeout(Duration::from_secs(1), async {
while started.load(Ordering::SeqCst) != BUCKET_OPERATION_CONCURRENCY {
tokio::task::yield_now().await;
}
})
.await
.expect("all admitted operations should start");
let ninth_started = Arc::new(AtomicBool::new(false));
let ninth_started_for_task = ninth_started.clone();
let ninth_result = tokio::time::timeout(
Duration::from_secs(1),
await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission.clone(), async move {
ninth_started_for_task.store(true, Ordering::SeqCst);
Ok(())
}),
)
.await
.expect("an excess bucket transaction must fail within the bounded timeout");
assert!(
!ninth_started.load(Ordering::SeqCst),
"the ninth bucket transaction must not start when admission is saturated"
);
assert_eq!(
ninth_result.expect_err("the ninth bucket transaction must fail fast").code(),
&S3ErrorCode::SlowDown
);
release.add_permits(BUCKET_OPERATION_CONCURRENCY);
for task in tasks {
task.await
.expect("bucket transaction parent should join")
.expect("bucket transaction should succeed");
}
await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission, async { Ok(()) })
.await
.expect("admission must recover after active transactions finish");
}
#[tokio::test]
async fn bucket_usecase_task_maps_panics_to_internal_errors() {
async fn panicking_bucket_operation() -> S3Result<()> {
panic!("injected bucket task panic");
}
let result = await_bucket_usecase_on_fresh_task("test bucket operation", panicking_bucket_operation()).await;
let err = result.expect_err("a bucket task panic must be returned as an error");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert!(err.message().is_some_and(|message| message.contains("task failed")));
}
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
@@ -2662,6 +2858,25 @@ mod tests {
&rest[..end]
}
#[test]
fn bucket_create_and_delete_use_admitted_fresh_tasks() {
let source = include_str!("bucket_usecase.rs");
for (method, operation, inner_method) in [
("execute_create_bucket", "bucket creation", "execute_create_bucket_inner"),
("execute_delete_bucket", "bucket deletion", "execute_delete_bucket_inner"),
] {
let body = usecase_method_source(source, method);
assert!(
body.contains(&format!("await_bucket_usecase_on_fresh_task(\"{operation}\"")),
"{method} must use bounded detached-task admission"
);
assert!(
body.contains(&format!("{inner_method}(req).await")),
"{method} must run its mutation inside the admitted task"
);
}
}
#[test]
fn bucket_metadata_config_changes_notify_peer_metadata_reload() {
let source = include_str!("bucket_usecase.rs");
rustfs/src/app/context/global.rs
+8
View File
@@ -354,6 +354,14 @@ impl AppContext {
self.storage_class = storage_class;
self
}
pub(crate) fn with_test_notification_system_interface(
mut self,
notification_system: Arc<dyn NotificationSystemInterface>,
) -> Self {
self.notification_system = notification_system;
self
}
}
static APP_CONTEXT_SINGLETON: OnceLock<Arc<AppContext>> = OnceLock::new();
rustfs/src/app/storage_api.rs
+1 -8
View File
@@ -110,13 +110,6 @@ pub(crate) mod data_usage {
)
.await;
}
pub(crate) async fn remove_bucket_usage_from_backend(
store: Arc<crate::storage::storage_api::ECStore>,
bucket: &str,
) -> Result<(), crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::remove_bucket_usage_from_backend(store, bucket).await
}
}
pub(crate) mod runtime {
@@ -956,7 +949,7 @@ pub(crate) mod bucket_usecase {
}
}
pub(crate) use super::{access, bucket, data_usage, error, helper, object_utils, request_context, s3_api};
pub(crate) use super::{access, bucket, error, helper, object_utils, request_context, s3_api};
pub(crate) use crate::storage::storage_api::{
ECStore, StorageObjectInfo, get_validated_store, process_lambda_configurations, process_queue_configurations,
process_topic_configurations, validate_list_object_unordered_with_delimiter,
rustfs/src/runtime_sources.rs
+1 -1
View File
@@ -61,7 +61,7 @@ pub(crate) fn set_test_outbound_tls_generation(generation: u64) {
pub(crate) use context::install_test_app_context;
#[cfg(test)]
pub(crate) use context::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
pub(crate) use context::{IamInterface, KmsInterface, NotificationSystemInterface, ServerConfigInterface, StorageClassInterface};
pub(crate) fn current_app_context() -> Option<Arc<AppContext>> {
context::get_global_app_context()
rustfs/src/storage/rpc/http_service.rs
+577 -17
View File
@@ -14,22 +14,26 @@
use crate::server::RPC_PREFIX;
use crate::storage::request_context::spawn_traced;
#[cfg(test)]
use crate::storage::storage_api::rpc_consumer::http_service::WALK_DIR_BODY_SHA256_QUERY;
use crate::storage::storage_api::rpc_consumer::http_service::{
DEFAULT_READ_BUFFER_SIZE, StorageDiskRpcExt as _, WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, find_local_disk_by_ref,
verify_rpc_signature,
DEFAULT_READ_BUFFER_SIZE, NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, StorageDiskRpcExt as _,
WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature,
};
#[cfg(test)]
use crate::storage::storage_api::rpc_consumer::http_service::{
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
WALK_DIR_BODY_SHA256_QUERY,
};
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use bytes::{Bytes, BytesMut};
use futures_util::{Stream, StreamExt, TryStreamExt, stream};
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, Uri};
use http_body_util::{BodyExt, Limited};
use hyper::body::Incoming;
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_OPERATION_WALK_DIR, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
};
use rustfs_utils::net::bytes_stream;
use s3s::Body;
@@ -39,11 +43,12 @@ use serde_urlencoded::from_bytes;
use sha2::{Digest, Sha256};
use std::future::Future;
use std::pin::Pin;
use std::sync::LazyLock;
use std::task::{Context, Poll};
use std::time::Instant;
use std::time::{Duration, Instant};
use tokio::io::{self, AsyncWriteExt};
use tokio::sync::oneshot;
use tokio_util::io::ReaderStream;
use tokio_util::{io::ReaderStream, sync::CancellationToken};
use tower::Service;
use tracing::{error, warn};
@@ -52,6 +57,7 @@ type RpcErrorResponse = Box<Response<Body>>;
const LOG_COMPONENT_INTERNODE_RPC: &str = "internode_rpc";
const LOG_SUBSYSTEM_FILE_TRANSFER: &str = "file_transfer";
const LOG_SUBSYSTEM_DIRECTORY_WALK: &str = "directory_walk";
const LOG_SUBSYSTEM_NAMESPACE_SCANNER: &str = "namespace_scanner";
const LOG_SUBSYSTEM_ROUTING: &str = "routing";
const EVENT_RPC_REQUEST_REJECTED: &str = "rpc_request_rejected";
const EVENT_RPC_REQUEST_FAILED: &str = "rpc_request_failed";
@@ -60,6 +66,10 @@ const RPC_OPERATION_UNKNOWN: &str = "unknown";
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner";
const NS_SCANNER_REQUEST_BODY_TIMEOUT: Duration = Duration::from_secs(15);
const NS_SCANNER_STREAM_BUFFER_SIZE: usize = 64 * 1024;
static NS_SCANNER_SERVER_EPOCH: LazyLock<uuid::Uuid> = LazyLock::new(uuid::Uuid::new_v4);
macro_rules! log_internode_rpc_response_failure {
($status:expr, $rpc_path:expr, $method:expr, $operation:expr, $reason:expr, $result:expr, Some(($context_key:expr, $context_value:expr)), Some($error_text:expr)) => {{
@@ -247,6 +257,34 @@ struct WalkDirQuery {
walk_dir_body_sha256: Option<String>,
}
#[derive(Debug, Default, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct NsScannerQuery {
disk: String,
ns_scanner_request_id: uuid::Uuid,
ns_scanner_server_epoch: uuid::Uuid,
ns_scanner_session_id: uuid::Uuid,
ns_scanner_session_sequence: u64,
ns_scanner_cycle: u64,
ns_scanner_leader_epoch: u64,
ns_scanner_body_sha256: Option<String>,
}
#[derive(Debug, Default, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct NsScannerCapabilityQuery {
ns_scanner_protocol: Option<u16>,
ns_scanner_challenge: Option<uuid::Uuid>,
}
fn verify_ns_scanner_body_digest(query: &NsScannerQuery, body: &[u8]) -> bool {
let Some(expected) = query.ns_scanner_body_sha256.as_deref() else {
return false;
};
let actual = hex_simd::encode_to_string(Sha256::digest(body), hex_simd::AsciiCase::Lower);
expected == actual
}
fn supports_walk_dir_stream_completion(query: &WalkDirQuery) -> bool {
query.walk_dir_stream_completion.as_deref() == Some(WALK_DIR_STREAM_COMPLETION_V1)
}
@@ -322,6 +360,15 @@ async fn handle_internode_rpc(req: Request<Incoming>) -> Response<Body> {
let response = match (method, path) {
(Method::GET, READ_FILE_STREAM_PATH) | (Method::HEAD, READ_FILE_STREAM_PATH) => handle_read_file(req).await,
(Method::GET, WALK_DIR_PATH) | (Method::HEAD, WALK_DIR_PATH) => handle_walk_dir(req).await,
(Method::GET, NS_SCANNER_PATH) => match parse_query::<NsScannerCapabilityQuery>(&req) {
Ok(query) if query.ns_scanner_protocol == Some(NS_SCANNER_PROTOCOL_VERSION) => match query.ns_scanner_challenge {
Some(challenge) if !challenge.is_nil() => ns_scanner_capability_response(challenge),
Some(_) | None => response_with_status(StatusCode::BAD_REQUEST, "namespace scanner challenge is invalid"),
},
Ok(_) => response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner protocol is unsupported"),
Err(response) => *response,
},
(Method::POST, NS_SCANNER_PATH) => handle_ns_scanner(req).await,
(Method::PUT, PUT_FILE_STREAM_PATH) => handle_put_file(req).await,
_ => response_with_status(StatusCode::NOT_FOUND, "internode rpc route not found"),
};
@@ -346,6 +393,7 @@ fn internode_http_operation(path: &str) -> Option<&'static str> {
READ_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_READ_FILE_STREAM),
PUT_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_PUT_FILE_STREAM),
WALK_DIR_PATH => Some(INTERNODE_OPERATION_WALK_DIR),
NS_SCANNER_PATH => Some(INTERNODE_OPERATION_NS_SCANNER),
_ => None,
}
}
@@ -358,6 +406,64 @@ fn record_internode_rpc_error(operation: Option<&'static str>) {
}
}
fn ns_scanner_capability_response(challenge: uuid::Uuid) -> Response<Body> {
let server_epoch = *NS_SCANNER_SERVER_EPOCH;
let proof = match sign_ns_scanner_capability(challenge, server_epoch) {
Ok(proof) => proof,
Err(err) => {
error!(
event = EVENT_RPC_REQUEST_FAILED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "failed",
status_code = StatusCode::UPGRADE_REQUIRED.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::GET,
reason = "capability_authentication_unavailable",
error = %err,
"internode rpc request failed"
);
return response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner RPC authentication is unavailable");
}
};
let body = match rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch,
proof,
}) {
Ok(body) => body,
Err(err) => {
error!(
event = EVENT_RPC_REQUEST_FAILED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "failed",
status_code = StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::GET,
reason = "capability_response_encode_failed",
error = %err,
"internode rpc request failed"
);
return response_with_status(
StatusCode::INTERNAL_SERVER_ERROR,
"namespace scanner capability response encoding failed",
);
}
};
let mut response = Response::new(Body::from(Bytes::from(body)));
response
.headers_mut()
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/msgpack"));
response
}
fn ns_scanner_server_epoch_matches(server_epoch: uuid::Uuid) -> bool {
server_epoch == *NS_SCANNER_SERVER_EPOCH
}
fn verify_internode_rpc_signature(uri: &Uri, method: &Method, headers: &HeaderMap) -> Result<(), RpcErrorResponse> {
if method == Method::HEAD {
return Ok(());
@@ -602,6 +708,304 @@ async fn handle_walk_dir(req: Request<Incoming>) -> Response<Body> {
.expect("failed to build walk dir response")
}
async fn handle_ns_scanner(req: Request<Incoming>) -> Response<Body> {
let query = match parse_query::<NsScannerQuery>(&req) {
Ok(query) => query,
Err(response) => return *response,
};
if !ns_scanner_server_epoch_matches(query.ns_scanner_server_epoch) {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::CONFLICT.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "server_epoch_mismatch",
"internode rpc request rejected"
);
return response_with_status(StatusCode::CONFLICT, "namespace scanner server epoch is stale");
}
let Some(disk) = find_local_disk_by_ref(&query.disk).await else {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::BAD_REQUEST.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "disk_not_found",
disk = %query.disk,
"internode rpc request rejected"
);
return response_with_status(StatusCode::BAD_REQUEST, "disk not found");
};
if let Err(err) = rustfs_scanner::preflight_remote_scanner_request(
disk.as_ref(),
query.ns_scanner_session_id,
query.ns_scanner_cycle,
query.ns_scanner_leader_epoch,
query.ns_scanner_session_sequence,
) {
let (status, reason, message) = remote_scanner_claim_rejection(&err);
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = status.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason,
disk = %query.disk,
error = %err,
"internode rpc request rejected"
);
return response_with_status(status, message);
}
if let Err(err) =
rustfs_scanner::validate_remote_scanner_request_fence(query.ns_scanner_cycle, query.ns_scanner_leader_epoch).await
{
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::CONFLICT.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "scanner_cycle_mismatch",
disk = %query.disk,
error = %err,
"internode rpc request rejected"
);
return response_with_status(StatusCode::CONFLICT, "namespace scanner cycle does not match persisted state");
}
// Acquire the per-disk permit before reading the request body. This bounds
// slow or duplicated authenticated uploads to one request per physical disk;
// dropping the request on timeout releases the permit without consuming its
// replay sequence.
let admission = match rustfs_scanner::admit_remote_scanner_request(disk.as_ref()) {
Ok(admission) => admission,
Err(rustfs_scanner::ScannerError::RemoteDiskBusy) => {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::TOO_MANY_REQUESTS.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "disk_scan_already_active",
disk = %query.disk,
"internode rpc request rejected"
);
return response_with_status(StatusCode::TOO_MANY_REQUESTS, "namespace scanner disk is already active");
}
Err(err) => {
let (status, reason, message) = remote_scanner_claim_rejection(&err);
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = status.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason,
disk = %query.disk,
error = %err,
"internode rpc request rejected"
);
return response_with_status(status, message);
}
};
let body = match tokio::time::timeout(
NS_SCANNER_REQUEST_BODY_TIMEOUT,
Limited::new(req.into_body(), rustfs_scanner::NS_SCANNER_MAX_REQUEST_BODY_SIZE).collect(),
)
.await
{
Ok(Ok(body)) => body.to_bytes(),
Ok(Err(err)) => {
log_internode_rpc_response_failure!(
StatusCode::PAYLOAD_TOO_LARGE,
NS_SCANNER_PATH,
&Method::POST,
Some(INTERNODE_OPERATION_NS_SCANNER),
"request_body_read_failed",
"rejected",
Some(("disk", query.disk.as_str())),
Some(&err)
);
return response_with_status(StatusCode::PAYLOAD_TOO_LARGE, "namespace scanner request body is too large");
}
Err(_) => {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::REQUEST_TIMEOUT.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "request_body_timeout",
disk = %query.disk,
"internode rpc request rejected"
);
return response_with_status(StatusCode::REQUEST_TIMEOUT, "namespace scanner request body timed out");
}
};
if !verify_ns_scanner_body_digest(&query, &body) {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::FORBIDDEN.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "request_body_digest_mismatch",
disk = %query.disk,
"internode rpc request rejected"
);
return response_with_status(StatusCode::FORBIDDEN, "invalid request body digest");
}
let request = match rustfs_scanner::decode_remote_scanner_request(&body) {
Ok(request) => request,
Err(err) => {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::BAD_REQUEST.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "request_body_decode_failed",
disk = %query.disk,
error = %err,
"internode rpc request rejected"
);
return response_with_status(StatusCode::BAD_REQUEST, "invalid namespace scanner request");
}
};
if !rustfs_scanner::remote_scanner_request_matches_envelope(
&request,
query.ns_scanner_request_id,
query.ns_scanner_server_epoch,
query.ns_scanner_session_id,
query.ns_scanner_session_sequence,
query.ns_scanner_cycle,
query.ns_scanner_leader_epoch,
) {
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = StatusCode::FORBIDDEN.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason = "request_envelope_mismatch",
disk = %query.disk,
"internode rpc request rejected"
);
return response_with_status(StatusCode::FORBIDDEN, "namespace scanner request envelope mismatch");
}
if let Err(err) = rustfs_scanner::claim_remote_scanner_request(
disk.as_ref(),
query.ns_scanner_session_id,
query.ns_scanner_cycle,
query.ns_scanner_leader_epoch,
query.ns_scanner_session_sequence,
) {
let (status, reason, message) = remote_scanner_claim_rejection(&err);
warn!(
event = EVENT_RPC_REQUEST_REJECTED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "rejected",
status_code = status.as_u16(),
rpc_path = NS_SCANNER_PATH,
method = %Method::POST,
reason,
disk = %query.disk,
error = %err,
"internode rpc request rejected"
);
return response_with_status(status, message);
}
let metrics = runtime_sources::current_internode_metrics();
metrics
.record_incoming_request_for_operation_and_backend(INTERNODE_OPERATION_NS_SCANNER, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP);
metrics.record_recv_bytes_for_operation_and_backend(
INTERNODE_OPERATION_NS_SCANNER,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
body.len(),
);
let log_disk = query.disk;
let response_body = ns_scanner_response_body(move |writer, disconnect| async move {
let _admission = admission;
rustfs_scanner::serve_remote_scanner_request(disk, request, writer, disconnect)
.await
.map_err(|err| {
error!(
event = EVENT_RPC_BACKGROUND_TASK_FAILED,
component = LOG_COMPONENT_INTERNODE_RPC,
subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER,
operation = INTERNODE_OPERATION_NS_SCANNER,
result = "failed",
disk = %log_disk,
error = %err,
"internode rpc background task failed"
);
io::Error::other("remote namespace scanner failed")
})
});
let mut response = Response::new(response_body);
response
.headers_mut()
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/msgpack"));
response
}
fn remote_scanner_claim_rejection(error: &rustfs_scanner::ScannerError) -> (StatusCode, &'static str, &'static str) {
match error {
rustfs_scanner::ScannerError::RemoteRequestReplay => {
(StatusCode::CONFLICT, "request_replay", "namespace scanner request was already accepted")
}
rustfs_scanner::ScannerError::RemoteReplayCapacity => (
StatusCode::TOO_MANY_REQUESTS,
"replay_capacity",
"namespace scanner request capacity is temporarily exhausted",
),
_ => (
StatusCode::SERVICE_UNAVAILABLE,
"replay_state_unavailable",
"namespace scanner replay state is unavailable",
),
}
}
fn walk_dir_response_body<F, Fut>(propagate_completion_errors: bool, producer: F) -> Body
where
F: FnOnce(tokio::io::DuplexStream) -> Fut + Send + 'static,
@@ -656,6 +1060,52 @@ where
)
}
fn ns_scanner_response_body<F, Fut>(producer: F) -> Body
where
F: FnOnce(tokio::io::DuplexStream, CancellationToken) -> Fut + Send + 'static,
Fut: Future<Output = io::Result<()>> + Send + 'static,
{
let (reader, writer) = tokio::io::duplex(NS_SCANNER_STREAM_BUFFER_SIZE);
let (mut completion_tx, completion_rx) = oneshot::channel();
let disconnect = CancellationToken::new();
spawn_traced(async move {
let producer = producer(writer, disconnect.clone());
tokio::pin!(producer);
tokio::select! {
biased;
result = &mut producer => {
let _ = completion_tx.send(result);
}
_ = completion_tx.closed() => {
disconnect.cancel();
let _ = producer.await;
}
}
});
let metrics = runtime_sources::current_internode_metrics();
let stream = ReaderStream::with_capacity(reader, NS_SCANNER_STREAM_BUFFER_SIZE).map_ok(move |bytes| {
metrics.record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_NS_SCANNER,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
bytes.len(),
);
bytes
});
let completion = stream::once(async move {
match completion_rx.await {
Ok(Ok(())) => None,
Ok(Err(err)) => Some(Err(err)),
Err(err) => Some(Err(io::Error::other(format!(
"remote namespace scanner task ended without a result: {err}"
)))),
}
})
.filter_map(std::future::ready);
Body::from(StreamingBlob::wrap(stream.chain(completion)))
}
async fn handle_put_file(req: Request<Incoming>) -> Response<Body> {
let method = req.method().clone();
let path = req.uri().path().to_string();
@@ -805,6 +1255,7 @@ fn response_with_status(status: StatusCode, message: impl Into<String>) -> Respo
fn internode_rpc_subsystem(operation: Option<&'static str>) -> &'static str {
match operation {
Some(INTERNODE_OPERATION_WALK_DIR) => LOG_SUBSYSTEM_DIRECTORY_WALK,
Some(INTERNODE_OPERATION_NS_SCANNER) => LOG_SUBSYSTEM_NAMESPACE_SCANNER,
Some(INTERNODE_OPERATION_READ_FILE_STREAM | INTERNODE_OPERATION_PUT_FILE_STREAM) => LOG_SUBSYSTEM_FILE_TRANSFER,
_ => LOG_SUBSYSTEM_ROUTING,
}
@@ -828,19 +1279,23 @@ fn put_file_stage_error_message(stage: &str, query: &PutFileQuery, err: &dyn std
#[cfg(test)]
mod tests {
use super::{
LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_ROUTING, PUT_FILE_STREAM_PATH, PutFileQuery,
LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_NAMESPACE_SCANNER, LOG_SUBSYSTEM_ROUTING,
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PATH, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerQuery, PUT_FILE_STREAM_PATH, PutFileQuery,
READ_FILE_STREAM_PATH, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_PATH, WalkDirQuery, append_walk_dir_completion,
internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, put_body_size_mismatch,
put_file_stage_error_message, read_file_body_stream, supports_walk_dir_stream_completion,
validate_walk_dir_completion_request, verify_internode_rpc_signature, verify_walk_dir_body_digest,
walk_dir_response_body, write_body_chunks_to_writer,
internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, ns_scanner_response_body,
ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_stage_error_message, read_file_body_stream,
remote_scanner_claim_rejection, supports_walk_dir_stream_completion, validate_walk_dir_completion_request,
verify_internode_rpc_signature, verify_ns_scanner_body_digest, verify_walk_dir_body_digest, walk_dir_response_body,
write_body_chunks_to_writer,
};
use bytes::Bytes;
use http::{HeaderMap, Method, StatusCode, Uri};
use http_body_util::BodyExt;
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
global_internode_metrics,
INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_OPERATION_WALK_DIR, global_internode_metrics,
};
use sha2::Digest as _;
use tokio::io;
@@ -862,6 +1317,7 @@ mod tests {
fn internode_rpc_path_matches_rpc_prefix() {
assert!(is_internode_rpc_path("/rustfs/rpc/read_file_stream"));
assert!(is_internode_rpc_path("/rustfs/rpc/walk_dir"));
assert!(is_internode_rpc_path("/rustfs/rpc/ns_scanner"));
assert!(!is_internode_rpc_path("/rustfs/admin/v3/info"));
}
@@ -873,16 +1329,37 @@ mod tests {
);
assert_eq!(internode_http_operation(PUT_FILE_STREAM_PATH), Some(INTERNODE_OPERATION_PUT_FILE_STREAM));
assert_eq!(internode_http_operation(WALK_DIR_PATH), Some(INTERNODE_OPERATION_WALK_DIR));
assert_eq!(internode_http_operation(NS_SCANNER_PATH), Some(INTERNODE_OPERATION_NS_SCANNER));
assert_eq!(internode_http_operation("/rustfs/rpc/unknown"), None);
}
#[test]
fn rpc_head_signature_verification_is_skipped() {
fn file_stream_head_signature_verification_is_skipped() {
let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri");
let headers = HeaderMap::new();
assert!(verify_internode_rpc_signature(&uri, &Method::HEAD, &headers).is_ok());
}
#[test]
fn namespace_scanner_capability_get_requires_signature() {
let challenge = uuid::Uuid::new_v4();
let uri: Uri = format!(
"{NS_SCANNER_PATH}?ns_scanner_protocol={}&{NS_SCANNER_CAPABILITY_CHALLENGE_QUERY}={challenge}",
super::NS_SCANNER_PROTOCOL_VERSION
)
.parse()
.expect("uri");
let headers = HeaderMap::new();
let response = verify_internode_rpc_signature(&uri, &Method::GET, &headers).expect_err("response");
assert_eq!(response.status(), StatusCode::FORBIDDEN);
}
#[test]
fn namespace_scanner_rejects_requests_from_a_prior_server_epoch() {
assert!(ns_scanner_server_epoch_matches(*super::NS_SCANNER_SERVER_EPOCH));
assert!(!ns_scanner_server_epoch_matches(uuid::Uuid::nil()));
}
#[test]
fn rpc_get_request_requires_signature() {
let uri: Uri = READ_FILE_STREAM_PATH.parse().expect("uri");
@@ -941,9 +1418,92 @@ mod tests {
LOG_SUBSYSTEM_FILE_TRANSFER
);
assert_eq!(internode_rpc_subsystem(Some(INTERNODE_OPERATION_WALK_DIR)), LOG_SUBSYSTEM_DIRECTORY_WALK);
assert_eq!(
internode_rpc_subsystem(Some(INTERNODE_OPERATION_NS_SCANNER)),
LOG_SUBSYSTEM_NAMESPACE_SCANNER
);
assert_eq!(internode_rpc_subsystem(None), LOG_SUBSYSTEM_ROUTING);
}
#[test]
fn namespace_scanner_requires_matching_signed_body_digest() {
let body = b"scanner-request";
let digest = hex_simd::encode_to_string(sha2::Sha256::digest(body), hex_simd::AsciiCase::Lower);
let request_id = uuid::Uuid::new_v4();
let server_epoch = uuid::Uuid::new_v4();
let session_id = uuid::Uuid::new_v4();
let valid: NsScannerQuery = serde_urlencoded::from_str(&format!(
"disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9&{NS_SCANNER_BODY_SHA256_QUERY}={digest}"
))
.expect("query should parse");
let missing: NsScannerQuery = serde_urlencoded::from_str(&format!(
"disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9"
))
.expect("query should parse");
assert!(verify_ns_scanner_body_digest(&valid, body));
assert!(!verify_ns_scanner_body_digest(&valid, b"tampered"));
assert!(!verify_ns_scanner_body_digest(&missing, body));
}
#[test]
fn namespace_scanner_queries_reject_unknown_fields() {
let request_id = uuid::Uuid::new_v4();
let server_epoch = uuid::Uuid::new_v4();
let session_id = uuid::Uuid::new_v4();
let query = format!(
"disk=disk-a&{NS_SCANNER_REQUEST_ID_QUERY}={request_id}&{NS_SCANNER_SERVER_EPOCH_QUERY}={server_epoch}&{NS_SCANNER_SESSION_ID_QUERY}={session_id}&{NS_SCANNER_SESSION_SEQUENCE_QUERY}=0&{NS_SCANNER_CYCLE_QUERY}=7&{NS_SCANNER_LEADER_EPOCH_QUERY}=9&{NS_SCANNER_BODY_SHA256_QUERY}=digest&unexpected=true"
);
assert!(serde_urlencoded::from_str::<NsScannerQuery>(&query).is_err());
assert!(serde_urlencoded::from_str::<super::NsScannerCapabilityQuery>("ns_scanner_protocol=1&unexpected=true").is_err());
}
#[test]
fn namespace_scanner_replay_capacity_is_retryable_without_weakening_replay_rejection() {
let (replay_status, replay_reason, _) =
remote_scanner_claim_rejection(&rustfs_scanner::ScannerError::RemoteRequestReplay);
let (capacity_status, capacity_reason, _) =
remote_scanner_claim_rejection(&rustfs_scanner::ScannerError::RemoteReplayCapacity);
assert_eq!(replay_status, StatusCode::CONFLICT);
assert_eq!(replay_reason, "request_replay");
assert_eq!(capacity_status, StatusCode::TOO_MANY_REQUESTS);
assert_eq!(capacity_reason, "replay_capacity");
}
#[tokio::test]
async fn namespace_scanner_body_surfaces_background_failure() {
let body = ns_scanner_response_body(|mut writer, _disconnect| async move {
writer.write_all(b"partial scanner frame").await?;
Err(io::Error::other("remote namespace scanner failed"))
});
let err = BodyExt::collect(body)
.await
.expect_err("failed completion must fail body collection");
assert!(err.to_string().contains("remote namespace scanner failed"));
}
#[tokio::test]
async fn dropping_namespace_scanner_body_cancels_producer() {
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel();
let body = ns_scanner_response_body(move |_writer, disconnect| async move {
let _drop_notifier = DropNotifier(Some(dropped_tx));
let _ = started_tx.send(());
disconnect.cancelled().await;
Ok(())
});
started_rx.await.expect("namespace scanner producer should start");
drop(body);
tokio::time::timeout(std::time::Duration::from_secs(1), dropped_rx)
.await
.expect("dropping the response body should cancel the namespace scanner producer")
.expect("drop notifier should send a cancellation signal");
}
#[tokio::test]
async fn write_body_chunks_to_writer_streams_all_chunks() {
let (mut reader, mut writer) = tokio::io::duplex(64);
rustfs/src/storage/rpc/node_service.rs
+361 -55
View File
@@ -23,8 +23,9 @@ use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_S
use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType};
use crate::storage::storage_api::rpc_consumer::node_service::{
DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path,
find_local_disk_by_ref, reload_transition_tier_config,
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref,
reload_transition_tier_config,
};
use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources};
use crate::storage::storage_api::{sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest};
@@ -78,6 +79,14 @@ const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1;
const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2;
const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3;
fn signal_service_response(success: bool, error_info: Option<String>) -> Response<SignalServiceResponse> {
Response::new(SignalServiceResponse {
success,
error_info,
protocol_version: rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
})
}
fn supports_dynamic_config_rpc(sub_system: &str) -> bool {
NOTIFY_SUB_SYSTEMS.contains(&sub_system)
|| matches!(
@@ -174,11 +183,54 @@ fn validate_admin_heal_control_start(request: &rustfs_common::heal_channel::Heal
Ok(())
}
fn scanner_activity_response(namespace_generation: u64) -> ScannerActivityResponse {
fn scanner_activity_response(
namespace_generation: u64,
topology_digest: [u8; 32],
data_movement_active: bool,
dirty_usage: rustfs_scanner::ScannerDirtyUsageState,
) -> ScannerActivityResponse {
ScannerActivityResponse {
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
namespace_generation,
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: topology_digest.to_vec().into(),
data_movement_active,
response_proof: Bytes::new(),
dirty_usage_generation: dirty_usage.generation,
dirty_usage_pending: dirty_usage.pending,
}
}
fn previous_scanner_activity_response(
namespace_generation: u64,
topology_digest: [u8; 32],
data_movement_active: bool,
) -> ScannerActivityResponse {
ScannerActivityResponse {
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
namespace_generation,
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
topology_digest: topology_digest.to_vec().into(),
data_movement_active,
response_proof: Bytes::new(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
}
}
fn legacy_scanner_activity_response(namespace_generation: u64) -> ScannerActivityResponse {
ScannerActivityResponse {
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
namespace_generation,
maintenance_generation: rustfs_scanner::scanner_maintenance_generation(),
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
topology_digest: Bytes::new(),
data_movement_active: false,
response_proof: Bytes::new(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
}
}
@@ -1559,73 +1611,149 @@ impl Node for NodeService {
Some(value) => match value.parse::<bool>() {
Ok(value) => value,
Err(_) => {
return Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("invalid dry-run value: {value}")),
}));
return Ok(signal_service_response(false, Some(format!("invalid dry-run value: {value}"))));
}
},
};
match signal {
Some(SERVICE_SIGNAL_REFRESH_CONFIG) => match reload_runtime_config_snapshot().await {
Ok(()) => Ok(Response::new(SignalServiceResponse {
success: true,
error_info: None,
})),
Err(_) => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some("runtime config snapshot reload failed".to_string()),
})),
Ok(()) => Ok(signal_service_response(true, None)),
Err(_) => Ok(signal_service_response(false, Some("runtime config snapshot reload failed".to_string()))),
},
Some(SERVICE_SIGNAL_RELOAD_DYNAMIC) => {
let supported = sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM || supports_dynamic_config_rpc(sub_system);
if !supported {
return Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("unsupported dynamic config subsystem: {sub_system}")),
}));
return Ok(signal_service_response(
false,
Some(format!("unsupported dynamic config subsystem: {sub_system}")),
));
}
if dry_run {
return Ok(Response::new(SignalServiceResponse {
success: true,
error_info: None,
}));
return Ok(signal_service_response(true, None));
}
match reload_dynamic_config_runtime_state(sub_system).await {
Ok(()) => Ok(Response::new(SignalServiceResponse {
success: true,
error_info: None,
})),
Err(_) => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("dynamic config reload failed for {sub_system}")),
})),
Ok(()) => Ok(signal_service_response(true, None)),
Err(_) => Ok(signal_service_response(
false,
Some(format!("dynamic config reload failed for {sub_system}")),
)),
}
}
Some(other) => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("unsupported service signal: {other}")),
})),
None if raw_signal.is_some() => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some(format!("invalid service signal value: {}", raw_signal.unwrap_or_default())),
})),
None => Ok(Response::new(SignalServiceResponse {
success: false,
error_info: Some("missing service signal".to_string()),
})),
Some(other) => Ok(signal_service_response(false, Some(format!("unsupported service signal: {other}")))),
None if raw_signal.is_some() => Ok(signal_service_response(
false,
Some(format!("invalid service signal value: {}", raw_signal.unwrap_or_default())),
)),
None => Ok(signal_service_response(false, Some("missing service signal".to_string()))),
}
}
async fn scanner_activity(
&self,
_request: Request<ScannerActivityRequest>,
request: Request<ScannerActivityRequest>,
) -> Result<Response<ScannerActivityResponse>, Status> {
let request_protocol = request.get_ref().protocol_version;
match request_protocol {
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy request body is unbound. Remove after protocol v0 peers are unsupported.
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
if !request.get_ref().acknowledge_instance_id.is_empty()
|| request.get_ref().acknowledge_dirty_usage_generation != 0
{
return Err(Status::invalid_argument("legacy scanner activity request cannot acknowledge dirty usage"));
}
}
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
verify_tonic_canonical_body_digest(&request, request.get_ref().challenge.as_ref())
.map_err(|err| Status::permission_denied(format!("scanner activity authentication failed: {err}")))?;
if !request.get_ref().acknowledge_instance_id.is_empty()
|| request.get_ref().acknowledge_dirty_usage_generation != 0
{
return Err(Status::invalid_argument("scanner activity protocol v4 cannot acknowledge dirty usage"));
}
}
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
let canonical = rustfs_protos::canonical_scanner_activity_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("scanner activity request is too large to authenticate"))?;
verify_tonic_canonical_body_digest(&request, &canonical)
.map_err(|err| Status::permission_denied(format!("scanner activity authentication failed: {err}")))?;
let has_acknowledgement = !request.get_ref().acknowledge_instance_id.is_empty();
if has_acknowledgement != (request.get_ref().acknowledge_dirty_usage_generation != 0) {
return Err(Status::invalid_argument(
"scanner dirty usage acknowledgement requires both instance ID and generation",
));
}
}
version => {
return Err(Status::failed_precondition(format!(
"unsupported scanner activity request protocol {version}"
)));
}
}
let challenge_len = request.get_ref().challenge.len();
if challenge_len != 16 && !(request_protocol == SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION && challenge_len == 0) {
return Err(Status::invalid_argument("scanner activity challenge must be 16 bytes"));
}
let store = self
.resolve_object_store()
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
Ok(Response::new(scanner_activity_response(store.scanner_namespace_mutation_generation())))
if request.get_ref().challenge.is_empty() {
// Older peers send an empty protocol-0 request and cannot establish
// the topology fence required for distributed usage publication.
return Ok(Response::new(legacy_scanner_activity_response(
store.scanner_namespace_mutation_generation(),
)));
}
let request = request.into_inner();
let challenge: [u8; 16] = request
.challenge
.as_ref()
.try_into()
.map_err(|_| Status::invalid_argument("scanner activity challenge must be 16 bytes"))?;
if !request.acknowledge_instance_id.is_empty() {
rustfs_scanner::acknowledge_dirty_usage_generation(
&request.acknowledge_instance_id,
request.acknowledge_dirty_usage_generation,
)
.map_err(|err| Status::failed_precondition(err.to_string()))?;
}
let namespace_generation = store.scanner_namespace_mutation_generation();
let topology_digest = rustfs_scanner::scanner_topology_digest(store.as_ref());
let data_movement_active = store.scanner_data_movement_active().await;
let mut response = match request_protocol {
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION | SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
previous_scanner_activity_response(namespace_generation, topology_digest, data_movement_active)
}
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => scanner_activity_response(
namespace_generation,
topology_digest,
data_movement_active,
rustfs_scanner::scanner_dirty_usage_state(),
),
version => {
return Err(Status::failed_precondition(format!(
"unsupported scanner activity request protocol {version}"
)));
}
};
let canonical = match response.protocol_version {
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
rustfs_protos::canonical_scanner_activity_v4_response_body(&challenge, &response)
}
rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION => {
rustfs_protos::canonical_scanner_activity_response_body(&challenge, &response)
}
version => {
return Err(Status::internal(format!(
"scanner activity response selected unsupported protocol {version}"
)));
}
}
.map_err(|_| Status::internal("scanner activity response is too large to authenticate"))?;
response.response_proof = sign_tonic_rpc_response_proof(&canonical)
.map_err(|_| Status::unavailable("scanner activity response authentication is unavailable"))?
.into();
Ok(Response::new(response))
}
async fn background_heal_status(
@@ -1882,11 +2010,13 @@ impl Node for NodeService {
mod tests {
use super::{
CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, MetricType, Node as _, NodeService,
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, make_heal_control_server,
make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context,
remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, legacy_scanner_activity_response,
make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay,
scanner_activity_response, stop_rebalance_response,
};
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint};
@@ -4296,24 +4426,199 @@ mod tests {
}
#[tokio::test]
async fn test_scanner_activity_requires_storage_layer() {
async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() {
let service = create_test_node_service();
let err = service
.scanner_activity(Request::new(ScannerActivityRequest {}))
let legacy = service
.scanner_activity(Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
}))
.await
.expect_err("activity queries must fail closed before storage is initialized");
.expect_err("a rolling-upgrade request should pass authentication before storage lookup");
assert_eq!(legacy.code(), tonic::Code::Unavailable);
assert_eq!(err.code(), tonic::Code::Unavailable);
let unsupported = service
.scanner_activity(Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION + 1,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
}))
.await
.expect_err("an unknown request protocol must fail before storage lookup");
assert_eq!(unsupported.code(), tonic::Code::FailedPrecondition);
let malformed_legacy = service
.scanner_activity(Request::new(ScannerActivityRequest {
challenge: vec![7; 15].into(),
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
}))
.await
.expect_err("a malformed legacy challenge must fail before storage lookup");
assert_eq!(malformed_legacy.code(), tonic::Code::InvalidArgument);
let unsigned = service
.scanner_activity(Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
}))
.await
.expect_err("unsigned activity queries must fail before storage lookup");
assert_eq!(unsigned.code(), tonic::Code::PermissionDenied);
let mut malformed_current = Request::new(ScannerActivityRequest {
challenge: vec![7; 15].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
});
let malformed_canonical = rustfs_protos::canonical_scanner_activity_request_body(malformed_current.get_ref())
.expect("scanner activity request should encode");
set_tonic_canonical_body_digest(&mut malformed_current, &malformed_canonical).expect("digest metadata should encode");
mark_v2_authenticated(&mut malformed_current);
let malformed_current = service
.scanner_activity(malformed_current)
.await
.expect_err("a signed malformed challenge must fail before storage lookup");
assert_eq!(malformed_current.code(), tonic::Code::InvalidArgument);
let mut tampered = Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
});
let other = ScannerActivityRequest {
challenge: vec![8; 16].into(),
..tampered.get_ref().clone()
};
let other_canonical =
rustfs_protos::canonical_scanner_activity_request_body(&other).expect("scanner activity request should encode");
set_tonic_canonical_body_digest(&mut tampered, &other_canonical).expect("digest metadata should encode");
mark_v2_authenticated(&mut tampered);
let tampered = service
.scanner_activity(tampered)
.await
.expect_err("tampered activity challenge must fail before storage lookup");
assert_eq!(tampered.code(), tonic::Code::PermissionDenied);
let mut downgraded = Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
});
let current_canonical = rustfs_protos::canonical_scanner_activity_request_body(downgraded.get_ref())
.expect("scanner activity request should encode");
set_tonic_canonical_body_digest(&mut downgraded, &current_canonical).expect("digest metadata should encode");
downgraded.get_mut().protocol_version = SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION;
mark_v2_authenticated(&mut downgraded);
let downgraded = service
.scanner_activity(downgraded)
.await
.expect_err("a signed current request must not be downgraded to protocol v4");
assert_eq!(downgraded.code(), tonic::Code::PermissionDenied);
let mut incomplete_acknowledgement = Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
acknowledge_dirty_usage_generation: 0,
});
let acknowledgement_canonical =
rustfs_protos::canonical_scanner_activity_request_body(incomplete_acknowledgement.get_ref())
.expect("scanner activity request should encode");
set_tonic_canonical_body_digest(&mut incomplete_acknowledgement, &acknowledgement_canonical)
.expect("digest metadata should encode");
mark_v2_authenticated(&mut incomplete_acknowledgement);
let incomplete_acknowledgement = service
.scanner_activity(incomplete_acknowledgement)
.await
.expect_err("dirty usage acknowledgements require an instance ID and generation");
assert_eq!(incomplete_acknowledgement.code(), tonic::Code::InvalidArgument);
let mut previous = Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
});
set_tonic_canonical_body_digest(&mut previous, &[7; 16]).expect("protocol v4 digest metadata should encode");
mark_v2_authenticated(&mut previous);
let previous = service
.scanner_activity(previous)
.await
.expect_err("an authenticated protocol v4 request should reach storage lookup during rolling upgrades");
assert_eq!(previous.code(), tonic::Code::Unavailable);
let mut signed = Request::new(ScannerActivityRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id: String::new(),
acknowledge_dirty_usage_generation: 0,
});
let signed_canonical = rustfs_protos::canonical_scanner_activity_request_body(signed.get_ref())
.expect("scanner activity request should encode");
set_tonic_canonical_body_digest(&mut signed, &signed_canonical).expect("digest metadata should encode");
mark_v2_authenticated(&mut signed);
let unavailable = service
.scanner_activity(signed)
.await
.expect_err("authenticated activity queries still require initialized storage");
assert_eq!(unavailable.code(), tonic::Code::Unavailable);
}
#[test]
fn test_scanner_activity_response_uses_process_epoch_and_generations() {
let response = scanner_activity_response(17);
let response = scanner_activity_response(
17,
[7; 32],
true,
rustfs_scanner::ScannerDirtyUsageState {
generation: 11,
pending: true,
},
);
assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch());
assert_eq!(response.namespace_generation, 17);
assert_eq!(response.maintenance_generation, rustfs_scanner::scanner_maintenance_generation());
assert_eq!(response.protocol_version, rustfs_scanner::SCANNER_ACTIVITY_PROTOCOL_VERSION);
assert_eq!(response.topology_digest.as_ref(), &[7; 32]);
assert!(response.data_movement_active);
assert_eq!(response.dirty_usage_generation, 11);
assert!(response.dirty_usage_pending);
}
#[test]
fn test_previous_scanner_activity_response_omits_dirty_usage_fields() {
let response = previous_scanner_activity_response(17, [7; 32], true);
assert_eq!(response.protocol_version, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION);
assert_eq!(response.topology_digest.as_ref(), &[7; 32]);
assert!(response.data_movement_active);
assert_eq!(response.dirty_usage_generation, 0);
assert!(!response.dirty_usage_pending);
}
#[test]
fn test_legacy_scanner_activity_response_omits_extended_fields() {
let response = legacy_scanner_activity_response(17);
assert_eq!(response.namespace_generation, 17);
assert_eq!(response.protocol_version, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION);
assert!(response.topology_digest.is_empty());
assert!(!response.data_movement_active);
assert!(response.response_proof.is_empty());
assert_eq!(response.dirty_usage_generation, 0);
assert!(!response.dirty_usage_pending);
}
#[tokio::test]
@@ -4373,6 +4678,7 @@ mod tests {
assert!(response.success, "new nodes must advertise notify lifecycle reload support");
assert!(response.error_info.is_none());
assert_eq!(response.protocol_version, rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION);
}
#[tokio::test]
rustfs/src/storage/rpc/node_service/disk.rs
+18 -7
View File
@@ -698,13 +698,24 @@ impl NodeService {
Ok(volume_infos) => {
let volume_infos = volume_infos
.into_iter()
.filter_map(|volume_info| serde_json::to_string(&volume_info).ok())
.collect();
Ok(Response::new(ListVolumesResponse {
success: true,
volume_infos,
error: None,
}))
.enumerate()
.map(|(index, volume_info)| {
serde_json::to_string(&volume_info)
.map_err(|err| DiskError::other(format!("encode list volumes entry {index} failed: {err}")))
})
.collect::<std::result::Result<Vec<_>, DiskError>>();
match volume_infos {
Ok(volume_infos) => Ok(Response::new(ListVolumesResponse {
success: true,
volume_infos,
error: None,
})),
Err(err) => Ok(Response::new(ListVolumesResponse {
success: false,
volume_infos: Vec::new(),
error: Some(err.into()),
})),
}
}
Err(err) => Ok(Response::new(ListVolumesResponse {
success: false,
rustfs/src/storage/storage_api.rs
+21 -5
View File
@@ -212,12 +212,23 @@ pub(crate) mod rpc_consumer {
pub(crate) mod http_service {
pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = super::super::DEFAULT_READ_BUFFER_SIZE;
#[cfg(test)]
pub(crate) use super::super::storage_contracts::WALK_DIR_BODY_SHA256_QUERY;
pub(crate) use super::super::storage_contracts::WALK_DIR_STREAM_COMPLETION_V1;
pub(crate) use super::super::{StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, verify_rpc_signature};
pub(crate) use super::super::storage_contracts::{
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, WALK_DIR_BODY_SHA256_QUERY,
};
pub(crate) use super::super::storage_contracts::{
NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, WALK_DIR_STREAM_COMPLETION_V1,
};
pub(crate) use super::super::{
StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature,
};
}
pub(crate) mod node_service {
pub(crate) use super::super::storage_contracts::{
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
};
pub(crate) use super::super::{
BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore,
ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
@@ -404,7 +415,7 @@ pub(crate) mod ecstore_data_usage {
apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_data_usage_from_backend,
load_data_usage_from_backend_cached, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, remove_bucket_usage_from_backend, store_compression_total_in_backend,
record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend,
};
// Test-only observables for the rustfs/backlog#1306 revert detector.
#[cfg(test)]
@@ -484,7 +495,8 @@ pub(crate) mod ecstore_rpc {
pub(crate) use rustfs_ecstore::api::rpc::{
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest,
verify_tonic_rpc_signature,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{
@@ -1543,6 +1555,10 @@ pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &h
ecstore_rpc::verify_rpc_signature(url, method, headers)
}
pub(crate) fn sign_ns_scanner_capability(challenge: uuid::Uuid, server_epoch: uuid::Uuid) -> std::io::Result<Vec<u8>> {
ecstore_rpc::sign_ns_scanner_capability(challenge, server_epoch)
}
pub(crate) fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &http::HeaderMap) -> std::io::Result<()> {
ecstore_rpc::verify_tonic_rpc_signature(audience, path, headers)
}