gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): make distributed usage convergence authoritative (#5151)

Browse Source 
* fix(scanner): make distributed usage cycles authoritative

* fix(scanner): close distributed refresh races

* fix(config): align scanner reload integration

* fix(admin): scope config test helpers

* fix(scanner): harden distributed usage convergence

* fix(scanner): preserve rolling activity compatibility

* fix(admin): expose non-secret optional config values

* fix(scanner): acknowledge distributed dirty usage

* fix(ecstore): make bucket mutations cancellation safe

* fix(scanner): preserve pending dirty acknowledgements

* test(obs): account for superseded scanner metric

* fix(api): reject excess detached bucket mutations

* test: close scanner convergence coverage gaps

* fix(scanner): make path tracking cleanup one-shot

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-07-25 18:45:16 +08:00
committed by GitHub
parent 0364523dad
commit a63b79004c
69 changed files with 18073 additions and 1585 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/common/src/metrics.rs
+113 -17
View File
@@ -768,6 +768,7 @@ pub struct Metrics {
last_scan_cycle_replication_checks: AtomicU64,
last_scan_cycle_usage_saves: AtomicU64,
failed_scan_cycles: AtomicU64,
superseded_scan_cycles: AtomicU64,
partial_scan_cycles_unknown: AtomicU64,
partial_scan_cycles_runtime: AtomicU64,
partial_scan_cycles_objects: AtomicU64,
@@ -833,10 +834,12 @@ const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4;
const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial";
const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded";
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum ScanCyclePartialReason {
@@ -1208,6 +1211,8 @@ pub struct ScannerMetricsReport {
pub last_cycle_usage_saves: u64,
pub failed_cycles: u64,
#[serde(default)]
pub superseded_cycles: u64,
#[serde(default)]
pub partial_cycles_unknown: u64,
#[serde(default)]
pub partial_cycles_runtime: u64,
@@ -1310,6 +1315,7 @@ fn scan_cycle_result_label(result: u8) -> &'static str {
SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL,
_ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
}
}
@@ -1633,6 +1639,11 @@ pub fn emit_scan_cycle_partial_with_source(
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1);
}
pub fn emit_scan_cycle_superseded(duration: Duration) {
global_metrics().record_scan_cycle_superseded(duration);
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1);
}
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
let result = if success { "success" } else { "error" };
metrics::counter!(
@@ -1726,6 +1737,7 @@ impl Metrics {
last_scan_cycle_replication_checks: AtomicU64::new(0),
last_scan_cycle_usage_saves: AtomicU64::new(0),
failed_scan_cycles: AtomicU64::new(0),
superseded_scan_cycles: AtomicU64::new(0),
partial_scan_cycles_unknown: AtomicU64::new(0),
partial_scan_cycles_runtime: AtomicU64::new(0),
partial_scan_cycles_objects: AtomicU64::new(0),
@@ -2349,6 +2361,18 @@ impl Metrics {
.store(duration_millis_saturated(duration), Ordering::Relaxed);
}
pub fn record_scan_cycle_superseded(&self, duration: Duration) {
self.record_scanner_cycle_end_time();
self.superseded_scan_cycles.fetch_add(1, Ordering::Relaxed);
self.last_scan_cycle_result
.store(SCAN_CYCLE_RESULT_SUPERSEDED, Ordering::Relaxed);
self.last_scan_cycle_partial_reason
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
self.last_scan_cycle_duration_millis
.store(duration_millis_saturated(duration), Ordering::Relaxed);
}
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
self.record_scan_cycle_partial_with_source(duration, reason, None);
}
@@ -2802,6 +2826,7 @@ impl Metrics {
m.last_cycle_replication_repair =
self.scanner_replication_repair_work_counter_snapshots(&self.last_scan_cycle_replication_repair_work);
m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
m.superseded_cycles = self.superseded_scan_cycles.load(Ordering::Relaxed);
m.partial_cycles_unknown = self.partial_scan_cycles_unknown.load(Ordering::Relaxed);
m.partial_cycles_runtime = self.partial_scan_cycles_runtime.load(Ordering::Relaxed);
m.partial_cycles_objects = self.partial_scan_cycles_objects.load(Ordering::Relaxed);
@@ -2950,19 +2975,15 @@ pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>>
/// Register a new disk in the global path tracker and return two callbacks:
/// one to update the current path and one to deregister the disk when done.
pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
pub async fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
let disk_name = disk.to_string();
let tracker_clone = Arc::clone(&tracker);
let disk_insert = disk_name.clone();
tokio::spawn(async move {
global_metrics()
.current_paths
.write()
.await
.insert(disk_insert, tracker_clone);
});
global_metrics()
.current_paths
.write()
.await
.insert(disk_name.clone(), Arc::clone(&tracker));
let update_fn: UpdateCurrentPathFn = {
let tracker = Arc::clone(&tracker);
@@ -2990,23 +3011,28 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn,
// CloseDiskGuard
// ---------------------------------------------------------------------------
pub struct CloseDiskGuard(CloseDiskFn);
pub struct CloseDiskGuard(Option<CloseDiskFn>);
impl CloseDiskGuard {
pub fn new(close_disk: CloseDiskFn) -> Self {
Self(close_disk)
Self(Some(close_disk))
}
pub async fn close(&self) {
self.0().await;
pub async fn close(&mut self) {
let Some(close_disk) = self.0.clone() else {
return;
};
close_disk().await;
self.0 = None;
}
}
impl Drop for CloseDiskGuard {
fn drop(&mut self) {
if let Ok(handle) = tokio::runtime::Handle::try_current() {
let close_fn = self.0.clone();
handle.spawn(async move { close_fn().await });
if let Some(close_disk) = self.0.take()
&& let Ok(handle) = tokio::runtime::Handle::try_current()
{
handle.spawn(close_disk());
}
// If there is no runtime we are in a test or shutdown path; skip cleanup.
}
@@ -3016,6 +3042,61 @@ impl Drop for CloseDiskGuard {
mod tests {
use super::*;
#[tokio::test]
async fn close_disk_guard_runs_cleanup_when_an_early_return_drops_it() {
let (closed_tx, closed_rx) = tokio::sync::oneshot::channel();
let closed_tx = Arc::new(std::sync::Mutex::new(Some(closed_tx)));
let close_disk: CloseDiskFn = {
let closed_tx = Arc::clone(&closed_tx);
Arc::new(move || {
let closed_tx = closed_tx.lock().expect("close callback lock").take();
Box::pin(async move {
if let Some(closed_tx) = closed_tx {
let _ = closed_tx.send(());
}
})
})
};
let guard = CloseDiskGuard::new(close_disk);
drop(guard);
tokio::time::timeout(std::time::Duration::from_secs(1), closed_rx)
.await
.expect("drop cleanup should run")
.expect("drop cleanup should signal");
}
#[tokio::test]
async fn close_disk_guard_runs_explicit_cleanup_once() {
let close_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let close_disk: CloseDiskFn = {
let close_count = Arc::clone(&close_count);
Arc::new(move || {
close_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
Box::pin(std::future::ready(()))
})
};
let mut guard = CloseDiskGuard::new(close_disk);
guard.close().await;
drop(guard);
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
assert_eq!(close_count.load(std::sync::atomic::Ordering::Relaxed), 1);
}
#[tokio::test]
async fn current_path_updater_registers_before_return() {
let disk = format!("test-disk-{}", uuid::Uuid::new_v4());
let (_update_path, close_disk) = current_path_updater(&disk, "bucket-a").await;
assert!(global_metrics().current_paths.read().await.contains_key(&disk));
close_disk().await;
assert!(!global_metrics().current_paths.read().await.contains_key(&disk));
}
#[tokio::test]
async fn report_counts_active_scan_paths() {
let metrics = Metrics::new();
@@ -3850,6 +3931,21 @@ mod tests {
assert_eq!(report.failed_cycles, 1);
}
#[tokio::test]
async fn report_tracks_superseded_cycle_without_failed_increment() {
let metrics = Metrics::new();
metrics.record_scan_cycle_superseded(Duration::from_millis(750));
let report = metrics.report().await;
assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_SUPERSEDED_LABEL);
assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_SUPERSEDED));
assert_eq!(report.last_cycle_duration_seconds, 0.75);
assert_eq!(report.failed_cycles, 0);
assert_eq!(report.superseded_cycles, 1);
assert_eq!(report.partial_cycles, 0);
}
#[tokio::test]
async fn report_tracks_successful_scan_cycle_without_failed_increment() {
let metrics = Metrics::new();
crates/config/src/constants/scanner.rs
+2 -4
View File
@@ -220,12 +220,10 @@ pub const ENV_SCANNER_YIELD_EVERY_N_OBJECTS: &str = "RUSTFS_SCANNER_YIELD_EVERY_
pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
/// Default set scan concurrency budget.
/// `0` means no additional limit beyond deployment topology.
pub const DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS: usize = 0;
pub const DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS: usize = 4;
/// Default disk scan concurrency budget.
/// `0` means no additional limit beyond available disks in the set.
pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 0;
pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 4;
/// Default object interval for cooperative scanner yields.
pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128;
crates/data-usage/src/data_usage.rs
+234 -14
View File
@@ -32,6 +32,20 @@ use std::{
/// save forever and freeze admin usage stats; callers must bypass the skip instead.
pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 * 60);
/// Authoritative cluster-wide usage snapshot written by coordinated scanners.
///
/// This object name is intentionally distinct from the legacy unfenced
/// snapshot. Older binaries can continue writing the legacy object during a
/// rolling upgrade without overwriting a snapshot produced by the current
/// scanner protocol.
pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json";
/// Usage snapshot written by scanner implementations predating distributed
/// leadership fencing. It is read only when neither authoritative snapshot
/// copy exists.
// RUSTFS_COMPAT_TODO(scanner-usage-v2): keep .usage.json readable and removable during rolling upgrades from pre-v2 scanners. Remove after supported direct-upgrade sources all write .usage.v2.json.
pub const LEGACY_DATA_USAGE_OBJECT_NAME: &str = ".usage.json";
/// Returns true when `existing_last_update` is ahead of `now` by more than
/// [`USAGE_LAST_UPDATE_FUTURE_TOLERANCE`], i.e. the persisted timestamp cannot be
/// trusted for staleness comparisons and a fresh snapshot save must be allowed.
@@ -95,7 +109,7 @@ impl AllTierStats {
}
/// Bucket target usage info provides replication statistics
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct BucketTargetUsageInfo {
pub replication_pending_size: u64,
pub replication_failed_size: u64,
@@ -107,7 +121,7 @@ pub struct BucketTargetUsageInfo {
}
/// Bucket usage info provides bucket-level statistics
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct BucketUsageInfo {
pub size: u64,
// Following five fields suffixed with V1 are here for backward compatibility
@@ -133,7 +147,7 @@ pub struct BucketUsageInfo {
}
/// DataUsageInfo represents data usage stats of the underlying storage
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DataUsageInfo {
/// Total capacity
pub total_capacity: u64,
@@ -145,6 +159,22 @@ pub struct DataUsageInfo {
/// LastUpdate is the timestamp of when the data usage info was last updated
pub last_update: Option<SystemTime>,
/// Monotonic scanner cycle that produced this complete snapshot.
///
/// Older snapshots omit this field and continue to use `last_update` for
/// compatibility. New scanner snapshots use the cycle to fence stale
/// leaders independently of wall-clock skew.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scanner_cycle: Option<u64>,
/// Persisted scanner leadership epoch that produced this snapshot.
///
/// The epoch is claimed through the cycle-state CAS before scanning. It
/// orders snapshots from different leaders even when their wall clocks or
/// cycle counters coincide.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scanner_epoch: Option<u64>,
/// Objects total count across all buckets
pub objects_total_count: u64,
/// Versions total count across all buckets
@@ -168,7 +198,7 @@ pub struct DataUsageInfo {
}
/// Metadata describing the status of a disk-level data usage snapshot.
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DiskUsageStatus {
pub disk_id: String,
pub pool_index: Option<usize>,
@@ -268,12 +298,30 @@ impl DataUsageHash {
pub type DataUsageHashMap = HashSet<String>;
/// Size histogram for object size distribution
#[derive(Clone, Debug, Serialize, Deserialize)]
const SIZE_HISTOGRAM_LEN: usize = 11;
#[derive(Clone, Debug, Serialize)]
pub struct SizeHistogram(Vec<u64>);
impl Default for SizeHistogram {
fn default() -> Self {
Self(vec![0; 11]) // DATA_USAGE_BUCKET_LEN = 11
Self(vec![0; SIZE_HISTOGRAM_LEN])
}
}
impl<'de> Deserialize<'de> for SizeHistogram {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let values = Vec::<u64>::deserialize(deserializer)?;
if values.len() != SIZE_HISTOGRAM_LEN {
return Err(serde::de::Error::invalid_length(
values.len(),
&"exactly 11 object-size histogram buckets",
));
}
Ok(Self(values))
}
}
@@ -343,7 +391,7 @@ impl SizeHistogram {
.zip(names.iter())
.filter(|((_, (start, end)), name)| name != &&"BETWEEN_1024B_AND_1_MB" && *start >= 1024 && *end < ONE_MIB)
.map(|((count, _), _)| *count)
.sum();
.fold(0, u64::saturating_add);
let mut res = HashMap::new();
for (count, name) in self.0.iter().zip(names.iter()) {
@@ -364,12 +412,30 @@ impl SizeHistogram {
}
/// Versions histogram for version count distribution
#[derive(Clone, Debug, Serialize, Deserialize)]
const VERSIONS_HISTOGRAM_LEN: usize = 7;
#[derive(Clone, Debug, Serialize)]
pub struct VersionsHistogram(Vec<u64>);
impl Default for VersionsHistogram {
fn default() -> Self {
Self(vec![0; 7]) // DATA_USAGE_VERSION_LEN = 7
Self(vec![0; VERSIONS_HISTOGRAM_LEN])
}
}
impl<'de> Deserialize<'de> for VersionsHistogram {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let values = Vec::<u64>::deserialize(deserializer)?;
if values.len() != VERSIONS_HISTOGRAM_LEN {
return Err(serde::de::Error::invalid_length(
values.len(),
&"exactly 7 object-version histogram buckets",
));
}
Ok(Self(values))
}
}
@@ -535,13 +601,74 @@ impl DataUsageEntry {
}
}
for (i, v) in other.obj_sizes.0.iter().enumerate() {
self.obj_sizes.0[i] += v;
}
self.obj_sizes.merge_from(&other.obj_sizes);
self.obj_versions.merge_from(&other.obj_versions);
}
for (i, v) in other.obj_versions.0.iter().enumerate() {
self.obj_versions.0[i] += v;
pub fn checked_merge(&mut self, other: &DataUsageEntry) -> bool {
let scalar_counts_fit = self.objects.checked_add(other.objects).is_some()
&& self.versions.checked_add(other.versions).is_some()
&& self.delete_markers.checked_add(other.delete_markers).is_some()
&& self.size.checked_add(other.size).is_some()
&& self.failed_objects.checked_add(other.failed_objects).is_some();
let histograms_fit = self.obj_sizes.0.len() == SIZE_HISTOGRAM_LEN
&& other.obj_sizes.0.len() == SIZE_HISTOGRAM_LEN
&& self.obj_versions.0.len() == VERSIONS_HISTOGRAM_LEN
&& other.obj_versions.0.len() == VERSIONS_HISTOGRAM_LEN
&& self
.obj_sizes
.0
.iter()
.zip(other.obj_sizes.0.iter())
.all(|(left, right)| left.checked_add(*right).is_some())
&& self
.obj_versions
.0
.iter()
.zip(other.obj_versions.0.iter())
.all(|(left, right)| left.checked_add(*right).is_some());
let replication_fits = match (&self.replication_stats, &other.replication_stats) {
(_, None) | (None, Some(_)) => true,
(Some(left), Some(right)) => {
left.replica_size.checked_add(right.replica_size).is_some()
&& left.replica_count.checked_add(right.replica_count).is_some()
&& right.targets.iter().all(|(target, right_stats)| {
left.targets.get(target).is_none_or(|left_stats| {
left_stats.pending_size.checked_add(right_stats.pending_size).is_some()
&& left_stats.replicated_size.checked_add(right_stats.replicated_size).is_some()
&& left_stats.failed_size.checked_add(right_stats.failed_size).is_some()
&& left_stats.failed_count.checked_add(right_stats.failed_count).is_some()
&& left_stats.pending_count.checked_add(right_stats.pending_count).is_some()
&& left_stats
.missed_threshold_size
.checked_add(right_stats.missed_threshold_size)
.is_some()
&& left_stats
.after_threshold_size
.checked_add(right_stats.after_threshold_size)
.is_some()
&& left_stats
.missed_threshold_count
.checked_add(right_stats.missed_threshold_count)
.is_some()
&& left_stats
.after_threshold_count
.checked_add(right_stats.after_threshold_count)
.is_some()
&& left_stats
.replicated_count
.checked_add(right_stats.replicated_count)
.is_some()
})
})
}
};
if !scalar_counts_fit || !histograms_fit || !replication_fits {
return false;
}
self.merge(other);
true
}
}
@@ -1395,6 +1522,17 @@ mod tests {
assert_eq!(map["BETWEEN_512_KB_AND_1_MB"], 1);
}
#[test]
fn test_size_histogram_compat_rollup_saturates_on_corrupt_counts() {
let mut hist = SizeHistogram::default();
hist.0[1] = u64::MAX;
hist.0[2] = 1;
let map = hist.to_map();
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], u64::MAX);
}
#[test]
fn test_data_usage_cache_merge_adds_missing_child() {
let mut base = DataUsageCache::default();
@@ -1708,4 +1846,86 @@ mod tests {
assert!(cache.find("bucket/large/a").is_some());
assert!(cache.find("bucket/large/b").is_some());
}
#[test]
fn checked_merge_rejects_scalar_and_replication_overflow_without_mutation() {
let mut entry = DataUsageEntry {
objects: usize::MAX,
replication_stats: Some(ReplicationAllStats {
replica_size: 7,
..Default::default()
}),
..Default::default()
};
let other = DataUsageEntry {
objects: 1,
replication_stats: Some(ReplicationAllStats {
replica_size: u64::MAX,
..Default::default()
}),
..Default::default()
};
assert!(!entry.checked_merge(&other));
assert_eq!(entry.objects, usize::MAX);
assert_eq!(entry.replication_stats.as_ref().map(|stats| stats.replica_size), Some(7));
}
#[test]
fn checked_merge_accepts_valid_usage() {
let mut entry = DataUsageEntry {
objects: 2,
size: 20,
..Default::default()
};
let other = DataUsageEntry {
objects: 3,
size: 30,
..Default::default()
};
assert!(entry.checked_merge(&other));
assert_eq!(entry.objects, 5);
assert_eq!(entry.size, 50);
}
#[test]
fn histogram_deserialization_rejects_noncanonical_lengths() {
let invalid_sizes =
rmp_serde::to_vec(&vec![0_u64; SIZE_HISTOGRAM_LEN + 1]).expect("encode invalid object-size histogram fixture");
let invalid_versions =
rmp_serde::to_vec(&vec![0_u64; VERSIONS_HISTOGRAM_LEN - 1]).expect("encode invalid object-version histogram fixture");
assert!(rmp_serde::from_slice::<SizeHistogram>(&invalid_sizes).is_err());
assert!(rmp_serde::from_slice::<VersionsHistogram>(&invalid_versions).is_err());
}
#[test]
fn replication_target_deserialization_preserves_large_historical_maps() {
let mut stats = ReplicationAllStats::default();
for index in 0..=1024 {
stats.targets.insert(format!("target-{index}"), ReplicationStats::default());
}
let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode");
let decoded = rmp_serde::from_slice::<ReplicationAllStats>(&encoded)
.expect("historical replication target maps must remain readable");
assert_eq!(decoded.targets.len(), stats.targets.len());
}
#[test]
fn checked_merge_rejects_noncanonical_histograms_without_mutation() {
let mut entry = DataUsageEntry {
objects: 2,
..Default::default()
};
let other = DataUsageEntry {
objects: 3,
obj_sizes: SizeHistogram(vec![0; SIZE_HISTOGRAM_LEN + 1]),
..Default::default()
};
assert!(!entry.checked_merge(&other));
assert_eq!(entry.objects, 2);
}
}
crates/e2e_test/src/content_encoding_test.rs
+19
View File
@@ -80,6 +80,25 @@ mod tests {
assert_eq!(head_resp.content_encoding(), Some("zstd"), "HEAD should return Content-Encoding: zstd");
assert_eq!(head_resp.content_type(), Some("text/plain"), "HEAD should return correct Content-Type");
client
.delete_object()
.bucket(bucket)
.key(key)
.send()
.await
.expect("DELETE object failed");
client
.delete_bucket()
.bucket(bucket)
.send()
.await
.expect("DELETE bucket failed");
client
.list_buckets()
.send()
.await
.expect("RustFS must remain available after deleting a bucket");
env.stop_server();
}
crates/ecstore/src/api/mod.rs
+15 -14
View File
@@ -244,11 +244,12 @@ pub mod config {
pub mod com {
pub use crate::config::com::{
COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
ServerConfigCorruptError, delete_config, is_server_config_corrupt_error, lookup_configs, read_config,
read_config_no_lock, read_config_with_metadata, read_config_without_migrate, read_config_without_migrate_no_lock,
read_existing_server_config_no_lock, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
save_server_config_no_lock, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
with_server_config_read_lock, with_server_config_write_lock,
ServerConfigCorruptError, ServerConfigSnapshot, delete_config, is_server_config_corrupt_error, lookup_configs,
read_config, read_config_no_lock, read_config_with_metadata, read_config_without_migrate,
read_config_without_migrate_no_lock, read_existing_server_config_no_lock, read_server_config_snapshot, save_config,
save_config_no_lock, save_config_with_opts, save_server_config, save_server_config_no_lock,
save_server_config_snapshot, server_config_path, try_migrate_server_config, with_config_object_read_lock,
with_config_object_write_lock, with_server_config_read_lock, with_server_config_write_lock,
};
}
@@ -287,9 +288,9 @@ pub mod disk {
pub use crate::disk::{
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, OldCurrentSize, RUSTFS_META_BUCKET, ReadMultipleReq,
ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
new_disk, validate_batch_read_version_item_count,
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, RUSTFS_META_BUCKET,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions, new_disk, validate_batch_read_version_item_count,
};
pub use bytes::Bytes;
pub use endpoint::Endpoint;
@@ -390,12 +391,12 @@ pub mod rio {
pub mod rpc {
pub use crate::cluster::rpc::{
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor,
gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature,
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
};
}
crates/ecstore/src/cluster/rpc/http_auth.rs
+52
View File
@@ -27,6 +27,7 @@
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
use crate::cluster::rpc::context_propagation::{inject_request_id_into_http_headers, inject_trace_context_into_http_headers};
use crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION;
use base64::Engine as _;
use base64::engine::general_purpose;
use hmac::{Hmac, KeyInit, Mac};
@@ -61,6 +62,7 @@ const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536;
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
get_env_bool(
@@ -209,6 +211,42 @@ fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, si
mac.verify_slice(&signature).is_ok()
}
fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
mac.update(challenge.as_bytes());
mac.update(server_epoch.as_bytes());
}
fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
Ok(mac.finalize().into_bytes().to_vec())
}
fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
mac.verify_slice(proof)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
}
pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch)
}
pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof)
}
#[derive(Clone, Copy)]
struct SignatureV2Scope<'a> {
audience: &'a str,
@@ -629,6 +667,20 @@ mod tests {
runtime_sources::ensure_test_rpc_secret();
}
#[test]
fn namespace_scanner_capability_proof_binds_challenge_and_server_epoch() {
let secret = "test-scanner-capability-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof =
generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err());
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err());
}
/// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
/// fixed in rustfs/rustfs#4402): secret resolution must never silently fall
/// back to a default/empty shared secret. Missing and default secrets both
crates/ecstore/src/cluster/rpc/internode_data_transport.rs
+220 -2
View File
@@ -12,11 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::build_auth_headers;
use crate::cluster::rpc::{build_auth_headers, verify_ns_scanner_capability};
use crate::disk::error::{Error, Result};
use crate::disk::{FileReader, FileWriter};
use crate::storage_api_contracts::internode::{
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY,
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
};
use async_trait::async_trait;
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
@@ -28,13 +31,18 @@ use rustfs_rio::{HttpReader, HttpWriter};
use sha2::{Digest, Sha256};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use tokio::io::AsyncReadExt;
use uuid::Uuid;
static INTERNODE_DATA_TRANSPORT: OnceLock<std::result::Result<Arc<dyn InternodeDataTransport>, String>> = OnceLock::new();
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner";
const NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE: usize = 1024;
const CONTENT_TYPE_JSON: &str = "application/json";
const CONTENT_TYPE_MSGPACK: &str = "application/msgpack";
fn unsupported_transport_message(transport: &str) -> String {
format!(
@@ -101,6 +109,25 @@ pub struct WalkDirStreamRequest {
pub stall_timeout: Option<Duration>,
}
#[derive(Debug, Clone)]
pub struct NsScannerStreamRequest {
pub endpoint: String,
pub disk: String,
pub request_id: Uuid,
pub server_epoch: Uuid,
pub session_id: Uuid,
pub session_sequence: u64,
pub next_cycle: u64,
pub leader_epoch: u64,
pub body: Vec<u8>,
pub stall_timeout: Option<Duration>,
}
#[derive(Debug, Clone)]
pub struct NsScannerCapabilityRequest {
pub endpoint: String,
}
/// Data-plane stream opener used by `RemoteDisk`.
///
/// This boundary is limited to remote disk streams that can move large payloads.
@@ -114,6 +141,12 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader>;
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter>;
async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader>;
async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result<FileReader> {
Err(Error::MethodNotAllowed)
}
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
Err(Error::MethodNotAllowed)
}
fn name(&self) -> &'static str;
fn capabilities(&self) -> InternodeDataTransportCapabilities;
}
@@ -148,6 +181,39 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
))
}
async fn open_ns_scanner(&self, request: NsScannerStreamRequest) -> Result<FileReader> {
let url = build_ns_scanner_url(&request);
let mut headers = msgpack_headers();
build_auth_headers(&url, &Method::POST, &mut headers)?;
Ok(Box::new(
HttpReader::new_with_stall_timeout(url, Method::POST, headers, Some(request.body), request.stall_timeout).await?,
))
}
async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
let challenge = Uuid::new_v4();
let url = build_ns_scanner_capability_url(&request, challenge);
let mut headers = msgpack_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
let reader = HttpReader::new(url, Method::GET, headers, None).await?;
let mut body = Vec::new();
reader
.take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
.read_to_end(&mut body)
.await?;
if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
return Err(Error::other("invalid remote namespace scanner capability response size"));
}
let response: NsScannerCapabilityResponse =
rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
return Err(Error::other("incompatible remote namespace scanner capability response"));
}
verify_ns_scanner_capability(challenge, response.server_epoch, &response.proof)
.map_err(|err| Error::other(format!("remote namespace scanner capability authentication failed: {err}")))?;
Ok(response.server_epoch)
}
fn name(&self) -> &'static str {
DEFAULT_INTERNODE_DATA_TRANSPORT
}
@@ -197,12 +263,54 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
)
}
fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
let body_sha256 = hex_simd::encode_to_string(Sha256::digest(&request.body), hex_simd::AsciiCase::Lower);
format!(
"{}{}?disk={}&{}={}&{}={}&{}={}&{}={}&{}={}&{}={}&{}={}",
request.endpoint,
NS_SCANNER_PATH,
urlencoding::encode(&request.disk),
NS_SCANNER_REQUEST_ID_QUERY,
request.request_id,
NS_SCANNER_SERVER_EPOCH_QUERY,
request.server_epoch,
NS_SCANNER_SESSION_ID_QUERY,
request.session_id,
NS_SCANNER_SESSION_SEQUENCE_QUERY,
request.session_sequence,
NS_SCANNER_CYCLE_QUERY,
request.next_cycle,
NS_SCANNER_LEADER_EPOCH_QUERY,
request.leader_epoch,
NS_SCANNER_BODY_SHA256_QUERY,
body_sha256
)
}
fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
format!(
"{}{}?{}={}&{}={}",
request.endpoint,
NS_SCANNER_PATH,
NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_PROTOCOL_VERSION,
NS_SCANNER_CAPABILITY_CHALLENGE_QUERY,
challenge
)
}
fn json_headers() -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static(CONTENT_TYPE_JSON));
headers
}
fn msgpack_headers() -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static(CONTENT_TYPE_MSGPACK));
headers
}
fn build_internode_data_transport_result(
configured_transport: Option<&str>,
) -> std::result::Result<Arc<dyn InternodeDataTransport>, String> {
@@ -241,6 +349,65 @@ pub fn build_internode_data_transport_from_env() -> Result<Arc<dyn InternodeData
mod tests {
use super::*;
#[derive(Debug)]
struct LegacyTestTransport;
#[async_trait::async_trait]
impl InternodeDataTransport for LegacyTestTransport {
async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
Ok(Box::new(tokio::io::empty()))
}
async fn open_write(&self, _request: WriteStreamRequest) -> Result<FileWriter> {
Ok(Box::new(tokio::io::sink()))
}
async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result<FileReader> {
Ok(Box::new(tokio::io::empty()))
}
fn name(&self) -> &'static str {
"legacy-test"
}
fn capabilities(&self) -> InternodeDataTransportCapabilities {
InternodeDataTransportCapabilities::tcp_http()
}
}
#[tokio::test]
async fn legacy_transport_defaults_namespace_scanner_to_unsupported() {
let transport = LegacyTestTransport;
let probe_err = transport
.probe_ns_scanner(NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
})
.await
.expect_err("legacy transport should report namespace scanner as unsupported");
assert!(matches!(probe_err, Error::MethodNotAllowed));
let open_result = transport
.open_ns_scanner(NsScannerStreamRequest {
endpoint: "http://node1:9000".to_string(),
disk: "http://node1:9000/data/rustfs0".to_string(),
request_id: Uuid::new_v4(),
server_epoch: Uuid::new_v4(),
session_id: Uuid::new_v4(),
session_sequence: 0,
next_cycle: 7,
leader_epoch: 9,
body: Vec::new(),
stall_timeout: None,
})
.await;
let open_err = match open_result {
Ok(_) => panic!("legacy transport should not open namespace scanner streams"),
Err(err) => err,
};
assert!(matches!(open_err, Error::MethodNotAllowed));
}
#[test]
fn tcp_http_capabilities_are_behavior_preserving() {
let transport = TcpHttpInternodeDataTransport;
@@ -322,6 +489,57 @@ mod tests {
);
}
#[test]
fn ns_scanner_url_binds_body_and_encodes_disk_ref() {
let request_id = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("request ID");
let server_epoch = Uuid::parse_str("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee").expect("server epoch");
let session_id = Uuid::parse_str("99999999-8888-4777-8666-555555555555").expect("session ID");
let url = build_ns_scanner_url(&NsScannerStreamRequest {
endpoint: "http://node1:9000".to_string(),
disk: "http://node1:9000/data/rustfs0".to_string(),
request_id,
server_epoch,
session_id,
session_sequence: 3,
next_cycle: 7,
leader_epoch: 9,
body: b"scanner-request".to_vec(),
stall_timeout: None,
});
assert_eq!(
url,
concat!(
"http://node1:9000/rustfs/rpc/ns_scanner?disk=http%3A%2F%2Fnode1%3A9000%2Fdata%2Frustfs0",
"&ns_scanner_request_id=11111111-2222-4333-8444-555555555555",
"&ns_scanner_server_epoch=aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee",
"&ns_scanner_session_id=99999999-8888-4777-8666-555555555555",
"&ns_scanner_session_sequence=3",
"&ns_scanner_cycle=7",
"&ns_scanner_leader_epoch=9",
"&ns_scanner_body_sha256=c958f15ca28422275c1245399f4c44eaba628ca453fcd77d6b3d4484573e4387"
)
);
}
#[test]
fn ns_scanner_capability_url_binds_version_and_challenge() {
let challenge = Uuid::parse_str("12345678-1234-4234-8234-123456789abc").expect("challenge");
let url = build_ns_scanner_capability_url(
&NsScannerCapabilityRequest {
endpoint: "http://node1:9000".to_string(),
},
challenge,
);
assert_eq!(
url,
format!(
"http://node1:9000/rustfs/rpc/ns_scanner?ns_scanner_protocol={NS_SCANNER_PROTOCOL_VERSION}&ns_scanner_challenge={challenge}"
)
);
}
#[test]
fn transport_config_defaults_to_tcp_http() {
let transport = build_internode_data_transport(None).unwrap();
crates/ecstore/src/cluster/rpc/mod.rs
+3 -3
View File
@@ -30,8 +30,8 @@ pub use client::{
};
pub use http_auth::{
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability,
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
};
#[cfg(test)]
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
@@ -41,6 +41,6 @@ pub use peer_rest_client::{
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
};
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys};
pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys, ScannerBucketListing, ScannerSetBucketListing};
pub use remote_disk::RemoteDisk;
pub use remote_locker::RemoteClient;
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+337 -12
View File
@@ -18,6 +18,9 @@ use crate::cluster::rpc::client::{
};
use crate::cluster::rpc::{set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof};
use crate::error::{Error, Result};
use crate::storage_api_contracts::internode::{
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
};
use crate::{
bucket::replication::BucketStats,
disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout},
@@ -27,6 +30,7 @@ use crate::{
};
use bytes::Bytes;
use rmp_serde::{Deserializer, Serializer};
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
use rustfs_madmin::{
ServerProperties,
health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysServices},
@@ -97,15 +101,34 @@ fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<
Ok(stats)
}
fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u32) -> Result<()> {
if sig == SERVICE_SIGNAL_RELOAD_DYNAMIC
&& matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS)
&& protocol_version < rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION
{
return Err(Error::other(format!("peer does not support dynamic {sub_sys} config convergence")));
}
Ok(())
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ScannerPeerActivity {
pub instance_id: String,
pub namespace_generation: u64,
pub maintenance_generation: u64,
pub protocol_version: u32,
pub topology_digest: Option<[u8; 32]>,
pub data_movement_active: Option<bool>,
pub dirty_usage_generation: Option<u64>,
pub dirty_usage_pending: Option<bool>,
}
fn decode_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerPeerActivity> {
let instance_id = response.instance_id;
fn decode_scanner_activity_with_verifier(
response: ScannerActivityResponse,
challenge: &[u8; 16],
verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>,
) -> Result<ScannerPeerActivity> {
let instance_id = &response.instance_id;
if instance_id.len() != 32
|| !instance_id
.as_bytes()
@@ -114,10 +137,80 @@ fn decode_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerP
{
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
}
let (topology_digest, data_movement_active, dirty_usage_generation, dirty_usage_pending) = match response.protocol_version {
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): legacy response fields are unauthenticated. Remove after protocol v0 peers are unsupported.
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION
if response.topology_digest.is_empty()
&& response.response_proof.is_empty()
&& !response.data_movement_active
&& response.dirty_usage_generation == 0
&& !response.dirty_usage_pending =>
{
(None, None, None, None)
}
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(Error::other("legacy scanner activity peer returned unexpected extended fields"));
}
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
if response.dirty_usage_generation != 0 || response.dirty_usage_pending {
return Err(Error::other("scanner activity protocol v4 peer returned unauthenticated v5 fields"));
}
let canonical = rustfs_protos::canonical_scanner_activity_v4_response_body(challenge, &response)
.map_err(|_| Error::other("peer scanner activity response is too large to authenticate"))?;
verify_proof(&canonical, &response.response_proof)?;
(
Some(
response
.topology_digest
.as_ref()
.try_into()
.map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?,
),
Some(response.data_movement_active),
None,
None,
)
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
}
let canonical = rustfs_protos::canonical_scanner_activity_response_body(challenge, &response)
.map_err(|_| Error::other("peer scanner activity response is too large to authenticate"))?;
verify_proof(&canonical, &response.response_proof)?;
(
Some(
response
.topology_digest
.as_ref()
.try_into()
.map_err(|_| Error::other("peer returned an invalid scanner topology digest"))?,
),
Some(response.data_movement_active),
Some(response.dirty_usage_generation),
Some(response.dirty_usage_pending),
)
}
version => {
return Err(Error::other(format!("peer returned unsupported scanner activity protocol {version}")));
}
};
Ok(ScannerPeerActivity {
instance_id,
instance_id: response.instance_id,
namespace_generation: response.namespace_generation,
maintenance_generation: response.maintenance_generation,
protocol_version: response.protocol_version,
topology_digest,
data_movement_active,
dirty_usage_generation,
dirty_usage_pending,
})
}
fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 16]) -> Result<ScannerPeerActivity> {
decode_scanner_activity_with_verifier(response, challenge, |canonical, proof| {
verify_tonic_rpc_response_proof(canonical, proof)
.map_err(|_| Error::other("peer returned an invalid scanner activity response proof"))
})
}
@@ -1331,6 +1424,7 @@ impl PeerRestClient {
}
return Err(Error::other(""));
}
validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
Ok(())
}
.await,
@@ -1338,25 +1432,44 @@ impl PeerRestClient {
.await
}
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
async fn scanner_activity_request(
&self,
acknowledge_instance_id: String,
acknowledge_dirty_usage_generation: u64,
) -> Result<ScannerPeerActivity> {
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self
.get_client()
.await?
.max_decoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE)
.max_encoding_message_size(SCANNER_ACTIVITY_MAX_MESSAGE_SIZE);
let response = client
.scanner_activity(Request::new(ScannerActivityRequest {}))
.await?
.into_inner();
decode_scanner_activity(response)
let mut request = Request::new(ScannerActivityRequest {
challenge: challenge.as_bytes().to_vec().into(),
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
acknowledge_instance_id,
acknowledge_dirty_usage_generation,
});
let canonical = rustfs_protos::canonical_scanner_activity_request_body(request.get_ref())
.map_err(|_| Error::other("scanner activity request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.scanner_activity(request).await?.into_inner();
decode_scanner_activity(response, challenge.as_bytes())
}
.await,
)
.await
}
pub async fn scanner_activity(&self) -> Result<ScannerPeerActivity> {
self.scanner_activity_request(String::new(), 0).await
}
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
self.scanner_activity_request(instance_id, generation).await
}
pub async fn get_metacache_listing(&self) -> Result<()> {
warn!("get_metacache_listing is not implemented in PeerRestClient");
Err(Error::NotImplemented)
@@ -1542,6 +1655,7 @@ impl PeerRestClient {
#[cfg(test)]
mod tests {
use super::*;
use crate::config::com::STORAGE_CLASS_SUB_SYS;
use serde_json::Value;
use std::io::{self, Write};
use std::sync::{Arc, Mutex};
@@ -1650,6 +1764,14 @@ mod tests {
)
}
fn decode_test_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerPeerActivity> {
decode_scanner_activity_with_verifier(response, &[9; 16], |_canonical, proof| {
(proof == b"proof")
.then_some(())
.ok_or_else(|| Error::other("peer returned an invalid scanner activity response proof"))
})
}
#[test]
fn build_clients_from_slots_preserves_missing_remote_topology_slots() {
let slots = vec![
@@ -1682,13 +1804,89 @@ mod tests {
#[test]
fn scanner_activity_requires_restart_safe_peer_identity() {
let legacy = decode_test_scanner_activity(ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
topology_digest: Vec::new().into(),
data_movement_active: false,
response_proof: Vec::new().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
})
.expect("legacy peers should retain their activity generations during a rolling upgrade");
assert_eq!(
legacy,
ScannerPeerActivity {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
topology_digest: None,
data_movement_active: None,
dirty_usage_generation: None,
dirty_usage_pending: None,
}
);
let previous = decode_test_scanner_activity(ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: true,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
})
.expect("protocol v4 peers should remain observable during a rolling upgrade");
assert_eq!(
previous,
ScannerPeerActivity {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
topology_digest: Some([7; 32]),
data_movement_active: Some(true),
dirty_usage_generation: None,
dirty_usage_pending: None,
}
);
let malformed_topology = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 31].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_test_scanner_activity(malformed_topology)
.expect_err("activity topology digests must have the protocol-defined length")
.to_string()
.contains("topology digest")
);
let missing_instance = ScannerActivityResponse {
instance_id: String::new(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_scanner_activity(missing_instance)
decode_test_scanner_activity(missing_instance)
.expect_err("an empty instance ID is not restart safe")
.to_string()
.contains("instance ID")
@@ -1698,18 +1896,30 @@ mod tests {
instance_id: "ABCDEF0123456789ABCDEF0123456789".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_scanner_activity(malformed_instance)
decode_test_scanner_activity(malformed_instance)
.expect_err("activity instance IDs must use the canonical lowercase hex form")
.to_string()
.contains("instance ID")
);
let activity = decode_scanner_activity(ScannerActivityResponse {
let activity = decode_test_scanner_activity(ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: true,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
})
.expect("complete activity responses should be accepted");
assert_eq!(
@@ -1718,8 +1928,123 @@ mod tests {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: Some([7; 32]),
data_movement_active: Some(true),
dirty_usage_generation: Some(11),
dirty_usage_pending: Some(true),
}
);
let pending_without_generation = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: true,
};
assert!(
decode_test_scanner_activity(pending_without_generation)
.expect_err("pending dirty usage must carry a nonzero generation")
.to_string()
.contains("without a generation")
);
let previous_with_dirty_usage = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_test_scanner_activity(previous_with_dirty_usage)
.expect_err("protocol v4 responses must not claim unauthenticated dirty usage fields")
.to_string()
.contains("unauthenticated v5 fields")
);
let legacy_with_topology = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
};
assert!(
decode_test_scanner_activity(legacy_with_topology)
.expect_err("legacy protocol responses must not claim extended fields")
.to_string()
.contains("unexpected extended fields")
);
let unsupported_protocol = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION + 1,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: b"proof".to_vec().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_test_scanner_activity(unsupported_protocol)
.expect_err("unknown activity protocols must fail closed")
.to_string()
.contains("unsupported scanner activity protocol")
);
let missing_proof = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: vec![7; 32].into(),
data_movement_active: false,
response_proof: Vec::new().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
assert!(
decode_test_scanner_activity(missing_proof)
.expect_err("unsigned scanner activity responses must fail closed")
.to_string()
.contains("response proof")
);
}
#[test]
fn dynamic_scanner_config_requires_versioned_peer_acknowledgement() {
for sub_system in [SCANNER_SUB_SYS, HEAL_SUB_SYS] {
let err = validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, sub_system, 0)
.expect_err("an unversioned peer must not claim scanner config convergence");
assert!(err.to_string().contains("does not support dynamic"));
validate_signal_service_protocol(
SERVICE_SIGNAL_RELOAD_DYNAMIC,
sub_system,
rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
)
.expect("a current peer should support dynamic scanner config");
}
validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, 0)
.expect("unrelated dynamic config keeps its existing compatibility contract");
validate_signal_service_protocol(SERVICE_SIGNAL_REFRESH_CONFIG, SCANNER_SUB_SYS, 0)
.expect("full refresh compatibility is guarded by its scanner preflight");
}
#[test]
crates/ecstore/src/cluster/rpc/peer_s3_client.rs
+108 -6
View File
@@ -49,6 +49,20 @@ use tracing::{debug, info, warn};
type Client = Arc<Box<dyn PeerS3Client>>;
#[derive(Clone, Debug)]
pub struct ScannerBucketListing {
pub buckets: Vec<BucketInfo>,
pub set_buckets: Vec<ScannerSetBucketListing>,
pub topology_complete: bool,
}
#[derive(Clone, Debug)]
pub struct ScannerSetBucketListing {
pub pool_index: usize,
pub set_index: usize,
pub buckets: Vec<BucketInfo>,
}
fn pool_participant_errors(clients: &[Client], errors: &[Option<Error>], pool_idx: usize) -> Vec<Option<Error>> {
clients
.iter()
@@ -216,6 +230,10 @@ impl S3PeerSys {
Ok(())
}
pub async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
Ok(self.list_bucket_for_scanner(opts).await?.buckets)
}
pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result<ScannerBucketListing> {
let mut futures = Vec::with_capacity(self.clients.len());
for cli in self.clients.iter() {
futures.push(cli.list_bucket(opts));
@@ -239,9 +257,12 @@ impl S3PeerSys {
}
let mut result_map: HashMap<&String, BucketInfo> = HashMap::new();
let mut topology_complete = true;
for i in 0..self.pools_count {
let per_pool_errs = pool_participant_errors(&self.clients, &errors, i);
let quorum = pool_write_quorum(per_pool_errs.len());
topology_complete &=
!per_pool_errs.is_empty() && per_pool_errs.iter().all(|participant_error| participant_error.is_none());
if let Some(pool_err) = reduce_pool_write_quorum_errs(&per_pool_errs) {
tracing::error!("list_bucket per_pool_errs: {per_pool_errs:?}");
@@ -261,20 +282,17 @@ impl S3PeerSys {
}
for bucket in buckets.iter() {
if result_map.contains_key(&bucket.name) {
continue;
}
// incr bucket_map count create if not exists
let count = bucket_map.entry(&bucket.name).or_insert(0usize);
*count += 1;
if *count >= quorum {
result_map.insert(&bucket.name, bucket.clone());
result_map.entry(&bucket.name).or_insert_with(|| bucket.clone());
}
}
}
}
topology_complete &= bucket_map.values().all(|count| *count >= quorum);
// TODO: MRF
}
@@ -282,7 +300,11 @@ impl S3PeerSys {
buckets.sort_by_key(|b| b.name.clone());
Ok(buckets)
Ok(ScannerBucketListing {
buckets,
set_buckets: Vec::new(),
topology_complete,
})
}
pub async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
let mut futures = Vec::with_capacity(self.clients.len());
@@ -1645,6 +1667,86 @@ mod tests {
assert_eq!(buckets[0].name, bucket.name);
}
#[tokio::test]
async fn scanner_bucket_listing_marks_quorum_result_incomplete_when_a_peer_is_missing() {
let bucket = BucketInfo {
name: "bucket-hidden-by-quorum".to_string(),
..Default::default()
};
let peer_sys = S3PeerSys {
clients: vec![
test_peer_with_list_bucket(&[0], Ok(vec![bucket])),
test_peer_with_list_bucket(&[0], Ok(Vec::new())),
test_peer_with_list_bucket(&[0], Ok(Vec::new())),
test_peer_with_list_bucket(&[0], Err(Error::DiskAccessDenied)),
],
pools_count: 1,
};
let listing = peer_sys
.list_bucket_for_scanner(&BucketOptions::default())
.await
.expect("peer quorum should still produce a scanner candidate listing");
assert!(listing.buckets.is_empty());
assert!(!listing.topology_complete);
}
#[tokio::test]
async fn scanner_bucket_listing_marks_divergent_successful_peers_incomplete() {
let bucket = BucketInfo {
name: "bucket-below-quorum".to_string(),
..Default::default()
};
let peer_sys = S3PeerSys {
clients: vec![
test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[0], Ok(vec![bucket])),
test_peer_with_list_bucket(&[0], Ok(Vec::new())),
test_peer_with_list_bucket(&[0], Ok(Vec::new())),
],
pools_count: 1,
};
let listing = peer_sys
.list_bucket_for_scanner(&BucketOptions::default())
.await
.expect("successful peer responses should still produce a scanner candidate listing");
assert!(listing.buckets.is_empty());
assert!(!listing.topology_complete);
}
#[tokio::test]
async fn scanner_bucket_listing_checks_same_bucket_in_every_pool() {
let bucket = BucketInfo {
name: "shared-bucket".to_string(),
..Default::default()
};
let peer_sys = S3PeerSys {
clients: vec![
test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[0], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[1], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[1], Ok(vec![bucket.clone()])),
test_peer_with_list_bucket(&[1], Ok(Vec::new())),
test_peer_with_list_bucket(&[1], Ok(Vec::new())),
],
pools_count: 2,
};
let listing = peer_sys
.list_bucket_for_scanner(&BucketOptions::default())
.await
.expect("a bucket visible in one pool should remain a scan candidate");
assert_eq!(listing.buckets.len(), 1);
assert_eq!(listing.buckets[0].name, bucket.name);
assert!(!listing.topology_complete);
}
#[tokio::test]
async fn test_delete_bucket_fails_when_any_pool_misses_write_quorum() {
let peer_sys = S3PeerSys {
crates/ecstore/src/cluster/rpc/remote_disk.rs
+279 -6
View File
@@ -17,7 +17,8 @@ use crate::cluster::rpc::client::{
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
};
use crate::cluster::rpc::internode_data_transport::{
InternodeDataTransport, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest,
InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
WriteStreamRequest,
};
use crate::disk::error::{Error, Result};
use crate::disk::{
@@ -81,6 +82,7 @@ const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2;
const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20);
const REMOTE_DISK_OPEN_READ_MAX_ATTEMPTS: usize = 2;
const REMOTE_DISK_OPEN_READ_RETRY_BACKOFF: Duration = Duration::from_millis(20);
const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
/// Base backoff for idempotent read-only RPC retries (grpc-optimization P3-3); doubles per attempt.
const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
@@ -97,6 +99,17 @@ const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
volume_infos
.into_iter()
.enumerate()
.map(|(index, json)| {
serde_json::from_str::<VolumeInfo>(&json)
.map_err(|err| Error::other(format!("decode list volumes entry {index} failed: {err}")))
})
.collect()
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum BatchMetadataRpcMode {
Off,
@@ -264,6 +277,71 @@ fn spawn_control_channel_prewarm(addr: String) {
}
impl RemoteDisk {
pub(crate) async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let probe = self.data_transport.probe_ns_scanner(NsScannerCapabilityRequest {
endpoint: self.endpoint.grid_host(),
});
let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
.await
.map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?;
match result {
Ok(server_epoch) => Ok(Some(server_epoch)),
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
Err(DiskError::MethodNotAllowed) => Ok(None),
Err(err)
if matches!(
err.internode_http_error_kind(),
Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status))
if matches!(status.as_u16(), 404 | 405)
) =>
{
Ok(None)
}
Err(err)
if matches!(
err.internode_http_error_kind(),
Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status)) if status.as_u16() == 426
) =>
{
Ok(None)
}
Err(err) => Err(err),
}
}
pub(crate) async fn open_ns_scanner_stream(&self, request: crate::disk::NsScannerOpenRequest) -> Result<FileReader> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let crate::disk::NsScannerOpenRequest {
request_id,
server_epoch,
session_id,
session_sequence,
next_cycle,
leader_epoch,
body,
stall_timeout,
} = request;
self.data_transport
.open_ns_scanner(NsScannerStreamRequest {
endpoint: self.endpoint.grid_host(),
disk: self.disk_ref().await,
request_id,
server_epoch,
session_id,
session_sequence,
next_cycle,
leader_epoch,
body,
stall_timeout,
})
.await
}
fn recovery_monitor_span(addr: &str, endpoint: &Endpoint) -> tracing::Span {
tracing::info_span!(
"recovery-monitor",
@@ -1335,11 +1413,7 @@ impl DiskAPI for RemoteDisk {
return Err(response.error.unwrap_or_default().into());
}
let infos = response
.volume_infos
.into_iter()
.filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
.collect();
let infos = decode_volume_infos(response.volume_infos)?;
Ok(infos)
},
@@ -2722,6 +2796,19 @@ mod tests {
static INIT: Once = Once::new();
#[test]
fn list_volumes_decode_rejects_a_malformed_entry() {
let valid = serde_json::to_string(&VolumeInfo {
name: "bucket".to_string(),
created: None,
})
.expect("volume info should serialize");
let err = decode_volume_infos(vec![valid, "{".to_string()])
.expect_err("a malformed volume entry must fail the complete response");
assert!(err.to_string().contains("entry 1"));
}
#[test]
fn decoded_remote_metadata_rejects_default_like_delete_marker() {
let forged = FileInfo {
@@ -2794,14 +2881,31 @@ mod tests {
Read(ReadStreamRequest),
Write(WriteStreamRequest),
WalkDir(WalkDirStreamRequest),
NsScanner(NsScannerStreamRequest),
NsScannerProbe(NsScannerCapabilityRequest),
}
#[derive(Debug, Clone, Default)]
struct RecordingInternodeDataTransport {
calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
ns_scanner_probe_status: Arc<StdMutex<Option<u16>>>,
}
impl RecordingInternodeDataTransport {
fn with_ns_scanner_probe_status(status: u16) -> Self {
Self {
calls: Arc::default(),
ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))),
}
}
fn set_ns_scanner_probe_status(&self, status: Option<u16>) {
*self
.ns_scanner_probe_status
.lock()
.expect("namespace scanner probe status lock poisoned") = status;
}
fn calls(&self) -> Vec<RecordedTransportCall> {
self.calls.lock().expect("recorded transport calls lock poisoned").clone()
}
@@ -3369,6 +3473,26 @@ mod tests {
Ok(Box::new(EmptyTestReader))
}
async fn open_ns_scanner(&self, request: NsScannerStreamRequest) -> Result<FileReader> {
self.record(RecordedTransportCall::NsScanner(request));
Ok(Box::new(EmptyTestReader))
}
async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
self.record(RecordedTransportCall::NsScannerProbe(request));
if let Some(status) = *self
.ns_scanner_probe_status
.lock()
.expect("namespace scanner probe status lock poisoned")
{
let status = reqwest::StatusCode::from_u16(status).expect("test status code should be valid");
return Err(
rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status)).into(),
);
}
Ok(Uuid::from_u128(1))
}
fn name(&self) -> &'static str {
"recording"
}
@@ -3401,6 +3525,14 @@ mod tests {
}
}
async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result<FileReader> {
panic!("open_ns_scanner should not be used in walk_dir retry test");
}
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
Ok(Uuid::from_u128(1))
}
fn name(&self) -> &'static str {
"retrying-walk-dir"
}
@@ -3429,6 +3561,14 @@ mod tests {
panic!("open_walk_dir should not be used in open_write retry test");
}
async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result<FileReader> {
panic!("open_ns_scanner should not be used in open_write retry test");
}
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
Ok(Uuid::from_u128(1))
}
fn name(&self) -> &'static str {
"retrying-open-write"
}
@@ -4092,6 +4232,139 @@ mod tests {
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_uses_configured_data_transport() {
let transport = RecordingInternodeDataTransport::default();
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let expected_disk = remote_disk.disk_ref().await;
let expected_body = b"namespace-scanner-request".to_vec();
let expected_request_id = Uuid::new_v4();
let expected_server_epoch = Uuid::new_v4();
let expected_session_id = Uuid::new_v4();
let _reader = remote_disk
.open_ns_scanner_stream(crate::disk::NsScannerOpenRequest {
request_id: expected_request_id,
server_epoch: expected_server_epoch,
session_id: expected_session_id,
session_sequence: 3,
next_cycle: 7,
leader_epoch: 9,
body: expected_body.clone(),
stall_timeout: Some(Duration::from_secs(15)),
})
.await
.expect("namespace scanner stream should open");
let calls = transport.calls();
assert_eq!(calls.len(), 1);
match &calls[0] {
RecordedTransportCall::NsScanner(request) => {
assert_eq!(request.endpoint, "http://remote-node:9000");
assert_eq!(request.disk, expected_disk);
assert_eq!(request.request_id, expected_request_id);
assert_eq!(request.server_epoch, expected_server_epoch);
assert_eq!(request.session_id, expected_session_id);
assert_eq!(request.session_sequence, 3);
assert_eq!(request.next_cycle, 7);
assert_eq!(request.leader_epoch, 9);
assert_eq!(request.body, expected_body);
assert_eq!(request.stall_timeout, Some(Duration::from_secs(15)));
}
other => panic!("expected namespace scanner transport call, got {other:?}"),
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_uses_configured_data_transport() {
let transport = RecordingInternodeDataTransport::default();
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("namespace scanner capability probe should succeed"),
Some(Uuid::from_u128(1))
);
let calls = transport.calls();
assert_eq!(calls.len(), 1);
match &calls[0] {
RecordedTransportCall::NsScannerProbe(request) => {
assert_eq!(request.endpoint, "http://remote-node:9000");
}
other => panic!("expected namespace scanner capability probe, got {other:?}"),
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_rejects_old_and_incompatible_peers() {
for status in [404, 405, 426] {
let transport = RecordingInternodeDataTransport::with_ns_scanner_probe_status(status);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("unsupported namespace scanner response should be classified"),
None
);
}
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_rejects_legacy_transport() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RetryingOpenReadInternodeDataTransport::default())).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("legacy transport should be classified as unsupported"),
None
);
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_reprobes_after_peer_upgrade() {
let transport = RecordingInternodeDataTransport::with_ns_scanner_probe_status(404);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("old peer should be classified as unsupported"),
None
);
transport.set_ns_scanner_probe_status(None);
assert_eq!(
remote_disk
.ns_scanner_server_epoch()
.await
.expect("upgraded peer should be re-probed"),
Some(Uuid::from_u128(1))
);
assert_eq!(transport.calls().len(), 2);
}
#[tokio::test]
async fn test_remote_disk_namespace_scanner_capability_propagates_transient_failure() {
let transport = RecordingInternodeDataTransport::with_ns_scanner_probe_status(503);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await;
let err = remote_disk
.ns_scanner_server_epoch()
.await
.expect_err("transient capability failure must not be reported as unsupported");
assert!(matches!(
err.internode_http_error_kind(),
Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(status)) if status.as_u16() == 503
));
}
#[tokio::test]
async fn test_remote_disk_walk_dir_preserves_skip_total_timeout_option() {
let transport = RecordingInternodeDataTransport::default();
crates/ecstore/src/config/com.rs
+1108 -49
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/data_movement/mod.rs
+16 -11
View File
@@ -560,17 +560,22 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
set.delete_object(
bucket,
cleanup_key.as_str(),
ObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
no_lock: true,
..Default::default()
},
)
.await
let result = set
.delete_object(
bucket,
cleanup_key.as_str(),
ObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
no_lock: true,
..Default::default()
},
)
.await;
if result.is_ok() {
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
result
}
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
crates/ecstore/src/data_usage/mod.rs
+1304 -62
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/disk/error.rs
+18
View File
@@ -231,6 +231,13 @@ impl DiskError {
}
}
pub fn is_internode_http_status(&self, status: u16) -> bool {
matches!(
self.internode_http_error_kind(),
Some(InternodeHttpErrorKind::HttpStatus(actual)) if actual.as_u16() == status
)
}
// /// If all errors are of the same fatal disk error type, returns the corresponding error.
// /// Otherwise, returns Ok.
// pub fn check_disk_fatal_errs(errs: &[Option<Error>]) -> Result<()> {
@@ -981,6 +988,17 @@ mod tests {
assert!(!disk_error.contains_io_error_kind(std::io::ErrorKind::TimedOut));
}
#[test]
fn test_internode_http_status_classification() {
let too_many_requests = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::TOO_MANY_REQUESTS),
));
assert!(too_many_requests.is_internode_http_status(429));
assert!(!too_many_requests.is_internode_http_status(500));
assert!(!DiskError::FileNotFound.is_internode_http_status(429));
}
#[test]
fn test_metacache_output_stream_closed_classification_survives_clone() {
let disk_error = DiskError::metacache_output_stream_closed();
crates/ecstore/src/disk/mod.rs
+27 -1
View File
@@ -52,7 +52,7 @@ use local::LocalDisk;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_madmin::info_commands::DiskMetrics;
use serde::{Deserialize, Serialize};
use std::{fmt::Debug, path::PathBuf, sync::Arc};
use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncWrite};
use uuid::Uuid;
@@ -453,6 +453,20 @@ impl DiskAPI for Disk {
}
impl Disk {
pub async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
match self {
Disk::Local(_) => Ok(None),
Disk::Remote(remote_disk) => remote_disk.ns_scanner_server_epoch().await,
}
}
pub async fn open_ns_scanner_stream(&self, request: NsScannerOpenRequest) -> Result<FileReader> {
match self {
Disk::Remote(remote_disk) => remote_disk.open_ns_scanner_stream(request).await,
Disk::Local(_) => Err(Error::other("namespace scanner stream requires a remote disk")),
}
}
pub fn runtime_state(&self) -> RuntimeDriveHealthState {
match self {
Disk::Local(local_disk) => local_disk.runtime_state(),
@@ -498,6 +512,18 @@ impl Disk {
}
}
#[derive(Debug)]
pub struct NsScannerOpenRequest {
pub request_id: Uuid,
pub server_epoch: Uuid,
pub session_id: Uuid,
pub session_sequence: u64,
pub next_cycle: u64,
pub leader_epoch: u64,
pub body: Vec<u8>,
pub stall_timeout: Option<Duration>,
}
impl Disk {
/// Reset drive health so `connect_load_init_formats` retries are not blocked by a prior
/// transient mark-faulty (same disk handles are reused across retries).
crates/ecstore/src/object_api/readers.rs
+8
View File
@@ -471,6 +471,14 @@ impl PutObjReader {
}
}
pub fn from_prehashed_bytes(data: Bytes, sha256hex: Option<String>) -> std::io::Result<Self> {
let content_length =
i64::try_from(data.len()).map_err(|_| std::io::Error::other("prehashed object payload exceeds i64 length"))?;
Ok(PutObjReader {
stream: HashReader::from_stream(Cursor::new(data), content_length, content_length, None, sha256hex, false)?,
})
}
pub fn size(&self) -> i64 {
self.stream.size()
}
crates/ecstore/src/services/metrics_realtime.rs
+3
View File
@@ -154,6 +154,7 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
failed_cycles: metrics.failed_cycles,
superseded_cycles: metrics.superseded_cycles,
partial_cycles_unknown: metrics.partial_cycles_unknown,
partial_cycles_runtime: metrics.partial_cycles_runtime,
partial_cycles_objects: metrics.partial_cycles_objects,
@@ -796,6 +797,7 @@ mod test {
last_cycle_replication_checks: 27,
last_cycle_usage_saves: 28,
failed_cycles: 29,
superseded_cycles: 30,
partial_cycles_unknown: 30,
partial_cycles_runtime: 31,
partial_cycles_objects: 32,
@@ -927,6 +929,7 @@ mod test {
assert_eq!(scanner.last_cycle_replication_checks, 27);
assert_eq!(scanner.last_cycle_usage_saves, 28);
assert_eq!(scanner.failed_cycles, 29);
assert_eq!(scanner.superseded_cycles, 30);
assert_eq!(scanner.partial_cycles_unknown, 30);
assert_eq!(scanner.partial_cycles_runtime, 31);
assert_eq!(scanner.partial_cycles_objects, 32);
crates/ecstore/src/services/notification_sys.rs
+124 -1
View File
@@ -29,7 +29,7 @@ use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_madmin::net::NetInfo;
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
use rustfs_utils::XHost;
use std::collections::hash_map::DefaultHasher;
use std::collections::{HashMap, hash_map::DefaultHasher};
use std::future::Future;
use std::hash::{Hash, Hasher};
use std::sync::{Arc, Mutex, OnceLock};
@@ -1041,6 +1041,40 @@ impl NotificationSys {
Ok(generations)
}
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<(String, String, u64)>) -> Result<bool> {
let mut by_host = HashMap::with_capacity(acknowledgements.len());
for (host, instance_id, generation) in acknowledgements {
if by_host.insert(host.clone(), (instance_id, generation)).is_some() {
return Err(Error::other(format!("duplicate scanner dirty usage acknowledgement target: {host}")));
}
}
let clients = self
.peer_clients
.iter()
.flatten()
.map(|client| (client.grid_host.clone(), client.clone()))
.collect::<HashMap<_, _>>();
let mut failures = Vec::new();
let mut futures = Vec::with_capacity(by_host.len());
for (host, (instance_id, generation)) in by_host {
let Some(client) = clients.get(&host).cloned() else {
failures.push(format!("peer {host} scanner dirty usage acknowledgement failed: peer is not reachable"));
continue;
};
futures.push(async move {
let result = scanner_activity_with_timeout(
SCANNER_ACTIVITY_PROBE_TIMEOUT,
&host,
client.acknowledge_scanner_dirty_usage(instance_id, generation),
)
.await;
(host, result)
});
}
aggregate_scanner_dirty_usage_acknowledgement_results(join_all(futures).await, failures)
}
pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
let mut futures = Vec::with_capacity(self.peer_clients.len());
for client in self.peer_clients.iter() {
@@ -1591,6 +1625,23 @@ fn aggregate_notification_failures(operation: &str, failures: Vec<String>) -> Re
)))
}
fn aggregate_scanner_dirty_usage_acknowledgement_results(
results: Vec<(String, Result<ScannerPeerActivity>)>,
mut failures: Vec<String>,
) -> Result<bool> {
let mut dirty_usage_pending = false;
for (host, result) in results {
match result {
Ok(activity) => {
dirty_usage_pending |= activity.dirty_usage_pending != Some(false);
}
Err(err) => failures.push(format!("peer {host} scanner dirty usage acknowledgement failed: {err}")),
}
}
aggregate_notification_failures("acknowledge_scanner_dirty_usage", failures)?;
Ok(dirty_usage_pending)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1842,6 +1893,78 @@ mod tests {
assert!(err.to_string().contains("peer-1"));
}
#[tokio::test]
async fn scanner_dirty_usage_acknowledgement_rejects_missing_and_duplicate_targets() {
let sys = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: Vec::new(),
peer_admin_caches: Vec::new(),
peer_topology_hosts: Vec::new(),
};
let missing = sys
.acknowledge_scanner_dirty_usage(vec![("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7)])
.await
.expect_err("a missing acknowledgement target must remain pending");
assert!(missing.to_string().contains("peer is not reachable"));
let duplicate = sys
.acknowledge_scanner_dirty_usage(vec![
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
])
.await
.expect_err("duplicate acknowledgement targets must be rejected");
assert!(
duplicate
.to_string()
.contains("duplicate scanner dirty usage acknowledgement target")
);
}
#[test]
fn scanner_dirty_usage_acknowledgement_preserves_newer_pending_work() {
let activity = |dirty_usage_pending| ScannerPeerActivity {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 1,
maintenance_generation: 1,
protocol_version: crate::storage_api_contracts::internode::SCANNER_ACTIVITY_PROTOCOL_VERSION,
topology_digest: Some([0; 32]),
data_movement_active: Some(false),
dirty_usage_generation: Some(2),
dirty_usage_pending,
};
let pending = aggregate_scanner_dirty_usage_acknowledgement_results(
vec![
("peer-1".to_string(), Ok(activity(Some(false)))),
("peer-2".to_string(), Ok(activity(Some(true)))),
],
Vec::new(),
)
.expect("successful acknowledgements should return their pending state");
assert!(pending, "new dirty usage reported by an acknowledged peer must remain pending");
let cleared = aggregate_scanner_dirty_usage_acknowledgement_results(
vec![("peer-1".to_string(), Ok(activity(Some(false))))],
Vec::new(),
)
.expect("a cleared acknowledgement should succeed");
assert!(!cleared, "an explicitly cleared peer must not remain pending");
let unknown =
aggregate_scanner_dirty_usage_acknowledgement_results(vec![("peer-1".to_string(), Ok(activity(None)))], Vec::new())
.expect("an acknowledgement without a pending field should remain retryable");
assert!(unknown, "a peer that cannot prove its dirty state is clear must remain pending");
let err = aggregate_scanner_dirty_usage_acknowledgement_results(
vec![("peer-1".to_string(), Err(Error::other("injected acknowledgement failure")))],
Vec::new(),
)
.expect_err("a reachable peer acknowledgement failure must be reported");
assert!(err.to_string().contains("peer-1"));
assert!(err.to_string().contains("injected acknowledgement failure"));
}
#[tokio::test]
async fn load_bucket_metadata_reports_unreachable_peers() {
let sys = NotificationSys {
crates/ecstore/src/set_disk/core/io_primitives.rs
+55 -13
View File
@@ -3862,6 +3862,24 @@ impl SetDisks {
Ok(removed)
}
fn write_precondition_lookup_error(
error: StorageError,
http_preconditions: &HTTPPreconditions,
bucket: &str,
object: &str,
) -> Option<StorageError> {
match error {
StorageError::VersionNotFound(_, _, _) | StorageError::ObjectNotFound(_, _) => {
if http_preconditions.if_match_value().is_some() {
Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()))
} else {
None
}
}
error => Some(error),
}
}
pub(in crate::set_disk) async fn check_write_precondition(
&self,
bucket: &str,
@@ -3892,19 +3910,8 @@ impl SetDisks {
}
}
Err(StorageError::VersionNotFound(_, _, _))
| Err(StorageError::ObjectNotFound(_, _))
| Err(StorageError::ErasureReadQuorum) => {
// When the object is not found,
// - if If-Match is set, we should return 404 NotFound
// - if If-None-Match is set, we should be able to proceed with the request
if http_preconditions.if_match_value().is_some() {
return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()));
}
}
Err(e) => {
return Some(e);
Err(error) => {
return Self::write_precondition_lookup_error(error, &http_preconditions, bucket, object);
}
}
@@ -4407,6 +4414,41 @@ mod tests {
use tempfile::TempDir;
use tokio::io::AsyncReadExt;
#[test]
fn write_precondition_lookup_errors_fail_closed_unless_absence_is_known() {
let create_only = HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
};
let replace_only = HTTPPreconditions {
if_match: Some("etag".to_string()),
..Default::default()
};
assert!(matches!(
SetDisks::write_precondition_lookup_error(StorageError::ErasureReadQuorum, &create_only, "bucket", "object",),
Some(StorageError::ErasureReadQuorum)
));
assert!(
SetDisks::write_precondition_lookup_error(
StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
&create_only,
"bucket",
"object",
)
.is_none()
);
assert!(matches!(
SetDisks::write_precondition_lookup_error(
StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
&replace_only,
"bucket",
"object",
),
Some(StorageError::ObjectNotFound(_, _))
));
}
fn metadata_test_fileinfo(object: &str) -> FileInfo {
let mut fi = FileInfo::new(object, 2, 2);
fi.volume = "bucket".to_string();
crates/ecstore/src/set_disk/mod.rs
+42
View File
@@ -9275,6 +9275,48 @@ mod tests {
));
}
#[tokio::test]
async fn set_level_if_none_match_fails_closed_without_read_quorum() {
let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
let bucket = "bucket-write-precondition-quorum";
let object = "existing-object.txt";
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before disk loss");
let mut reader = PutObjReader::from_vec(b"existing object body".to_vec());
set_disks
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("object should be written before disk loss");
{
let mut disks = set_disks.disks.write().await;
disks[1..].fill(None);
}
let create_only = ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
}),
..Default::default()
};
let result = set_disks.check_write_precondition(bucket, object, &create_only).await;
assert!(
matches!(result, Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))),
"expected read-quorum failure, got {result:?}"
);
}
#[tokio::test]
async fn set_level_versioned_delete_marker_hides_object_without_corrupting_version_metadata() {
let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
crates/ecstore/src/set_disk/ops/bucket.rs
+67 -59
View File
@@ -21,6 +21,71 @@
use super::super::*;
impl SetDisks {
pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec<BucketInfo>, bool)> {
let disks = self.disk_inventory().await;
let write_quorum = (disks.len() / 2) + 1;
let mut futures = Vec::with_capacity(disks.len());
for disk in disks {
futures.push(async move {
match disk {
Some(disk) => disk.list_volumes().await,
None => Err(DiskError::DiskNotFound),
}
});
}
let results = join_all(futures).await;
let mut topology_complete = results.iter().all(|result| result.is_ok());
let mut infos = Vec::with_capacity(results.len());
let mut errs = Vec::with_capacity(results.len());
for result in results {
match result {
Ok(volumes) => {
infos.push(Some(volumes));
errs.push(None);
}
Err(err) => {
infos.push(None);
errs.push(Some(err));
}
}
}
if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
return Err(err.into());
}
let mut counts: HashMap<String, (usize, BucketInfo)> = HashMap::new();
for volumes in infos.into_iter().flatten() {
for volume in volumes {
if is_reserved_or_invalid_bucket(&volume.name, false) {
continue;
}
let entry = counts.entry(volume.name.clone()).or_insert((
0,
BucketInfo {
name: volume.name.clone(),
created: volume.created,
..Default::default()
},
));
entry.0 += 1;
}
}
topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum);
let mut buckets = counts
.into_values()
.filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket))
.collect::<Vec<_>>();
buckets.sort_by(|left, right| left.name.cmp(&right.name));
Ok((buckets, topology_complete))
}
}
#[async_trait::async_trait]
impl BucketOperations for SetDisks {
type Error = Error;
@@ -117,65 +182,8 @@ impl BucketOperations for SetDisks {
}
#[tracing::instrument(skip(self))]
async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
let disks = self.disk_inventory().await;
let write_quorum = (disks.len() / 2) + 1;
let mut futures = Vec::with_capacity(disks.len());
for disk in disks {
futures.push(async move {
match disk {
Some(disk) => disk.list_volumes().await,
None => Err(DiskError::DiskNotFound),
}
});
}
let results = join_all(futures).await;
let mut infos = Vec::with_capacity(results.len());
let mut errs = Vec::with_capacity(results.len());
for result in results {
match result {
Ok(volumes) => {
infos.push(Some(volumes));
errs.push(None);
}
Err(err) => {
infos.push(None);
errs.push(Some(err));
}
}
}
if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
return Err(err.into());
}
let mut counts: HashMap<String, (usize, BucketInfo)> = HashMap::new();
for volumes in infos.into_iter().flatten() {
for volume in volumes {
if is_reserved_or_invalid_bucket(&volume.name, false) {
continue;
}
let entry = counts.entry(volume.name.clone()).or_insert((
0,
BucketInfo {
name: volume.name.clone(),
created: volume.created,
..Default::default()
},
));
entry.0 += 1;
}
}
let mut buckets = counts
.into_values()
.filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket))
.collect::<Vec<_>>();
buckets.sort_by(|left, right| left.name.cmp(&right.name));
Ok(buckets)
async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
Ok(self.list_bucket_for_scanner(opts).await?.0)
}
#[tracing::instrument(skip(self))]
crates/ecstore/src/storage_api_contracts/mod.rs
+6 -1
View File
@@ -23,7 +23,12 @@ pub(crate) mod heal {
pub(crate) mod internode {
pub(crate) use rustfs_storage_api::{
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY,
NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY,
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
};
}
crates/ecstore/src/store/bucket.rs
+746 -30
View File
@@ -17,12 +17,21 @@ use crate::bucket::{
metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix},
utils::is_meta_bucketname,
};
use crate::error::is_err_bucket_not_found;
use crate::runtime::sources as runtime_sources;
use crate::set_disk::get_lock_acquire_timeout;
use crate::storage_api_contracts::bucket::SRBucketDeleteOp;
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use futures::stream::{self, StreamExt};
use std::collections::BTreeMap;
use std::future::Future;
const DELETED_BUCKETS_PREFIX: &str = ".deleted";
const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
fn scanner_bucket_list_set_concurrency(set_count: usize) -> usize {
set_count.clamp(1, SCANNER_BUCKET_LIST_SET_CONCURRENCY)
}
fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
created != OffsetDateTime::UNIX_EPOCH
@@ -78,6 +87,49 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
}
async fn await_bucket_namespace_operation<T, F>(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
operation: &'static str,
future: F,
) -> Result<T>
where
F: Future<Output = Result<T>>,
{
let Some(guard) = guard else {
return future.await;
};
if guard.is_lock_lost() {
return Err(StorageError::other(format!(
"bucket namespace lock was lost before {operation}: {bucket}"
)));
}
tokio::select! {
biased;
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
"bucket namespace lock was lost during {operation}: {bucket}"
))),
result = future => result,
}
}
async fn run_bucket_usage_cleanup<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
where
F: Future<Output = Result<()>>,
{
await_bucket_namespace_operation(guard, bucket, "bucket usage cleanup", future).await
}
async fn run_physical_bucket_deletion<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
where
F: Future<Output = Result<()>>,
{
// Fence before polling deletion: the physical namespace may become
// invisible at any await point inside the storage operation.
list_objects::observe_scanner_namespace_mutations(bucket, 1);
await_bucket_namespace_operation(guard, bucket, "physical bucket deletion", future).await
}
impl ECStore {
async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> {
let marker_volume = bucket_deleted_marker_volume(bucket);
@@ -96,21 +148,84 @@ impl ECStore {
Ok(())
}
async fn cleanup_deleted_bucket_metadata(&self, bucket: &str, include_deleted_marker: bool) -> Result<()> {
async fn cleanup_deleted_bucket_metadata(
&self,
bucket: &str,
include_deleted_marker: bool,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<()> {
for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()).await?;
await_bucket_namespace_operation(
guard,
bucket,
"deleted bucket metadata cleanup",
self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()),
)
.await?;
}
if include_deleted_marker {
let marker_prefix = bucket_deleted_marker_prefix(bucket);
self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()).await?;
await_bucket_namespace_operation(
guard,
bucket,
"deleted bucket marker cleanup",
self.delete_all(RUSTFS_META_BUCKET, marker_prefix.as_str()),
)
.await?;
}
metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket).await?;
await_bucket_namespace_operation(
guard,
bucket,
"deleted bucket metadata cache cleanup",
metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket),
)
.await?;
runtime_sources::delete_bucket_monitor_entry(bucket);
Ok(())
}
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
run_bucket_usage_cleanup(guard, bucket, async {
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await
})
.await
}
async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
if let Err(err) = self.cleanup_bucket_usage(bucket, guard).await {
warn!(
bucket = %bucket,
error = ?err,
"bucket data usage cleanup deferred to scanner reconciliation"
);
}
}
async fn rollback_failed_bucket_creation(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
let rollback_opts = DeleteBucketOptions {
no_lock: true,
no_recreate: true,
..Default::default()
};
if let Err(err) = await_bucket_namespace_operation(guard, bucket, "failed bucket creation rollback", async {
self.peer_sys
.delete_bucket(bucket, &rollback_opts)
.await
.map_err(|rollback_err| to_object_err(rollback_err.into(), vec![bucket]))
})
.await
{
warn!(
bucket = %bucket,
error = ?err,
"failed bucket creation rollback did not remove every physical bucket volume"
);
}
}
#[instrument(skip(self))]
pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
if !is_meta_bucketname(bucket)
@@ -119,7 +234,7 @@ impl ECStore {
return Err(StorageError::BucketNameInvalid(err.to_string()));
}
let _ns_guard = if !opts.no_lock {
let ns_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
Some(
ns_lock
@@ -142,8 +257,34 @@ impl ECStore {
None
};
if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await {
let err = to_object_err(err.into(), vec![bucket]);
let confirmed_missing = match self.peer_sys.get_bucket_info(bucket, &BucketOptions::default()).await {
Ok(_) => false,
Err(err) => {
let err: StorageError = err.into();
if is_err_bucket_not_found(&err) {
true
} else {
return Err(to_object_err(err, vec![bucket]));
}
}
};
if confirmed_missing && !is_meta_bucketname(bucket) {
// Fence every scanner cycle that could have observed the namespace
// before physical creation. Creation may become visible even when a
// later metadata write or namespace-lock check fails.
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?;
}
if let Err(err) = await_bucket_namespace_operation(ns_guard.as_ref(), bucket, "physical bucket creation", async {
self.peer_sys
.make_bucket(bucket, opts)
.await
.map_err(|err| to_object_err(err.into(), vec![bucket]))
})
.await
{
if is_err_bucket_exists(&err)
&& let Err(heal_err) = self
.handle_heal_bucket(
@@ -157,18 +298,9 @@ impl ECStore {
{
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
}
if !is_err_bucket_exists(&err) {
if !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
error!("make bucket failed: {err}");
let _ = self
.delete_bucket(
bucket,
&DeleteBucketOptions {
no_lock: true,
no_recreate: true,
..Default::default()
},
)
.await;
self.rollback_failed_bucket_creation(bucket, ns_guard.as_ref()).await;
}
return Err(err);
};
@@ -186,7 +318,13 @@ impl ECStore {
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
}
metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await?;
await_bucket_namespace_operation(
ns_guard.as_ref(),
bucket,
"bucket metadata initialization",
metadata_sys::set_bucket_metadata_in(&self.ctx, meta),
)
.await?;
Ok(())
}
@@ -224,6 +362,80 @@ impl ECStore {
Ok(buckets)
}
pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result<crate::cluster::rpc::ScannerBucketListing> {
let sets = self
.pools
.iter()
.flat_map(|pool| {
pool.disk_set
.iter()
.map(|set| (set.pool_index, set.set_index, Arc::clone(set)))
})
.collect::<Vec<_>>();
let set_count = sets.len();
let deleted = opts.deleted;
let cached = opts.cached;
let no_metadata = opts.no_metadata;
let mut set_listings = stream::iter(sets.into_iter().map(move |(pool_index, set_index, set)| {
let opts = BucketOptions {
deleted,
cached,
no_metadata,
};
async move {
set.list_bucket_for_scanner(&opts)
.await
.map(|(buckets, complete)| (pool_index, set_index, buckets, complete))
}
}))
.buffer_unordered(scanner_bucket_list_set_concurrency(set_count));
let mut topology_complete = set_count != 0;
let mut bucket_map = BTreeMap::<String, BucketInfo>::new();
let mut scoped_buckets = Vec::with_capacity(set_count);
while let Some(set_listing) = set_listings.next().await {
let (pool_index, set_index, buckets, set_complete) = set_listing?;
topology_complete &= set_complete;
for bucket in &buckets {
bucket_map.entry(bucket.name.clone()).or_insert_with(|| bucket.clone());
}
scoped_buckets.push(crate::cluster::rpc::ScannerSetBucketListing {
pool_index,
set_index,
buckets,
});
}
scoped_buckets.sort_unstable_by_key(|scope| (scope.pool_index, scope.set_index));
let mut listing = crate::cluster::rpc::ScannerBucketListing {
buckets: bucket_map.into_values().collect(),
set_buckets: scoped_buckets,
topology_complete,
};
if !opts.no_metadata {
for bucket in &mut listing.buckets {
if let Ok(created) = metadata_sys::created_at_in(&self.ctx, &bucket.name).await
&& should_override_created_from_metadata(created)
{
bucket.created = Some(created);
}
}
let created_by_bucket = listing
.buckets
.iter()
.map(|bucket| (bucket.name.as_str(), bucket.created))
.collect::<BTreeMap<_, _>>();
for scope in &mut listing.set_buckets {
for bucket in &mut scope.buckets {
if let Some(created) = created_by_bucket.get(bucket.name.as_str()) {
bucket.created = *created;
}
}
}
}
Ok(listing)
}
#[instrument(skip(self))]
pub(super) async fn handle_delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
if is_meta_bucketname(bucket) {
@@ -234,7 +446,7 @@ impl ECStore {
return Err(StorageError::BucketNameInvalid(err.to_string()));
}
let _ns_guard = if !opts.no_lock {
let ns_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, bucket).await?;
Some(
ns_lock
@@ -299,17 +511,40 @@ impl ECStore {
}
if sr_mark_delete {
self.mark_bucket_deleted(bucket).await?;
await_bucket_namespace_operation(
ns_guard.as_ref(),
bucket,
"bucket delete marker creation",
self.mark_bucket_deleted(bucket),
)
.await?;
}
if let Err(err) = self.peer_sys.delete_bucket(bucket, &delete_opts).await {
let storage_err = to_object_err(err.into(), vec![bucket]);
if !sr_delete || !is_err_strict_volume_not_found(&storage_err) {
return Err(storage_err);
}
let delete_result = run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
self.peer_sys
.delete_bucket(bucket, &delete_opts)
.await
.map_err(|err| to_object_err(err.into(), vec![bucket]))
})
.await;
if let Err(err) = delete_result
&& (!sr_delete || !is_err_strict_volume_not_found(&err))
{
return Err(err);
}
self.cleanup_deleted_bucket_metadata(bucket, sr_purge).await?;
self.cleanup_bucket_usage_best_effort(bucket, ns_guard.as_ref()).await;
if let Err(err) = self
.cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_ref())
.await
{
warn!(
bucket = %bucket,
error = ?err,
"physical bucket deletion succeeded but metadata cleanup remains pending"
);
}
Ok(())
}
}
@@ -317,8 +552,9 @@ impl ECStore {
#[cfg(test)]
mod tests {
use super::{
bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix, bucket_deleted_marker_volume,
should_override_created_from_metadata, validate_table_bucket_delete_allowed,
SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes,
bucket_deleted_marker_prefix, bucket_deleted_marker_volume, run_bucket_usage_cleanup, run_physical_bucket_deletion,
scanner_bucket_list_set_concurrency, should_override_created_from_metadata, validate_table_bucket_delete_allowed,
};
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
use crate::bucket::metadata_sys;
@@ -335,16 +571,146 @@ mod tests {
disk::endpoint::Endpoint,
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
};
use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
use serial_test::serial;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, SystemTime};
use time::OffsetDateTime;
use tokio::sync::OnceCell;
use tokio::sync::{Notify, OnceCell};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
#[tokio::test(start_paused = true)]
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
let ttl = Duration::from_millis(20);
let lock = NamespaceLock::new("bucket-operation-test".to_string(), Arc::new(LocalClient::new()));
let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
.with_acquire_timeout(Duration::from_secs(1))
.with_ttl(ttl)
.with_refresh_interval(ttl);
let guard = lock
.acquire_guard(&request)
.await
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired");
tokio::time::advance(ttl + Duration::from_millis(1)).await;
let operation_ran = Arc::new(AtomicBool::new(false));
let operation_ran_for_future = operation_ran.clone();
let result = await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
operation_ran_for_future.store(true, Ordering::SeqCst);
Ok(())
})
.await;
assert!(result.is_err(), "an expired namespace lease must fence the operation");
assert!(
!operation_ran.load(Ordering::SeqCst),
"a fenced namespace operation must not poll its mutation future"
);
}
#[tokio::test]
#[serial]
async fn physical_bucket_delete_fences_scanner_before_polling_storage() {
let generation_before = crate::store::list_objects::scanner_namespace_mutation_generation();
let storage_polled = Arc::new(AtomicBool::new(false));
let storage_polled_for_future = storage_polled.clone();
run_physical_bucket_deletion(None, "generation-order-bucket", async move {
assert!(
crate::store::list_objects::scanner_namespace_mutation_generation() > generation_before,
"scanner generation must advance before physical deletion is polled"
);
storage_polled_for_future.store(true, Ordering::SeqCst);
Ok(())
})
.await
.expect("synthetic physical deletion should succeed");
assert!(storage_polled.load(Ordering::SeqCst));
}
#[tokio::test]
async fn bucket_usage_cleanup_stops_after_parent_cancellation() {
let started = Arc::new(Notify::new());
let started_wait = started.notified();
let release = Arc::new(Notify::new());
let completed = Arc::new(AtomicBool::new(false));
let started_for_cleanup = started.clone();
let release_for_cleanup = release.clone();
let completed_for_cleanup = completed.clone();
let parent = tokio::spawn(run_bucket_usage_cleanup(None, "bucket", async move {
started_for_cleanup.notify_one();
release_for_cleanup.notified().await;
completed_for_cleanup.store(true, Ordering::SeqCst);
Ok(())
}));
started_wait.await;
parent.abort();
let _ = parent.await;
tokio::task::yield_now().await;
release.notify_waiters();
tokio::task::yield_now().await;
assert!(
!completed.load(Ordering::SeqCst),
"a cancelled cleanup future must not continue in a detached task"
);
}
#[tokio::test(start_paused = true)]
#[serial]
async fn bucket_namespace_operation_stops_in_flight_work_after_lock_loss() {
let ttl = Duration::from_millis(20);
let lock = NamespaceLock::new("bucket-operation-in-flight-loss-test".to_string(), Arc::new(LocalClient::new()));
let request = LockRequest::new(ObjectKey::new("bucket", ""), LockType::Exclusive, "test-owner")
.with_acquire_timeout(Duration::from_secs(1))
.with_ttl(ttl)
.with_refresh_interval(ttl);
let guard = lock
.acquire_guard(&request)
.await
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired");
let operation_started = Arc::new(Notify::new());
let started_wait = operation_started.notified();
let operation_release = Arc::new(Notify::new());
let operation_completed = Arc::new(AtomicBool::new(false));
let started_for_operation = operation_started.clone();
let release_for_operation = operation_release.clone();
let completed_for_operation = operation_completed.clone();
let task = tokio::spawn(async move {
await_bucket_namespace_operation(Some(&guard), "bucket", "test operation", async move {
started_for_operation.notify_one();
release_for_operation.notified().await;
completed_for_operation.store(true, Ordering::SeqCst);
Ok(())
})
.await
});
started_wait.await;
tokio::time::advance(ttl + Duration::from_millis(1)).await;
let err = task
.await
.expect("operation task should join")
.expect_err("an operation still running after lease loss must be fenced");
assert!(err.to_string().contains("namespace lock was lost during test operation"));
operation_release.notify_waiters();
tokio::task::yield_now().await;
assert!(
!operation_completed.load(Ordering::SeqCst),
"lock loss must stop the old owner before a successor can acquire the namespace"
);
}
async fn setup_bucket_delete_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
BUCKET_DELETE_TEST_ENV
.get_or_init(|| async {
@@ -413,6 +779,58 @@ mod tests {
.clone()
}
async fn setup_multi_pool_scanner_listing_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
let temp_dir = tempfile::tempdir().expect("multi-pool scanner test directory should be created");
let mut pools = Vec::new();
for pool_index in 0..2 {
let mut endpoints = Vec::new();
for disk_index in 0..4 {
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
tokio::fs::create_dir_all(&disk_path)
.await
.expect("multi-pool scanner test disk should be created");
let mut endpoint =
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
endpoints.push(endpoint);
}
pools.push(PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("scanner-listing-pool-{pool_index}"),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
});
}
let endpoint_pools = EndpointServerPools(pools);
let instance_ctx = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("multi-pool local disks should initialize");
let ecstore = ECStore::new_with_instance_ctx(
"127.0.0.1:0".parse().expect("test address"),
endpoint_pools,
CancellationToken::new(),
instance_ctx,
)
.await
.expect("multi-pool ECStore should initialize");
let storage_class =
crate::config::storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4, 4])
.expect("multi-pool storage class should match both four-disk pools");
for pool in &ecstore.pools {
for set in &pool.disk_set {
set.set_test_storage_class_config(storage_class.clone());
}
}
(temp_dir, ecstore)
}
async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
let generation_before_make = ecstore.scanner_namespace_mutation_generation();
ecstore
@@ -515,6 +933,98 @@ mod tests {
);
}
#[test]
fn scanner_bucket_listing_bounds_set_fanout() {
assert_eq!(scanner_bucket_list_set_concurrency(0), 1);
assert_eq!(scanner_bucket_list_set_concurrency(2), 2);
assert_eq!(scanner_bucket_list_set_concurrency(100), SCANNER_BUCKET_LIST_SET_CONCURRENCY);
}
#[tokio::test]
#[serial]
async fn scanner_bucket_listing_unions_every_erasure_set() {
let (_temp_dir, ecstore) = setup_multi_pool_scanner_listing_test_env().await;
let bucket = format!("second-pool-only-{}", Uuid::new_v4().simple());
ecstore.pools[1].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created in the second pool only");
let listing = ecstore
.list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("scanner should enumerate every pool and set");
assert!(listing.topology_complete);
assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
assert_eq!(listing.set_buckets.len(), 2);
assert!(
listing
.set_buckets
.iter()
.find(|scope| scope.pool_index == 0 && scope.set_index == 0)
.is_some_and(|scope| scope.buckets.is_empty())
);
assert!(
listing
.set_buckets
.iter()
.find(|scope| scope.pool_index == 1 && scope.set_index == 0)
.is_some_and(|scope| scope.buckets.iter().any(|entry| entry.name == bucket))
);
}
#[tokio::test]
async fn scanner_bucket_listing_marks_degraded_set_incomplete() {
let (_temp_dir, ecstore) = setup_multi_pool_scanner_listing_test_env().await;
let bucket = format!("degraded-set-{}", Uuid::new_v4().simple());
let set = &ecstore.pools[0].disk_set[0];
set.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before a disk is removed");
set.disks.write().await[0] = None;
let listing = ecstore
.list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("a degraded set with quorum should still return candidate buckets");
assert!(listing.buckets.iter().any(|entry| entry.name == bucket));
assert!(!listing.topology_complete);
}
#[tokio::test]
async fn scanner_bucket_listing_marks_divergent_disk_views_incomplete() {
let (temp_dir, ecstore) = setup_multi_pool_scanner_listing_test_env().await;
let bucket = format!("divergent-set-{}", Uuid::new_v4().simple());
ecstore.pools[0].disk_set[0]
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before disk views diverge");
for disk_index in 0..2 {
tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool0-disk{disk_index}")).join(&bucket))
.await
.expect("test bucket directory should be removed from a minority disk view");
}
let listing = ecstore
.list_bucket_for_scanner(&crate::storage_api_contracts::bucket::BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.expect("responsive disks should still produce a scanner candidate listing");
assert!(listing.buckets.iter().all(|entry| entry.name != bucket));
assert!(!listing.topology_complete);
}
// These tests share one isolated instance and mutate its bucket metadata;
// serialize them so their assertions cannot observe each other's operations.
#[tokio::test]
@@ -736,4 +1246,210 @@ mod tests {
"failed default S3 DeleteBucket must keep metadata cache"
);
}
#[tokio::test]
#[serial]
async fn bucket_delete_finishes_usage_cleanup_before_same_name_recreation() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-usage-generation-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut snapshot = DataUsageInfo {
last_update: Some(SystemTime::now()),
buckets_count: 1,
..Default::default()
};
snapshot.buckets_usage.insert(
bucket.clone(),
BucketUsageInfo {
size: 42,
objects_count: 1,
versions_count: 1,
..Default::default()
},
);
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
.await
.expect("usage snapshot should be stored");
ecstore
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("empty bucket should be deleted");
let deleted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("usage snapshot should remain readable");
assert!(!deleted.buckets_usage.contains_key(&bucket));
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("same bucket name should be recreated after delete returns");
crate::data_usage::record_bucket_object_write_memory(&bucket, None, 84).await;
let mut recreated = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("recreated bucket usage base should load");
crate::data_usage::apply_bucket_usage_memory_overlay(&mut recreated).await;
assert_eq!(
recreated
.buckets_usage
.get(&bucket)
.map(|usage| (usage.objects_count, usage.versions_count, usage.size)),
Some((1, 1, 84))
);
}
#[tokio::test]
#[serial]
async fn bucket_create_removes_stale_usage_before_physical_creation() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-create-stale-usage-{}", Uuid::new_v4().simple());
let mut snapshot = DataUsageInfo {
last_update: Some(SystemTime::now()),
buckets_count: 1,
..Default::default()
};
snapshot.buckets_usage.insert(
bucket.clone(),
BucketUsageInfo {
size: 42,
objects_count: 1,
versions_count: 1,
..Default::default()
},
);
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
.await
.expect("stale usage fixture should be stored");
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("CreateBucket should succeed");
let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("usage snapshot should remain readable");
assert!(
!persisted.buckets_usage.contains_key(&bucket),
"a newly created bucket must not inherit the predecessor generation's usage"
);
assert!(
ecstore.get_bucket_info(&bucket, &BucketOptions::default()).await.is_ok(),
"physical creation should happen after the usage fence succeeds"
);
}
#[tokio::test]
#[serial]
async fn failed_create_rollback_does_not_run_unfenced_usage_cleanup() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-create-rollback-{}", Uuid::new_v4().simple());
ecstore
.peer_sys
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("the partial-create fixture should expose a physical bucket");
let mut snapshot = DataUsageInfo {
last_update: Some(SystemTime::now()),
buckets_count: 1,
..Default::default()
};
snapshot.buckets_usage.insert(
bucket.clone(),
BucketUsageInfo {
size: 42,
objects_count: 1,
versions_count: 1,
..Default::default()
},
);
snapshot.bucket_sizes.insert(bucket.clone(), 42);
snapshot.calculate_totals();
crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone())
.await
.expect("the usage fixture should be stored");
ecstore.rollback_failed_bucket_creation(&bucket, None).await;
assert!(
ecstore
.peer_sys
.get_bucket_info(&bucket, &BucketOptions::default())
.await
.is_err(),
"failed-create rollback should remove the partial physical bucket"
);
let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone())
.await
.expect("the usage snapshot should remain readable");
assert!(
persisted.buckets_usage.contains_key(&bucket),
"failed-create rollback must not start an unfenced usage cleanup"
);
crate::data_usage::store_data_usage_in_backend(DataUsageInfo::default(), ecstore.clone())
.await
.expect("the rollback usage fixture should be cleared");
}
#[tokio::test]
#[serial]
async fn bucket_create_fails_closed_when_usage_snapshot_cannot_be_fenced() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let deleted_bucket = format!("bucket-delete-corrupt-usage-{}", Uuid::new_v4().simple());
ecstore
.make_bucket(&deleted_bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created before corrupting usage");
let usage_path = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}");
crate::config::com::save_config(ecstore.clone(), &usage_path, b"{".to_vec())
.await
.expect("corrupt usage fixture should be stored");
ecstore
.delete_bucket(&deleted_bucket, &DeleteBucketOptions::default())
.await
.expect("usage snapshot corruption must not block DeleteBucket");
assert!(
ecstore
.get_bucket_info(&deleted_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
.await
.is_err(),
"successful DeleteBucket must remove the physical bucket"
);
let new_bucket = format!("bucket-create-corrupt-usage-{}", Uuid::new_v4().simple());
let create_err = ecstore
.make_bucket(&new_bucket, &MakeBucketOptions::default())
.await
.expect_err("MakeBucket must fail when stale usage cannot be fenced");
assert!(!create_err.to_string().is_empty(), "the usage snapshot failure should be surfaced");
assert!(
ecstore
.get_bucket_info(&new_bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
.await
.is_err(),
"failed MakeBucket must not expose a new physical bucket"
);
let restored = serde_json::to_vec(&DataUsageInfo {
last_update: Some(SystemTime::now()),
..Default::default()
})
.expect("restored usage fixture should encode");
crate::config::com::save_config(ecstore, &usage_path, restored)
.await
.expect("usage fixture should be restored after the failure-path test");
}
}
crates/ecstore/src/store/list_objects.rs
+1 -1
View File
@@ -619,7 +619,7 @@ pub(super) fn scanner_namespace_mutation_generation() -> u64 {
SCANNER_NAMESPACE_MUTATION_GENERATION.load(Ordering::Acquire)
}
pub(super) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) {
pub(crate) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) {
if bucket == RUSTFS_META_BUCKET {
return;
}
crates/ecstore/src/store/mod.rs
+10 -13
View File
@@ -322,6 +322,11 @@ impl ECStore {
pub fn scanner_namespace_mutation_generation(&self) -> u64 {
list_objects::scanner_namespace_mutation_generation()
}
pub async fn scanner_data_movement_active(&self) -> bool {
let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
decommission || rebalance
}
}
// impl Clone for ECStore {
@@ -440,11 +445,7 @@ impl BucketOperations for ECStore {
#[instrument(skip(self))]
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
let result = self.handle_make_bucket(bucket, opts).await;
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
result
Box::pin(self.handle_make_bucket(bucket, opts)).await
}
#[instrument(skip(self))]
@@ -457,11 +458,7 @@ impl BucketOperations for ECStore {
}
#[instrument(skip(self))]
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
let result = self.handle_delete_bucket(bucket, opts).await;
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
result
Box::pin(self.handle_delete_bucket(bucket, opts)).await
}
}
@@ -871,9 +868,9 @@ mod tests {
// Build a minimal ECStore carrying an explicit instance context. Empty
// pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> ECStore {
fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
let endpoint_pools = EndpointServerPools::default();
ECStore {
Arc::new(ECStore {
id: uuid::Uuid::new_v4(),
disk_map: std::collections::HashMap::new(),
pools: Vec::new(),
@@ -884,7 +881,7 @@ mod tests {
start_gate: Mutex::new(()),
pool_meta_save_gate: Mutex::new(()),
ctx,
}
})
}
// The object graph is the isolation carrier: two ECStore instances holding
crates/io-metrics/src/internode_metrics.rs
+1
View File
@@ -23,6 +23,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
pub const INTERNODE_OPERATION_READ_FILE_STREAM: &str = "read_file_stream";
pub const INTERNODE_OPERATION_PUT_FILE_STREAM: &str = "put_file_stream";
pub const INTERNODE_OPERATION_WALK_DIR: &str = "walk_dir";
pub const INTERNODE_OPERATION_NS_SCANNER: &str = "ns_scanner";
pub const INTERNODE_OPERATION_GRPC_READ_ALL: &str = "grpc_read_all";
pub const INTERNODE_OPERATION_GRPC_WRITE_ALL: &str = "grpc_write_all";
pub const INTERNODE_OPERATION_GRPC_READ_MULTIPLE: &str = "grpc_read_multiple";
crates/lock/src/distributed_lock.rs
+399 -118
View File
@@ -18,16 +18,20 @@ use crate::{
error::{LockError, Result},
types::{LockId, LockInfo, LockRequest, LockResponse, LockStatus, LockType},
};
use futures::future::join_all;
use futures::{
future::join_all,
stream::{FuturesUnordered, StreamExt},
};
use rustfs_io_metrics::{
record_lock_refresh_quorum_lost, record_read_lock_held_acquire, record_read_lock_held_release,
record_write_lock_held_acquire, record_write_lock_held_release,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::sync::Notify;
use tokio::task::{JoinHandle, JoinSet};
use tokio::time::Instant;
use tracing::{debug, warn};
use uuid::Uuid;
@@ -80,10 +84,10 @@ fn is_unrecoverable_quorum_error(error: &str) -> bool {
/// Deliberately avoids `Duration::clamp` (which asserts `min <= max` and would panic for
/// sub-second ttls where `ttl - 1s` underflows to zero). Returns `None` for degenerate cases so
/// no heartbeat is spawned:
/// - `entries_len <= 1`: single/degenerate path (matches a local, non-distributed lock),
/// - `entries_len == 0`: no acquired backend lease to renew,
/// - `interval.is_zero()` or `interval >= ttl`: too small a ttl / too large an interval to renew.
fn derive_refresh_interval(entries_len: usize, ttl: Duration, injected: Option<Duration>) -> Option<Duration> {
if entries_len <= 1 {
if entries_len == 0 {
return None;
}
let interval = injected.unwrap_or(ttl / 3);
@@ -130,10 +134,29 @@ fn should_warn_lock_failure(error: &str) -> bool {
#[derive(Debug, Default)]
pub struct LockLostSignal {
lost: AtomicBool,
valid_until: Mutex<Option<Instant>>,
notify: Notify,
}
impl LockLostSignal {
fn set_valid_until(&self, valid_until: Instant) {
if self.lost.load(Ordering::SeqCst) {
return;
}
match self.valid_until.lock() {
Ok(mut deadline) => {
if deadline.is_some_and(|current| Instant::now() >= current) {
drop(deadline);
self.mark_lost();
return;
}
*deadline = Some(valid_until);
self.notify.notify_waiters();
}
Err(_) => self.mark_lost(),
}
}
fn mark_lost(&self) {
self.lost.store(true, Ordering::SeqCst);
self.notify.notify_waiters();
@@ -141,16 +164,81 @@ impl LockLostSignal {
/// Whether refresh quorum has been lost for the associated guard.
pub fn is_lost(&self) -> bool {
self.lost.load(Ordering::SeqCst)
if self.lost.load(Ordering::SeqCst) {
return true;
}
let expired = match self.valid_until.lock() {
Ok(deadline) => deadline.is_some_and(|valid_until| Instant::now() >= valid_until),
Err(_) => true,
};
if expired {
self.mark_lost();
}
expired
}
/// Resolves once the lock is declared lost (immediately if already lost).
pub async fn notified(&self) {
if self.is_lost() {
return;
}
self.notify.notified().await;
self.notified_after_registration(|| {}).await;
}
async fn notified_after_registration(&self, after_registration: impl FnOnce()) {
let mut after_registration = Some(after_registration);
loop {
let notified = self.notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if let Some(after_registration) = after_registration.take() {
after_registration();
}
if self.is_lost() {
return;
}
let valid_until = match self.valid_until.lock() {
Ok(deadline) => *deadline,
Err(_) => return,
};
match valid_until {
Some(valid_until) => {
tokio::select! {
_ = tokio::time::sleep_until(valid_until) => {}
_ = &mut notified => {}
}
}
None => notified.await,
}
}
}
}
#[derive(Clone, Debug)]
struct HeldLockEntry {
lock_id: LockId,
client: Arc<dyn LockClient>,
valid_until: Instant,
}
impl HeldLockEntry {
fn release_entry(&self) -> (LockId, Arc<dyn LockClient>) {
(self.lock_id.clone(), self.client.clone())
}
}
#[derive(Clone, Debug)]
struct LockHeartbeatConfig {
ttl: Duration,
interval: Option<Duration>,
refresh_quorum: usize,
owner: String,
resource: ObjectKey,
}
#[derive(Clone, Copy, Debug, Default)]
struct LockRefreshStats {
refreshed: usize,
not_found: usize,
refresh_errors: usize,
}
/// A RAII guard for distributed locks that releases the lock asynchronously when dropped.
@@ -159,14 +247,12 @@ pub struct DistributedLockGuard {
/// The public-facing lock id. For multi-client scenarios this is typically
/// an aggregate id; for single-client it is the only id.
lock_id: LockId,
/// All underlying (LockId, client) entries that should be released when the
/// guard is dropped.
entries: Vec<(LockId, Arc<dyn LockClient>)>,
/// All underlying leases that should be refreshed and released with the guard.
entries: Vec<HeldLockEntry>,
lock_type: LockType,
/// If true, Drop will not try to release (used if user manually released).
disarmed: bool,
/// Background heartbeat task that periodically refreshes the per-client leases.
/// `None` for degenerate/single-client paths that do not need renewal.
/// Background lease task for renewable leases.
refresh_task: Option<JoinHandle<()>>,
/// Lock-loss signal, shared with the heartbeat task.
lock_lost: Arc<LockLostSignal>,
@@ -176,31 +262,22 @@ impl DistributedLockGuard {
/// Create a new guard.
///
/// - `lock_id` is the id returned to the caller (`lock_id()`).
/// - `entries` is the full list of underlying (LockId, client) pairs
/// that should be released when this guard is dropped.
/// - `refresh_interval`: `Some` spawns a heartbeat that refreshes every entry on that
/// cadence; `None` (degenerate/single-client, or interval derived away) spawns nothing.
/// - `refresh_quorum`: minimum refreshes that must keep succeeding; if `not_found` exceeds
/// `entries.len() - refresh_quorum` the guard is declared lost.
/// - `owner`/`resource`: diagnostics only.
pub(crate) fn new(
lock_id: LockId,
entries: Vec<(LockId, Arc<dyn LockClient>)>,
lock_type: LockType,
refresh_interval: Option<Duration>,
refresh_quorum: usize,
owner: String,
resource: ObjectKey,
) -> Self {
/// - `entries` is the full list of underlying leases that should be refreshed and released
/// with this guard.
/// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics.
fn new(lock_id: LockId, entries: Vec<HeldLockEntry>, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self {
record_lock_held_acquire(lock_type);
let lock_lost = Arc::new(LockLostSignal::default());
let refresh_task = refresh_interval.and_then(|interval| {
// Only spawn when a tokio runtime is available; guard construction off-runtime
// (e.g. some tests) simply skips the heartbeat.
match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) {
Some(valid_until) => lock_lost.set_valid_until(valid_until),
None => lock_lost.mark_lost(),
}
// Non-renewable guards are fenced directly by LockLostSignal's deadline checks.
let refresh_task = heartbeat.interval.and_then(|_| {
tokio::runtime::Handle::try_current().ok().map(|handle| {
let entries = entries.clone();
let lock_lost = lock_lost.clone();
handle.spawn(Self::run_heartbeat(entries, interval, refresh_quorum, lock_lost, owner, resource))
handle.spawn(Self::run_heartbeat(entries, heartbeat, lock_lost))
})
});
Self {
@@ -213,59 +290,107 @@ impl DistributedLockGuard {
}
}
/// Heartbeat loop: every `interval`, refresh all entries and classify the outcomes.
/// `Ok(true)` = refreshed, `Ok(false)` = not_found, `Err` = RPC jitter (ignored, absorbed by
/// the ttl > interval margin and retried next tick). Declares the lock lost when
/// `not_found > entries.len() - refresh_quorum`. Phase 1 does not release on loss: the guarded
/// operation is still running, and tearing the lock down here would only widen the window.
async fn run_heartbeat(
entries: Vec<(LockId, Arc<dyn LockClient>)>,
interval: Duration,
refresh_quorum: usize,
lock_lost: Arc<LockLostSignal>,
owner: String,
resource: ObjectKey,
fn quorum_valid_until(entries: &[HeldLockEntry], refresh_quorum: usize) -> Option<Instant> {
if refresh_quorum == 0 || entries.len() < refresh_quorum {
return None;
}
let mut deadlines: Vec<_> = entries.iter().map(|entry| entry.valid_until).collect();
deadlines.sort_unstable();
deadlines.get(deadlines.len() - refresh_quorum).copied()
}
fn signal_refresh_quorum_lost(
lock_lost: &LockLostSignal,
config: &LockHeartbeatConfig,
entries: usize,
stats: LockRefreshStats,
) {
let tolerable_not_found = entries.len().saturating_sub(refresh_quorum);
let mut ticker = tokio::time::interval(interval);
warn!(
resource = %config.resource,
owner = config.owner,
refreshed = stats.refreshed,
not_found = stats.not_found,
refresh_errors = stats.refresh_errors,
entries,
refresh_quorum = config.refresh_quorum,
"lock refresh lost quorum"
);
record_lock_refresh_quorum_lost();
lock_lost.mark_lost();
}
/// Refresh every tracked lease while a quorum is still known to be valid.
///
/// A successful refresh extends only that entry's conservative local deadline. RPC errors
/// retain the previous deadline, so transient jitter is tolerated but an unconfirmed lease
/// can never remain valid beyond its backend TTL. Waiting for a slow refresh is also fenced
/// by the current quorum deadline.
async fn run_heartbeat(mut entries: Vec<HeldLockEntry>, config: LockHeartbeatConfig, lock_lost: Arc<LockLostSignal>) {
let Some(interval) = config.interval else {
return;
};
let now = Instant::now();
let first_refresh = now.checked_add(interval).unwrap_or(now);
let mut ticker = tokio::time::interval_at(first_refresh, interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
// The first tick fires immediately; skip it so the first refresh lands one interval
// after acquisition rather than right away.
ticker.tick().await;
loop {
ticker.tick().await;
let Some(valid_until) = Self::quorum_valid_until(&entries, config.refresh_quorum) else {
Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), LockRefreshStats::default());
return;
};
lock_lost.set_valid_until(valid_until);
let results = join_all(entries.iter().map(|(lock_id, client)| {
let lock_id = lock_id.clone();
let client = client.clone();
async move { client.refresh(&lock_id).await }
}))
.await;
let mut refreshed = 0usize;
let mut not_found = 0usize;
for result in &results {
match result {
Ok(true) => refreshed += 1,
Ok(false) => not_found += 1,
// RPC jitter: count as neither; the ttl > interval margin covers a transient
// dip and the next tick retries.
Err(_) => {}
tokio::select! {
biased;
_ = tokio::time::sleep_until(valid_until) => {
Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), LockRefreshStats::default());
return;
}
_ = ticker.tick() => {}
}
if not_found > tolerable_not_found {
warn!(
resource = %resource,
owner = %owner,
refreshed,
not_found,
entries = entries.len(),
refresh_quorum,
"lock refresh lost quorum"
);
record_lock_refresh_quorum_lost();
lock_lost.mark_lost();
let mut stats = LockRefreshStats::default();
let mut pending = FuturesUnordered::new();
for (idx, entry) in entries.iter().enumerate() {
let refresh_started = Instant::now();
let lock_id = entry.lock_id.clone();
let client = entry.client.clone();
pending.push(async move { (idx, refresh_started, client.refresh(&lock_id).await) });
}
while !pending.is_empty() {
let Some(valid_until) = Self::quorum_valid_until(&entries, config.refresh_quorum) else {
Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), stats);
return;
};
lock_lost.set_valid_until(valid_until);
let next = tokio::select! {
biased;
_ = tokio::time::sleep_until(valid_until) => {
Self::signal_refresh_quorum_lost(&lock_lost, &config, entries.len(), stats);
return;
}
next = pending.next() => next
};
let Some((idx, refresh_started, result)) = next else {
break;
};
match result {
Ok(true) => {
stats.refreshed += 1;
entries[idx].valid_until = refresh_started.checked_add(config.ttl).unwrap_or(refresh_started);
}
Ok(false) => {
stats.not_found += 1;
entries[idx].valid_until = Instant::now();
}
Err(_) => stats.refresh_errors += 1,
}
}
}
}
@@ -326,7 +451,7 @@ impl DistributedLockGuard {
return true;
}
let entries = self.entries.clone();
let entries = self.entries.iter().map(HeldLockEntry::release_entry).collect();
DistributedLock::spawn_release_cleanup(entries, "distributed_lock_guard_release");
// Disarm to prevent double-release on drop
@@ -361,7 +486,7 @@ type LockAcquireTaskResult = (usize, Result<LockResponse>);
struct LockAcquireQuorumResult {
response: LockResponse,
individual_locks: Vec<(LockId, Arc<dyn LockClient>)>,
individual_locks: Vec<HeldLockEntry>,
failure_kind: Option<LockAcquireFailureKind>,
quorum_impossible: bool,
}
@@ -438,15 +563,18 @@ impl DistributedLock {
// Heartbeat operates on the per-client individual locks (their per-client ids), never
// the aggregate id, so refreshes round-trip to the exact backend entries.
let refresh_interval = derive_refresh_interval(individual_locks.len(), request.ttl, request.refresh_interval);
let heartbeat = LockHeartbeatConfig {
ttl: request.ttl,
interval: derive_refresh_interval(individual_locks.len(), request.ttl, request.refresh_interval),
refresh_quorum: required_quorum,
owner: request.owner.clone(),
resource: request.resource.clone(),
};
Ok(Some(DistributedLockGuard::new(
aggregate_lock_id,
individual_locks,
request.lock_type,
refresh_interval,
required_quorum,
request.owner.clone(),
request.resource.clone(),
heartbeat,
)))
} else {
// Check if it's a timeout or quorum failure
@@ -738,7 +866,7 @@ impl DistributedLock {
fn lock_acquire_attempt_timeout_result(
timeout: Duration,
individual_locks: Vec<(LockId, Arc<dyn LockClient>)>,
individual_locks: Vec<HeldLockEntry>,
last_failure: Option<String>,
last_failure_kind: Option<LockAcquireFailureKind>,
) -> LockAcquireQuorumResult {
@@ -770,19 +898,23 @@ impl DistributedLock {
/// Returns the LockResponse with aggregate lock_id and individual lock mappings.
async fn acquire_lock_quorum_once(&self, request: &LockRequest) -> Result<LockAcquireQuorumResult> {
let required_quorum = self.required_quorum(request.lock_type);
let attempt_started = Instant::now();
let mut pending = self.spawn_lock_requests(request);
let mut individual_locks: Vec<(LockId, Arc<dyn LockClient>)> = Vec::new();
let mut individual_locks: Vec<HeldLockEntry> = Vec::new();
let fallback_lock_id = request.lock_id.clone();
let mut last_failure = None;
let mut last_failure_kind = None;
let mut last_hard_failure_kind = None;
let mut hard_failures = 0usize;
let start = tokio::time::Instant::now();
let start = Instant::now();
while !pending.is_empty() {
let remaining = request.acquire_timeout.saturating_sub(start.elapsed());
if remaining.is_zero() {
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_attempt_timeout");
Self::spawn_release_cleanup(
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
"distributed_lock_attempt_timeout",
);
Self::spawn_pending_cleanup(
pending,
self.clients.clone(),
@@ -801,7 +933,10 @@ impl DistributedLock {
Ok(Some(join_result)) => join_result,
Ok(None) => break,
Err(_) => {
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_attempt_timeout");
Self::spawn_release_cleanup(
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
"distributed_lock_attempt_timeout",
);
Self::spawn_pending_cleanup(
pending,
self.clients.clone(),
@@ -827,7 +962,11 @@ impl DistributedLock {
.unwrap_or_else(|| fallback_lock_id.clone());
if let Some(client) = self.clients.get(idx) {
individual_locks.push((lock_id, client.clone()));
individual_locks.push(HeldLockEntry {
lock_id,
client: client.clone(),
valid_until: attempt_started.checked_add(request.ttl).unwrap_or(attempt_started),
});
} else {
tracing::warn!("Missing lock client at index {} while recording success", idx);
}
@@ -862,7 +1001,10 @@ impl DistributedLock {
if self.clients.len().saturating_sub(hard_failures) < required_quorum {
let rollback_count = individual_locks.len();
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
Self::spawn_release_cleanup(
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
"distributed_lock_quorum_rollback",
);
if !pending.is_empty() {
Self::spawn_pending_cleanup(
pending,
@@ -933,7 +1075,10 @@ impl DistributedLock {
if individual_locks.len() + pending.len() < required_quorum {
let rollback_count = individual_locks.len();
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
Self::spawn_release_cleanup(
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
"distributed_lock_quorum_rollback",
);
if !pending.is_empty() {
Self::spawn_pending_cleanup(
pending,
@@ -967,7 +1112,10 @@ impl DistributedLock {
}
let rollback_count = individual_locks.len();
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
Self::spawn_release_cleanup(
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
"distributed_lock_quorum_rollback",
);
let mut error = format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required");
if let Some(last_failure) = &last_failure {
error.push_str("; last failure: ");
@@ -1006,7 +1154,7 @@ fn record_lock_held_release(lock_type: LockType) {
#[cfg(test)]
mod tests {
use super::{
DistributedLock, LOCK_ACQUIRE_ATTEMPT_TIMEOUT, LockAcquireFailureKind, is_remote_lock_rpc_failure,
DistributedLock, LOCK_ACQUIRE_ATTEMPT_TIMEOUT, LockAcquireFailureKind, LockLostSignal, is_remote_lock_rpc_failure,
should_warn_lock_failure,
};
use crate::{LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockType, ObjectKey, client::LockClient};
@@ -1020,9 +1168,10 @@ mod tests {
#[derive(Clone, Copy, Debug)]
enum RefreshOutcome {
Alive, // Ok(true) refresh succeeded
NotFound, // Ok(false) remote already lost the lock (reclaimed / never held)
RpcError, // Err RPC jitter
Alive, // Ok(true) refresh succeeded
SlowAlive(Duration), // Ok(true) after a delayed response
NotFound, // Ok(false) remote already lost the lock (reclaimed / never held)
RpcError, // Err RPC jitter
}
/// Counting test client: acquires successfully and echoes back request.lock_id as
@@ -1071,6 +1220,10 @@ mod tests {
self.refresh_calls.fetch_add(1, Ordering::SeqCst);
match self.outcome {
RefreshOutcome::Alive => Ok(true),
RefreshOutcome::SlowAlive(delay) => {
tokio::time::sleep(delay).await;
Ok(true)
}
RefreshOutcome::NotFound => Ok(false),
RefreshOutcome::RpcError => Err(LockError::internal("refresh rpc jitter")),
}
@@ -1140,19 +1293,37 @@ mod tests {
drop(guard);
}
// A2 -- single client (degenerate path) spawns no heartbeat (matches localLockInstance).
// A2 -- a quorum-one distributed lease still requires renewal.
#[tokio::test]
async fn single_client_guard_spawns_no_heartbeat() {
async fn single_client_guard_keeps_backend_lease_refreshed() {
let (clients, counters) = counting_clients(&[RefreshOutcome::Alive]);
let lock = DistributedLock::new("test".to_string(), clients, 1);
let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner")
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock.acquire_guard(&request).await.unwrap().unwrap();
tokio::time::sleep(Duration::from_millis(90)).await;
let guard = lock
.acquire_guard(&request)
.await
.expect("single-client lock acquisition should not error")
.expect("single-client lock quorum should be reached");
assert!(
guard.refresh_task.is_some(),
"single-client distributed guard must spawn a background task"
);
tokio::time::timeout(Duration::from_secs(2), async {
while counters[0].load(Ordering::SeqCst) < 2 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("single-client distributed guard should renew before the test deadline");
assert_eq!(counters[0].load(Ordering::SeqCst), 0, "single-client guard must not run a heartbeat");
assert!(
counters[0].load(Ordering::SeqCst) >= 2,
"single-client distributed guard must renew beyond the original backend TTL"
);
assert!(!guard.is_lock_lost(), "successful quorum-one refresh must preserve the lease");
drop(guard);
}
@@ -1170,7 +1341,11 @@ mod tests {
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock.acquire_guard(&request).await.unwrap().unwrap();
let guard = lock
.acquire_guard(&request)
.await
.expect("distributed lock acquisition should not error")
.expect("distributed lock quorum should be reached");
tokio::time::sleep(Duration::from_millis(50)).await;
drop(guard); // should abort the heartbeat first
let after_drop: Vec<usize> = counters.iter().map(|c| c.load(Ordering::SeqCst)).collect();
@@ -1200,7 +1375,11 @@ mod tests {
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock.acquire_guard(&request).await.unwrap().unwrap();
let guard = lock
.acquire_guard(&request)
.await
.expect("distributed lock acquisition should not error")
.expect("distributed lock quorum should be reached");
tokio::time::sleep(Duration::from_millis(60)).await; // at least one tick
assert!(
@@ -1214,6 +1393,52 @@ mod tests {
drop(guard);
}
#[tokio::test]
async fn lock_lost_notification_cannot_be_missed_after_waiter_registration() {
let signal = LockLostSignal::default();
tokio::time::timeout(Duration::from_secs(2), signal.notified_after_registration(|| signal.mark_lost()))
.await
.expect("loss between waiter registration and state recheck must still resolve");
}
#[tokio::test]
async fn expired_lease_is_fenced_without_waiting_for_monitor_scheduling() {
let signal = LockLostSignal::default();
signal.set_valid_until(tokio::time::Instant::now());
assert!(signal.is_lost(), "an elapsed backend deadline must fence the owner immediately");
tokio::time::timeout(Duration::from_secs(2), signal.notified())
.await
.expect("an elapsed backend deadline must resolve lock-loss waiters immediately");
}
#[tokio::test]
async fn deadline_notification_permanently_fences_the_signal() {
let signal = LockLostSignal::default();
signal.set_valid_until(tokio::time::Instant::now() + Duration::from_millis(10));
tokio::time::timeout(Duration::from_secs(2), signal.notified())
.await
.expect("the backend deadline must resolve lock-loss waiters");
assert!(
signal.lost.load(Ordering::SeqCst),
"deadline notification must persist the lock-loss fence"
);
}
#[test]
fn expired_lease_cannot_be_revived_by_a_later_deadline() {
let signal = LockLostSignal::default();
signal.set_valid_until(tokio::time::Instant::now());
assert!(signal.is_lost(), "an elapsed lease must become permanently fenced");
signal.set_valid_until(tokio::time::Instant::now() + Duration::from_secs(60));
assert!(signal.is_lost(), "a later heartbeat must not revive an expired lease");
}
// Phase 2 passthrough: NamespaceLockGuard::Standard must forward the distributed
// guard's lock-loss signal so the write path can fence its commit on it.
#[tokio::test]
@@ -1229,7 +1454,11 @@ mod tests {
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock.acquire_guard(&request).await.unwrap().unwrap();
let guard = lock
.acquire_guard(&request)
.await
.expect("namespace lock acquisition should not error")
.expect("namespace lock quorum should be reached");
tokio::time::sleep(Duration::from_millis(60)).await; // at least one tick -> lost
let ns_guard = crate::namespace::NamespaceLockGuard::Standard(guard);
@@ -1239,9 +1468,10 @@ mod tests {
);
}
// A5 -- refresh RPC jitter (Err) is not counted as not_found; the lock is not declared lost.
// A5 -- refresh RPC jitter is tolerated within the lease, but an unconfirmed lease cannot
// remain valid after its backend TTL.
#[tokio::test]
async fn heartbeat_rpc_error_not_counted_as_lock_lost() {
async fn heartbeat_rpc_error_expires_at_backend_ttl() {
let (clients, _counters) = counting_clients(&[
RefreshOutcome::RpcError,
RefreshOutcome::RpcError,
@@ -1253,17 +1483,53 @@ mod tests {
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock.acquire_guard(&request).await.unwrap().unwrap();
let guard = lock
.acquire_guard(&request)
.await
.expect("distributed lock acquisition should not error")
.expect("distributed lock quorum should be reached");
tokio::time::sleep(Duration::from_millis(60)).await;
assert!(
!guard.is_lock_lost(),
"RPC jitter (Err) must NOT be counted as not_found; lock must not be declared lost"
"transient RPC jitter before the backend TTL must not immediately signal lock loss"
);
let signal = guard.lock_lost();
tokio::time::timeout(Duration::from_secs(2), signal.notified())
.await
.expect("unconfirmed leases must signal lock loss when their backend TTL expires");
assert!(guard.is_lock_lost(), "the guard must fail closed once refresh quorum expires");
drop(guard);
}
// A6 -- a refresh call that remains in flight cannot keep an expired quorum alive.
#[tokio::test]
async fn heartbeat_slow_refresh_is_fenced_by_backend_ttl() {
let (clients, _counters) =
counting_clients(&[RefreshOutcome::SlowAlive(Duration::from_millis(200)), RefreshOutcome::Alive]);
let lock = DistributedLock::new("test".to_string(), clients, 2);
let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner")
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let guard = lock
.acquire_guard(&request)
.await
.expect("distributed lock acquisition should not error")
.expect("distributed lock quorum should be reached");
let signal = guard.lock_lost();
tokio::time::timeout(Duration::from_secs(2), signal.notified())
.await
.expect("an in-flight refresh must not extend an unconfirmed lease past its TTL");
assert!(
guard.is_lock_lost(),
"slow refresh must fail closed at the last confirmed quorum deadline"
);
drop(guard);
}
// A6 -- sub-second ttl boundary: no panic, and skip heartbeat when interval>=ttl
// A7 -- sub-second ttl boundary: no panic, and skip heartbeat when interval>=ttl
// (fixes clamp panic; aligns with the 50ms-ttl distributed path in namespace tests).
#[tokio::test]
async fn subsecond_ttl_guard_does_not_panic_and_skips_heartbeat() {
@@ -1286,15 +1552,26 @@ mod tests {
let request2 = LockRequest::new(ObjectKey::new("bucket", "object2"), LockType::Exclusive, "owner")
.with_ttl(Duration::from_millis(50))
.with_refresh_interval(Duration::from_millis(50)); // interval >= ttl -> None
let guard2 = lock2.acquire_guard(&request2).await.unwrap().unwrap();
tokio::time::sleep(Duration::from_millis(60)).await;
let guard2 = lock2
.acquire_guard(&request2)
.await
.expect("subsecond lock acquisition should not error")
.expect("subsecond lock quorum should be reached");
assert!(
guard2.refresh_task.is_none(),
"non-renewable subsecond guard must not spawn a background task"
);
tokio::time::timeout(Duration::from_secs(2), guard2.lock_lost().notified())
.await
.expect("non-renewable subsecond guard must signal loss at its backend TTL");
for c in &counters2 {
assert_eq!(c.load(Ordering::SeqCst), 0, "interval>=ttl must not spawn heartbeat");
assert_eq!(c.load(Ordering::SeqCst), 0, "interval>=ttl must not refresh the backend lease");
}
assert!(guard2.is_lock_lost(), "interval>=ttl guard must fail closed after backend TTL");
drop(guard2);
}
// A7 -- disarm() must also stop the heartbeat (public API gap).
// A8 -- disarm() must also stop the heartbeat (public API gap).
#[tokio::test]
async fn disarm_stops_heartbeat() {
let (clients, counters) = counting_clients(&[
@@ -1308,7 +1585,11 @@ mod tests {
.with_ttl(Duration::from_millis(120))
.with_refresh_interval(Duration::from_millis(20));
let mut guard = lock.acquire_guard(&request).await.unwrap().unwrap();
let mut guard = lock
.acquire_guard(&request)
.await
.expect("distributed lock acquisition should not error")
.expect("distributed lock quorum should be reached");
tokio::time::sleep(Duration::from_millis(50)).await;
guard.disarm(); // should abort the heartbeat
let after_disarm: Vec<usize> = counters.iter().map(|c| c.load(Ordering::SeqCst)).collect();
crates/lock/src/namespace/mod.rs
+13
View File
@@ -128,6 +128,19 @@ impl NamespaceLockGuard {
Self::Fast(_) => false,
}
}
/// Resolves when a distributed guard loses refresh quorum.
///
/// Local fast locks cannot lose distributed quorum, so their future remains pending.
pub async fn lock_lost_notified(&self) {
match self {
Self::Standard(guard) => {
let signal = guard.lock_lost();
signal.notified().await;
}
Self::Fast(_) => std::future::pending().await,
}
}
}
/// Namespace lock for managing locks by resource namespaces
crates/madmin/src/metrics.rs
+6
View File
@@ -640,6 +640,8 @@ pub struct ScannerMetrics {
pub last_cycle_usage_saves: u64,
#[serde(rename = "failed_cycles", default)]
pub failed_cycles: u64,
#[serde(rename = "superseded_cycles", default)]
pub superseded_cycles: u64,
#[serde(rename = "partial_cycles_unknown", default)]
pub partial_cycles_unknown: u64,
#[serde(rename = "partial_cycles_runtime", default)]
@@ -820,6 +822,7 @@ impl ScannerMetrics {
.saturating_add(other.last_cycle_replication_checks);
self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves);
self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles);
self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles);
self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown);
self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime);
self.partial_cycles_objects = self.partial_cycles_objects.saturating_add(other.partial_cycles_objects);
@@ -1862,6 +1865,7 @@ mod tests {
last_cycle_replication_checks: 7,
last_cycle_usage_saves: 8,
failed_cycles: 1,
superseded_cycles: 2,
partial_cycles_unknown: 2,
partial_cycles_runtime: 3,
partial_cycles_objects: 4,
@@ -1933,6 +1937,7 @@ mod tests {
last_cycle_replication_checks: 70,
last_cycle_usage_saves: 80,
failed_cycles: 10,
superseded_cycles: 20,
partial_cycles_unknown: 20,
partial_cycles_runtime: 30,
partial_cycles_objects: 40,
@@ -2014,6 +2019,7 @@ mod tests {
assert_eq!(scanner.last_cycle_replication_checks, 77);
assert_eq!(scanner.last_cycle_usage_saves, 88);
assert_eq!(scanner.failed_cycles, 11);
assert_eq!(scanner.superseded_cycles, 22);
assert_eq!(scanner.partial_cycles_unknown, 22);
assert_eq!(scanner.partial_cycles_runtime, 33);
assert_eq!(scanner.partial_cycles_objects, 44);
crates/obs/src/metrics/collectors/scanner.rs
+15 -5
View File
@@ -44,8 +44,9 @@ use crate::metrics::schema::scanner::{
SCANNER_LAST_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_THROTTLE_SLEEP_EVENTS_MD,
SCANNER_LAST_CYCLE_USAGE_SAVES_MD, SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD,
SCANNER_OBJECTS_SCANNED_MD, SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD, SCANNER_PARTIAL_CYCLES_BY_REASON_MD,
SCANNER_PARTIAL_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD,
SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD,
SCANNER_PARTIAL_CYCLES_MD, SCANNER_SUPERSEDED_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD,
SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD,
SCANNER_YIELD_EVERY_N_OBJECTS_MD,
};
/// Scanner statistics.
@@ -139,7 +140,7 @@ pub struct ScannerStats {
pub current_cycle_usage_saves: u64,
/// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
pub current_scan_mode: u64,
/// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial
/// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded
pub last_cycle_result: u64,
/// Last scanner partial cycle reason: 0 unknown, 1 runtime, 2 objects, 3 directories
pub last_cycle_partial_reason: u64,
@@ -177,6 +178,8 @@ pub struct ScannerStats {
pub last_cycle_usage_saves: u64,
/// Number of scanner cycles that failed since server start
pub failed_cycles: u64,
/// Number of scanner cycles superseded by concurrent namespace activity
pub superseded_cycles: u64,
/// Number of scanner cycles stopped by runtime budget since server start
pub partial_cycles: u64,
/// Number of scanner cycles stopped by an unknown budget reason
@@ -318,6 +321,7 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, stats.last_cycle_replication_checks as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_USAGE_SAVES_MD, stats.last_cycle_usage_saves as f64),
PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
PrometheusMetric::from_descriptor(&SCANNER_SUPERSEDED_CYCLES_MD, stats.superseded_cycles as f64),
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_MD, stats.partial_cycles as f64),
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_unknown as f64)
.with_label("reason", "unknown"),
@@ -405,6 +409,7 @@ mod tests {
last_cycle_replication_checks: 12,
last_cycle_usage_saves: 9,
failed_cycles: 3,
superseded_cycles: 5,
partial_cycles: 10,
partial_cycles_unknown: 1,
partial_cycles_runtime: 2,
@@ -415,7 +420,7 @@ mod tests {
let metrics = collect_scanner_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 68);
assert_eq!(metrics.len(), 69);
let objects = metrics.iter().find(|m| m.value == 1000000.0);
assert!(objects.is_some());
@@ -708,6 +713,11 @@ mod tests {
.find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
assert_eq!(failed_cycles.map(|m| m.value), Some(3.0));
let superseded_cycles = metrics
.iter()
.find(|m| m.name == SCANNER_SUPERSEDED_CYCLES_MD.get_full_metric_name());
assert_eq!(superseded_cycles.map(|m| m.value), Some(5.0));
let partial_cycles = metrics
.iter()
.find(|m| m.name == SCANNER_PARTIAL_CYCLES_MD.get_full_metric_name());
@@ -743,7 +753,7 @@ mod tests {
let stats = ScannerStats::default();
let metrics = collect_scanner_metrics(&stats);
assert_eq!(metrics.len(), 68);
assert_eq!(metrics.len(), 69);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
if metric.name == SCANNER_PARTIAL_CYCLES_BY_REASON_MD.get_full_metric_name() {
crates/obs/src/metrics/schema/entry/metric_name.rs
+2
View File
@@ -337,6 +337,7 @@ pub enum MetricName {
ScannerLastCycleReplicationChecks,
ScannerLastCycleUsageSaves,
ScannerFailedCycles,
ScannerSupersededCycles,
ScannerPartialCycles,
ScannerPartialCyclesByReason,
@@ -745,6 +746,7 @@ impl MetricName {
Self::ScannerLastCycleReplicationChecks => "last_cycle_replication_checks".to_string(),
Self::ScannerLastCycleUsageSaves => "last_cycle_usage_saves".to_string(),
Self::ScannerFailedCycles => "failed_cycles".to_string(),
Self::ScannerSupersededCycles => "superseded_cycles".to_string(),
Self::ScannerPartialCycles => "partial_cycles".to_string(),
Self::ScannerPartialCyclesByReason => "partial_cycles_by_reason".to_string(),
crates/obs/src/metrics/schema/scanner.rs
+10 -1
View File
@@ -416,7 +416,7 @@ pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::
pub static SCANNER_LAST_CYCLE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleResult,
"Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial.",
"Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded.",
&[],
subsystems::SCANNER,
)
@@ -584,6 +584,15 @@ pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(
)
});
pub static SCANNER_SUPERSEDED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::ScannerSupersededCycles,
"Total number of clean scanner cycles superseded by concurrent cluster, configuration, topology, or namespace activity since server start.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_PARTIAL_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::ScannerPartialCycles,
crates/obs/src/metrics/stats_collector.rs
+1
View File
@@ -1135,6 +1135,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
failed_cycles: metrics.failed_cycles,
superseded_cycles: metrics.superseded_cycles,
partial_cycles: metrics.partial_cycles,
partial_cycles_unknown: metrics.partial_cycles_unknown,
partial_cycles_runtime: metrics.partial_cycles_runtime,
crates/protos/src/generated/proto_gen/node_service.rs
+25 -2
View File
@@ -1076,9 +1076,20 @@ pub struct SignalServiceResponse {
pub success: bool,
#[prost(string, optional, tag = "2")]
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
#[prost(uint32, tag = "3")]
pub protocol_version: u32,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerActivityRequest {
#[prost(bytes = "bytes", tag = "1")]
pub challenge: ::prost::bytes::Bytes,
#[prost(uint32, tag = "2")]
pub protocol_version: u32,
#[prost(string, tag = "3")]
pub acknowledge_instance_id: ::prost::alloc::string::String,
#[prost(uint64, tag = "4")]
pub acknowledge_dirty_usage_generation: u64,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerActivityRequest {}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerActivityResponse {
#[prost(string, tag = "1")]
@@ -1087,6 +1098,18 @@ pub struct ScannerActivityResponse {
pub namespace_generation: u64,
#[prost(uint64, tag = "3")]
pub maintenance_generation: u64,
#[prost(uint32, tag = "4")]
pub protocol_version: u32,
#[prost(bytes = "bytes", tag = "5")]
pub topology_digest: ::prost::bytes::Bytes,
#[prost(bool, tag = "6")]
pub data_movement_active: bool,
#[prost(bytes = "bytes", tag = "7")]
pub response_proof: ::prost::bytes::Bytes,
#[prost(uint64, tag = "8")]
pub dirty_usage_generation: u64,
#[prost(bool, tag = "9")]
pub dirty_usage_pending: bool,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BackgroundHealStatusRequest {}
crates/protos/src/lib.rs
+225
View File
@@ -169,6 +169,7 @@ pub fn internode_rpc_max_message_size() -> usize {
pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SIZE + 1024;
pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
@@ -353,6 +354,230 @@ pub fn canonical_tier_mutation_rpc_response_body(
Ok(body)
}
/// Builds the stable byte representation authenticated for a scanner activity request.
pub fn canonical_scanner_activity_request_body(
request: &proto_gen::node_service::ScannerActivityRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-activity-request-v1\0";
let challenge = request.challenge.as_ref();
let acknowledge_instance_id = request.acknowledge_instance_id.as_bytes();
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + acknowledge_instance_id.len() + 4 + 8 * 3);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&request.protocol_version.to_be_bytes());
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.extend_from_slice(&u64::try_from(acknowledge_instance_id.len())?.to_be_bytes());
body.extend_from_slice(acknowledge_instance_id);
body.extend_from_slice(&request.acknowledge_dirty_usage_generation.to_be_bytes());
Ok(body)
}
/// Builds the protocol-v4 byte representation authenticated for a scanner activity response.
pub fn canonical_scanner_activity_v4_response_body(
challenge: &[u8],
response: &proto_gen::node_service::ScannerActivityResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-activity-response-v1\0";
let instance_id = response.instance_id.as_bytes();
let topology_digest = response.topology_digest.as_ref();
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + instance_id.len() + topology_digest.len() + 4 + 8 * 5 + 1);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.extend_from_slice(&u64::try_from(instance_id.len())?.to_be_bytes());
body.extend_from_slice(instance_id);
body.extend_from_slice(&response.namespace_generation.to_be_bytes());
body.extend_from_slice(&response.maintenance_generation.to_be_bytes());
body.extend_from_slice(&response.protocol_version.to_be_bytes());
body.extend_from_slice(&u64::try_from(topology_digest.len())?.to_be_bytes());
body.extend_from_slice(topology_digest);
body.push(u8::from(response.data_movement_active));
Ok(body)
}
/// Builds the stable byte representation authenticated for a scanner activity response.
pub fn canonical_scanner_activity_response_body(
challenge: &[u8],
response: &proto_gen::node_service::ScannerActivityResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-scanner-activity-response-v2\0";
let instance_id = response.instance_id.as_bytes();
let topology_digest = response.topology_digest.as_ref();
let mut body = Vec::with_capacity(DOMAIN.len() + challenge.len() + instance_id.len() + topology_digest.len() + 4 + 8 * 6 + 2);
body.extend_from_slice(DOMAIN);
body.extend_from_slice(&u64::try_from(challenge.len())?.to_be_bytes());
body.extend_from_slice(challenge);
body.extend_from_slice(&u64::try_from(instance_id.len())?.to_be_bytes());
body.extend_from_slice(instance_id);
body.extend_from_slice(&response.namespace_generation.to_be_bytes());
body.extend_from_slice(&response.maintenance_generation.to_be_bytes());
body.extend_from_slice(&response.protocol_version.to_be_bytes());
body.extend_from_slice(&u64::try_from(topology_digest.len())?.to_be_bytes());
body.extend_from_slice(topology_digest);
body.push(u8::from(response.data_movement_active));
body.extend_from_slice(&response.dirty_usage_generation.to_be_bytes());
body.push(u8::from(response.dirty_usage_pending));
Ok(body)
}
#[cfg(test)]
mod scanner_activity_tests {
use super::{
canonical_scanner_activity_request_body, canonical_scanner_activity_response_body,
canonical_scanner_activity_v4_response_body,
proto_gen::node_service::{ScannerActivityRequest, ScannerActivityResponse},
};
#[test]
fn canonical_scanner_activity_request_binds_every_field() {
let request = ScannerActivityRequest {
challenge: vec![1; 16].into(),
protocol_version: 5,
acknowledge_instance_id: "0123456789abcdef0123456789abcdef".to_string(),
acknowledge_dirty_usage_generation: 11,
};
let baseline = canonical_scanner_activity_request_body(&request).expect("scanner activity request should encode");
let variants = [
ScannerActivityRequest {
challenge: vec![2; 16].into(),
..request.clone()
},
ScannerActivityRequest {
protocol_version: 4,
..request.clone()
},
ScannerActivityRequest {
acknowledge_instance_id: "1123456789abcdef0123456789abcdef".to_string(),
..request.clone()
},
ScannerActivityRequest {
acknowledge_dirty_usage_generation: 12,
..request
},
];
for variant in variants {
assert_ne!(
baseline,
canonical_scanner_activity_request_body(&variant).expect("scanner activity request variant should encode")
);
}
}
#[test]
fn canonical_scanner_activity_response_binds_challenge_and_every_status_field() {
let response = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: 4,
topology_digest: vec![9; 32].into(),
data_movement_active: true,
response_proof: Vec::new().into(),
dirty_usage_generation: 11,
dirty_usage_pending: true,
};
let baseline =
canonical_scanner_activity_response_body(&[1; 16], &response).expect("scanner activity response should encode");
let variants = [
ScannerActivityResponse {
instance_id: "1123456789abcdef0123456789abcdef".to_string(),
..response.clone()
},
ScannerActivityResponse {
namespace_generation: 8,
..response.clone()
},
ScannerActivityResponse {
maintenance_generation: 4,
..response.clone()
},
ScannerActivityResponse {
protocol_version: 5,
..response.clone()
},
ScannerActivityResponse {
topology_digest: vec![8; 32].into(),
..response.clone()
},
ScannerActivityResponse {
data_movement_active: false,
..response.clone()
},
ScannerActivityResponse {
dirty_usage_generation: 12,
..response.clone()
},
ScannerActivityResponse {
dirty_usage_pending: false,
..response.clone()
},
];
for variant in variants {
assert_ne!(
baseline,
canonical_scanner_activity_response_body(&[1; 16], &variant)
.expect("scanner activity response variant should encode")
);
}
assert_ne!(
baseline,
canonical_scanner_activity_response_body(&[2; 16], &response)
.expect("scanner activity response with a different challenge should encode")
);
}
#[test]
fn scanner_activity_v4_response_canonicalization_ignores_v5_fields() {
let response = ScannerActivityResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
namespace_generation: 7,
maintenance_generation: 3,
protocol_version: 4,
topology_digest: vec![9; 32].into(),
data_movement_active: true,
response_proof: Vec::new().into(),
dirty_usage_generation: 0,
dirty_usage_pending: false,
};
let baseline =
canonical_scanner_activity_v4_response_body(&[1; 16], &response).expect("scanner activity v4 response should encode");
let expected = [
b"rustfs-scanner-activity-response-v1\0".as_slice(),
16u64.to_be_bytes().as_slice(),
[1u8; 16].as_slice(),
32u64.to_be_bytes().as_slice(),
b"0123456789abcdef0123456789abcdef".as_slice(),
7u64.to_be_bytes().as_slice(),
3u64.to_be_bytes().as_slice(),
4u32.to_be_bytes().as_slice(),
32u64.to_be_bytes().as_slice(),
[9u8; 32].as_slice(),
[1u8].as_slice(),
]
.concat();
assert_eq!(
baseline, expected,
"protocol v4 response bytes must remain stable during rolling upgrades"
);
let extended = ScannerActivityResponse {
dirty_usage_generation: 11,
dirty_usage_pending: true,
..response
};
assert_eq!(
baseline,
canonical_scanner_activity_v4_response_body(&[1; 16], &extended)
.expect("scanner activity v4 response should ignore v5 fields")
);
}
}
#[cfg(test)]
mod heal_control_tests {
use super::{
crates/protos/src/node.proto
+13 -1
View File
@@ -756,14 +756,26 @@ message SignalServiceRequest {
message SignalServiceResponse {
bool success = 1;
optional string error_info = 2;
uint32 protocol_version = 3;
}
message ScannerActivityRequest {}
message ScannerActivityRequest {
bytes challenge = 1;
uint32 protocol_version = 2;
string acknowledge_instance_id = 3;
uint64 acknowledge_dirty_usage_generation = 4;
}
message ScannerActivityResponse {
string instance_id = 1;
uint64 namespace_generation = 2;
uint64 maintenance_generation = 3;
uint32 protocol_version = 4;
bytes topology_digest = 5;
bool data_movement_active = 6;
bytes response_proof = 7;
uint64 dirty_usage_generation = 8;
bool dirty_usage_pending = 9;
}
message BackgroundHealStatusRequest {}
crates/rio/src/http_reader.rs
+8 -1
View File
@@ -19,7 +19,8 @@ use http::{HeaderMap, Version};
use pin_project_lite::pin_project;
use reqwest::{Certificate, Client, Identity, Method, RequestBuilder};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_OPERATION_WALK_DIR,
};
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_opt_u64, get_env_opt_usize};
@@ -43,6 +44,7 @@ use tracing::{error, warn};
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner";
const HTTP_VERSION_09_LABEL: &str = "http/0.9";
const HTTP_VERSION_10_LABEL: &str = "http/1.0";
const HTTP_VERSION_11_LABEL: &str = "http/1.1";
@@ -1120,6 +1122,7 @@ fn internode_rpc_operation(url: &str) -> Option<&'static str> {
READ_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_READ_FILE_STREAM),
PUT_FILE_STREAM_PATH => Some(INTERNODE_OPERATION_PUT_FILE_STREAM),
WALK_DIR_PATH => Some(INTERNODE_OPERATION_WALK_DIR),
NS_SCANNER_PATH => Some(INTERNODE_OPERATION_NS_SCANNER),
_ => None,
}
}
@@ -1770,6 +1773,10 @@ mod tests {
internode_rpc_operation(&format!("http://node:9000{WALK_DIR_PATH}?disk=d")),
Some(INTERNODE_OPERATION_WALK_DIR)
);
assert_eq!(
internode_rpc_operation(&format!("http://node:9000{NS_SCANNER_PATH}?disk=d")),
Some(INTERNODE_OPERATION_NS_SCANNER)
);
assert_eq!(internode_rpc_operation("http://node:9000/rustfs/rpc/unknown"), None);
assert_eq!(
internode_rpc_operation("http://node:9000/rustfs/rpc/unknown?next=/rustfs/rpc/read_file_stream"),
crates/scanner/Cargo.toml
+7
View File
@@ -32,6 +32,7 @@ workspace = true
[dependencies]
rustfs-config = { workspace = true, features = ["server-config-model"] }
rustfs-common = { workspace = true }
rustfs-credentials = { workspace = true }
rustfs-utils = { workspace = true }
tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] }
tracing = { workspace = true }
@@ -43,6 +44,8 @@ futures = { workspace = true }
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
chrono = { workspace = true, features = ["serde"] }
rmp-serde = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
rustfs-filemeta = { workspace = true }
tokio-util = { workspace = true, features = ["io", "compat"] }
rustfs-ecstore = { workspace = true }
@@ -52,11 +55,15 @@ rand = { workspace = true, features = ["serde"] }
s3s = { workspace = true, features = ["minio"] }
metrics = { workspace = true }
rustfs-data-usage = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde", "fast-rng"] }
bytes.workspace = true
hex-simd.workspace = true
[dev-dependencies]
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
serial_test = { workspace = true }
temp-env = { workspace = true }
tempfile = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
# Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via
crates/scanner/src/data_usage_define.rs
+1485 -128
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/error.rs
+16
View File
@@ -36,7 +36,23 @@ pub enum ScannerError {
#[error("Scanner error: {0}")]
Other(String),
/// A remote namespace scanner request ID was already accepted.
#[error("Remote namespace scanner request replay detected")]
RemoteRequestReplay,
/// The bounded remote namespace scanner replay cache cannot accept more IDs yet.
#[error("Remote namespace scanner replay cache capacity exceeded")]
RemoteReplayCapacity,
/// A remote namespace scanner request is already active for the target disk.
#[error("Remote namespace scanner disk is already active")]
RemoteDiskBusy,
/// Partial data usage cache produced before the scanner stopped.
#[error("Scanner stopped with partial data usage cache")]
PartialCache(Box<DataUsageCache>),
/// Partial cache retained because the bucket or scan root disappeared.
#[error("Scanner namespace path disappeared during the scan")]
NamespaceNotFoundCache(Box<DataUsageCache>),
}
crates/scanner/src/lib.rs
+104 -8
View File
@@ -20,6 +20,7 @@
rust_2018_idioms
)]
use bytes::Bytes;
use http::HeaderMap;
use rustfs_config::server_config::{Config as ServerConfig, get_global_server_config as config_get_global_server_config};
use std::path::PathBuf;
@@ -29,11 +30,11 @@ use storage_api::owner::{
ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk,
EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, EcstoreDiskLocation,
EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc, EcstoreLifecycle,
EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge,
EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig,
EcstoreVersioningApi, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, ScannerReplicationHealObject,
ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule,
ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, EcstoreReplicationConfigurationExt,
EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore,
EcstoreTierConfig, EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete,
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
@@ -41,11 +42,16 @@ use storage_api::owner::{
scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
use storage_api::owner::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
use storage_api::owner::{
EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints, EcstoreInstanceContext,
EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
ecstore_new_disk,
};
use tokio_util::sync::CancellationToken;
pub mod data_usage_define;
pub mod error;
mod remote_scanner;
pub mod runtime_config;
pub mod scanner;
pub mod scanner_budget;
@@ -56,16 +62,23 @@ pub(crate) mod storage_api;
pub use data_usage_define::*;
pub use error::ScannerError;
pub use remote_scanner::{
NS_SCANNER_MAX_REQUEST_BODY_SIZE, RemoteScannerAdmission, RemoteScannerRequest, admit_remote_scanner_request,
claim_remote_scanner_request, decode_remote_scanner_request, preflight_remote_scanner_request,
remote_scanner_request_matches_envelope, serve_remote_scanner_request, validate_remote_scanner_request_fence,
};
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
pub use rustfs_common::last_minute;
pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status};
pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status, scanner_topology_digest};
pub use scanner_io::{
clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch,
ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,
scanner_maintenance_generation,
};
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
use std::sync::atomic::{AtomicU64, Ordering};
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
pub use storage_api::scan::SCANNER_ACTIVITY_PROTOCOL_VERSION;
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
static SCANNER_FOREGROUND_READ_ACTIVITY: AtomicU64 = AtomicU64::new(0);
@@ -150,6 +163,7 @@ pub(crate) type BucketTargetSys = EcstoreBucketTargetSys;
pub(crate) type BucketVersioningSys = EcstoreBucketVersioningSys;
pub(crate) type DiskInfo = EcstoreDiskInfo;
pub(crate) type DiskInfoOptions = EcstoreDiskInfoOptions;
pub(crate) type NsScannerOpenRequest = EcstoreNsScannerOpenRequest;
pub(crate) type DiskBytes = EcstoreDiskBytes;
pub(crate) type Evaluator = EcstoreEvaluator;
pub(crate) type Event = EcstoreEvent;
@@ -185,6 +199,27 @@ pub(crate) fn init_ecstore_config_for_scanner_tests() {
pub(crate) type DiskOption = EcstoreDiskOption;
#[cfg(test)]
pub(crate) type Endpoint = EcstoreEndpoint;
#[cfg(test)]
pub(crate) type EndpointServerPools = EcstoreEndpointServerPools;
#[cfg(test)]
pub(crate) type Endpoints = EcstoreEndpoints;
#[cfg(test)]
pub(crate) type InstanceContext = EcstoreInstanceContext;
#[cfg(test)]
pub(crate) type PoolEndpoints = EcstorePoolEndpoints;
#[cfg(test)]
pub(crate) async fn init_local_disks_with_instance_ctx(
ctx: &Arc<InstanceContext>,
pools: EndpointServerPools,
) -> EcstoreResult<()> {
ecstore_init_local_disks_with_instance_ctx(ctx, pools).await
}
#[cfg(test)]
pub(crate) async fn init_bucket_metadata_sys_for_scanner_tests(store: Arc<ECStore>) {
ecstore_init_bucket_metadata_sys(store, Vec::new()).await;
}
#[cfg(test)]
pub(crate) async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> DiskResult<DiskStore> {
@@ -247,6 +282,8 @@ impl ScannerVersioningConfigExt for s3s::dto::VersioningConfiguration {
pub(crate) trait ScannerDiskExt {
async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult<DiskInfo>;
async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult<DiskBytes>;
fn is_local(&self) -> bool;
fn host_name(&self) -> String;
fn path(&self) -> PathBuf;
fn get_disk_location(&self) -> DiskLocation;
fn start_scan(&self) -> ScanGuard;
@@ -264,6 +301,14 @@ where
EcstoreDiskAPI::read_metadata(self, volume, path).await
}
fn is_local(&self) -> bool {
EcstoreDiskAPI::is_local(self)
}
fn host_name(&self) -> String {
EcstoreDiskAPI::host_name(self)
}
fn path(&self) -> PathBuf {
EcstoreDiskAPI::path(self)
}
@@ -344,6 +389,10 @@ pub(crate) async fn scanner_is_erasure_sd() -> bool {
ecstore_is_erasure_sd().await
}
pub(crate) async fn scanner_disk_is_online(disk: &Disk) -> bool {
EcstoreDiskAPI::is_online(disk).await
}
pub(crate) async fn read_config<S>(api: Arc<S>, file: &str) -> EcstoreResult<Vec<u8>>
where
S: ScannerObjectIO,
@@ -358,6 +407,53 @@ where
ecstore_save_config(api, file, data).await
}
pub(crate) async fn save_config_with_preconditions<S>(
api: Arc<S>,
file: &str,
data: Vec<u8>,
preconditions: HTTPPreconditions,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO,
{
let mut reader = ScannerPutObjReader::from_vec(data);
api.put_object(
RUSTFS_META_BUCKET,
file,
&mut reader,
&ScannerObjectOptions {
max_parity: true,
http_preconditions: Some(preconditions),
..Default::default()
},
)
.await
}
pub(crate) async fn save_config_shared_with_preconditions<S>(
api: Arc<S>,
file: &str,
data: Bytes,
sha256hex: Option<String>,
preconditions: HTTPPreconditions,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO,
{
let mut reader = ScannerPutObjReader::from_prehashed_bytes(data, sha256hex)?;
api.put_object(
RUSTFS_META_BUCKET,
file,
&mut reader,
&ScannerObjectOptions {
max_parity: true,
http_preconditions: Some(preconditions),
..Default::default()
},
)
.await
}
pub(crate) async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> std::result::Result<(), DiskError> {
ecstore_list_path_raw(rx, opts).await
}
crates/scanner/src/remote_scanner.rs
+3011
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/scanner.rs
+3089 -167
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/scanner_budget.rs
+223 -12
View File
@@ -16,6 +16,7 @@ use std::sync::{
Arc,
atomic::{AtomicU8, AtomicU64, Ordering},
};
use std::time::Instant;
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
@@ -61,15 +62,26 @@ impl ScannerCycleBudgetReason {
pub struct ScannerCycleBudget {
token: CancellationToken,
reason: Arc<AtomicU8>,
started_at: Instant,
max_duration: Option<Duration>,
max_objects: Option<u64>,
max_directories: Option<u64>,
track_progress: bool,
objects_scanned: AtomicU64,
directories_started: AtomicU64,
entries_visited: AtomicU64,
}
impl ScannerCycleBudget {
pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
Self::new_inner(parent, config, false)
}
pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
Self::new_inner(parent, config, true)
}
fn new_inner(parent: &CancellationToken, config: ScannerCycleBudgetConfig, track_progress: bool) -> Arc<Self> {
let token = parent.child_token();
let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE));
@@ -92,11 +104,14 @@ impl ScannerCycleBudget {
Arc::new(Self {
token,
reason,
started_at: Instant::now(),
max_duration: config.max_duration,
max_objects: config.max_objects,
max_directories: config.max_directories,
track_progress,
objects_scanned: AtomicU64::new(0),
directories_started: AtomicU64::new(0),
entries_visited: AtomicU64::new(0),
})
}
@@ -124,27 +139,97 @@ impl ScannerCycleBudget {
self.max_directories
}
pub(crate) fn try_start_directory(&self) -> bool {
let Some(max_directories) = self.max_directories else {
return true;
};
pub(crate) fn requires_serial_progress_accounting(&self) -> bool {
self.max_objects.is_some() || self.max_directories.is_some()
}
let directories = self.directories_started.fetch_add(1, Ordering::Relaxed) + 1;
if directories <= max_directories {
pub(crate) fn remaining_config(&self) -> ScannerCycleBudgetConfig {
let max_duration = self
.max_duration
.map(|duration| duration.saturating_sub(self.started_at.elapsed()));
if max_duration.is_some_and(|duration| duration.is_zero()) {
self.cancel_for(ScannerCycleBudgetReason::Runtime);
}
ScannerCycleBudgetConfig {
max_duration,
max_objects: self
.max_objects
.map(|max| max.saturating_sub(self.objects_scanned.load(Ordering::Relaxed))),
max_directories: self
.max_directories
.map(|max| max.saturating_sub(self.directories_started.load(Ordering::Relaxed))),
}
}
pub(crate) fn progress(&self) -> (u64, u64) {
(
self.objects_scanned.load(Ordering::Relaxed),
self.directories_started.load(Ordering::Relaxed),
)
}
pub(crate) fn entries_visited(&self) -> u64 {
self.entries_visited.load(Ordering::Relaxed)
}
pub(crate) fn record_entries_visited(&self, entries_visited: u64) {
if self.track_progress {
saturating_fetch_add(&self.entries_visited, entries_visited);
}
}
pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) {
if self.track_progress || self.max_objects.is_some() {
let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned);
if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
self.cancel_for(ScannerCycleBudgetReason::Objects);
}
}
if self.track_progress || self.max_directories.is_some() {
let directories = saturating_fetch_add(&self.directories_started, directories_started);
if self
.max_directories
.is_some_and(|max_directories| directories > max_directories)
{
self.cancel_for(ScannerCycleBudgetReason::Directories);
}
}
}
pub(crate) fn cancel_after_unreported_remote_progress(&self) {
if self.max_objects.is_some() {
self.cancel_for(ScannerCycleBudgetReason::Objects);
} else if self.max_directories.is_some() {
self.cancel_for(ScannerCycleBudgetReason::Directories);
}
}
pub(crate) fn try_start_directory(&self) -> bool {
if !self.track_progress && self.max_directories.is_none() {
return true;
}
self.cancel_for(ScannerCycleBudgetReason::Directories);
false
let directories = saturating_fetch_add(&self.directories_started, 1);
if self
.max_directories
.is_some_and(|max_directories| directories > max_directories)
{
self.cancel_for(ScannerCycleBudgetReason::Directories);
return false;
}
true
}
pub(crate) fn record_object_scanned(&self) {
let Some(max_objects) = self.max_objects else {
if !self.track_progress && self.max_objects.is_none() {
return;
};
}
let objects = self.objects_scanned.fetch_add(1, Ordering::Relaxed) + 1;
if objects >= max_objects {
let objects = saturating_fetch_add(&self.objects_scanned, 1);
if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
self.cancel_for(ScannerCycleBudgetReason::Objects);
}
}
@@ -163,6 +248,17 @@ impl ScannerCycleBudget {
}
}
fn saturating_fetch_add(value: &AtomicU64, delta: u64) -> u64 {
let mut current = value.load(Ordering::Relaxed);
loop {
let next = current.saturating_add(delta);
match value.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return next,
Err(observed) => current = observed,
}
}
}
impl Drop for ScannerCycleBudget {
fn drop(&mut self) {
self.token.cancel();
@@ -250,4 +346,119 @@ mod tests {
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
assert!(budget.token().is_cancelled());
}
#[tokio::test]
async fn remaining_config_accounts_for_local_and_remote_progress() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_duration: Some(Duration::from_secs(60)),
max_objects: Some(10),
max_directories: Some(5),
},
);
budget.record_object_scanned();
assert!(budget.try_start_directory());
budget.record_remote_progress(3, 2);
let remaining = budget.remaining_config();
assert!(
remaining
.max_duration
.is_some_and(|duration| duration <= Duration::from_secs(60))
);
assert_eq!(remaining.max_objects, Some(6));
assert_eq!(remaining.max_directories, Some(2));
assert_eq!(budget.progress(), (4, 3));
assert!(!budget.budget_elapsed());
}
#[test]
fn remote_progress_preserves_object_and_directory_limit_semantics() {
let parent = CancellationToken::new();
let object_budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(2),
..Default::default()
},
);
object_budget.record_remote_progress(2, 0);
assert_eq!(object_budget.reason(), Some(ScannerCycleBudgetReason::Objects));
let directory_budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_directories: Some(2),
..Default::default()
},
);
directory_budget.record_remote_progress(0, 2);
assert!(!directory_budget.budget_elapsed());
directory_budget.record_remote_progress(0, 1);
assert_eq!(directory_budget.reason(), Some(ScannerCycleBudgetReason::Directories));
}
#[test]
fn explicit_progress_tracking_counts_unbounded_remote_work_without_cancelling() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default());
assert!(budget.try_start_directory());
budget.record_object_scanned();
budget.record_remote_progress(2, 3);
assert_eq!(budget.progress(), (3, 4));
assert!(!budget.budget_elapsed());
assert!(!budget.token().is_cancelled());
}
#[test]
fn unreported_remote_progress_cancels_count_budget() {
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(10),
..Default::default()
},
);
budget.cancel_after_unreported_remote_progress();
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
assert!(budget.token().is_cancelled());
}
#[tokio::test]
async fn count_budgets_require_serial_progress_accounting() {
let parent = CancellationToken::new();
let runtime_only = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_duration: Some(Duration::from_secs(1)),
..Default::default()
},
);
let object_limited = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(1),
..Default::default()
},
);
let directory_limited = ScannerCycleBudget::new(
&parent,
ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
assert!(!runtime_only.requires_serial_progress_accounting());
assert!(object_limited.requires_serial_progress_accounting());
assert!(directory_limited.requires_serial_progress_accounting());
}
}
crates/scanner/src/scanner_folder.rs
+341 -49
View File
@@ -38,8 +38,8 @@ use rustfs_common::heal_channel::{
HealRequestSource, HealScanMode, send_heal_request_with_admission,
};
use rustfs_common::metrics::{
IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn,
current_path_updater, global_metrics,
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
UpdateCurrentPathFn, current_path_updater, global_metrics,
};
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
@@ -76,6 +76,8 @@ const DATA_SCANNER_COMPACT_AT_CHILDREN: usize = 10000;
const DATA_SCANNER_COMPACT_AT_FOLDERS: usize = DATA_SCANNER_COMPACT_AT_CHILDREN / 4;
const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000;
const SCANNER_LIST_PATH_RAW_TIMEOUT: Duration = Duration::from_secs(60);
const SCANNER_ENTRY_PROGRESS_BATCH: u64 = 32;
const SCANNER_ENTRY_PROGRESS_INTERVAL: Duration = Duration::from_secs(30);
const DEFAULT_HEAL_OBJECT_SELECT_PROB: u32 = 1024;
const ENV_DATA_USAGE_UPDATE_DIR_CYCLES: &str = "RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES";
const ENV_HEAL_OBJECT_SELECT_PROB: &str = "RUSTFS_HEAL_OBJECT_SELECT_PROB";
@@ -411,24 +413,36 @@ fn non_negative_i64_to_u64(value: i64) -> u64 {
}
fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary) {
into.size += summary.total_size;
into.versions += summary.versions;
into.delete_markers += summary.delete_markers;
into.obj_sizes.add(summary.total_size as u64);
into.obj_versions.add(summary.versions as u64);
into.size = into.size.saturating_add(summary.total_size);
into.versions = into.versions.saturating_add(summary.versions);
into.delete_markers = into.delete_markers.saturating_add(summary.delete_markers);
into.obj_sizes.add(u64::try_from(summary.total_size).unwrap_or(u64::MAX));
into.obj_versions.add(u64::try_from(summary.versions).unwrap_or(u64::MAX));
let replication_stats = into.replication_stats.get_or_insert_with(Default::default);
replication_stats.replica_size += non_negative_i64_to_u64(summary.replica_size);
replication_stats.replica_count += summary.replica_count as u64;
replication_stats.replica_size = replication_stats
.replica_size
.saturating_add(non_negative_i64_to_u64(summary.replica_size));
replication_stats.replica_count = replication_stats
.replica_count
.saturating_add(u64::try_from(summary.replica_count).unwrap_or(u64::MAX));
for (arn, st) in &summary.repl_target_stats {
let tgt_stat = replication_stats.targets.entry(arn.clone()).or_default();
tgt_stat.pending_size += non_negative_i64_to_u64(st.pending_size);
tgt_stat.failed_size += non_negative_i64_to_u64(st.failed_size);
tgt_stat.replicated_size += non_negative_i64_to_u64(st.replicated_size);
tgt_stat.replicated_count += st.replicated_count as u64;
tgt_stat.failed_count += st.failed_count as u64;
tgt_stat.pending_count += st.pending_count as u64;
tgt_stat.pending_size = tgt_stat.pending_size.saturating_add(non_negative_i64_to_u64(st.pending_size));
tgt_stat.failed_size = tgt_stat.failed_size.saturating_add(non_negative_i64_to_u64(st.failed_size));
tgt_stat.replicated_size = tgt_stat
.replicated_size
.saturating_add(non_negative_i64_to_u64(st.replicated_size));
tgt_stat.replicated_count = tgt_stat
.replicated_count
.saturating_add(u64::try_from(st.replicated_count).unwrap_or(u64::MAX));
tgt_stat.failed_count = tgt_stat
.failed_count
.saturating_add(u64::try_from(st.failed_count).unwrap_or(u64::MAX));
tgt_stat.pending_count = tgt_stat
.pending_count
.saturating_add(u64::try_from(st.pending_count).unwrap_or(u64::MAX));
}
}
@@ -1036,22 +1050,23 @@ impl ScannerItem {
if let Some(repl_target_size_summary) = size_summary.repl_target_stats.get_mut(arn.as_str()) {
match target_status {
ReplicationStatusType::Pending => {
repl_target_size_summary.pending_size += roi.size;
repl_target_size_summary.pending_count += 1;
size_summary.pending_size += roi.size;
size_summary.pending_count += 1;
repl_target_size_summary.pending_size = repl_target_size_summary.pending_size.saturating_add(roi.size);
repl_target_size_summary.pending_count = repl_target_size_summary.pending_count.saturating_add(1);
size_summary.pending_size = size_summary.pending_size.saturating_add(roi.size);
size_summary.pending_count = size_summary.pending_count.saturating_add(1);
}
ReplicationStatusType::Failed => {
repl_target_size_summary.failed_size += roi.size;
repl_target_size_summary.failed_count += 1;
size_summary.failed_size += roi.size;
size_summary.failed_count += 1;
repl_target_size_summary.failed_size = repl_target_size_summary.failed_size.saturating_add(roi.size);
repl_target_size_summary.failed_count = repl_target_size_summary.failed_count.saturating_add(1);
size_summary.failed_size = size_summary.failed_size.saturating_add(roi.size);
size_summary.failed_count = size_summary.failed_count.saturating_add(1);
}
ReplicationStatusType::Completed | ReplicationStatusType::CompletedLegacy => {
repl_target_size_summary.replicated_size += roi.size;
repl_target_size_summary.replicated_count += 1;
size_summary.replicated_size += roi.size;
size_summary.replicated_count += 1;
repl_target_size_summary.replicated_size =
repl_target_size_summary.replicated_size.saturating_add(roi.size);
repl_target_size_summary.replicated_count = repl_target_size_summary.replicated_count.saturating_add(1);
size_summary.replicated_size = size_summary.replicated_size.saturating_add(roi.size);
size_summary.replicated_count = size_summary.replicated_count.saturating_add(1);
}
_ => {}
}
@@ -1059,8 +1074,8 @@ impl ScannerItem {
}
if oi.replication_status == ReplicationStatusType::Replica {
size_summary.replica_size += roi.size;
size_summary.replica_count += 1;
size_summary.replica_size = size_summary.replica_size.saturating_add(roi.size);
size_summary.replica_count = size_summary.replica_count.saturating_add(1);
}
}
@@ -1835,21 +1850,10 @@ impl FolderScanner {
let mut dir_reader = match tokio::fs::read_dir(&dir_path).await {
Ok(dir_reader) => dir_reader,
Err(e) if e.kind() == ErrorKind::NotFound => {
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
dir_path = %dir_path,
state = "dir_missing_before_read",
error = %e,
"Scanner folder state updated"
);
return Ok(());
}
Err(e) => return Err(ScannerError::Io(e)),
};
let mut pending_entry_progress = 0_u64;
let mut last_entry_progress = Instant::now();
loop {
let entry = match dir_reader.next_entry().await {
@@ -1883,6 +1887,14 @@ impl FolderScanner {
}
Err(e) => return Err(ScannerError::Io(e)),
};
pending_entry_progress = pending_entry_progress.saturating_add(1);
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
{
self.budget.record_entries_visited(pending_entry_progress);
pending_entry_progress = 0;
last_entry_progress = Instant::now();
}
let file_name = entry.file_name().to_string_lossy().to_string();
if file_name.is_empty() || file_name == "." || file_name == ".." {
continue;
@@ -2122,6 +2134,7 @@ impl FolderScanner {
global_metrics().record_scanner_yield(yield_start.elapsed());
}
}
self.budget.record_entries_visited(pending_entry_progress);
if ctx.is_cancelled() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
@@ -2759,7 +2772,8 @@ pub async fn scan_data_folder(
// Get disk path
let base_path = local_disk.path().to_string_lossy().to_string();
let (update_current_path, close_disk) = current_path_updater(&base_path, &cache.info.name);
let (update_current_path, close_disk) = current_path_updater(&base_path, &cache.info.name).await;
let mut close_disk_guard = CloseDiskGuard::new(close_disk);
// Create skip_heal flag
let is_erasure_mode = scanner_is_erasure().await;
@@ -2829,6 +2843,8 @@ pub async fn scan_data_folder(
new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
new_cache.info.last_update = Some(SystemTime::now());
new_cache.info.next_cycle = cache.info.next_cycle;
let unresolved_objects = root.failed_objects > 0 || !new_cache.info.failed_objects.is_empty();
new_cache.info.snapshot_complete = !unresolved_objects;
let had_scan_checkpoint = cache.info.scan_checkpoint.is_some() || new_cache.info.scan_checkpoint.is_some();
new_cache.info.scan_resume_after = None;
new_cache.info.scan_checkpoint = None;
@@ -2836,8 +2852,12 @@ pub async fn scan_data_folder(
global_metrics().record_scanner_checkpoint_cleared();
}
close_disk().await;
Ok(new_cache.clone())
close_disk_guard.close().await;
if unresolved_objects {
Err(ScannerError::PartialCache(Box::new(new_cache.clone())))
} else {
Ok(new_cache.clone())
}
}
Err(e) => {
if ctx.is_cancelled() {
@@ -2857,14 +2877,23 @@ pub async fn scan_data_folder(
}
new_cache.info.last_update = Some(SystemTime::now());
new_cache.info.next_cycle = cache.info.next_cycle;
new_cache.info.snapshot_complete = false;
if root_has_progress {
set_scan_checkpoint(new_cache, checkpoint_reason_from_budget(budget.reason()));
}
close_disk().await;
close_disk_guard.close().await;
return Err(ScannerError::PartialCache(Box::new(new_cache.clone())));
}
}
close_disk().await;
if matches!(&e, ScannerError::Io(io) if io.kind() == ErrorKind::NotFound) {
let mut partial_cache = scanner.old_cache.clone();
partial_cache.info.last_update = Some(SystemTime::now());
partial_cache.info.next_cycle = cache.info.next_cycle;
partial_cache.info.snapshot_complete = false;
close_disk_guard.close().await;
return Err(ScannerError::NamespaceNotFoundCache(Box::new(partial_cache)));
}
close_disk_guard.close().await;
// No useful information, return original cache
Err(e)
}
@@ -2877,7 +2906,7 @@ mod tests {
use super::*;
use crate::storage_api::VersionPurgeStatusType;
use crate::{DiskOption, Endpoint, new_disk};
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, new_disk};
use rustfs_filemeta::{FileInfo, FileMeta};
use serial_test::serial;
#[cfg(unix)]
@@ -2886,6 +2915,76 @@ mod tests {
use temp_env::{with_var, with_var_unset};
use uuid::Uuid;
#[test]
fn scanner_size_summary_application_saturates_usage_counters() {
let target = "arn:minio:replication::target".to_string();
let mut entry = DataUsageEntry {
size: usize::MAX,
versions: usize::MAX,
delete_markers: usize::MAX,
replication_stats: Some(Default::default()),
..Default::default()
};
let replication_stats = entry
.replication_stats
.as_mut()
.expect("replication statistics should be initialized");
replication_stats.replica_size = u64::MAX;
replication_stats.replica_count = u64::MAX;
let target_stats = replication_stats.targets.entry(target.clone()).or_default();
target_stats.pending_size = u64::MAX;
target_stats.failed_size = u64::MAX;
target_stats.replicated_size = u64::MAX;
target_stats.replicated_count = u64::MAX;
target_stats.failed_count = u64::MAX;
target_stats.pending_count = u64::MAX;
let mut summary = SizeSummary {
total_size: 1,
versions: 1,
delete_markers: 1,
replica_size: 1,
replica_count: 1,
..Default::default()
};
summary.repl_target_stats.insert(
target.clone(),
ReplTargetSizeSummary {
replicated_size: 1,
replicated_count: 1,
pending_size: 1,
failed_size: 1,
pending_count: 1,
failed_count: 1,
},
);
apply_scanner_size_summary(&mut entry, &summary);
assert_eq!(entry.size, usize::MAX);
assert_eq!(entry.versions, usize::MAX);
assert_eq!(entry.delete_markers, usize::MAX);
assert_eq!(entry.obj_sizes.to_map()["LESS_THAN_1024_B"], 1);
assert_eq!(entry.obj_versions.to_map()["SINGLE_VERSION"], 1);
let replication_stats = entry
.replication_stats
.as_ref()
.expect("replication statistics should remain present");
assert_eq!(replication_stats.replica_size, u64::MAX);
assert_eq!(replication_stats.replica_count, u64::MAX);
let target_stats = replication_stats
.targets
.get(&target)
.expect("replication target statistics should remain present");
assert_eq!(target_stats.pending_size, u64::MAX);
assert_eq!(target_stats.failed_size, u64::MAX);
assert_eq!(target_stats.replicated_size, u64::MAX);
assert_eq!(target_stats.replicated_count, u64::MAX);
assert_eq!(target_stats.failed_count, u64::MAX);
assert_eq!(target_stats.pending_count, u64::MAX);
}
async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
let temp_dir = std::env::temp_dir().join(format!("rustfs-scanner-test-{}", Uuid::new_v4()));
tokio::fs::create_dir_all(&temp_dir)
@@ -4145,7 +4244,7 @@ mod tests {
scanner.update_cache.info.name = "bucket".to_string();
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(1),
@@ -4168,6 +4267,7 @@ mod tests {
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
assert!(budget.token().is_cancelled());
assert!(budget.entries_visited() >= 1);
}
#[tokio::test]
@@ -4212,6 +4312,51 @@ mod tests {
assert!(update.compacted, "partial update should preserve compacted state");
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_cancelled_before_scan_clears_current_path() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir),
};
let parent = CancellationToken::new();
parent.cancel();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let cache = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
let disk_path = scanner.local_disk.path().to_string_lossy().to_string();
let result = scan_data_folder(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
scanner.local_disk,
cache,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
assert!(matches!(result, Err(ScannerError::Other(message)) if message == "Operation cancelled"));
tokio::time::timeout(Duration::from_secs(1), async {
loop {
let report = global_metrics().report().await;
if report.active_paths.iter().all(|path| !path.starts_with(&disk_path)) {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("cancelled scan should deregister its active path");
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
@@ -4262,6 +4407,7 @@ mod tests {
assert!(partial_cache.info.last_update.is_some());
assert_eq!(partial_cache.info.next_cycle, 7);
assert!(!partial_cache.info.snapshot_complete);
assert!(partial_cache.root().is_some(), "partial cache should keep completed scan progress");
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
@@ -4386,6 +4532,104 @@ mod tests {
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
}
#[tokio::test]
#[serial]
async fn scan_data_folder_missing_bucket_returns_partial() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir),
};
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let mut cache = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "missing-bucket".to_string(),
next_cycle: 9,
..Default::default()
},
..Default::default()
};
cache.replace(
"missing-bucket",
crate::data_usage_define::DATA_USAGE_ROOT,
DataUsageEntry {
objects: 7,
size: 70,
..Default::default()
},
);
let result = scan_data_folder(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
scanner.local_disk,
cache,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
let partial = match result {
Err(ScannerError::NamespaceNotFoundCache(partial)) => partial,
other => panic!("missing bucket should keep the scan incomplete, got {other:?}"),
};
assert!(!partial.info.snapshot_complete);
assert_eq!(partial.info.next_cycle, 9);
let root = partial
.checked_flatten("missing-bucket")
.expect("missing bucket partial must retain the last durable usage");
assert_eq!(root.objects, 7);
assert_eq!(root.size, 70);
}
#[tokio::test]
#[serial]
async fn scan_data_folder_missing_scan_root_returns_partial() {
let (scanner, temp_dir) = build_test_scanner().await;
tokio::fs::remove_dir_all(&temp_dir)
.await
.expect("failed to remove scanner root");
let _guard = TestGuard {
temp_dir: Some(temp_dir),
};
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let cache = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "missing-bucket".to_string(),
next_cycle: 9,
..Default::default()
},
..Default::default()
};
let result = scan_data_folder(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
scanner.local_disk,
cache,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
let partial = match result {
Err(ScannerError::NamespaceNotFoundCache(partial)) => partial,
other => panic!("missing scan root should keep the scan incomplete, got {other:?}"),
};
assert!(!partial.info.snapshot_complete);
assert_eq!(partial.info.next_cycle, 9);
let root = partial
.checked_flatten("missing-bucket")
.expect("missing scan root partial should retain a non-publishable root");
assert_eq!(root.objects, 0);
assert_eq!(root.size, 0);
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folders() {
@@ -4533,6 +4777,7 @@ mod tests {
.size_recursive("bucket")
.expect("completed cache should retain bucket usage");
assert_eq!(root.objects, 5);
assert!(result.info.snapshot_complete);
assert!(result.info.scan_resume_after.is_none());
assert!(result.info.scan_checkpoint.is_none());
}
@@ -4626,6 +4871,53 @@ mod tests {
assert_eq!(result.info.next_cycle, 11);
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_keeps_unresolved_objects_partial() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir.clone()),
};
write_test_object_metadata(&temp_dir, "bucket", "object").await;
let failed_path = temp_dir
.join("bucket")
.join("object")
.join(STORAGE_FORMAT_FILE)
.to_string_lossy()
.into_owned();
let mut cache = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 12,
..Default::default()
},
..Default::default()
};
cache.info.failed_objects.insert(failed_path, FolderScanner::now_secs());
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let result = scan_data_folder(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
scanner.local_disk.clone(),
cache,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
let partial = match result {
Err(ScannerError::PartialCache(partial)) => partial,
other => panic!("expected unresolved object to keep the cache partial, got {other:?}"),
};
assert!(!partial.info.snapshot_complete);
assert!(!partial.info.failed_objects.is_empty());
}
#[tokio::test]
#[serial]
#[cfg(unix)]
crates/scanner/src/scanner_io.rs
+2407 -360
View File
File diff suppressed because it is too large Load Diff
crates/scanner/src/storage_api.rs
+38 -15
View File
@@ -27,6 +27,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE, object_opts_from_object_info as ecstore_object_opts_from_object_info,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
get_replication_config as ecstore_get_replication_config,
@@ -62,8 +64,8 @@ pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError,
pub(crate) use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI,
DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation,
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE,
ScanGuard as EcstoreScanGuard,
NsScannerOpenRequest as EcstoreNsScannerOpenRequest, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE, ScanGuard as EcstoreScanGuard,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{
@@ -72,6 +74,12 @@ pub(crate) use rustfs_ecstore::api::disk::{
pub(crate) use rustfs_ecstore::api::error::{
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::layout::{
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
pub(crate) use rustfs_ecstore::api::runtime::{
expiry_state_handle as ecstore_expiry_state_handle, global_tier_config_mgr as ecstore_get_global_tier_config_mgr,
object_store_handle as ecstore_resolve_object_store_handle, setup_is_erasure as ecstore_is_erasure,
@@ -79,30 +87,39 @@ pub(crate) use rustfs_ecstore::api::runtime::{
};
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx;
pub(crate) use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
use rustfs_storage_api as storage_contracts;
pub(crate) mod owner {
pub(crate) use super::storage_contracts::{HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete};
pub(crate) use super::storage_contracts::{
HTTPPreconditions, HTTPRangeSpec, NS_SCANNER_PROTOCOL_VERSION, ObjectIO, ObjectOperations, ObjectToDelete,
};
pub(crate) use super::{
ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS,
ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys,
EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions,
EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc,
EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfigurationExt,
EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore,
EcstoreTierConfig, EcstoreVersioningApi, ScannerReplicationHealObject, ScannerReplicationHealResult,
ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle,
ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket,
ecstore_list_path_raw, ecstore_object_opts_from_object_info, ecstore_path2_bucket_object,
ecstore_path2_bucket_object_with_base_path, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info,
ecstore_resolve_object_store_handle, ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts,
EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard,
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi,
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure,
ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle,
ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
};
#[cfg(test)]
pub(crate) use super::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
pub(crate) use super::{
EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints,
EcstoreInstanceContext, EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys,
ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
};
}
#[derive(Clone, Debug, Serialize, Deserialize)]
@@ -244,11 +261,17 @@ impl From<EcstoreReplicationHealQueueResult> for ScannerReplicationHealResult {
}
pub(crate) mod scan {
pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, NamespaceLocking};
pub use super::storage_contracts::SCANNER_ACTIVITY_PROTOCOL_VERSION;
pub(crate) use super::storage_contracts::{
BucketOperations, BucketOptions, NamespaceLocking, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
};
#[cfg(test)]
pub(crate) use super::storage_contracts::{MakeBucketOptions, ObjectIO};
}
pub(crate) mod scanner_io {
pub(crate) use super::storage_contracts::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
pub(crate) use super::storage_contracts::{BucketInfo, BucketOptions};
#[cfg(test)]
pub(crate) use super::storage_contracts::{HTTPRangeSpec, ObjectIO};
}
crates/storage-api/src/lib.rs
+21
View File
@@ -17,6 +17,27 @@
pub const WALK_DIR_STREAM_COMPLETION_QUERY: &str = "walk_dir_stream_completion";
pub const WALK_DIR_STREAM_COMPLETION_V1: &str = "error-v1";
pub const WALK_DIR_BODY_SHA256_QUERY: &str = "walk_dir_body_sha256";
pub const NS_SCANNER_BODY_SHA256_QUERY: &str = "ns_scanner_body_sha256";
pub const NS_SCANNER_CAPABILITY_CHALLENGE_QUERY: &str = "ns_scanner_challenge";
pub const NS_SCANNER_CYCLE_QUERY: &str = "ns_scanner_cycle";
pub const NS_SCANNER_LEADER_EPOCH_QUERY: &str = "ns_scanner_leader_epoch";
pub const NS_SCANNER_REQUEST_ID_QUERY: &str = "ns_scanner_request_id";
pub const NS_SCANNER_SERVER_EPOCH_QUERY: &str = "ns_scanner_server_epoch";
pub const NS_SCANNER_SESSION_ID_QUERY: &str = "ns_scanner_session_id";
pub const NS_SCANNER_SESSION_SEQUENCE_QUERY: &str = "ns_scanner_session_sequence";
pub const NS_SCANNER_PROTOCOL_VERSION_QUERY: &str = "ns_scanner_protocol";
pub const NS_SCANNER_PROTOCOL_VERSION: u16 = 3;
pub const SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION: u32 = 0;
pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 4;
pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 5;
#[derive(Debug, serde::Deserialize, serde::Serialize)]
#[serde(deny_unknown_fields)]
pub struct NsScannerCapabilityResponse {
pub version: u16,
pub server_epoch: uuid::Uuid,
pub proof: Vec<u8>,
}
pub mod admin;
pub mod bucket;
crates/test-utils/src/data_usage_snapshot_tests.rs
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
use std::time::{Duration, SystemTime};
use crate::TestECStoreEnv;
@@ -48,7 +48,7 @@ async fn data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache()
.init_bucket_metadata(false)
.build()
.await;
let primary = format!("{BUCKET_META_PREFIX}/.usage.json");
let primary = format!("{BUCKET_META_PREFIX}/{DATA_USAGE_OBJECT_NAME}");
let backup = format!("{primary}.bkp");
let last_update = SystemTime::UNIX_EPOCH + Duration::from_secs(100);
let expected = snapshot("bucket-a", last_update);