mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-18 22:14:42 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dada3276a3 |
@@ -1623,6 +1623,8 @@ pub struct LocalDiskWrapper {
|
||||
health_check: bool,
|
||||
/// Cancellation token for monitoring tasks
|
||||
cancel_token: CancellationToken,
|
||||
/// Retirement belongs to this wrapper instance, not its monitoring policy.
|
||||
retired: CancellationToken,
|
||||
/// Disk ID for stale checking
|
||||
disk_id: Arc<RwLock<Option<Uuid>>>,
|
||||
/// Timeout policy for scanner-sensitive operations, loaded once on wrapper initialization.
|
||||
@@ -1776,6 +1778,7 @@ impl LocalDiskWrapper {
|
||||
physical_admission: Arc::new(crate::disk::io_admission::PhysicalDiskBinding::default()),
|
||||
health_check: health_check && env_health_check,
|
||||
cancel_token: CancellationToken::new(),
|
||||
retired: CancellationToken::new(),
|
||||
disk_id: Arc::new(RwLock::new(None)),
|
||||
timeout_health_policy: get_drive_timeout_health_policy(),
|
||||
};
|
||||
@@ -1983,6 +1986,11 @@ impl LocalDiskWrapper {
|
||||
self.cancel_token.cancel();
|
||||
}
|
||||
|
||||
/// Observe retirement without granting callers authority to retire this wrapper.
|
||||
pub fn retirement_token(&self) -> CancellationToken {
|
||||
self.retired.child_token()
|
||||
}
|
||||
|
||||
fn spawn_recovery_monitor_if_needed(&self) {
|
||||
if !self.health_check {
|
||||
return;
|
||||
@@ -2980,6 +2988,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<()> {
|
||||
self.retired.cancel();
|
||||
self.stop_monitoring().await;
|
||||
self.disk.close().await
|
||||
}
|
||||
|
||||
@@ -104,6 +104,7 @@ use sha2::Digest as _;
|
||||
use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Independent admission and physical ownership for one disk rename.
|
||||
@@ -1418,6 +1419,15 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe a local wrapper's retirement. Remote handles have their own RPC lifecycle.
|
||||
/// Cancelling the returned child token does not retire the underlying wrapper.
|
||||
pub fn local_retirement_token(&self) -> Option<CancellationToken> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => Some(local_disk.retirement_token()),
|
||||
Disk::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn offline_duration_secs(&self) -> Option<u64> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.offline_duration_secs(),
|
||||
|
||||
@@ -61,6 +61,12 @@ struct FillRequest {
|
||||
response: oneshot::Sender<FillResult>,
|
||||
}
|
||||
|
||||
#[cfg_attr(test, derive(Debug))]
|
||||
struct FilledStripes {
|
||||
primary: Vec<u8>,
|
||||
queued: VecDeque<Vec<u8>>,
|
||||
}
|
||||
|
||||
struct FillResult {
|
||||
result: io::Result<Option<Vec<u8>>>,
|
||||
queued_buffers: VecDeque<Vec<u8>>,
|
||||
@@ -293,7 +299,7 @@ where
|
||||
Self::new_with_fill_policy_inner(source, engine, total_length, metrics_path, fill_policy)
|
||||
}
|
||||
|
||||
fn poll_fill_result(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<Option<Vec<u8>>>> {
|
||||
fn poll_fill_result(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<Option<FilledStripes>>> {
|
||||
if self.fill.is_none() {
|
||||
let fill_worker_tx = match self.fill_worker_tx() {
|
||||
Ok(tx) => tx,
|
||||
@@ -345,15 +351,6 @@ where
|
||||
if let Some(deferred_error) = deferred_error {
|
||||
self.prefetch_error = Some(deferred_error);
|
||||
}
|
||||
// Queued stripes are delivered to the client via `prefetched_bufs.pop_front()`
|
||||
// in `poll_read` without touching `self.remaining`, so their bytes must be
|
||||
// accounted for here. Otherwise, under multi-in-flight fill policies (e.g. the
|
||||
// default `DualInFlight`), `remaining` never reaches 0 and a fully delivered
|
||||
// multi-block object still terminates the GET with `LessData`. Buffers in
|
||||
// `queued_buffers` come only from `Ok(true)` decodes, so they are non-empty and
|
||||
// their total is bounded by `remaining - main_buf.len()`, ruling out underflow.
|
||||
self.remaining -= queued_buffers.iter().map(Vec::len).sum::<usize>();
|
||||
self.prefetched_bufs.extend(queued_buffers);
|
||||
|
||||
match result {
|
||||
Ok(Some(buf)) => {
|
||||
@@ -362,8 +359,17 @@ where
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_reader_stripe(self.metrics_path);
|
||||
rustfs_io_metrics::record_get_object_reader_bytes(self.metrics_path, buf.len());
|
||||
// Queued stripes are delivered to the client via `prefetched_bufs.pop_front()`
|
||||
// in `poll_read` without touching `self.remaining`, so their bytes must be
|
||||
// accounted for here alongside the primary stripe. Buffers in `queued_buffers`
|
||||
// come only from `Ok(true)` decodes, so they are non-empty and their total is
|
||||
// bounded by `remaining - buf.len()`, ruling out underflow.
|
||||
self.remaining -= buf.len();
|
||||
Poll::Ready(Ok(Some(buf)))
|
||||
self.remaining -= queued_buffers.iter().map(Vec::len).sum::<usize>();
|
||||
Poll::Ready(Ok(Some(FilledStripes {
|
||||
primary: buf,
|
||||
queued: queued_buffers,
|
||||
})))
|
||||
}
|
||||
Ok(None) => {
|
||||
if self.remaining == 0 {
|
||||
@@ -401,8 +407,8 @@ where
|
||||
};
|
||||
|
||||
match fill {
|
||||
Ok(Some(buf)) => {
|
||||
let output_has_remaining = self.output_pos < self.output_buf.len();
|
||||
Ok(Some(FilledStripes { primary: buf, queued })) => {
|
||||
let output_has_remaining = self.output_pos < self.output_buf.len() || !self.prefetched_bufs.is_empty();
|
||||
let mut queued_count = 0usize;
|
||||
let mut queued_bytes = 0usize;
|
||||
|
||||
@@ -420,6 +426,7 @@ where
|
||||
self.output_buf = buf;
|
||||
self.output_pos = 0;
|
||||
}
|
||||
self.prefetched_bufs.extend(queued);
|
||||
|
||||
if !self.prefetched_bufs.is_empty() {
|
||||
let total_prefetched_bytes = self.prefetched_bufs.iter().map(Vec::len).sum::<usize>();
|
||||
@@ -451,7 +458,7 @@ where
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
Err(err) => {
|
||||
if self.output_pos < self.output_buf.len() {
|
||||
if self.output_pos < self.output_buf.len() || !self.prefetched_bufs.is_empty() {
|
||||
rustfs_io_metrics::record_get_object_reader_prefetch(self.metrics_path, GET_READER_PREFETCH_ERROR_DEFERRED);
|
||||
self.prefetch_error = Some(err);
|
||||
Poll::Ready(Ok(()))
|
||||
@@ -1860,6 +1867,42 @@ mod tests {
|
||||
.expect("dual inflight policy should prefetch two future stripes before current output drains");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_decode_reader_dual_inflight_preserves_stripe_order_when_prefetch_completes_before_output_drains() {
|
||||
let erasure = Erasure::new(4, 2, 16);
|
||||
let data = (0..96u8).collect::<Vec<_>>();
|
||||
let read_count = Arc::new(AtomicUsize::new(0));
|
||||
let mut source = source_from_data(&erasure, &data, &[]);
|
||||
source.read_count = Some(Arc::clone(&read_count));
|
||||
let engine = LegacyEcDecodeEngine::new(erasure);
|
||||
let mut reader = ErasureDecodeReader::new_with_fill_policy(
|
||||
source,
|
||||
engine,
|
||||
data.len(),
|
||||
GET_OBJECT_PATH_CODEC_STREAMING,
|
||||
FillPolicy::DualInFlight,
|
||||
)
|
||||
.expect("reader should be constructed");
|
||||
|
||||
let mut decoded = Vec::with_capacity(data.len());
|
||||
let mut chunk = [0u8; 5];
|
||||
loop {
|
||||
let n = reader.read(&mut chunk).await.expect("chunk read should succeed");
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
decoded.extend_from_slice(&chunk[..n]);
|
||||
// Yield between partial reads so the DualInFlight worker completes
|
||||
// a 2-stripe fill (primary + queued_buffers) while output_buf still
|
||||
// has unconsumed bytes.
|
||||
for _ in 0..4 {
|
||||
yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(decoded, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_decode_reader_dual_inflight_reads_multi_block_object_to_end() {
|
||||
// Regression for the byte-accounting bug: under DualInFlight each fill
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
mod storage_api;
|
||||
use storage_api::local_drive_retirement::{Disk, DiskAPI, DiskOption, Endpoint, new_disk};
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_wrapper_retirement_is_shared_and_independent_of_monitoring() {
|
||||
let dir = tempfile::tempdir().expect("retirement fixture directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 fixture path")).expect("local endpoint");
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("local disk");
|
||||
let Disk::Local(wrapper) = disk.as_ref() else { panic!("fixture must use a local disk") };
|
||||
let cloned = wrapper.clone();
|
||||
let observer = disk.local_retirement_token().expect("local retirement observer");
|
||||
let cancelled_observer = disk.local_retirement_token().expect("another observer");
|
||||
cancelled_observer.cancel();
|
||||
assert!(!observer.is_cancelled(), "observers cannot retire the wrapper");
|
||||
wrapper.stop_monitoring().await;
|
||||
assert!(!observer.is_cancelled(), "monitoring policy is not retirement");
|
||||
cloned.close().await.expect("close shared wrapper");
|
||||
assert!(observer.is_cancelled(), "clones share retirement");
|
||||
assert!(
|
||||
disk.local_retirement_token().expect("late observer").is_cancelled(),
|
||||
"late observers see retirement"
|
||||
);
|
||||
disk.close().await.expect("retirement is idempotent");
|
||||
}
|
||||
@@ -94,3 +94,8 @@ pub(crate) mod pinned_erasure_fixtures {
|
||||
storage::init_local_disks_with_instance_ctx,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod local_drive_retirement {
|
||||
pub(crate) use super::{DiskAPI, DiskOption, Endpoint, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::disk::Disk;
|
||||
}
|
||||
|
||||
@@ -1093,6 +1093,7 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
scan_mode,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: Some(checkpoint_tx),
|
||||
local_stop: None,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
@@ -613,6 +613,7 @@ async fn remote_worker_rechecks_real_producers_before_persisting_complete_usage(
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
local_stop: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -641,6 +642,7 @@ async fn remote_worker_rechecks_real_producers_before_persisting_complete_usage(
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
local_stop: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -835,6 +835,7 @@ pub struct FolderScanner {
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
skip_heal: Arc<std::sync::atomic::AtomicBool>,
|
||||
local_disk: Arc<Disk>,
|
||||
local_stop: Option<Arc<crate::scanner_io::ScannerLocalStop>>,
|
||||
/// Tier registry frozen for this folder scan. A refresh applies to the
|
||||
/// next scan and cannot mix generations in one aggregate.
|
||||
tier_registry: TierRegistrySnapshot,
|
||||
@@ -2415,6 +2416,17 @@ impl FolderScanner {
|
||||
self.record_failed(&item.path);
|
||||
}
|
||||
|
||||
if self.local_disk.is_local() && matches!(e, StorageError::FaultyDisk | StorageError::DiskNotFound) {
|
||||
if let Some(stop) = &self.local_stop {
|
||||
stop.stop(crate::scanner_io::ScannerLocalStopReason::Unavailable, &ctx);
|
||||
} else {
|
||||
ctx.cancel();
|
||||
}
|
||||
// Cancellation preserves this unresolved object and
|
||||
// completed siblings through every recursive path.
|
||||
return Err(ScannerError::Other("Local scanner drive unavailable".to_string()));
|
||||
}
|
||||
|
||||
if !retry_suppressed && should_log_failed_object(into.failed_objects) {
|
||||
if let GetSizeFailureAction::HealMetadata { object } = &failure_action {
|
||||
error!(
|
||||
@@ -3345,7 +3357,20 @@ pub async fn scan_data_folder(
|
||||
scan_mode: HealScanMode,
|
||||
sleeper: DynamicSleeper,
|
||||
) -> Result<DataUsageCache, ScannerError> {
|
||||
scan_data_folder_scoped(ctx, budget, disks, local_disk, cache, updates, scan_mode, sleeper, None, None).await
|
||||
scan_data_folder_scoped(
|
||||
ctx.child_token(),
|
||||
budget,
|
||||
disks,
|
||||
local_disk,
|
||||
cache,
|
||||
updates,
|
||||
scan_mode,
|
||||
sleeper,
|
||||
None,
|
||||
None,
|
||||
Some(Arc::new(crate::scanner_io::ScannerLocalStop::default())),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -3360,6 +3385,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
sleeper: DynamicSleeper,
|
||||
prefix_scan_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
checkpoint_tx: Option<mpsc::Sender<DataUsageCache>>,
|
||||
local_stop: Option<Arc<crate::scanner_io::ScannerLocalStop>>,
|
||||
) -> Result<DataUsageCache, ScannerError> {
|
||||
use crate::data_usage_define::DATA_USAGE_ROOT;
|
||||
|
||||
@@ -3457,6 +3483,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
budget: budget.clone(),
|
||||
skip_heal,
|
||||
local_disk,
|
||||
local_stop,
|
||||
tier_registry,
|
||||
pending_heals_changed: false,
|
||||
coverage_frontier: clone_cache_value(&resume_frontier),
|
||||
@@ -3481,11 +3508,13 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
prune_size_reconciliation(&mut scanner.new_cache.info, now);
|
||||
|
||||
// Check if context is cancelled
|
||||
if ctx.is_cancelled() {
|
||||
if ctx.is_cancelled() && scanner.local_stop.as_ref().is_none_or(|stop| stop.reason().is_none()) {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
|
||||
scanner.retry_pending_scanner_heals().await?;
|
||||
if !ctx.is_cancelled() {
|
||||
scanner.retry_pending_scanner_heals().await?;
|
||||
}
|
||||
|
||||
// Read top level in bucket
|
||||
let mut root = DataUsageEntry::default();
|
||||
@@ -3667,7 +3696,9 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
close_disk_guard.close().await;
|
||||
return Err(ScannerError::NamespaceNotFoundCache(Box::new(partial_cache)));
|
||||
}
|
||||
if matches!(&e, ScannerError::ResourceDeferred) {
|
||||
if matches!(&e, ScannerError::ResourceDeferred)
|
||||
|| scanner.local_stop.as_ref().is_some_and(|stop| stop.reason().is_some())
|
||||
{
|
||||
// Admission pressure is not disk failure or an empty census.
|
||||
// Keep the previous tree when no new useful progress exists.
|
||||
let mut partial_cache = scanner.old_cache;
|
||||
@@ -3677,7 +3708,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
.insert(hash_path(&bucket_name).0, DataUsageEntry::default());
|
||||
}
|
||||
partial_cache.info.snapshot_complete = false;
|
||||
partial_cache.info.scan_resource_deferred = true;
|
||||
partial_cache.info.scan_resource_deferred = matches!(&e, ScannerError::ResourceDeferred);
|
||||
close_disk_guard.close().await;
|
||||
return Err(ScannerError::PartialCache(Box::new(partial_cache)));
|
||||
}
|
||||
|
||||
@@ -392,6 +392,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
budget: ScannerCycleBudget::new(&CancellationToken::new(), Default::default()),
|
||||
skip_heal: Arc::new(AtomicBool::new(false)),
|
||||
local_disk: disk,
|
||||
local_stop: None,
|
||||
tier_registry: crate::TierRegistrySnapshot {
|
||||
generation: 0,
|
||||
names: Arc::new([]),
|
||||
|
||||
@@ -40,6 +40,7 @@ fn scan_options(scan_mode: HealScanMode) -> ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
local_stop: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1307,6 +1308,7 @@ where
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
local_stop: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1390,6 +1392,7 @@ where
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
local_stop: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -1699,6 +1699,8 @@ pub struct ScannerDiskScanOptions {
|
||||
pub scan_mode: HealScanMode,
|
||||
pub prefix_scan_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
pub checkpoint_tx: Option<mpsc::Sender<DataUsageCache>>,
|
||||
/// Local interruption state shared with the coordinator; not part of RPC or cache formats.
|
||||
pub local_stop: Option<Arc<ScannerLocalStop>>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -2131,6 +2133,9 @@ pub(crate) use usage_watch::ScannerUsageWatchReleaseGuard;
|
||||
use usage_watch::{ScannerUsageWatchCohort, admit_scanner_usage_watch, record_completed_usage_watch};
|
||||
pub(crate) use usage_watch::{bind_usage_watch_to_checkpoint, invalidate_completed_usage_watch_cache};
|
||||
mod io_disk;
|
||||
mod local_drive;
|
||||
use local_drive::*;
|
||||
pub use local_drive::{ScannerLocalStop, ScannerLocalStopReason};
|
||||
#[cfg(test)]
|
||||
mod publish_gate_tests;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -989,6 +989,7 @@ impl ScannerIOCache for SetDisks {
|
||||
if turn_bucket_count == 0 {
|
||||
bucket_work_complete.cancel();
|
||||
}
|
||||
let local_recovery_attempts = Arc::new(Mutex::new(HashSet::<String>::new()));
|
||||
for NamespaceScannerWalker {
|
||||
disk,
|
||||
mode: worker_mode,
|
||||
@@ -1011,6 +1012,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let bucket_result_tx_clone = bucket_result_tx.clone();
|
||||
let bucket_observations_clone = bucket_observations.clone();
|
||||
let set_disk_inventory_clone = set_disk_inventory.clone();
|
||||
let local_recovery_attempts_clone = local_recovery_attempts.clone();
|
||||
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
|
||||
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
|
||||
let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone();
|
||||
@@ -1078,6 +1080,34 @@ impl ScannerIOCache for SetDisks {
|
||||
pool_label_clone.clone(),
|
||||
set_label_clone.clone(),
|
||||
);
|
||||
let assigned_disk = &disk;
|
||||
let disk = if assigned_disk.is_local() {
|
||||
match current_local_scanner_disk(&store_clone_clone, assigned_disk, &ctx_clone, &budget_clone).await {
|
||||
Some(current) => current,
|
||||
None => {
|
||||
drop(_active_guard);
|
||||
drop(_permit);
|
||||
let retry_available = {
|
||||
local_recovery_attempts_clone.lock().await.insert(bucket.name.clone())
|
||||
};
|
||||
if retry_available
|
||||
&& wait_for_local_scanner_recovery(
|
||||
&store_clone_clone, assigned_disk, &ctx_clone, &budget_clone,
|
||||
).await.is_some()
|
||||
&& requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await
|
||||
{
|
||||
increment_disk_bucket_scans_queued(
|
||||
&queued_disk_bucket_scans_clone, &pool_label_clone, &set_label_clone,
|
||||
);
|
||||
} else {
|
||||
record_partial_dirty_bucket(&partial_dirty_buckets_clone, &bucket.name).await;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
assigned_disk.clone()
|
||||
};
|
||||
if ctx_clone.is_cancelled() || budget_clone.budget_elapsed() {
|
||||
break;
|
||||
}
|
||||
@@ -1651,6 +1681,14 @@ impl ScannerIOCache for SetDisks {
|
||||
|
||||
let resumed_watch_sweep = cache.has_scan_position();
|
||||
let scan_position_before = cache.durable_scan_progress();
|
||||
let local_stop = Arc::new(ScannerLocalStop::default());
|
||||
#[cfg(test)]
|
||||
super::tests::local_drive_recovery::before_attempt(&bucket.name, &disk).await;
|
||||
let current_set_disks = if disk.is_local() {
|
||||
scanner_set_disk_inventory(&store_clone_clone).await
|
||||
} else {
|
||||
set_disk_inventory_clone.as_ref().clone()
|
||||
};
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
{
|
||||
let disk_uuid = if crate::e2e_barrier::worker_observation::enabled() {
|
||||
@@ -1671,13 +1709,14 @@ impl ScannerIOCache for SetDisks {
|
||||
let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
|
||||
scan_ctx.clone(),
|
||||
bucket_budget.clone(),
|
||||
set_disk_inventory_clone.as_ref().clone(),
|
||||
current_set_disks,
|
||||
cache,
|
||||
None,
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope,
|
||||
checkpoint_tx: Some(checkpoint_tx),
|
||||
local_stop: Some(local_stop.clone()),
|
||||
},
|
||||
));
|
||||
tokio::pin!(scan);
|
||||
@@ -1711,6 +1750,9 @@ impl ScannerIOCache for SetDisks {
|
||||
await_scanner_disk_shutdown(scan.as_mut()).await;
|
||||
break Err(Error::other("scanner bucket cache lock was lost during bucket scan"));
|
||||
}
|
||||
if disk.is_local() && let Some(reason) = local_scanner_stop_reason(&store_clone_clone, &disk).await {
|
||||
local_stop.stop(reason, &scan_ctx);
|
||||
}
|
||||
}
|
||||
checkpoint = checkpoint_rx.recv(), if !checkpoint_channel_closed => {
|
||||
let Some(checkpoint) = checkpoint else {
|
||||
@@ -1818,6 +1860,9 @@ impl ScannerIOCache for SetDisks {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if disk.is_local() && let Some(reason) = local_scanner_stop_reason(&store_clone_clone, &disk).await {
|
||||
local_stop.stop(reason, &scan_ctx);
|
||||
}
|
||||
let scan_outcome = match scan_result {
|
||||
Ok(scan_outcome) => scan_outcome,
|
||||
Err(e) if bucket_budget.quantum_exhausted() || bucket_budget.source_turn_exhausted() => {
|
||||
@@ -1864,6 +1909,15 @@ impl ScannerIOCache for SetDisks {
|
||||
}
|
||||
};
|
||||
|
||||
let scan_outcome = match scan_outcome {
|
||||
ScannerDiskScanOutcome::Complete(mut cache) | ScannerDiskScanOutcome::Partial(mut cache)
|
||||
if local_stop.reason().is_some() => {
|
||||
retain_interrupted_local_cache(&mut cache);
|
||||
ScannerDiskScanOutcome::Partial(cache)
|
||||
}
|
||||
other => other,
|
||||
};
|
||||
|
||||
let (cache, partial) = match scan_outcome {
|
||||
ScannerDiskScanOutcome::Complete(mut completed_cache) => {
|
||||
if resumed_watch_sweep && let Some(proof) = completed_cache.info.scan_usage_watch.as_ref()
|
||||
@@ -1987,6 +2041,31 @@ impl ScannerIOCache for SetDisks {
|
||||
bucket_budget.record_durable_turn_progress();
|
||||
}
|
||||
}
|
||||
if local_stop.reason().is_some() {
|
||||
// Recovery is independent of durable forward progress, but
|
||||
// cannot retry an unsaved result or renew its own cache lease.
|
||||
drop(cache_guard);
|
||||
drop(_local_admission);
|
||||
drop(_active_guard);
|
||||
drop(_permit);
|
||||
let retry_available = {
|
||||
local_recovery_attempts_clone.lock().await.insert(bucket.name.clone())
|
||||
};
|
||||
if partial_progress_proven
|
||||
&& retry_available
|
||||
&& wait_for_local_scanner_recovery(
|
||||
&store_clone_clone, &disk, &ctx_clone, &budget_clone,
|
||||
).await.is_some()
|
||||
&& requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await
|
||||
{
|
||||
increment_disk_bucket_scans_queued(
|
||||
&queued_disk_bucket_scans_clone, &pool_label_clone, &set_label_clone,
|
||||
);
|
||||
} else {
|
||||
record_partial_dirty_bucket(&partial_dirty_buckets_clone, &bucket.name).await;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Only saved forward motion earns another turn in this
|
||||
// cycle; otherwise the bucket waits for the next one.
|
||||
if partial_progress_proven
|
||||
|
||||
@@ -64,12 +64,19 @@ impl ScannerIODisk for Disk {
|
||||
let metadata_object_path = item.object_path();
|
||||
let metadata_result =
|
||||
crate::workload_admission::with_scanner_metadata_io(self.read_metadata(&item.bucket, &metadata_object_path)).await;
|
||||
#[cfg(test)]
|
||||
let metadata_result = if super::tests::local_drive_recovery::after_metadata_read(&item.bucket).await {
|
||||
Err(DiskError::FaultyDisk)
|
||||
} else {
|
||||
metadata_result
|
||||
};
|
||||
ensure_scanner_node_operation_active(cancellation.as_ref())?;
|
||||
let data = match metadata_result {
|
||||
Ok(data) => data,
|
||||
Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
|
||||
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
|
||||
}
|
||||
Err(e @ (DiskError::FaultyDisk | DiskError::DiskNotFound)) if self.is_local() => return Err(e.into()),
|
||||
Err(e) => {
|
||||
return Err(scanner_metadata_transient_error(
|
||||
format!("failed to read metadata: {e}"),
|
||||
@@ -196,7 +203,15 @@ impl ScannerIODisk for Disk {
|
||||
scan_mode,
|
||||
prefix_scan_scope,
|
||||
checkpoint_tx,
|
||||
local_stop,
|
||||
} = options;
|
||||
let local_stop = local_stop.unwrap_or_default();
|
||||
let retirement = self.local_retirement_token();
|
||||
// Only this attempt stops; cycle budgets and the leader fence remain owned by the caller.
|
||||
let ctx = ctx.child_token();
|
||||
if retirement.as_ref().is_some_and(CancellationToken::is_cancelled) {
|
||||
local_stop.stop(ScannerLocalStopReason::Retired, &ctx);
|
||||
}
|
||||
let done_drive = Metrics::time(Metric::ScanBucketDrive);
|
||||
let drive_start = std::time::Instant::now();
|
||||
let bucket = clone_cache_value(&cache.info.name);
|
||||
@@ -256,9 +271,8 @@ impl ScannerIODisk for Disk {
|
||||
.then_some(prefix_scan_scope)
|
||||
.flatten();
|
||||
|
||||
let result = crate::storage_api::ecstore_with_background_disk_io(
|
||||
ctx.clone(),
|
||||
scan_data_folder_scoped(
|
||||
let result = crate::storage_api::ecstore_with_background_disk_io(ctx.clone(), async {
|
||||
let scan = scan_data_folder_scoped(
|
||||
ctx.clone(),
|
||||
budget,
|
||||
set_disks,
|
||||
@@ -270,12 +284,53 @@ impl ScannerIODisk for Disk {
|
||||
SCANNER_SLEEPER.for_disk(&self.path()),
|
||||
prefix_scan_scope,
|
||||
checkpoint_tx,
|
||||
),
|
||||
)
|
||||
Some(local_stop.clone()),
|
||||
);
|
||||
tokio::pin!(scan);
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = &mut scan => break result,
|
||||
_ = async {
|
||||
match &retirement {
|
||||
Some(token) => token.cancelled().await,
|
||||
None => std::future::pending().await,
|
||||
}
|
||||
}, if local_stop.reason().is_none() => {
|
||||
local_stop.stop(ScannerLocalStopReason::Retired, &ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
#[cfg(test)]
|
||||
super::tests::local_drive_recovery::after_walk(&bucket, &self).await;
|
||||
if retirement.as_ref().is_some_and(CancellationToken::is_cancelled) {
|
||||
local_stop.stop(ScannerLocalStopReason::Retired, &ctx);
|
||||
}
|
||||
if let Some(reason) = local_stop.reason() {
|
||||
warn!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_DISK_BUCKET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "local_drive_interrupted",
|
||||
bucket = %bucket,
|
||||
endpoint = %crate::storage_api::owner::EcstoreDiskAPI::endpoint(self.as_ref()),
|
||||
reason = reason.as_str(),
|
||||
"Scanner retained unfinished work after a local drive interruption"
|
||||
);
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(mut data_usage_info) => {
|
||||
if local_stop.reason().is_some() {
|
||||
retain_interrupted_local_cache(&mut data_usage_info);
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
|
||||
failure_guard.mark_not_failed();
|
||||
return Ok(ScannerDiskScanOutcome::Partial(data_usage_info));
|
||||
}
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_complete(source, true, &bucket, &disk_path, drive_start.elapsed());
|
||||
data_usage_info.info.last_update = Some(SystemTime::now());
|
||||
@@ -283,6 +338,9 @@ impl ScannerIODisk for Disk {
|
||||
Ok(ScannerDiskScanOutcome::Complete(data_usage_info))
|
||||
}
|
||||
Err(ScannerError::PartialCache(mut partial_cache)) => {
|
||||
if local_stop.reason().is_some() {
|
||||
retain_interrupted_local_cache(&mut partial_cache);
|
||||
}
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
|
||||
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api::owner::EcstoreDiskAPI;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
/// A local scan interruption is distinct from a cycle budget or leader fence.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ScannerLocalStopReason {
|
||||
Retired,
|
||||
Unavailable,
|
||||
Replaced,
|
||||
}
|
||||
|
||||
impl ScannerLocalStopReason {
|
||||
pub(super) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Retired => "retired",
|
||||
Self::Unavailable => "unavailable",
|
||||
Self::Replaced => "replaced",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Process-local outcome of one scan attempt; never serialized into a checkpoint.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ScannerLocalStop {
|
||||
reason: OnceLock<ScannerLocalStopReason>,
|
||||
}
|
||||
|
||||
impl ScannerLocalStop {
|
||||
/// The first local interruption, independent of ordinary cancellation.
|
||||
pub fn reason(&self) -> Option<ScannerLocalStopReason> {
|
||||
self.reason.get().copied()
|
||||
}
|
||||
|
||||
pub(crate) fn stop(&self, reason: ScannerLocalStopReason, scan: &CancellationToken) {
|
||||
let _ = self.reason.set(reason);
|
||||
scan.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn retain_interrupted_local_cache(cache: &mut DataUsageCache) {
|
||||
cache.info.snapshot_complete = false;
|
||||
// An EOF racing with retirement must be verified again, not published as
|
||||
// a completed companion or reused under an old coverage proof.
|
||||
cache.info.scan_plan_digest = None;
|
||||
cache.info.scan_bucket_observations.clear();
|
||||
cache.info.scan_bucket_value_origins.clear();
|
||||
}
|
||||
|
||||
const LOCAL_SCANNER_LOOKUP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Bind the assigned local topology slot to its currently usable instance.
|
||||
/// The format identity fences resume across a physical drive replacement.
|
||||
pub(super) async fn current_local_scanner_disk(
|
||||
set: &SetDisks,
|
||||
assigned: &Arc<Disk>,
|
||||
ctx: &CancellationToken,
|
||||
budget: &ScannerCycleBudget,
|
||||
) -> Option<Arc<Disk>> {
|
||||
// This also bounds the first binding, while the coordinator owns a scan
|
||||
// permit. Waiting for a format probe must respect source cancellation and its turn.
|
||||
let lookup = async {
|
||||
let location = EcstoreDiskAPI::get_disk_location(assigned.as_ref());
|
||||
let expected_id = location
|
||||
.disk_idx
|
||||
.and_then(|index| set.format.erasure.sets.get(set.set_index)?.get(index))
|
||||
.copied()
|
||||
.filter(|id| !id.is_nil());
|
||||
let expected_id = expected_id?;
|
||||
let candidate = scanner_set_disk_inventory(set).await.into_iter().find(|candidate| {
|
||||
EcstoreDiskAPI::is_local(candidate.as_ref())
|
||||
&& EcstoreDiskAPI::endpoint(candidate.as_ref()) == EcstoreDiskAPI::endpoint(assigned.as_ref())
|
||||
});
|
||||
let candidate = candidate?;
|
||||
#[cfg(test)]
|
||||
super::tests::local_drive_recovery::before_binding_probe().await;
|
||||
if candidate.local_retirement_token().is_some_and(|token| token.is_cancelled()) || !candidate.is_online().await {
|
||||
return None;
|
||||
}
|
||||
match candidate.get_disk_id().await {
|
||||
Ok(Some(id)) if id == expected_id => Some(candidate),
|
||||
Ok(_) | Err(_) => None,
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = ctx.cancelled() => None,
|
||||
_ = budget.wait_for_stop() => None,
|
||||
_ = tokio::time::sleep(LOCAL_SCANNER_LOOKUP_TIMEOUT) => None,
|
||||
disk = lookup => disk,
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait without a cache lease or disk scan permit. Each bucket gets at most
|
||||
/// one such recovery opportunity per source turn, including zero-progress stops.
|
||||
pub(super) async fn wait_for_local_scanner_recovery(
|
||||
set: &SetDisks,
|
||||
assigned: &Arc<Disk>,
|
||||
ctx: &CancellationToken,
|
||||
budget: &ScannerCycleBudget,
|
||||
) -> Option<Arc<Disk>> {
|
||||
const MAX_WAIT: Duration = Duration::from_secs(5);
|
||||
let deadline = tokio::time::Instant::now() + MAX_WAIT;
|
||||
let mut interval = tokio::time::interval(SCANNER_CACHE_LOCK_POLL_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = ctx.cancelled() => return None,
|
||||
_ = budget.wait_for_stop() => return None,
|
||||
_ = tokio::time::sleep_until(deadline) => return None,
|
||||
_ = interval.tick() => {
|
||||
if budget.budget_elapsed() {
|
||||
return None;
|
||||
}
|
||||
let current = tokio::select! {
|
||||
biased;
|
||||
_ = ctx.cancelled() => return None,
|
||||
_ = budget.wait_for_stop() => return None,
|
||||
_ = tokio::time::sleep_until(deadline) => return None,
|
||||
disk = current_local_scanner_disk(set, assigned, ctx, budget) => disk,
|
||||
};
|
||||
if let Some(disk) = current {
|
||||
return Some(disk);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Inspect runtime health and ownership without probing the drive or holding inventory guards during I/O.
|
||||
pub(super) async fn local_scanner_stop_reason(set: &SetDisks, disk: &Arc<Disk>) -> Option<ScannerLocalStopReason> {
|
||||
if disk.local_retirement_token().is_some_and(|token| token.is_cancelled()) {
|
||||
return Some(ScannerLocalStopReason::Retired);
|
||||
}
|
||||
if !disk.runtime_state().is_snapshot_eligible() {
|
||||
return Some(ScannerLocalStopReason::Unavailable);
|
||||
}
|
||||
(!scanner_set_disk_inventory(set)
|
||||
.await
|
||||
.iter()
|
||||
.any(|current| Arc::ptr_eq(current, disk)))
|
||||
.then_some(ScannerLocalStopReason::Replaced)
|
||||
}
|
||||
@@ -52,6 +52,7 @@ mod collector_lifecycle;
|
||||
mod cycle_collector_diagnostics;
|
||||
mod expiry_lock;
|
||||
mod listing_window_scope;
|
||||
pub(super) mod local_drive_recovery;
|
||||
mod node_admission;
|
||||
mod partial_bucket_carry;
|
||||
mod per_bucket_certification;
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api::owner::EcstoreDiskAPI as _;
|
||||
|
||||
struct Probe {
|
||||
set: Arc<SetDisks>,
|
||||
before: bool,
|
||||
replace: bool,
|
||||
faulty: bool,
|
||||
eof: bool,
|
||||
attempts: AtomicUsize,
|
||||
reads: AtomicUsize,
|
||||
}
|
||||
|
||||
static PROBES: std::sync::LazyLock<StdMutex<HashMap<String, Arc<Probe>>>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
|
||||
fn probe(bucket: &str) -> Option<Arc<Probe>> {
|
||||
PROBES.lock().expect("probe registry").get(bucket).cloned()
|
||||
}
|
||||
|
||||
async fn retire(probe: &Probe, disk: &Arc<Disk>) {
|
||||
let replacement = if probe.replace {
|
||||
let next = new_disk(
|
||||
&disk.endpoint(),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("replacement wrapper");
|
||||
assert_eq!(
|
||||
next.get_disk_id().await.expect("replacement identity"),
|
||||
disk.get_disk_id().await.expect("old identity")
|
||||
);
|
||||
Some(next)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
disk.close().await.expect("retire old wrapper");
|
||||
if let Some(next) = replacement {
|
||||
let mut disks = probe.set.disks.write().await;
|
||||
let slot = disks
|
||||
.iter_mut()
|
||||
.find(|slot| slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, disk)))
|
||||
.expect("old wrapper in authoritative inventory");
|
||||
*slot = Some(next);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn before_attempt(bucket: &str, disk: &Arc<Disk>) {
|
||||
let Some(probe) = probe(bucket) else { return };
|
||||
let attempt = probe.attempts.fetch_add(1, Ordering::SeqCst);
|
||||
if probe.before && !probe.faulty && attempt == 0 {
|
||||
retire(&probe, disk).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn after_metadata_read(bucket: &str) -> bool {
|
||||
let Some(probe) = probe(bucket) else { return false };
|
||||
if probe.faulty {
|
||||
return probe.reads.fetch_add(1, Ordering::SeqCst) == 0;
|
||||
}
|
||||
if !probe.before && !probe.eof && probe.reads.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
let disk = scanner_set_disk_inventory(&probe.set)
|
||||
.await
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("local wrapper");
|
||||
retire(&probe, &disk).await;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) async fn after_walk(bucket: &str, disk: &Arc<Disk>) {
|
||||
let Some(probe) = probe(bucket) else { return };
|
||||
if probe.eof && probe.reads.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
retire(&probe, disk).await;
|
||||
}
|
||||
}
|
||||
|
||||
struct RegisteredProbe(String);
|
||||
impl Drop for RegisteredProbe {
|
||||
fn drop(&mut self) {
|
||||
PROBES.lock().expect("probe registry").remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
async fn recovery_fixture(bucket: &str, before: bool, replace: bool, faulty: bool, eof: bool) {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 1).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
for name in [bucket, "unaffected-bucket"] {
|
||||
store
|
||||
.make_bucket(name, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
for path in ["a/deep/first", "a/deep/second", "z/last"] {
|
||||
let mut reader = ScannerPutObjReader::from_vec(b"payload".to_vec());
|
||||
store
|
||||
.put_object(name, path, &mut reader, &ScannerObjectOptions::default())
|
||||
.await
|
||||
.expect("write object");
|
||||
}
|
||||
}
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
let probe = Arc::new(Probe {
|
||||
set: set.clone(),
|
||||
before,
|
||||
replace,
|
||||
faulty,
|
||||
eof,
|
||||
attempts: AtomicUsize::new(0),
|
||||
reads: AtomicUsize::new(0),
|
||||
});
|
||||
PROBES
|
||||
.lock()
|
||||
.expect("probe registry")
|
||||
.insert(bucket.to_string(), probe.clone());
|
||||
let _registered = RegisteredProbe(bucket.to_string());
|
||||
let status = run_full_cycle(&store, 1).await;
|
||||
if replace || faulty {
|
||||
assert_eq!(status, ScannerCycleStatus::Complete);
|
||||
assert_eq!(
|
||||
probe.attempts.load(Ordering::SeqCst),
|
||||
2,
|
||||
"one recovery attempt including zero-progress interruption"
|
||||
);
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache
|
||||
.load(set, &path_join_buf(&[bucket, DATA_USAGE_CACHE_NAME]))
|
||||
.await
|
||||
.expect("reload durable cache");
|
||||
assert!(cache.info.snapshot_complete, "recovery must finish a full scan");
|
||||
let total = cache.checked_flatten(bucket).expect("complete recovered statistics");
|
||||
assert_eq!(total.objects, 3);
|
||||
assert_eq!(total.size, 21);
|
||||
} else {
|
||||
assert_ne!(status, ScannerCycleStatus::Complete, "retired drive cannot certify completion");
|
||||
assert_eq!(probe.attempts.load(Ordering::SeqCst), 1, "no repeated scans on a retired wrapper");
|
||||
}
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_9469_recovers_retired_wrapper_before_first_object() {
|
||||
recovery_fixture("recovery-first", true, true, false, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_9469_recovers_retired_wrapper_in_deep_subtree() {
|
||||
recovery_fixture("recovery-deep", false, true, false, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_9469_persistent_retirement_is_bounded() {
|
||||
recovery_fixture("recovery-offline", true, false, false, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_9469_faulty_metadata_stops_and_retries_without_losing_object() {
|
||||
recovery_fixture("recovery-faulty", false, false, true, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_9469_retirement_at_eof_cannot_certify_old_handle() {
|
||||
recovery_fixture("recovery-eof", false, true, false, true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_9469_binding_rejects_changed_format_identity_and_cancelled_recovery() {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 1).await;
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
let disk = scanner_set_disk_inventory(&set)
|
||||
.await
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("local drive");
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
||||
let current = current_local_scanner_disk(&set, &disk, &ctx, &budget)
|
||||
.await
|
||||
.expect("matching drive binding");
|
||||
assert!(Arc::ptr_eq(¤t, &disk));
|
||||
let mut different_format = set.as_ref().clone();
|
||||
different_format.format.erasure.sets[0][0] = Uuid::new_v4();
|
||||
assert!(
|
||||
current_local_scanner_disk(&different_format, &disk, &ctx, &budget)
|
||||
.await
|
||||
.is_none(),
|
||||
"different physical identity cannot resume this slot"
|
||||
);
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
||||
ctx.cancel();
|
||||
assert!(
|
||||
wait_for_local_scanner_recovery(&set, &disk, &ctx, &budget).await.is_none(),
|
||||
"cancelled source must not admit recovery"
|
||||
);
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static BLOCK_BINDING_PROBE: Arc<tokio::sync::Notify>;
|
||||
}
|
||||
|
||||
pub(crate) async fn before_binding_probe() {
|
||||
if let Ok(entered) = BLOCK_BINDING_PROBE.try_with(Arc::clone) {
|
||||
entered.notify_one();
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum BindingStop {
|
||||
Timeout,
|
||||
Cancellation,
|
||||
Budget,
|
||||
}
|
||||
|
||||
async fn blocked_binding_fixture(stop: BindingStop) {
|
||||
let (_dir, store) = setup_local_scanner_store(1, 1).await;
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
let disk = scanner_set_disk_inventory(&set)
|
||||
.await
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("local drive");
|
||||
// Real filesystem setup completes before using controlled time.
|
||||
tokio::time::pause();
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: matches!(stop, BindingStop::Budget).then_some(Duration::from_secs(1)),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let started = tokio::time::Instant::now();
|
||||
BLOCK_BINDING_PROBE
|
||||
.scope(entered.clone(), async {
|
||||
let lookup = current_local_scanner_disk(&set, &disk, &ctx, &budget);
|
||||
tokio::pin!(lookup);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut lookup => panic!("the controlled probe must block before binding finishes"),
|
||||
_ = entered.notified() => {},
|
||||
}
|
||||
if matches!(stop, BindingStop::Cancellation) {
|
||||
ctx.cancel();
|
||||
}
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(6), &mut lookup)
|
||||
.await
|
||||
.expect("initial binding must stop even when metadata I/O never returns")
|
||||
.is_none()
|
||||
);
|
||||
})
|
||||
.await;
|
||||
let expected = match stop {
|
||||
BindingStop::Timeout => Duration::from_secs(5),
|
||||
BindingStop::Cancellation => Duration::ZERO,
|
||||
BindingStop::Budget => Duration::from_secs(1),
|
||||
};
|
||||
let elapsed = started.elapsed();
|
||||
// Tokio timers round to millisecond ticks; no wall-clock sleeps are used.
|
||||
assert!(
|
||||
(expected..=expected + Duration::from_millis(1)).contains(&elapsed),
|
||||
"the binding must respect the earliest bound: expected {expected:?}, elapsed {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_9469_initial_binding_metadata_probe_has_a_timeout() {
|
||||
blocked_binding_fixture(BindingStop::Timeout).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_9469_initial_binding_metadata_probe_observes_cancellation() {
|
||||
blocked_binding_fixture(BindingStop::Cancellation).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_9469_initial_binding_metadata_probe_observes_budget() {
|
||||
blocked_binding_fixture(BindingStop::Budget).await;
|
||||
}
|
||||
@@ -61,6 +61,7 @@ async fn disk_scan_withdraws_persisted_lifecycle_policy() {
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
local_stop: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
Reference in New Issue
Block a user