Compare commits

..

1 Commits

Author SHA1 Message Date
Chris An dada3276a3 fix(storage): backport scanner recovery and ordered shard reads (#9512)
* fix(scanner): recover interrupted local drive scans

(cherry picked from commit f18171d5cb)

* fix(scanner): bound initial local drive binding

(cherry picked from commit e56ce9d5e8)

* fix(ecstore): preserve DualInFlight stripe order in ErasureDecodeReader

In ErasureDecodeReader, poll_fill_result() previously extended
self.prefetched_bufs with lookahead stripes (queued_buffers, stripe
2k+1) before returning the primary stripe (buf, stripe 2k) to
poll_prefetch(). When a DualInFlight fill completed while output_buf
still had unconsumed bytes, poll_prefetch() pushed stripe 2k onto the
back of self.prefetched_bufs after stripe 2k+1, swapping adjacent
(2k, 2k+1) stripes on multi-block GetObject reads.

Return (buf, queued_buffers) from poll_fill_result() without mutating
self.prefetched_bufs, and extend self.prefetched_bufs with
queued_buffers in poll_prefetch() only after buf has been installed in
output_buf or pushed to self.prefetched_bufs. Also account for
non-empty self.prefetched_bufs when checking output_has_remaining and
deferring prefetch errors, and add a unit regression test.

(cherry picked from commit cd1e92921b)

* refactor(ecstore): name completed erasure stripes

(cherry picked from commit 9a3043fbb0)

* test(ecstore): keep completed stripes debuggable in tests

(cherry picked from commit f8662ec759)

---------

Co-authored-by: cxymds <cxymds@gmail.com>
Co-authored-by: Jinhao Hu <jinhaohu@google.com>
2026-11-19 04:47:17 +08:00
17 changed files with 765 additions and 25 deletions
+9
View File
@@ -1623,6 +1623,8 @@ pub struct LocalDiskWrapper {
health_check: bool,
/// Cancellation token for monitoring tasks
cancel_token: CancellationToken,
/// Retirement belongs to this wrapper instance, not its monitoring policy.
retired: CancellationToken,
/// Disk ID for stale checking
disk_id: Arc<RwLock<Option<Uuid>>>,
/// Timeout policy for scanner-sensitive operations, loaded once on wrapper initialization.
@@ -1776,6 +1778,7 @@ impl LocalDiskWrapper {
physical_admission: Arc::new(crate::disk::io_admission::PhysicalDiskBinding::default()),
health_check: health_check && env_health_check,
cancel_token: CancellationToken::new(),
retired: CancellationToken::new(),
disk_id: Arc::new(RwLock::new(None)),
timeout_health_policy: get_drive_timeout_health_policy(),
};
@@ -1983,6 +1986,11 @@ impl LocalDiskWrapper {
self.cancel_token.cancel();
}
/// Observe retirement without granting callers authority to retire this wrapper.
pub fn retirement_token(&self) -> CancellationToken {
self.retired.child_token()
}
fn spawn_recovery_monitor_if_needed(&self) {
if !self.health_check {
return;
@@ -2980,6 +2988,7 @@ impl DiskAPI for LocalDiskWrapper {
}
async fn close(&self) -> Result<()> {
self.retired.cancel();
self.stop_monitoring().await;
self.disk.close().await
}
+10
View File
@@ -104,6 +104,7 @@ use sha2::Digest as _;
use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncWrite};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
/// Independent admission and physical ownership for one disk rename.
@@ -1418,6 +1419,15 @@ impl Disk {
}
}
/// Observe a local wrapper's retirement. Remote handles have their own RPC lifecycle.
/// Cancelling the returned child token does not retire the underlying wrapper.
pub fn local_retirement_token(&self) -> Option<CancellationToken> {
match self {
Disk::Local(local_disk) => Some(local_disk.retirement_token()),
Disk::Remote(_) => None,
}
}
pub fn offline_duration_secs(&self) -> Option<u64> {
match self {
Disk::Local(local_disk) => local_disk.offline_duration_secs(),
@@ -61,6 +61,12 @@ struct FillRequest {
response: oneshot::Sender<FillResult>,
}
#[cfg_attr(test, derive(Debug))]
struct FilledStripes {
primary: Vec<u8>,
queued: VecDeque<Vec<u8>>,
}
struct FillResult {
result: io::Result<Option<Vec<u8>>>,
queued_buffers: VecDeque<Vec<u8>>,
@@ -293,7 +299,7 @@ where
Self::new_with_fill_policy_inner(source, engine, total_length, metrics_path, fill_policy)
}
fn poll_fill_result(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<Option<Vec<u8>>>> {
fn poll_fill_result(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<Option<FilledStripes>>> {
if self.fill.is_none() {
let fill_worker_tx = match self.fill_worker_tx() {
Ok(tx) => tx,
@@ -345,15 +351,6 @@ where
if let Some(deferred_error) = deferred_error {
self.prefetch_error = Some(deferred_error);
}
// Queued stripes are delivered to the client via `prefetched_bufs.pop_front()`
// in `poll_read` without touching `self.remaining`, so their bytes must be
// accounted for here. Otherwise, under multi-in-flight fill policies (e.g. the
// default `DualInFlight`), `remaining` never reaches 0 and a fully delivered
// multi-block object still terminates the GET with `LessData`. Buffers in
// `queued_buffers` come only from `Ok(true)` decodes, so they are non-empty and
// their total is bounded by `remaining - main_buf.len()`, ruling out underflow.
self.remaining -= queued_buffers.iter().map(Vec::len).sum::<usize>();
self.prefetched_bufs.extend(queued_buffers);
match result {
Ok(Some(buf)) => {
@@ -362,8 +359,17 @@ where
}
rustfs_io_metrics::record_get_object_reader_stripe(self.metrics_path);
rustfs_io_metrics::record_get_object_reader_bytes(self.metrics_path, buf.len());
// Queued stripes are delivered to the client via `prefetched_bufs.pop_front()`
// in `poll_read` without touching `self.remaining`, so their bytes must be
// accounted for here alongside the primary stripe. Buffers in `queued_buffers`
// come only from `Ok(true)` decodes, so they are non-empty and their total is
// bounded by `remaining - buf.len()`, ruling out underflow.
self.remaining -= buf.len();
Poll::Ready(Ok(Some(buf)))
self.remaining -= queued_buffers.iter().map(Vec::len).sum::<usize>();
Poll::Ready(Ok(Some(FilledStripes {
primary: buf,
queued: queued_buffers,
})))
}
Ok(None) => {
if self.remaining == 0 {
@@ -401,8 +407,8 @@ where
};
match fill {
Ok(Some(buf)) => {
let output_has_remaining = self.output_pos < self.output_buf.len();
Ok(Some(FilledStripes { primary: buf, queued })) => {
let output_has_remaining = self.output_pos < self.output_buf.len() || !self.prefetched_bufs.is_empty();
let mut queued_count = 0usize;
let mut queued_bytes = 0usize;
@@ -420,6 +426,7 @@ where
self.output_buf = buf;
self.output_pos = 0;
}
self.prefetched_bufs.extend(queued);
if !self.prefetched_bufs.is_empty() {
let total_prefetched_bytes = self.prefetched_bufs.iter().map(Vec::len).sum::<usize>();
@@ -451,7 +458,7 @@ where
Poll::Ready(Ok(()))
}
Err(err) => {
if self.output_pos < self.output_buf.len() {
if self.output_pos < self.output_buf.len() || !self.prefetched_bufs.is_empty() {
rustfs_io_metrics::record_get_object_reader_prefetch(self.metrics_path, GET_READER_PREFETCH_ERROR_DEFERRED);
self.prefetch_error = Some(err);
Poll::Ready(Ok(()))
@@ -1860,6 +1867,42 @@ mod tests {
.expect("dual inflight policy should prefetch two future stripes before current output drains");
}
#[tokio::test]
async fn erasure_decode_reader_dual_inflight_preserves_stripe_order_when_prefetch_completes_before_output_drains() {
let erasure = Erasure::new(4, 2, 16);
let data = (0..96u8).collect::<Vec<_>>();
let read_count = Arc::new(AtomicUsize::new(0));
let mut source = source_from_data(&erasure, &data, &[]);
source.read_count = Some(Arc::clone(&read_count));
let engine = LegacyEcDecodeEngine::new(erasure);
let mut reader = ErasureDecodeReader::new_with_fill_policy(
source,
engine,
data.len(),
GET_OBJECT_PATH_CODEC_STREAMING,
FillPolicy::DualInFlight,
)
.expect("reader should be constructed");
let mut decoded = Vec::with_capacity(data.len());
let mut chunk = [0u8; 5];
loop {
let n = reader.read(&mut chunk).await.expect("chunk read should succeed");
if n == 0 {
break;
}
decoded.extend_from_slice(&chunk[..n]);
// Yield between partial reads so the DualInFlight worker completes
// a 2-stripe fill (primary + queued_buffers) while output_buf still
// has unconsumed bytes.
for _ in 0..4 {
yield_now().await;
}
}
assert_eq!(decoded, data);
}
#[tokio::test]
async fn erasure_decode_reader_dual_inflight_reads_multi_block_object_to_end() {
// Regression for the byte-accounting bug: under DualInFlight each fill
@@ -0,0 +1,35 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
mod storage_api;
use storage_api::local_drive_retirement::{Disk, DiskAPI, DiskOption, Endpoint, new_disk};
#[tokio::test]
async fn local_wrapper_retirement_is_shared_and_independent_of_monitoring() {
let dir = tempfile::tempdir().expect("retirement fixture directory");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 fixture path")).expect("local endpoint");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("local disk");
let Disk::Local(wrapper) = disk.as_ref() else { panic!("fixture must use a local disk") };
let cloned = wrapper.clone();
let observer = disk.local_retirement_token().expect("local retirement observer");
let cancelled_observer = disk.local_retirement_token().expect("another observer");
cancelled_observer.cancel();
assert!(!observer.is_cancelled(), "observers cannot retire the wrapper");
wrapper.stop_monitoring().await;
assert!(!observer.is_cancelled(), "monitoring policy is not retirement");
cloned.close().await.expect("close shared wrapper");
assert!(observer.is_cancelled(), "clones share retirement");
assert!(
disk.local_retirement_token().expect("late observer").is_cancelled(),
"late observers see retirement"
);
disk.close().await.expect("retirement is idempotent");
}
+5
View File
@@ -94,3 +94,8 @@ pub(crate) mod pinned_erasure_fixtures {
storage::init_local_disks_with_instance_ctx,
};
}
pub(crate) mod local_drive_retirement {
pub(crate) use super::{DiskAPI, DiskOption, Endpoint, new_disk};
pub(crate) use rustfs_ecstore::api::disk::Disk;
}
@@ -1093,6 +1093,7 @@ async fn scan_and_persist_local_bucket_with_store(
scan_mode,
prefix_scan_scope: None,
checkpoint_tx: Some(checkpoint_tx),
local_stop: None,
},
),
);
@@ -613,6 +613,7 @@ async fn remote_worker_rechecks_real_producers_before_persisting_complete_usage(
scan_mode: HealScanMode::Normal,
prefix_scan_scope: None,
checkpoint_tx: None,
local_stop: None,
},
)
.await
@@ -641,6 +642,7 @@ async fn remote_worker_rechecks_real_producers_before_persisting_complete_usage(
scan_mode: HealScanMode::Normal,
prefix_scan_scope: None,
checkpoint_tx: None,
local_stop: None,
},
)
.await
+36 -5
View File
@@ -835,6 +835,7 @@ pub struct FolderScanner {
budget: Arc<ScannerCycleBudget>,
skip_heal: Arc<std::sync::atomic::AtomicBool>,
local_disk: Arc<Disk>,
local_stop: Option<Arc<crate::scanner_io::ScannerLocalStop>>,
/// Tier registry frozen for this folder scan. A refresh applies to the
/// next scan and cannot mix generations in one aggregate.
tier_registry: TierRegistrySnapshot,
@@ -2415,6 +2416,17 @@ impl FolderScanner {
self.record_failed(&item.path);
}
if self.local_disk.is_local() && matches!(e, StorageError::FaultyDisk | StorageError::DiskNotFound) {
if let Some(stop) = &self.local_stop {
stop.stop(crate::scanner_io::ScannerLocalStopReason::Unavailable, &ctx);
} else {
ctx.cancel();
}
// Cancellation preserves this unresolved object and
// completed siblings through every recursive path.
return Err(ScannerError::Other("Local scanner drive unavailable".to_string()));
}
if !retry_suppressed && should_log_failed_object(into.failed_objects) {
if let GetSizeFailureAction::HealMetadata { object } = &failure_action {
error!(
@@ -3345,7 +3357,20 @@ pub async fn scan_data_folder(
scan_mode: HealScanMode,
sleeper: DynamicSleeper,
) -> Result<DataUsageCache, ScannerError> {
scan_data_folder_scoped(ctx, budget, disks, local_disk, cache, updates, scan_mode, sleeper, None, None).await
scan_data_folder_scoped(
ctx.child_token(),
budget,
disks,
local_disk,
cache,
updates,
scan_mode,
sleeper,
None,
None,
Some(Arc::new(crate::scanner_io::ScannerLocalStop::default())),
)
.await
}
#[allow(clippy::too_many_arguments)]
@@ -3360,6 +3385,7 @@ pub(crate) async fn scan_data_folder_scoped(
sleeper: DynamicSleeper,
prefix_scan_scope: Option<ScannerBucketPrefixScanScope>,
checkpoint_tx: Option<mpsc::Sender<DataUsageCache>>,
local_stop: Option<Arc<crate::scanner_io::ScannerLocalStop>>,
) -> Result<DataUsageCache, ScannerError> {
use crate::data_usage_define::DATA_USAGE_ROOT;
@@ -3457,6 +3483,7 @@ pub(crate) async fn scan_data_folder_scoped(
budget: budget.clone(),
skip_heal,
local_disk,
local_stop,
tier_registry,
pending_heals_changed: false,
coverage_frontier: clone_cache_value(&resume_frontier),
@@ -3481,11 +3508,13 @@ pub(crate) async fn scan_data_folder_scoped(
prune_size_reconciliation(&mut scanner.new_cache.info, now);
// Check if context is cancelled
if ctx.is_cancelled() {
if ctx.is_cancelled() && scanner.local_stop.as_ref().is_none_or(|stop| stop.reason().is_none()) {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
scanner.retry_pending_scanner_heals().await?;
if !ctx.is_cancelled() {
scanner.retry_pending_scanner_heals().await?;
}
// Read top level in bucket
let mut root = DataUsageEntry::default();
@@ -3667,7 +3696,9 @@ pub(crate) async fn scan_data_folder_scoped(
close_disk_guard.close().await;
return Err(ScannerError::NamespaceNotFoundCache(Box::new(partial_cache)));
}
if matches!(&e, ScannerError::ResourceDeferred) {
if matches!(&e, ScannerError::ResourceDeferred)
|| scanner.local_stop.as_ref().is_some_and(|stop| stop.reason().is_some())
{
// Admission pressure is not disk failure or an empty census.
// Keep the previous tree when no new useful progress exists.
let mut partial_cache = scanner.old_cache;
@@ -3677,7 +3708,7 @@ pub(crate) async fn scan_data_folder_scoped(
.insert(hash_path(&bucket_name).0, DataUsageEntry::default());
}
partial_cache.info.snapshot_complete = false;
partial_cache.info.scan_resource_deferred = true;
partial_cache.info.scan_resource_deferred = matches!(&e, ScannerError::ResourceDeferred);
close_disk_guard.close().await;
return Err(ScannerError::PartialCache(Box::new(partial_cache)));
}
@@ -392,6 +392,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
budget: ScannerCycleBudget::new(&CancellationToken::new(), Default::default()),
skip_heal: Arc::new(AtomicBool::new(false)),
local_disk: disk,
local_stop: None,
tier_registry: crate::TierRegistrySnapshot {
generation: 0,
names: Arc::new([]),
@@ -40,6 +40,7 @@ fn scan_options(scan_mode: HealScanMode) -> ScannerDiskScanOptions {
scan_mode,
prefix_scan_scope: None,
checkpoint_tx: None,
local_stop: None,
}
}
@@ -1307,6 +1308,7 @@ where
scan_mode: HealScanMode::Normal,
prefix_scan_scope: None,
checkpoint_tx: None,
local_stop: None,
},
)
.await
@@ -1390,6 +1392,7 @@ where
scan_mode: HealScanMode::Normal,
prefix_scan_scope: None,
checkpoint_tx: None,
local_stop: None,
},
)
.await;
+5
View File
@@ -1699,6 +1699,8 @@ pub struct ScannerDiskScanOptions {
pub scan_mode: HealScanMode,
pub prefix_scan_scope: Option<ScannerBucketPrefixScanScope>,
pub checkpoint_tx: Option<mpsc::Sender<DataUsageCache>>,
/// Local interruption state shared with the coordinator; not part of RPC or cache formats.
pub local_stop: Option<Arc<ScannerLocalStop>>,
}
#[async_trait::async_trait]
@@ -2131,6 +2133,9 @@ pub(crate) use usage_watch::ScannerUsageWatchReleaseGuard;
use usage_watch::{ScannerUsageWatchCohort, admit_scanner_usage_watch, record_completed_usage_watch};
pub(crate) use usage_watch::{bind_usage_watch_to_checkpoint, invalidate_completed_usage_watch_cache};
mod io_disk;
mod local_drive;
use local_drive::*;
pub use local_drive::{ScannerLocalStop, ScannerLocalStopReason};
#[cfg(test)]
mod publish_gate_tests;
#[cfg(test)]
+80 -1
View File
@@ -989,6 +989,7 @@ impl ScannerIOCache for SetDisks {
if turn_bucket_count == 0 {
bucket_work_complete.cancel();
}
let local_recovery_attempts = Arc::new(Mutex::new(HashSet::<String>::new()));
for NamespaceScannerWalker {
disk,
mode: worker_mode,
@@ -1011,6 +1012,7 @@ impl ScannerIOCache for SetDisks {
let bucket_result_tx_clone = bucket_result_tx.clone();
let bucket_observations_clone = bucket_observations.clone();
let set_disk_inventory_clone = set_disk_inventory.clone();
let local_recovery_attempts_clone = local_recovery_attempts.clone();
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone();
@@ -1078,6 +1080,34 @@ impl ScannerIOCache for SetDisks {
pool_label_clone.clone(),
set_label_clone.clone(),
);
let assigned_disk = &disk;
let disk = if assigned_disk.is_local() {
match current_local_scanner_disk(&store_clone_clone, assigned_disk, &ctx_clone, &budget_clone).await {
Some(current) => current,
None => {
drop(_active_guard);
drop(_permit);
let retry_available = {
local_recovery_attempts_clone.lock().await.insert(bucket.name.clone())
};
if retry_available
&& wait_for_local_scanner_recovery(
&store_clone_clone, assigned_disk, &ctx_clone, &budget_clone,
).await.is_some()
&& requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await
{
increment_disk_bucket_scans_queued(
&queued_disk_bucket_scans_clone, &pool_label_clone, &set_label_clone,
);
} else {
record_partial_dirty_bucket(&partial_dirty_buckets_clone, &bucket.name).await;
}
continue;
}
}
} else {
assigned_disk.clone()
};
if ctx_clone.is_cancelled() || budget_clone.budget_elapsed() {
break;
}
@@ -1651,6 +1681,14 @@ impl ScannerIOCache for SetDisks {
let resumed_watch_sweep = cache.has_scan_position();
let scan_position_before = cache.durable_scan_progress();
let local_stop = Arc::new(ScannerLocalStop::default());
#[cfg(test)]
super::tests::local_drive_recovery::before_attempt(&bucket.name, &disk).await;
let current_set_disks = if disk.is_local() {
scanner_set_disk_inventory(&store_clone_clone).await
} else {
set_disk_inventory_clone.as_ref().clone()
};
#[cfg(feature = "e2e-test-hooks")]
{
let disk_uuid = if crate::e2e_barrier::worker_observation::enabled() {
@@ -1671,13 +1709,14 @@ impl ScannerIOCache for SetDisks {
let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
scan_ctx.clone(),
bucket_budget.clone(),
set_disk_inventory_clone.as_ref().clone(),
current_set_disks,
cache,
None,
ScannerDiskScanOptions {
scan_mode,
prefix_scan_scope,
checkpoint_tx: Some(checkpoint_tx),
local_stop: Some(local_stop.clone()),
},
));
tokio::pin!(scan);
@@ -1711,6 +1750,9 @@ impl ScannerIOCache for SetDisks {
await_scanner_disk_shutdown(scan.as_mut()).await;
break Err(Error::other("scanner bucket cache lock was lost during bucket scan"));
}
if disk.is_local() && let Some(reason) = local_scanner_stop_reason(&store_clone_clone, &disk).await {
local_stop.stop(reason, &scan_ctx);
}
}
checkpoint = checkpoint_rx.recv(), if !checkpoint_channel_closed => {
let Some(checkpoint) = checkpoint else {
@@ -1818,6 +1860,9 @@ impl ScannerIOCache for SetDisks {
continue;
}
}
if disk.is_local() && let Some(reason) = local_scanner_stop_reason(&store_clone_clone, &disk).await {
local_stop.stop(reason, &scan_ctx);
}
let scan_outcome = match scan_result {
Ok(scan_outcome) => scan_outcome,
Err(e) if bucket_budget.quantum_exhausted() || bucket_budget.source_turn_exhausted() => {
@@ -1864,6 +1909,15 @@ impl ScannerIOCache for SetDisks {
}
};
let scan_outcome = match scan_outcome {
ScannerDiskScanOutcome::Complete(mut cache) | ScannerDiskScanOutcome::Partial(mut cache)
if local_stop.reason().is_some() => {
retain_interrupted_local_cache(&mut cache);
ScannerDiskScanOutcome::Partial(cache)
}
other => other,
};
let (cache, partial) = match scan_outcome {
ScannerDiskScanOutcome::Complete(mut completed_cache) => {
if resumed_watch_sweep && let Some(proof) = completed_cache.info.scan_usage_watch.as_ref()
@@ -1987,6 +2041,31 @@ impl ScannerIOCache for SetDisks {
bucket_budget.record_durable_turn_progress();
}
}
if local_stop.reason().is_some() {
// Recovery is independent of durable forward progress, but
// cannot retry an unsaved result or renew its own cache lease.
drop(cache_guard);
drop(_local_admission);
drop(_active_guard);
drop(_permit);
let retry_available = {
local_recovery_attempts_clone.lock().await.insert(bucket.name.clone())
};
if partial_progress_proven
&& retry_available
&& wait_for_local_scanner_recovery(
&store_clone_clone, &disk, &ctx_clone, &budget_clone,
).await.is_some()
&& requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await
{
increment_disk_bucket_scans_queued(
&queued_disk_bucket_scans_clone, &pool_label_clone, &set_label_clone,
);
} else {
record_partial_dirty_bucket(&partial_dirty_buckets_clone, &bucket.name).await;
}
continue;
}
// Only saved forward motion earns another turn in this
// cycle; otherwise the bucket waits for the next one.
if partial_progress_proven
+63 -5
View File
@@ -64,12 +64,19 @@ impl ScannerIODisk for Disk {
let metadata_object_path = item.object_path();
let metadata_result =
crate::workload_admission::with_scanner_metadata_io(self.read_metadata(&item.bucket, &metadata_object_path)).await;
#[cfg(test)]
let metadata_result = if super::tests::local_drive_recovery::after_metadata_read(&item.bucket).await {
Err(DiskError::FaultyDisk)
} else {
metadata_result
};
ensure_scanner_node_operation_active(cancellation.as_ref())?;
let data = match metadata_result {
Ok(data) => data,
Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
}
Err(e @ (DiskError::FaultyDisk | DiskError::DiskNotFound)) if self.is_local() => return Err(e.into()),
Err(e) => {
return Err(scanner_metadata_transient_error(
format!("failed to read metadata: {e}"),
@@ -196,7 +203,15 @@ impl ScannerIODisk for Disk {
scan_mode,
prefix_scan_scope,
checkpoint_tx,
local_stop,
} = options;
let local_stop = local_stop.unwrap_or_default();
let retirement = self.local_retirement_token();
// Only this attempt stops; cycle budgets and the leader fence remain owned by the caller.
let ctx = ctx.child_token();
if retirement.as_ref().is_some_and(CancellationToken::is_cancelled) {
local_stop.stop(ScannerLocalStopReason::Retired, &ctx);
}
let done_drive = Metrics::time(Metric::ScanBucketDrive);
let drive_start = std::time::Instant::now();
let bucket = clone_cache_value(&cache.info.name);
@@ -256,9 +271,8 @@ impl ScannerIODisk for Disk {
.then_some(prefix_scan_scope)
.flatten();
let result = crate::storage_api::ecstore_with_background_disk_io(
ctx.clone(),
scan_data_folder_scoped(
let result = crate::storage_api::ecstore_with_background_disk_io(ctx.clone(), async {
let scan = scan_data_folder_scoped(
ctx.clone(),
budget,
set_disks,
@@ -270,12 +284,53 @@ impl ScannerIODisk for Disk {
SCANNER_SLEEPER.for_disk(&self.path()),
prefix_scan_scope,
checkpoint_tx,
),
)
Some(local_stop.clone()),
);
tokio::pin!(scan);
loop {
tokio::select! {
result = &mut scan => break result,
_ = async {
match &retirement {
Some(token) => token.cancelled().await,
None => std::future::pending().await,
}
}, if local_stop.reason().is_none() => {
local_stop.stop(ScannerLocalStopReason::Retired, &ctx);
}
}
}
})
.await;
#[cfg(test)]
super::tests::local_drive_recovery::after_walk(&bucket, &self).await;
if retirement.as_ref().is_some_and(CancellationToken::is_cancelled) {
local_stop.stop(ScannerLocalStopReason::Retired, &ctx);
}
if let Some(reason) = local_stop.reason() {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DISK_BUCKET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "local_drive_interrupted",
bucket = %bucket,
endpoint = %crate::storage_api::owner::EcstoreDiskAPI::endpoint(self.as_ref()),
reason = reason.as_str(),
"Scanner retained unfinished work after a local drive interruption"
);
}
match result {
Ok(mut data_usage_info) => {
if local_stop.reason().is_some() {
retain_interrupted_local_cache(&mut data_usage_info);
done_drive();
emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
failure_guard.mark_not_failed();
return Ok(ScannerDiskScanOutcome::Partial(data_usage_info));
}
done_drive();
emit_scan_bucket_drive_complete(source, true, &bucket, &disk_path, drive_start.elapsed());
data_usage_info.info.last_update = Some(SystemTime::now());
@@ -283,6 +338,9 @@ impl ScannerIODisk for Disk {
Ok(ScannerDiskScanOutcome::Complete(data_usage_info))
}
Err(ScannerError::PartialCache(mut partial_cache)) => {
if local_stop.reason().is_some() {
retain_interrupted_local_cache(&mut partial_cache);
}
done_drive();
emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
@@ -0,0 +1,158 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::storage_api::owner::EcstoreDiskAPI;
use std::sync::OnceLock;
/// A local scan interruption is distinct from a cycle budget or leader fence.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ScannerLocalStopReason {
Retired,
Unavailable,
Replaced,
}
impl ScannerLocalStopReason {
pub(super) fn as_str(self) -> &'static str {
match self {
Self::Retired => "retired",
Self::Unavailable => "unavailable",
Self::Replaced => "replaced",
}
}
}
/// Process-local outcome of one scan attempt; never serialized into a checkpoint.
#[derive(Debug, Default)]
pub struct ScannerLocalStop {
reason: OnceLock<ScannerLocalStopReason>,
}
impl ScannerLocalStop {
/// The first local interruption, independent of ordinary cancellation.
pub fn reason(&self) -> Option<ScannerLocalStopReason> {
self.reason.get().copied()
}
pub(crate) fn stop(&self, reason: ScannerLocalStopReason, scan: &CancellationToken) {
let _ = self.reason.set(reason);
scan.cancel();
}
}
pub(super) fn retain_interrupted_local_cache(cache: &mut DataUsageCache) {
cache.info.snapshot_complete = false;
// An EOF racing with retirement must be verified again, not published as
// a completed companion or reused under an old coverage proof.
cache.info.scan_plan_digest = None;
cache.info.scan_bucket_observations.clear();
cache.info.scan_bucket_value_origins.clear();
}
const LOCAL_SCANNER_LOOKUP_TIMEOUT: Duration = Duration::from_secs(5);
/// Bind the assigned local topology slot to its currently usable instance.
/// The format identity fences resume across a physical drive replacement.
pub(super) async fn current_local_scanner_disk(
set: &SetDisks,
assigned: &Arc<Disk>,
ctx: &CancellationToken,
budget: &ScannerCycleBudget,
) -> Option<Arc<Disk>> {
// This also bounds the first binding, while the coordinator owns a scan
// permit. Waiting for a format probe must respect source cancellation and its turn.
let lookup = async {
let location = EcstoreDiskAPI::get_disk_location(assigned.as_ref());
let expected_id = location
.disk_idx
.and_then(|index| set.format.erasure.sets.get(set.set_index)?.get(index))
.copied()
.filter(|id| !id.is_nil());
let expected_id = expected_id?;
let candidate = scanner_set_disk_inventory(set).await.into_iter().find(|candidate| {
EcstoreDiskAPI::is_local(candidate.as_ref())
&& EcstoreDiskAPI::endpoint(candidate.as_ref()) == EcstoreDiskAPI::endpoint(assigned.as_ref())
});
let candidate = candidate?;
#[cfg(test)]
super::tests::local_drive_recovery::before_binding_probe().await;
if candidate.local_retirement_token().is_some_and(|token| token.is_cancelled()) || !candidate.is_online().await {
return None;
}
match candidate.get_disk_id().await {
Ok(Some(id)) if id == expected_id => Some(candidate),
Ok(_) | Err(_) => None,
}
};
tokio::select! {
biased;
_ = ctx.cancelled() => None,
_ = budget.wait_for_stop() => None,
_ = tokio::time::sleep(LOCAL_SCANNER_LOOKUP_TIMEOUT) => None,
disk = lookup => disk,
}
}
/// Wait without a cache lease or disk scan permit. Each bucket gets at most
/// one such recovery opportunity per source turn, including zero-progress stops.
pub(super) async fn wait_for_local_scanner_recovery(
set: &SetDisks,
assigned: &Arc<Disk>,
ctx: &CancellationToken,
budget: &ScannerCycleBudget,
) -> Option<Arc<Disk>> {
const MAX_WAIT: Duration = Duration::from_secs(5);
let deadline = tokio::time::Instant::now() + MAX_WAIT;
let mut interval = tokio::time::interval(SCANNER_CACHE_LOCK_POLL_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
biased;
_ = ctx.cancelled() => return None,
_ = budget.wait_for_stop() => return None,
_ = tokio::time::sleep_until(deadline) => return None,
_ = interval.tick() => {
if budget.budget_elapsed() {
return None;
}
let current = tokio::select! {
biased;
_ = ctx.cancelled() => return None,
_ = budget.wait_for_stop() => return None,
_ = tokio::time::sleep_until(deadline) => return None,
disk = current_local_scanner_disk(set, assigned, ctx, budget) => disk,
};
if let Some(disk) = current {
return Some(disk);
}
}
}
}
}
/// Inspect runtime health and ownership without probing the drive or holding inventory guards during I/O.
pub(super) async fn local_scanner_stop_reason(set: &SetDisks, disk: &Arc<Disk>) -> Option<ScannerLocalStopReason> {
if disk.local_retirement_token().is_some_and(|token| token.is_cancelled()) {
return Some(ScannerLocalStopReason::Retired);
}
if !disk.runtime_state().is_snapshot_eligible() {
return Some(ScannerLocalStopReason::Unavailable);
}
(!scanner_set_disk_inventory(set)
.await
.iter()
.any(|current| Arc::ptr_eq(current, disk)))
.then_some(ScannerLocalStopReason::Replaced)
}
+1
View File
@@ -52,6 +52,7 @@ mod collector_lifecycle;
mod cycle_collector_diagnostics;
mod expiry_lock;
mod listing_window_scope;
pub(super) mod local_drive_recovery;
mod node_admission;
mod partial_bucket_carry;
mod per_bucket_certification;
@@ -0,0 +1,298 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
use super::*;
use crate::storage_api::owner::EcstoreDiskAPI as _;
struct Probe {
set: Arc<SetDisks>,
before: bool,
replace: bool,
faulty: bool,
eof: bool,
attempts: AtomicUsize,
reads: AtomicUsize,
}
static PROBES: std::sync::LazyLock<StdMutex<HashMap<String, Arc<Probe>>>> =
std::sync::LazyLock::new(|| StdMutex::new(HashMap::new()));
fn probe(bucket: &str) -> Option<Arc<Probe>> {
PROBES.lock().expect("probe registry").get(bucket).cloned()
}
async fn retire(probe: &Probe, disk: &Arc<Disk>) {
let replacement = if probe.replace {
let next = new_disk(
&disk.endpoint(),
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("replacement wrapper");
assert_eq!(
next.get_disk_id().await.expect("replacement identity"),
disk.get_disk_id().await.expect("old identity")
);
Some(next)
} else {
None
};
disk.close().await.expect("retire old wrapper");
if let Some(next) = replacement {
let mut disks = probe.set.disks.write().await;
let slot = disks
.iter_mut()
.find(|slot| slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, disk)))
.expect("old wrapper in authoritative inventory");
*slot = Some(next);
}
}
pub(crate) async fn before_attempt(bucket: &str, disk: &Arc<Disk>) {
let Some(probe) = probe(bucket) else { return };
let attempt = probe.attempts.fetch_add(1, Ordering::SeqCst);
if probe.before && !probe.faulty && attempt == 0 {
retire(&probe, disk).await;
}
}
pub(crate) async fn after_metadata_read(bucket: &str) -> bool {
let Some(probe) = probe(bucket) else { return false };
if probe.faulty {
return probe.reads.fetch_add(1, Ordering::SeqCst) == 0;
}
if !probe.before && !probe.eof && probe.reads.fetch_add(1, Ordering::SeqCst) == 0 {
let disk = scanner_set_disk_inventory(&probe.set)
.await
.into_iter()
.next()
.expect("local wrapper");
retire(&probe, &disk).await;
}
false
}
pub(crate) async fn after_walk(bucket: &str, disk: &Arc<Disk>) {
let Some(probe) = probe(bucket) else { return };
if probe.eof && probe.reads.fetch_add(1, Ordering::SeqCst) == 0 {
retire(&probe, disk).await;
}
}
struct RegisteredProbe(String);
impl Drop for RegisteredProbe {
fn drop(&mut self) {
PROBES.lock().expect("probe registry").remove(&self.0);
}
}
async fn recovery_fixture(bucket: &str, before: bool, replace: bool, faulty: bool, eof: bool) {
let (_dir, store) = setup_local_scanner_store(1, 1).await;
clear_dirty_usage_buckets_for_tests();
for name in [bucket, "unaffected-bucket"] {
store
.make_bucket(name, &MakeBucketOptions::default())
.await
.expect("create bucket");
for path in ["a/deep/first", "a/deep/second", "z/last"] {
let mut reader = ScannerPutObjReader::from_vec(b"payload".to_vec());
store
.put_object(name, path, &mut reader, &ScannerObjectOptions::default())
.await
.expect("write object");
}
}
wait_for_namespace_commit_tails(store.as_ref()).await;
let set = store.pools[0].disk_set[0].clone();
let probe = Arc::new(Probe {
set: set.clone(),
before,
replace,
faulty,
eof,
attempts: AtomicUsize::new(0),
reads: AtomicUsize::new(0),
});
PROBES
.lock()
.expect("probe registry")
.insert(bucket.to_string(), probe.clone());
let _registered = RegisteredProbe(bucket.to_string());
let status = run_full_cycle(&store, 1).await;
if replace || faulty {
assert_eq!(status, ScannerCycleStatus::Complete);
assert_eq!(
probe.attempts.load(Ordering::SeqCst),
2,
"one recovery attempt including zero-progress interruption"
);
let mut cache = DataUsageCache::default();
cache
.load(set, &path_join_buf(&[bucket, DATA_USAGE_CACHE_NAME]))
.await
.expect("reload durable cache");
assert!(cache.info.snapshot_complete, "recovery must finish a full scan");
let total = cache.checked_flatten(bucket).expect("complete recovered statistics");
assert_eq!(total.objects, 3);
assert_eq!(total.size, 21);
} else {
assert_ne!(status, ScannerCycleStatus::Complete, "retired drive cannot certify completion");
assert_eq!(probe.attempts.load(Ordering::SeqCst), 1, "no repeated scans on a retired wrapper");
}
clear_dirty_usage_buckets_for_tests();
}
#[tokio::test]
#[serial]
async fn issue_9469_recovers_retired_wrapper_before_first_object() {
recovery_fixture("recovery-first", true, true, false, false).await;
}
#[tokio::test]
#[serial]
async fn issue_9469_recovers_retired_wrapper_in_deep_subtree() {
recovery_fixture("recovery-deep", false, true, false, false).await;
}
#[tokio::test]
#[serial]
async fn issue_9469_persistent_retirement_is_bounded() {
recovery_fixture("recovery-offline", true, false, false, false).await;
}
#[tokio::test]
#[serial]
async fn issue_9469_faulty_metadata_stops_and_retries_without_losing_object() {
recovery_fixture("recovery-faulty", false, false, true, false).await;
}
#[tokio::test]
#[serial]
async fn issue_9469_retirement_at_eof_cannot_certify_old_handle() {
recovery_fixture("recovery-eof", false, true, false, true).await;
}
#[tokio::test]
#[serial]
async fn issue_9469_binding_rejects_changed_format_identity_and_cancelled_recovery() {
let (_dir, store) = setup_local_scanner_store(1, 1).await;
let set = store.pools[0].disk_set[0].clone();
let disk = scanner_set_disk_inventory(&set)
.await
.into_iter()
.next()
.expect("local drive");
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
let current = current_local_scanner_disk(&set, &disk, &ctx, &budget)
.await
.expect("matching drive binding");
assert!(Arc::ptr_eq(&current, &disk));
let mut different_format = set.as_ref().clone();
different_format.format.erasure.sets[0][0] = Uuid::new_v4();
assert!(
current_local_scanner_disk(&different_format, &disk, &ctx, &budget)
.await
.is_none(),
"different physical identity cannot resume this slot"
);
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
ctx.cancel();
assert!(
wait_for_local_scanner_recovery(&set, &disk, &ctx, &budget).await.is_none(),
"cancelled source must not admit recovery"
);
}
tokio::task_local! {
static BLOCK_BINDING_PROBE: Arc<tokio::sync::Notify>;
}
pub(crate) async fn before_binding_probe() {
if let Ok(entered) = BLOCK_BINDING_PROBE.try_with(Arc::clone) {
entered.notify_one();
std::future::pending::<()>().await;
}
}
#[derive(Clone, Copy)]
enum BindingStop {
Timeout,
Cancellation,
Budget,
}
async fn blocked_binding_fixture(stop: BindingStop) {
let (_dir, store) = setup_local_scanner_store(1, 1).await;
let set = store.pools[0].disk_set[0].clone();
let disk = scanner_set_disk_inventory(&set)
.await
.into_iter()
.next()
.expect("local drive");
// Real filesystem setup completes before using controlled time.
tokio::time::pause();
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(
&ctx,
ScannerCycleBudgetConfig {
max_duration: matches!(stop, BindingStop::Budget).then_some(Duration::from_secs(1)),
..Default::default()
},
);
let entered = Arc::new(tokio::sync::Notify::new());
let started = tokio::time::Instant::now();
BLOCK_BINDING_PROBE
.scope(entered.clone(), async {
let lookup = current_local_scanner_disk(&set, &disk, &ctx, &budget);
tokio::pin!(lookup);
tokio::select! {
biased;
_ = &mut lookup => panic!("the controlled probe must block before binding finishes"),
_ = entered.notified() => {},
}
if matches!(stop, BindingStop::Cancellation) {
ctx.cancel();
}
assert!(
tokio::time::timeout(Duration::from_secs(6), &mut lookup)
.await
.expect("initial binding must stop even when metadata I/O never returns")
.is_none()
);
})
.await;
let expected = match stop {
BindingStop::Timeout => Duration::from_secs(5),
BindingStop::Cancellation => Duration::ZERO,
BindingStop::Budget => Duration::from_secs(1),
};
let elapsed = started.elapsed();
// Tokio timers round to millisecond ticks; no wall-clock sleeps are used.
assert!(
(expected..=expected + Duration::from_millis(1)).contains(&elapsed),
"the binding must respect the earliest bound: expected {expected:?}, elapsed {elapsed:?}"
);
}
#[tokio::test]
#[serial]
async fn issue_9469_initial_binding_metadata_probe_has_a_timeout() {
blocked_binding_fixture(BindingStop::Timeout).await;
}
#[tokio::test]
#[serial]
async fn issue_9469_initial_binding_metadata_probe_observes_cancellation() {
blocked_binding_fixture(BindingStop::Cancellation).await;
}
#[tokio::test]
#[serial]
async fn issue_9469_initial_binding_metadata_probe_observes_budget() {
blocked_binding_fixture(BindingStop::Budget).await;
}
@@ -61,6 +61,7 @@ async fn disk_scan_withdraws_persisted_lifecycle_policy() {
scan_mode: HealScanMode::Normal,
prefix_scan_scope: None,
checkpoint_tx: None,
local_stop: None,
},
)
.await