gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 05:26:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): fence movement generation publication (#6461)

Browse Source 
* feat(scanner): add movement generation fencing

* fix(scanner): prioritize unverified cycle deferral

* feat(ecstore): add scanner publication lease fence

* feat(rpc): add scanner publication lease protocol

* feat(scanner): hold remote leases through usage publish

* test(scanner): cover publication lease fencing

* fix(scanner): fence remote leases across restart and delay

* feat(rpc): fence scanner publication rename writes

* fix(scanner): fence observed cleanup deletes

* fix(proto): qualify lease release test types

* fix(scanner): pin movement notifications

* fix(scanner): clean publication imports

* fix(ecstore): satisfy scanner fence clippy

* refactor(scanner): group wait and publication options

* fix(scanner): satisfy final lint and facade guards

* fix(rpc): resolve facade export conflicts

* fix(ci): remove unused decommission and healing facades

* fix(ci): cfg-gate test-only usage overlay import

* fix(scanner): wake on remote scanner restart
This commit is contained in:
cxymds
2026-08-24 14:17:35 +08:00
committed by GitHub
parent 1c5c28842a
commit eec0e0e056
30 changed files with 2926 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/lib.rs
+11 -2
View File
@@ -92,7 +92,7 @@ pub use scanner_io::{
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
use std::sync::atomic::{AtomicU64, Ordering};
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
pub use storage_api::scan::SCANNER_ACTIVITY_PROTOCOL_VERSION;
pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
@@ -796,17 +796,25 @@ where
Some(admission)
}
pub(crate) async fn save_config_shared_with_preconditions<S>(
pub(crate) async fn save_config_shared_with_preconditions_and_lease_fence<S>(
api: Arc<S>,
file: &str,
data: Bytes,
sha256hex: Option<String>,
preconditions: HTTPPreconditions,
scanner_publication_lease_fence: Option<&str>,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO,
{
let mut reader = ScannerPutObjReader::from_prehashed_bytes(data, sha256hex)?;
let mut user_defined = HashMap::new();
if let Some(fence) = scanner_publication_lease_fence {
user_defined.insert(
storage_api::owner::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
fence.to_string(),
);
}
api.put_object(
RUSTFS_META_BUCKET,
file,
@@ -814,6 +822,7 @@ where
&ScannerObjectOptions {
max_parity: true,
http_preconditions: Some(preconditions),
user_defined,
..Default::default()
},
)
crates/scanner/src/scanner.rs
+159 -13
View File
@@ -63,6 +63,7 @@ use tokio_util::sync::CancellationToken;
use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, error, info, instrument, warn};
use crate::storage_api::owner::SCANNER_PUBLICATION_LEASE_TTL_MS;
use crate::storage_api::scan::{
BucketOperations, BucketOptions, NamespaceLocking as _, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
@@ -71,9 +72,9 @@ use crate::{
ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions,
save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd, scanner_publication_admission_for_epoch,
scanner_publication_epoch, scanner_publication_epoch_changed,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence,
save_config_with_preconditions, save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd,
scanner_publication_admission_for_epoch, scanner_publication_epoch, scanner_publication_epoch_changed,
};
const LOG_COMPONENT_SCANNER: &str = "scanner";
@@ -125,6 +126,8 @@ const MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL: Duration = Duration::fr
const MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS: usize = 2;
const SCANNER_PERSIST_CAS_RETRIES: usize = 2;
const DATA_USAGE_BACKUP_INTERVAL_CYCLES: u64 = 10;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
const SCANNER_CYCLE_STATE_MAGIC: &[u8; 8] = b"RSCYC001";
const SCANNER_CYCLE_STATE_HEADER_LEN: usize = 24;
#[cfg(test)]
@@ -137,6 +140,10 @@ static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCy
static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock<Notify> = LazyLock::new(Notify::new);
fn remote_publication_lease_fence_targets_are_required(target_count: usize, grants_present: bool, fence_present: bool) -> bool {
target_count > 0 && (!grants_present || !fence_present)
}
pub(super) fn notify_scanner_cycle_recovery_wake() {
SCANNER_CYCLE_RECOVERY_WAKE.notify_one();
}
@@ -407,19 +414,24 @@ async fn sync_data_usage_backup_from_primary(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> Result<(), EcstoreError> {
sync_data_usage_backup_from_primary_for_epoch(ctx, storeapi, None).await
sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(ctx, storeapi, None, None, None).await
}
async fn sync_data_usage_backup_from_primary_for_epoch(
async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<&str>,
) -> Result<(), EcstoreError> {
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() {
return Ok(());
}
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
@@ -446,6 +458,10 @@ async fn sync_data_usage_backup_from_primary_for_epoch(
}
let primary = Bytes::from(primary);
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?;
if backup.as_deref() == Some(primary.as_ref()) {
if scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
@@ -462,18 +478,22 @@ async fn sync_data_usage_backup_from_primary_for_epoch(
let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower));
let save_result = {
if remote_lease_deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
save_config_shared_with_preconditions(
save_config_shared_with_preconditions_and_lease_fence(
storeapi.clone(),
&backup_path,
primary.clone(),
sha256hex,
revision.preconditions(),
scanner_publication_lease_fence,
)
.await
};
@@ -1377,8 +1397,89 @@ async fn run_data_scanner_cycle_with_budget(
};
let publication_deferred = publication_defer_reason.is_some();
let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch);
let remote_publication_lease_targets = if publication_defer_reason.is_none() {
scan_result
.as_ref()
.ok()
.map(|result| result.remote_publication_lease_targets().to_vec())
.unwrap_or_default()
} else {
Vec::new()
};
let mut remote_publication_leases = None;
let remote_lease_defer_reason = if remote_publication_lease_targets.is_empty() {
None
} else if usage_persist_timeout >= Duration::from_millis(SCANNER_PUBLICATION_LEASE_TTL_MS) {
// The lease is intentionally fixed-duration and has no renewal path.
// Refuse a persistence budget that could outlive it instead of
// allowing the peer to admit movement while a local PUT is in flight.
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
} else if let Some(notification_system) = storeapi.notification_system() {
match notification_system
.acquire_scanner_publication_leases(remote_publication_lease_targets.clone())
.await
{
Ok(grants) => {
remote_publication_leases = Some((notification_system, grants));
None
}
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
}
} else {
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
};
let remote_lease_deadline = remote_publication_leases
.as_ref()
.and_then(|(_, grants)| grants.iter().map(|grant| grant.lease.expires_at).min());
// The transient fence is carried only to the SetDisks rename boundary;
// it is never inserted into FileInfo metadata. Keep the representation
// bounded and require one authenticated token per remote target so a
// partial grant can never silently fall back to an unfenced rename.
let remote_lease_fence = remote_publication_leases.as_ref().and_then(|(_, grants)| {
if grants.len() != remote_publication_lease_targets.len() || grants.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES {
return None;
}
let mut fence = BTreeMap::new();
for grant in grants {
if grant.host.is_empty() || grant.host.len() > 1024 {
return None;
}
if fence.insert(grant.host.clone(), grant.lease.token.to_string()).is_some() {
return None;
}
}
if remote_publication_lease_targets
.iter()
.any(|(host, _, _)| !fence.contains_key(host))
{
return None;
}
serde_json::to_string(&fence)
.ok()
.filter(|encoded| encoded.len() <= SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES)
});
let remote_lease_fence_defer_reason = (remote_publication_lease_fence_targets_are_required(
remote_publication_lease_targets.len(),
remote_publication_leases.is_some(),
remote_lease_fence.is_some(),
))
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable);
let remote_lease_covers_persistence = remote_lease_deadline.is_none_or(|deadline| {
std::time::Instant::now()
.checked_add(usage_persist_timeout)
.is_some_and(|latest_finish| latest_finish < deadline)
});
let publication_defer_reason = publication_defer_reason
.or(remote_lease_defer_reason)
.or(remote_lease_fence_defer_reason);
let publication_defer_reason = (!remote_lease_covers_persistence)
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
.or(publication_defer_reason);
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
let usage_persist_outcome = match publication_defer_reason {
let remote_lease_probe = remote_publication_leases
.as_ref()
.map(|(notification_system, grants)| (Arc::clone(notification_system), grants.clone()));
let mut usage_persist_outcome = match publication_defer_reason {
Some(reason) => {
drop(receiver);
DataUsagePersistOutcome::Deferred(reason)
@@ -1390,17 +1491,33 @@ async fn run_data_scanner_cycle_with_budget(
let storeapi_clone = storeapi.clone();
let ctx_clone = ctx.clone();
let route_probe_store = storeapi.clone();
let remote_lease_fence = remote_lease_fence.clone();
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
ctx_clone,
storeapi_clone,
receiver,
Some(leader_epoch),
Some(usage_persist_baseline),
publication_epoch,
ScannerPublicationFence::new(
publication_epoch,
remote_lease_deadline,
remote_lease_fence,
),
move || {
let storeapi = route_probe_store.clone();
async move { storeapi.scanner_data_usage_publication_blocked().await }
let remote_lease_probe = remote_lease_probe.clone();
async move {
if let Some((notification_system, grants)) = remote_lease_probe.as_ref()
&& notification_system.validate_scanner_publication_leases(grants).await.is_err()
{
// A remote restart or movement flip invalidates
// the token proof; usage_store interprets this
// as a publication barrier and performs no PUT.
return true;
}
storeapi.scanner_data_usage_publication_blocked().await
}
},
)
.await
@@ -1448,6 +1565,22 @@ async fn run_data_scanner_cycle_with_budget(
}
}
};
let lease_expired = remote_publication_leases
.as_ref()
.is_some_and(|(_, grants)| grants.iter().any(|grant| !grant.lease.is_valid()));
if let Some((notification_system, grants)) = remote_publication_leases.take() {
let release_result = notification_system.release_scanner_publication_leases(grants).await;
if lease_expired || release_result.is_err() {
// A lease that expired or could not be released is never treated
// as a successful authoritative publication. The peer may have
// admitted movement immediately after the lease ended.
usage_persist_outcome = if usage_persist_outcome == DataUsagePersistOutcome::Failed {
DataUsagePersistOutcome::Failed
} else {
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
};
}
}
let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
let scan_cycle_result = match scan_result {
@@ -2199,7 +2332,16 @@ async fn run_data_scanner_with_maintenance_state(
);
let activity_poll_interval = backoff_enabled.then_some(runtime_config.cycle_interval.max(Duration::from_secs(1)));
let wake_reason = wait_for_next_scanner_cycle_with_activity(
let movement_generation_before_wait = storeapi.scanner_data_movement_generation();
let movement_changed = storeapi.scanner_data_movement_changed();
let movement_store = storeapi.clone();
let movement = ScannerMovementWaitContext {
movement_generation_seen: Some(movement_generation_before_wait),
movement_changed,
current_movement_generation: move || movement_store.scanner_data_movement_generation(),
is_lock_lost: || guard.is_lock_lost(),
};
let wake_reason = wait_for_next_scanner_cycle_with_activity_and_movement(
&ctx,
wait_plan.delay,
activity_poll_interval,
@@ -2211,7 +2353,7 @@ async fn run_data_scanner_with_maintenance_state(
runtime_config_generation_seen,
maintenance_generation_before_wait,
),
|| guard.is_lock_lost(),
movement,
|| probe_scanner_activity(storeapi.as_ref(), distributed),
)
.await;
@@ -2239,6 +2381,10 @@ async fn run_data_scanner_with_maintenance_state(
ScannerCycleWakeReason::ClusterMaintenance => {
clean_idle_backoff.reset();
}
ScannerCycleWakeReason::MovementGeneration => {
scanner_activity_seen = None;
clean_idle_backoff.reset();
}
ScannerCycleWakeReason::Timer
| ScannerCycleWakeReason::DirtyUsage
| ScannerCycleWakeReason::ClusterActivity
@@ -2612,7 +2758,7 @@ use usage_store::*;
pub use activity::scanner_topology_digest;
pub(crate) use activity::{
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
};
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
#[cfg(test)]
crates/scanner/src/scanner/activity.rs
+225 -57
View File
@@ -13,11 +13,13 @@
// limitations under the License.
/// Cycle wake/backoff policy and scanner activity observation (probing, generations, topology digest).
use super::*;
use crate::storage_api::scan::SCANNER_ACTIVITY_V6_PROTOCOL_VERSION;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum ScannerCycleWakeReason {
Timer,
DirtyUsage,
MovementGeneration,
ClusterActivity,
ClusterMaintenance,
ClusterActivityUnavailable,
@@ -250,6 +252,17 @@ impl ScannerCycleObservedGenerations {
}
}
/// Movement state observed while a scanner waits for the next cycle.
///
/// Keeping the movement inputs together makes it harder for callers to pair a
/// generation with the wrong notification or lock predicate.
pub(super) struct ScannerMovementWaitContext<G, F> {
pub(super) movement_generation_seen: Option<u64>,
pub(super) movement_changed: Arc<Notify>,
pub(super) current_movement_generation: G,
pub(super) is_lock_lost: F,
}
pub(super) const LOCAL_SCANNER_ACTIVITY_NODE: &str = "<local>";
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -262,6 +275,8 @@ pub(crate) struct ScannerNodeActivity {
pub(super) data_movement_active: bool,
pub(super) dirty_usage_generation: u64,
pub(super) dirty_usage_pending: bool,
pub(super) movement_generation: u64,
pub(super) publication_blocked: bool,
}
pub(crate) type ScannerActivitySnapshot = BTreeMap<String, ScannerNodeActivity>;
@@ -278,6 +293,14 @@ pub(super) enum ScannerActivityObservation {
NotRequired,
Unchanged,
Changed,
/// A storage-owned movement generation changed. This wake must bypass the
/// ordinary deferred cluster-activity backoff so publication can retry
/// after a transition reaches its terminal state.
MovementChanged,
/// A remote scanner process restarted. Publication leases are bound to the
/// process instance, so this must bypass deferred cluster-activity backoff
/// even when the restarted peer reports otherwise ordinary activity.
RemoteRestarted,
MaintenanceChanged,
Unverified,
}
@@ -373,6 +396,8 @@ pub(super) fn scanner_activity_observed_work(observation: ScannerActivityObserva
matches!(
observation,
ScannerActivityObservation::Changed
| ScannerActivityObservation::MovementChanged
| ScannerActivityObservation::RemoteRestarted
| ScannerActivityObservation::MaintenanceChanged
| ScannerActivityObservation::Unverified
)
@@ -386,6 +411,7 @@ pub(super) fn scanner_activity_backoff_blocked_after_wake(currently_blocked: boo
}
}
#[cfg(test)]
pub(super) async fn wait_for_next_scanner_cycle<F>(
ctx: &CancellationToken,
delay: Duration,
@@ -396,6 +422,36 @@ pub(super) async fn wait_for_next_scanner_cycle<F>(
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
{
let movement = ScannerMovementWaitContext {
movement_generation_seen: None,
movement_changed: Arc::new(Notify::new()),
current_movement_generation: || 0,
is_lock_lost,
};
wait_for_next_scanner_cycle_with_movement(
ctx,
delay,
ScannerCycleObservedGenerations {
dirty_usage: dirty_usage_generation_seen,
runtime_config: runtime_config_generation,
maintenance: maintenance_generation,
defer_cluster_activity: false,
},
&movement,
)
.await
}
pub(super) async fn wait_for_next_scanner_cycle_with_movement<G, F>(
ctx: &CancellationToken,
delay: Duration,
generations: ScannerCycleObservedGenerations,
movement: &ScannerMovementWaitContext<G, F>,
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
G: Fn() -> u64,
{
let sleep = tokio::time::sleep(delay);
tokio::pin!(sleep);
@@ -403,55 +459,86 @@ where
tokio::pin!(lock_poll);
loop {
if is_lock_lost() {
if (movement.is_lock_lost)() {
return ScannerCycleWakeReason::LeaderLockLost;
}
if scanner_runtime_config_generation() != runtime_config_generation {
if scanner_runtime_config_generation() != generations.runtime_config {
return ScannerCycleWakeReason::RuntimeConfig;
}
if scanner_maintenance_generation() != maintenance_generation {
if scanner_maintenance_generation() != generations.maintenance {
return ScannerCycleWakeReason::MaintenanceConfig;
}
if dirty_usage_generation_seen.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen) {
if generations
.dirty_usage
.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen)
{
return ScannerCycleWakeReason::DirtyUsage;
}
if movement
.movement_generation_seen
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
{
return ScannerCycleWakeReason::MovementGeneration;
}
let movement_notification = movement.movement_changed.notified();
tokio::pin!(movement_notification);
movement_notification.as_mut().enable();
// A transition may finish between the initial generation read and
// registration with Notify. Re-check after `enable()` so that such a
// transition cannot be lost when it used `notify_waiters()`.
if movement
.movement_generation_seen
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
{
return ScannerCycleWakeReason::MovementGeneration;
}
tokio::select! {
_ = ctx.cancelled() => return ScannerCycleWakeReason::Cancelled,
_ = &mut sleep => return ScannerCycleWakeReason::Timer,
_ = &mut lock_poll => {
if is_lock_lost() {
if (movement.is_lock_lost)() {
return ScannerCycleWakeReason::LeaderLockLost;
}
lock_poll.as_mut().reset(Instant::now() + SCANNER_LEADER_LOCK_POLL_INTERVAL);
}
_ = dirty_usage_bucket_notified() => {
if scanner_runtime_config_generation() != runtime_config_generation {
if scanner_runtime_config_generation() != generations.runtime_config {
return ScannerCycleWakeReason::RuntimeConfig;
}
if scanner_maintenance_generation() != maintenance_generation {
if scanner_maintenance_generation() != generations.maintenance {
return ScannerCycleWakeReason::MaintenanceConfig;
}
if dirty_usage_generation_seen
if generations
.dirty_usage
.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen)
{
return ScannerCycleWakeReason::DirtyUsage;
}
}
_ = scanner_runtime_config_changed() => {
if scanner_runtime_config_generation() != runtime_config_generation {
if scanner_runtime_config_generation() != generations.runtime_config {
return ScannerCycleWakeReason::RuntimeConfig;
}
}
_ = scanner_maintenance_changed() => {
if scanner_maintenance_generation() != maintenance_generation {
if scanner_maintenance_generation() != generations.maintenance {
return ScannerCycleWakeReason::MaintenanceConfig;
}
}
_ = &mut movement_notification => {
if movement
.movement_generation_seen
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
{
return ScannerCycleWakeReason::MovementGeneration;
}
}
}
}
}
#[cfg(test)]
pub(super) async fn wait_for_next_scanner_cycle_with_activity<F, Probe, ProbeFuture>(
ctx: &CancellationToken,
delay: Duration,
@@ -459,10 +546,43 @@ pub(super) async fn wait_for_next_scanner_cycle_with_activity<F, Probe, ProbeFut
activity_seen: &mut Option<ScannerActivitySnapshot>,
generations: ScannerCycleObservedGenerations,
is_lock_lost: F,
probe_activity: Probe,
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
Probe: FnMut() -> ProbeFuture,
ProbeFuture: Future<Output = Result<ScannerActivitySnapshot, String>>,
{
let movement = ScannerMovementWaitContext {
movement_generation_seen: None,
movement_changed: Arc::new(Notify::new()),
current_movement_generation: || 0,
is_lock_lost,
};
wait_for_next_scanner_cycle_with_activity_and_movement(
ctx,
delay,
activity_poll_interval,
activity_seen,
generations,
movement,
probe_activity,
)
.await
}
pub(super) async fn wait_for_next_scanner_cycle_with_activity_and_movement<F, G, Probe, ProbeFuture>(
ctx: &CancellationToken,
delay: Duration,
activity_poll_interval: Option<Duration>,
activity_seen: &mut Option<ScannerActivitySnapshot>,
generations: ScannerCycleObservedGenerations,
movement: ScannerMovementWaitContext<G, F>,
mut probe_activity: Probe,
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
G: Fn() -> u64,
Probe: FnMut() -> ProbeFuture,
ProbeFuture: Future<Output = Result<ScannerActivitySnapshot, String>>,
{
@@ -475,15 +595,7 @@ where
let wait_slice = activity_poll_interval
.map(|interval| interval.max(Duration::from_secs(1)).min(remaining))
.unwrap_or(remaining);
let wake_reason = wait_for_next_scanner_cycle(
ctx,
wait_slice,
generations.dirty_usage,
generations.runtime_config,
generations.maintenance,
&is_lock_lost,
)
.await;
let wake_reason = wait_for_next_scanner_cycle_with_movement(ctx, wait_slice, generations, &movement).await;
if wake_reason != ScannerCycleWakeReason::Timer || Instant::now() >= deadline {
return wake_reason;
}
@@ -491,7 +603,7 @@ where
let Some(_) = activity_poll_interval else {
return ScannerCycleWakeReason::Timer;
};
if is_lock_lost() {
if (movement.is_lock_lost)() {
return ScannerCycleWakeReason::LeaderLockLost;
}
@@ -500,7 +612,7 @@ where
let lock_lost = async {
loop {
tokio::time::sleep(SCANNER_LEADER_LOCK_POLL_INTERVAL).await;
if is_lock_lost() {
if (movement.is_lock_lost)() {
break;
}
}
@@ -519,6 +631,9 @@ where
}
match observation {
ScannerActivityObservation::Unchanged | ScannerActivityObservation::NotRequired => {}
ScannerActivityObservation::MovementChanged | ScannerActivityObservation::RemoteRestarted => {
return ScannerCycleWakeReason::ClusterActivity;
}
ScannerActivityObservation::Changed if !generations.defer_cluster_activity => {
return ScannerCycleWakeReason::ClusterActivity;
}
@@ -568,12 +683,23 @@ pub(super) fn compare_scanner_activity(
let Some(previous_activity) = previous.get(host) else {
continue;
};
if host != LOCAL_SCANNER_ACTIVITY_NODE && previous_activity.instance_id != current_activity.instance_id {
return ScannerActivityObservation::RemoteRestarted;
}
if host != LOCAL_SCANNER_ACTIVITY_NODE
&& previous_activity.instance_id == current_activity.instance_id
&& previous_activity.maintenance_generation != current_activity.maintenance_generation
{
return ScannerActivityObservation::MaintenanceChanged;
}
if previous_activity.instance_id == current_activity.instance_id
&& (previous_activity.data_movement_active != current_activity.data_movement_active
|| previous_activity.movement_generation != current_activity.movement_generation
|| previous_activity.publication_blocked != current_activity.publication_blocked)
{
return ScannerActivityObservation::MovementChanged;
}
}
ScannerActivityObservation::Changed
@@ -630,12 +756,28 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
hasher.update([u8::from(activity.data_movement_active)]);
hasher.update(activity.dirty_usage_generation.to_be_bytes());
hasher.update([u8::from(activity.dirty_usage_pending)]);
hasher.update(activity.movement_generation.to_be_bytes());
hasher.update([u8::from(activity.publication_blocked)]);
}
hasher.finalize().into()
}
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
snapshot.values().all(|activity| !activity.data_movement_active)
!snapshot.is_empty()
&& snapshot.values().all(|activity| {
activity.protocol_version == SCANNER_ACTIVITY_PROTOCOL_VERSION
&& activity.movement_generation != u64::MAX
&& !activity.data_movement_active
&& !activity.publication_blocked
})
}
pub(crate) fn scanner_activity_publication_lease_targets(snapshot: &ScannerActivitySnapshot) -> Vec<(String, String, u64)> {
snapshot
.iter()
.filter(|(host, _)| host.as_str() != LOCAL_SCANNER_ACTIVITY_NODE)
.map(|(host, activity)| (host.clone(), activity.instance_id.clone(), activity.movement_generation))
.collect()
}
pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySnapshot) -> Vec<ScannerDirtyUsageAcknowledgement> {
@@ -695,11 +837,15 @@ pub(super) fn record_scanner_activity_instance(
pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool) -> Result<ScannerActivitySnapshot, String> {
let topology_digest = scanner_topology_digest(storeapi);
let data_movement_active = storeapi.scanner_data_movement_active().await;
let (data_movement_active, publication_blocked, movement_generation) = storeapi.scanner_data_movement_activity().await;
let namespace_generation = storeapi.scanner_namespace_mutation_generation();
let maintenance_generation = scanner_maintenance_generation();
let dirty_usage = scanner_dirty_usage_state();
if namespace_generation == u64::MAX || maintenance_generation == u64::MAX || dirty_usage.generation == u64::MAX {
if namespace_generation == u64::MAX
|| maintenance_generation == u64::MAX
|| dirty_usage.generation == u64::MAX
|| movement_generation == u64::MAX
{
return Err("local scanner activity generation is exhausted".to_string());
}
let local_instance_id = crate::scanner_io::scanner_activity_epoch().to_string();
@@ -715,6 +861,8 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
data_movement_active,
dirty_usage_generation: dirty_usage.generation,
dirty_usage_pending: dirty_usage.pending,
movement_generation,
publication_blocked,
},
)]);
if !distributed {
@@ -732,39 +880,57 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
if activity.namespace_generation == u64::MAX || activity.maintenance_generation == u64::MAX {
return Err(format!("scanner activity peer {host} exhausted its activity generation"));
}
let (peer_topology_digest, peer_data_movement_active, peer_dirty_usage_generation, peer_dirty_usage_pending) =
match activity.protocol_version {
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing"));
}
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
return Err(format!(
"scanner activity peer {host} cannot safely share scanner cache locks with protocol {}",
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION
));
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => (
activity
.topology_digest
.ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?,
activity
.data_movement_active
.ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?,
activity
.dirty_usage_generation
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?,
activity
.dirty_usage_pending
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?,
),
version => {
return Err(format!(
"scanner activity peer {host} uses protocol {version}, expected {}",
SCANNER_ACTIVITY_PROTOCOL_VERSION
));
}
};
if peer_dirty_usage_generation == u64::MAX {
let (
peer_topology_digest,
peer_data_movement_active,
peer_dirty_usage_generation,
peer_dirty_usage_pending,
peer_movement_generation,
peer_publication_blocked,
) = match activity.protocol_version {
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing"));
}
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
return Err(format!(
"scanner activity peer {host} cannot safely share scanner cache locks with protocol {}",
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION
));
}
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
return Err(format!(
"scanner activity peer {host} cannot verify terminal movement state with protocol {}",
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION
));
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => (
activity
.topology_digest
.ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?,
activity
.data_movement_active
.ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?,
activity
.dirty_usage_generation
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?,
activity
.dirty_usage_pending
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?,
activity
.movement_generation
.ok_or_else(|| format!("scanner activity peer {host} omitted its movement generation"))?,
activity
.publication_blocked
.ok_or_else(|| format!("scanner activity peer {host} omitted its publication blocked state"))?,
),
version => {
return Err(format!(
"scanner activity peer {host} uses protocol {version}, expected {}",
SCANNER_ACTIVITY_PROTOCOL_VERSION
));
}
};
if peer_dirty_usage_generation == u64::MAX || peer_movement_generation == u64::MAX {
return Err(format!("scanner activity peer {host} exhausted its dirty usage generation"));
}
if peer_topology_digest != topology_digest {
@@ -783,6 +949,8 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
data_movement_active: peer_data_movement_active,
dirty_usage_generation: peer_dirty_usage_generation,
dirty_usage_pending: peer_dirty_usage_pending,
movement_generation: peer_movement_generation,
publication_blocked: peer_publication_blocked,
},
)
.is_some()
crates/scanner/src/scanner/tests.rs
+179 -2
View File
@@ -23,7 +23,7 @@ use crate::{
};
use std::collections::{HashMap, HashSet};
use std::io::Cursor;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::task::Poll;
use temp_env::{with_var, with_var_unset};
use tokio::io::AsyncReadExt;
@@ -2888,6 +2888,47 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() {
assert_eq!(store.put_counts.lock().await.get(&key), None);
}
#[tokio::test]
async fn coordinator_does_not_put_after_remote_generation_flip() {
let store = Arc::new(MemoryConfigStore::default());
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let (sender, receiver) = mpsc::channel(1);
sender
.send(complete_usage_with_bucket_count(
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
1,
))
.await
.expect("usage snapshot should enqueue");
drop(sender);
let route_store = store.clone();
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
CancellationToken::new(),
store.clone(),
receiver,
None,
Some(DataUsagePersistBaseline {
data: None,
revision: DataUsageCacheRevision::Missing,
}),
ScannerPublicationFence::new(Some(0), None, None),
move || {
let route_store = route_store.clone();
async move {
// Model the remote lease holder flipping its movement generation
// after the activity probe but before the coordinator's PUT.
route_store.publication_admission_blocked.store(true, Ordering::Release);
false
}
},
)
.await;
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert_eq!(store.put_counts.lock().await.get(&key), None);
}
#[tokio::test]
async fn test_deferred_usage_save_keeps_last_real_save_metric() {
let metrics = global_metrics();
@@ -4974,6 +5015,41 @@ async fn test_wait_for_next_scanner_cycle_stops_after_leader_lock_loss() {
assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost);
}
#[tokio::test]
async fn movement_generation_wakes_deferred_wait_without_dirty_bucket() {
let ctx = CancellationToken::new();
let movement_generation = Arc::new(AtomicU64::new(7));
let movement_changed = Arc::new(Notify::new());
let next_generation = Arc::clone(&movement_generation);
let next_changed = Arc::clone(&movement_changed);
tokio::spawn(async move {
tokio::task::yield_now().await;
next_generation.store(8, Ordering::Release);
next_changed.notify_waiters();
});
let movement = ScannerMovementWaitContext {
movement_generation_seen: Some(7),
movement_changed,
current_movement_generation: move || movement_generation.load(Ordering::Acquire),
is_lock_lost: || false,
};
let reason = wait_for_next_scanner_cycle_with_movement(
&ctx,
Duration::from_secs(60),
ScannerCycleObservedGenerations {
dirty_usage: None,
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
maintenance: crate::scanner_io::scanner_maintenance_generation(),
defer_cluster_activity: false,
},
&movement,
)
.await;
assert_eq!(reason, ScannerCycleWakeReason::MovementGeneration);
}
fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_generation: u64) -> ScannerNodeActivity {
ScannerNodeActivity {
instance_id: epoch.to_string(),
@@ -4984,6 +5060,8 @@ fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_gen
data_movement_active: false,
dirty_usage_generation: 5,
dirty_usage_pending: false,
movement_generation: 9,
publication_blocked: false,
}
}
@@ -5024,6 +5102,7 @@ fn scanner_activity_snapshot_fences_data_movement() {
let mut moving = idle.clone();
moving.get_mut("node-2").expect("node should exist").data_movement_active = true;
assert!(!scanner_activity_allows_usage_publication(&BTreeMap::new()));
assert!(scanner_activity_allows_usage_publication(&idle));
assert!(!scanner_activity_allows_usage_publication(&moving));
assert_ne!(scanner_activity_snapshot_digest(&idle), scanner_activity_snapshot_digest(&moving));
@@ -5107,7 +5186,7 @@ fn scanner_activity_observation_requires_a_complete_baseline() {
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 8, 0))]);
let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(restarted));
assert_eq!(observation, ScannerActivityObservation::Changed);
assert_eq!(observation, ScannerActivityObservation::RemoteRestarted);
assert!(error.is_none());
let (observation, error) =
@@ -5142,6 +5221,36 @@ fn remote_maintenance_change_is_distinct_from_namespace_activity() {
);
}
#[test]
fn remote_movement_generation_change_is_distinct_from_cluster_activity() {
let previous = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote", 7, 3))]);
let movement_changed = BTreeMap::from([(
"node-2".to_string(),
ScannerNodeActivity {
movement_generation: 10,
..scanner_node_activity("remote", 7, 3)
},
)]);
assert_eq!(
compare_scanner_activity(&previous, &movement_changed),
ScannerActivityObservation::MovementChanged
);
assert!(scanner_activity_observed_work(ScannerActivityObservation::MovementChanged));
}
#[test]
fn remote_restart_is_distinct_from_deferred_cluster_activity() {
let previous = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-a", 7, 3))]);
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-b", 7, 3))]);
assert_eq!(
compare_scanner_activity(&previous, &restarted),
ScannerActivityObservation::RemoteRestarted
);
assert!(scanner_activity_observed_work(ScannerActivityObservation::RemoteRestarted));
}
#[test]
fn local_maintenance_wakeup_releases_a_remote_maintenance_block() {
let blocked = scanner_activity_backoff_blocked_after_wake(false, ScannerCycleWakeReason::ClusterMaintenance);
@@ -5231,6 +5340,74 @@ async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
assert_eq!(seen, Some(changed));
}
#[tokio::test(start_paused = true)]
async fn superseded_retry_wait_wakes_for_remote_movement_generation() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote", 7, 3))]));
let changed = BTreeMap::from([(
"node-2".to_string(),
ScannerNodeActivity {
movement_generation: 10,
..scanner_node_activity("remote", 7, 3)
},
)]);
let reason = wait_for_next_scanner_cycle_with_activity(
&ctx,
Duration::from_secs(120),
Some(Duration::from_secs(60)),
&mut seen,
ScannerCycleObservedGenerations {
dirty_usage: None,
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
maintenance: crate::scanner_io::scanner_maintenance_generation(),
defer_cluster_activity: true,
},
|| false,
|| std::future::ready(Ok(changed.clone())),
)
.await;
assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity);
assert_eq!(seen, Some(changed));
}
#[tokio::test(start_paused = true)]
async fn superseded_retry_wait_wakes_when_remote_restart_clears_movement_state() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
let blocked = BTreeMap::from([(
"node-2".to_string(),
ScannerNodeActivity {
data_movement_active: true,
publication_blocked: true,
..scanner_node_activity("remote-a", 7, 3)
},
)]);
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-b", 7, 3))]);
let mut seen = Some(blocked);
let reason = wait_for_next_scanner_cycle_with_activity(
&ctx,
Duration::from_secs(120),
Some(Duration::from_secs(60)),
&mut seen,
ScannerCycleObservedGenerations {
dirty_usage: None,
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
maintenance: crate::scanner_io::scanner_maintenance_generation(),
defer_cluster_activity: true,
},
|| false,
|| std::future::ready(Ok(restarted.clone())),
)
.await;
assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity);
assert_eq!(seen, Some(restarted));
}
#[tokio::test(start_paused = true)]
async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
crates/scanner/src/scanner/usage_store.rs
+117 -14
View File
@@ -13,6 +13,7 @@
// limitations under the License.
/// Data-usage snapshot persistence: CAS store pipeline, epoch baselines, and observed-snapshot cleanup.
use super::*;
use std::collections::HashMap;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(super) enum DataUsagePersistOutcome {
@@ -29,12 +30,40 @@ pub(super) enum DataUsagePersistOutcome {
Failed,
}
fn remote_lease_expired(deadline: Option<std::time::Instant>) -> bool {
deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline)
}
#[derive(Clone, Debug)]
pub(super) struct DataUsagePersistBaseline {
pub(super) data: Option<Bytes>,
pub(super) revision: DataUsageCacheRevision,
}
/// Short-lived publication inputs captured for one usage persistence attempt.
/// Keeping the movement epoch, lease deadline, and target fence together makes
/// it explicit that they are one proof rather than independent options.
#[derive(Clone, Debug, Default)]
pub(super) struct ScannerPublicationFence {
pub(super) expected_publication_epoch: Option<u64>,
pub(super) remote_lease_deadline: Option<std::time::Instant>,
pub(super) scanner_publication_lease_fence: Option<String>,
}
impl ScannerPublicationFence {
pub(super) fn new(
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<String>,
) -> Self {
Self {
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence,
}
}
}
#[derive(Debug)]
pub(super) enum DataUsagePersistTaskResult {
Completed(DataUsagePersistOutcome),
@@ -129,7 +158,7 @@ where
receiver,
leader_epoch,
initial_baseline,
None,
ScannerPublicationFence::default(),
route_probe,
)
.await
@@ -141,16 +170,49 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
mut receiver: mpsc::Receiver<DataUsageInfo>,
receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
expected_publication_epoch: Option<u64>,
publication_fence: ScannerPublicationFence,
route_probe: F,
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
{
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
ctx,
storeapi,
receiver,
leader_epoch,
initial_baseline,
publication_fence,
route_probe,
)
.await
}
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence<
F,
Fut,
>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
mut receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
publication_fence: ScannerPublicationFence,
route_probe: F,
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
{
let ScannerPublicationFence {
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence,
} = publication_fence;
let mut outcome = DataUsagePersistOutcome::NoUpdate;
let mut next_baseline = initial_baseline;
@@ -162,6 +224,10 @@ where
if let Some(leader_epoch) = leader_epoch {
data_usage_info.scanner_epoch = Some(leader_epoch);
}
if remote_lease_expired(remote_lease_deadline) {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
break 'updates;
}
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
@@ -430,6 +496,9 @@ where
);
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
if remote_lease_expired(remote_lease_deadline) {
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
}
let done_save = Metrics::time(Metric::SaveUsage);
let save_result = {
@@ -439,12 +508,17 @@ where
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
save_config_shared_with_preconditions(
if remote_lease_expired(remote_lease_deadline) {
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
}
save_config_shared_with_preconditions_and_lease_fence(
storeapi.clone(),
target_path,
data.clone(),
sha256hex.clone(),
revision.preconditions(),
scanner_publication_lease_fence.as_deref(),
)
.await
};
@@ -538,10 +612,12 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -559,10 +635,12 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -599,10 +677,12 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -620,8 +700,14 @@ where
if backup_due {
let done_save = Metrics::time(Metric::SaveUsage);
let backup_result =
sync_data_usage_backup_from_primary_for_epoch(&ctx, storeapi.clone(), expected_publication_epoch).await;
let backup_result = sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
&ctx,
storeapi.clone(),
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
done_save();
if let Err(e) = backup_result {
warn!(
@@ -647,11 +733,16 @@ where
outcome
}
pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
authoritative: &DataUsageInfo,
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<&str>,
) -> bool {
if remote_lease_expired(remote_lease_deadline) {
return false;
}
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
@@ -667,10 +758,11 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
None => return false,
},
};
if expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
if remote_lease_expired(remote_lease_deadline)
|| expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
{
return false;
}
@@ -711,6 +803,9 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
if observed_data_usage_is_newer(&observed, authoritative) {
return true;
}
if remote_lease_expired(remote_lease_deadline) {
return false;
}
let result = delete_config_with_publication_admission_for_epoch(
storeapi,
@@ -720,6 +815,14 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
delete_prefix: true,
delete_prefix_object: true,
http_preconditions: Some(revision.preconditions()),
user_defined: scanner_publication_lease_fence
.map(|fence| {
HashMap::from([(
crate::storage_api::owner::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
fence.to_string(),
)])
})
.unwrap_or_default(),
..Default::default()
},
read_epoch,
crates/scanner/src/scanner_io.rs
+28 -5
View File
@@ -239,6 +239,13 @@ fn classify_nsscanner_cycle(
dirty_usage_status: DirtyUsageSnapshotStatus,
activity_status: ScannerCycleActivityStatus,
) -> ScannerCycleStatus {
// The post-scan activity proof is required regardless of why the scan was
// incomplete. Returning Incomplete first would apply the long ordinary
// retry/backoff path to an unverifiable publication and could acknowledge
// a cycle without a movement-generation proof.
if activity_status == ScannerCycleActivityStatus::Unverified {
return ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
}
if budget_elapsed
|| cancelled
|| !matches!(bucket_scan_status, ScannerBucketScanStatus::Complete)
@@ -252,7 +259,6 @@ fn classify_nsscanner_cycle(
match (activity_status, dirty_usage_status) {
(ScannerCycleActivityStatus::Unchanged, DirtyUsageSnapshotStatus::Current) => ScannerCycleStatus::Complete,
(ScannerCycleActivityStatus::Unverified, _) => ScannerCycleStatus::Incomplete,
_ => ScannerCycleStatus::Superseded,
}
}
@@ -307,10 +313,16 @@ async fn scanner_cycle_activity_status(
store: &ECStore,
distributed: bool,
before: &crate::scanner::ScannerActivitySnapshot,
) -> ScannerCycleActivityStatus {
) -> (ScannerCycleActivityStatus, Vec<(String, String, u64)>) {
match crate::scanner::probe_scanner_activity(store, distributed).await {
Ok(after) if after == *before => ScannerCycleActivityStatus::Unchanged,
Ok(_) => ScannerCycleActivityStatus::Changed,
Ok(after) => {
let status = if after == *before {
ScannerCycleActivityStatus::Unchanged
} else {
ScannerCycleActivityStatus::Changed
};
(status, crate::scanner::scanner_activity_publication_lease_targets(&after))
}
Err(err) => {
warn!(
target: "rustfs::scanner::io",
@@ -321,7 +333,7 @@ async fn scanner_cycle_activity_status(
error = %err,
"Scanner cycle activity verification failed"
);
ScannerCycleActivityStatus::Unverified
(ScannerCycleActivityStatus::Unverified, Vec::new())
}
}
}
@@ -599,6 +611,7 @@ pub(crate) struct ScannerCycleResult {
publication_epoch: Option<u64>,
dirty_usage_clear: Option<DirtyUsageBuckets>,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
remote_publication_lease_targets: Vec<(String, String, u64)>,
failed_dirty_usage: bool,
pending_maintenance_work: bool,
required_cycle_floor: Option<u64>,
@@ -611,6 +624,7 @@ impl ScannerCycleResult {
publication_epoch: None,
dirty_usage_clear,
remote_dirty_usage_acknowledgements: Vec::new(),
remote_publication_lease_targets: Vec::new(),
failed_dirty_usage: false,
pending_maintenance_work: false,
required_cycle_floor: None,
@@ -649,6 +663,15 @@ impl ScannerCycleResult {
self
}
pub(crate) fn with_remote_publication_lease_targets(mut self, targets: Vec<(String, String, u64)>) -> Self {
self.remote_publication_lease_targets = targets;
self
}
pub(crate) fn remote_publication_lease_targets(&self) -> &[(String, String, u64)] {
&self.remote_publication_lease_targets
}
pub(crate) fn acknowledge_durable_usage(self) -> Vec<crate::scanner::ScannerDirtyUsageAcknowledgement> {
if let Some(snapshot) = self.dirty_usage_clear {
clear_dirty_usage_buckets(&snapshot);
crates/scanner/src/scanner_io/io_cycle.rs
+6 -2
View File
@@ -151,7 +151,8 @@ impl ScannerIOCycle for ECStore {
ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)
);
}
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
let (activity_status, remote_publication_lease_targets) =
scanner_cycle_activity_status(self, distributed, &activity_before).await;
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
let status = classify_nsscanner_cycle(
true,
@@ -187,6 +188,7 @@ impl ScannerIOCycle for ECStore {
};
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_remote_publication_lease_targets(remote_publication_lease_targets)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
}
@@ -400,7 +402,8 @@ impl ScannerIOCycle for ECStore {
let budget_elapsed = budget.budget_elapsed();
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
let dirty_usage_current = dirty_usage_status == DirtyUsageSnapshotStatus::Current;
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
let (activity_status, remote_publication_lease_targets) =
scanner_cycle_activity_status(self, distributed, &activity_before).await;
let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::<Vec<_>>();
let completed_usage = completed_data_usage_info(
&results,
@@ -460,6 +463,7 @@ impl ScannerIOCycle for ECStore {
};
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_remote_publication_lease_targets(remote_publication_lease_targets)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
.with_failed_dirty_usage(!failed_buckets.is_empty())
.with_pending_maintenance_work(pending_maintenance_work)
crates/scanner/src/scanner_io/tests.rs
+50 -6
View File
@@ -766,7 +766,7 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unverified,
),
ScannerCycleStatus::Incomplete
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
for status in [
@@ -815,6 +815,34 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
}
}
#[test]
fn unverified_activity_defers_partial_and_floor_cycles() {
let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
assert_eq!(
classify_nsscanner_cycle(
true,
false,
false,
ScannerBucketScanStatus::Partial,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unverified,
),
expected
);
assert_eq!(
classify_nsscanner_cycle(
false,
false,
false,
ScannerBucketScanStatus::Complete,
DirtyUsageSnapshotStatus::Current,
ScannerCycleActivityStatus::Unverified,
),
expected
);
}
#[tokio::test]
async fn structurally_complete_superseded_cycles_publish_without_claiming_convergence() {
let (updates, mut receiver) = mpsc::channel(2);
@@ -834,6 +862,15 @@ async fn structurally_complete_superseded_cycles_publish_without_claiming_conver
.await
.expect("incomplete snapshot suppression should succeed")
);
assert!(
!publish_usage_snapshot(
&updates,
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
DataUsageInfo::default(),
)
.await
.expect("unverified activity suppression should succeed")
);
assert_eq!(
receiver
@@ -855,11 +892,7 @@ async fn structurally_complete_superseded_cycles_publish_without_claiming_conver
#[test]
fn scanner_cycle_fails_closed_for_namespace_disappearance() {
for activity_status in [
ScannerCycleActivityStatus::Changed,
ScannerCycleActivityStatus::Unchanged,
ScannerCycleActivityStatus::Unverified,
] {
for activity_status in [ScannerCycleActivityStatus::Changed, ScannerCycleActivityStatus::Unchanged] {
assert_eq!(
classify_nsscanner_cycle(
false,
@@ -872,6 +905,17 @@ fn scanner_cycle_fails_closed_for_namespace_disappearance() {
ScannerCycleStatus::Incomplete
);
}
assert_eq!(
classify_nsscanner_cycle(
false,
false,
false,
ScannerBucketScanStatus::NamespaceNotFound,
DirtyUsageSnapshotStatus::Changed,
ScannerCycleActivityStatus::Unverified,
),
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
assert_eq!(
classify_nsscanner_cycle(
true,
crates/scanner/src/storage_api.rs
+8 -6
View File
@@ -85,6 +85,7 @@ pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_
pub(crate) use rustfs_ecstore::api::layout::{
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
};
pub(crate) use rustfs_ecstore::api::object::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rebalance::{
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
@@ -98,9 +99,9 @@ pub(crate) use rustfs_ecstore::api::runtime::{
setup_is_erasure_sd as ecstore_is_erasure_sd,
};
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx;
pub(crate) use rustfs_ecstore::api::storage::{ECStore as EcstoreStore, SCANNER_PUBLICATION_LEASE_TTL_MS};
use rustfs_storage_api as storage_contracts;
pub(crate) mod owner {
@@ -115,10 +116,11 @@ pub(crate) mod owner {
EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreEventArgs,
EcstoreLcEventSrc, EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts,
EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard,
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreVersioningApi, ScannerReplicationHealObject,
ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule,
ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreVersioningApi, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerReplicationHealObject, ScannerReplicationHealResult,
ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle,
ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
@@ -274,13 +276,13 @@ impl From<EcstoreReplicationHealQueueResult> for ScannerReplicationHealResult {
}
pub(crate) mod scan {
pub use super::storage_contracts::SCANNER_ACTIVITY_PROTOCOL_VERSION;
pub(crate) use super::storage_contracts::{
BucketOperations, BucketOptions, NamespaceLocking, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
};
#[cfg(test)]
pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO};
pub use super::storage_contracts::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
}
pub(crate) mod scanner_io {