gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-02 02:08:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): fence movement generation publication (#6461)

Browse Source 
* feat(scanner): add movement generation fencing

* fix(scanner): prioritize unverified cycle deferral

* feat(ecstore): add scanner publication lease fence

* feat(rpc): add scanner publication lease protocol

* feat(scanner): hold remote leases through usage publish

* test(scanner): cover publication lease fencing

* fix(scanner): fence remote leases across restart and delay

* feat(rpc): fence scanner publication rename writes

* fix(scanner): fence observed cleanup deletes

* fix(proto): qualify lease release test types

* fix(scanner): pin movement notifications

* fix(scanner): clean publication imports

* fix(ecstore): satisfy scanner fence clippy

* refactor(scanner): group wait and publication options

* fix(scanner): satisfy final lint and facade guards

* fix(rpc): resolve facade export conflicts

* fix(ci): remove unused decommission and healing facades

* fix(ci): cfg-gate test-only usage overlay import

* fix(scanner): wake on remote scanner restart
This commit is contained in:
cxymds
2026-08-24 14:17:35 +08:00
committed by GitHub
parent 1c5c28842a
commit eec0e0e056
30 changed files with 2926 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner/activity.rs
+225 -57
View File
@@ -13,11 +13,13 @@
// limitations under the License.
/// Cycle wake/backoff policy and scanner activity observation (probing, generations, topology digest).
use super::*;
use crate::storage_api::scan::SCANNER_ACTIVITY_V6_PROTOCOL_VERSION;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum ScannerCycleWakeReason {
Timer,
DirtyUsage,
MovementGeneration,
ClusterActivity,
ClusterMaintenance,
ClusterActivityUnavailable,
@@ -250,6 +252,17 @@ impl ScannerCycleObservedGenerations {
}
}
/// Movement state observed while a scanner waits for the next cycle.
///
/// Keeping the movement inputs together makes it harder for callers to pair a
/// generation with the wrong notification or lock predicate.
pub(super) struct ScannerMovementWaitContext<G, F> {
pub(super) movement_generation_seen: Option<u64>,
pub(super) movement_changed: Arc<Notify>,
pub(super) current_movement_generation: G,
pub(super) is_lock_lost: F,
}
pub(super) const LOCAL_SCANNER_ACTIVITY_NODE: &str = "<local>";
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -262,6 +275,8 @@ pub(crate) struct ScannerNodeActivity {
pub(super) data_movement_active: bool,
pub(super) dirty_usage_generation: u64,
pub(super) dirty_usage_pending: bool,
pub(super) movement_generation: u64,
pub(super) publication_blocked: bool,
}
pub(crate) type ScannerActivitySnapshot = BTreeMap<String, ScannerNodeActivity>;
@@ -278,6 +293,14 @@ pub(super) enum ScannerActivityObservation {
NotRequired,
Unchanged,
Changed,
/// A storage-owned movement generation changed. This wake must bypass the
/// ordinary deferred cluster-activity backoff so publication can retry
/// after a transition reaches its terminal state.
MovementChanged,
/// A remote scanner process restarted. Publication leases are bound to the
/// process instance, so this must bypass deferred cluster-activity backoff
/// even when the restarted peer reports otherwise ordinary activity.
RemoteRestarted,
MaintenanceChanged,
Unverified,
}
@@ -373,6 +396,8 @@ pub(super) fn scanner_activity_observed_work(observation: ScannerActivityObserva
matches!(
observation,
ScannerActivityObservation::Changed
| ScannerActivityObservation::MovementChanged
| ScannerActivityObservation::RemoteRestarted
| ScannerActivityObservation::MaintenanceChanged
| ScannerActivityObservation::Unverified
)
@@ -386,6 +411,7 @@ pub(super) fn scanner_activity_backoff_blocked_after_wake(currently_blocked: boo
}
}
#[cfg(test)]
pub(super) async fn wait_for_next_scanner_cycle<F>(
ctx: &CancellationToken,
delay: Duration,
@@ -396,6 +422,36 @@ pub(super) async fn wait_for_next_scanner_cycle<F>(
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
{
let movement = ScannerMovementWaitContext {
movement_generation_seen: None,
movement_changed: Arc::new(Notify::new()),
current_movement_generation: || 0,
is_lock_lost,
};
wait_for_next_scanner_cycle_with_movement(
ctx,
delay,
ScannerCycleObservedGenerations {
dirty_usage: dirty_usage_generation_seen,
runtime_config: runtime_config_generation,
maintenance: maintenance_generation,
defer_cluster_activity: false,
},
&movement,
)
.await
}
pub(super) async fn wait_for_next_scanner_cycle_with_movement<G, F>(
ctx: &CancellationToken,
delay: Duration,
generations: ScannerCycleObservedGenerations,
movement: &ScannerMovementWaitContext<G, F>,
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
G: Fn() -> u64,
{
let sleep = tokio::time::sleep(delay);
tokio::pin!(sleep);
@@ -403,55 +459,86 @@ where
tokio::pin!(lock_poll);
loop {
if is_lock_lost() {
if (movement.is_lock_lost)() {
return ScannerCycleWakeReason::LeaderLockLost;
}
if scanner_runtime_config_generation() != runtime_config_generation {
if scanner_runtime_config_generation() != generations.runtime_config {
return ScannerCycleWakeReason::RuntimeConfig;
}
if scanner_maintenance_generation() != maintenance_generation {
if scanner_maintenance_generation() != generations.maintenance {
return ScannerCycleWakeReason::MaintenanceConfig;
}
if dirty_usage_generation_seen.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen) {
if generations
.dirty_usage
.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen)
{
return ScannerCycleWakeReason::DirtyUsage;
}
if movement
.movement_generation_seen
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
{
return ScannerCycleWakeReason::MovementGeneration;
}
let movement_notification = movement.movement_changed.notified();
tokio::pin!(movement_notification);
movement_notification.as_mut().enable();
// A transition may finish between the initial generation read and
// registration with Notify. Re-check after `enable()` so that such a
// transition cannot be lost when it used `notify_waiters()`.
if movement
.movement_generation_seen
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
{
return ScannerCycleWakeReason::MovementGeneration;
}
tokio::select! {
_ = ctx.cancelled() => return ScannerCycleWakeReason::Cancelled,
_ = &mut sleep => return ScannerCycleWakeReason::Timer,
_ = &mut lock_poll => {
if is_lock_lost() {
if (movement.is_lock_lost)() {
return ScannerCycleWakeReason::LeaderLockLost;
}
lock_poll.as_mut().reset(Instant::now() + SCANNER_LEADER_LOCK_POLL_INTERVAL);
}
_ = dirty_usage_bucket_notified() => {
if scanner_runtime_config_generation() != runtime_config_generation {
if scanner_runtime_config_generation() != generations.runtime_config {
return ScannerCycleWakeReason::RuntimeConfig;
}
if scanner_maintenance_generation() != maintenance_generation {
if scanner_maintenance_generation() != generations.maintenance {
return ScannerCycleWakeReason::MaintenanceConfig;
}
if dirty_usage_generation_seen
if generations
.dirty_usage
.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen)
{
return ScannerCycleWakeReason::DirtyUsage;
}
}
_ = scanner_runtime_config_changed() => {
if scanner_runtime_config_generation() != runtime_config_generation {
if scanner_runtime_config_generation() != generations.runtime_config {
return ScannerCycleWakeReason::RuntimeConfig;
}
}
_ = scanner_maintenance_changed() => {
if scanner_maintenance_generation() != maintenance_generation {
if scanner_maintenance_generation() != generations.maintenance {
return ScannerCycleWakeReason::MaintenanceConfig;
}
}
_ = &mut movement_notification => {
if movement
.movement_generation_seen
.is_some_and(|seen| (movement.current_movement_generation)() != seen)
{
return ScannerCycleWakeReason::MovementGeneration;
}
}
}
}
}
#[cfg(test)]
pub(super) async fn wait_for_next_scanner_cycle_with_activity<F, Probe, ProbeFuture>(
ctx: &CancellationToken,
delay: Duration,
@@ -459,10 +546,43 @@ pub(super) async fn wait_for_next_scanner_cycle_with_activity<F, Probe, ProbeFut
activity_seen: &mut Option<ScannerActivitySnapshot>,
generations: ScannerCycleObservedGenerations,
is_lock_lost: F,
probe_activity: Probe,
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
Probe: FnMut() -> ProbeFuture,
ProbeFuture: Future<Output = Result<ScannerActivitySnapshot, String>>,
{
let movement = ScannerMovementWaitContext {
movement_generation_seen: None,
movement_changed: Arc::new(Notify::new()),
current_movement_generation: || 0,
is_lock_lost,
};
wait_for_next_scanner_cycle_with_activity_and_movement(
ctx,
delay,
activity_poll_interval,
activity_seen,
generations,
movement,
probe_activity,
)
.await
}
pub(super) async fn wait_for_next_scanner_cycle_with_activity_and_movement<F, G, Probe, ProbeFuture>(
ctx: &CancellationToken,
delay: Duration,
activity_poll_interval: Option<Duration>,
activity_seen: &mut Option<ScannerActivitySnapshot>,
generations: ScannerCycleObservedGenerations,
movement: ScannerMovementWaitContext<G, F>,
mut probe_activity: Probe,
) -> ScannerCycleWakeReason
where
F: Fn() -> bool,
G: Fn() -> u64,
Probe: FnMut() -> ProbeFuture,
ProbeFuture: Future<Output = Result<ScannerActivitySnapshot, String>>,
{
@@ -475,15 +595,7 @@ where
let wait_slice = activity_poll_interval
.map(|interval| interval.max(Duration::from_secs(1)).min(remaining))
.unwrap_or(remaining);
let wake_reason = wait_for_next_scanner_cycle(
ctx,
wait_slice,
generations.dirty_usage,
generations.runtime_config,
generations.maintenance,
&is_lock_lost,
)
.await;
let wake_reason = wait_for_next_scanner_cycle_with_movement(ctx, wait_slice, generations, &movement).await;
if wake_reason != ScannerCycleWakeReason::Timer || Instant::now() >= deadline {
return wake_reason;
}
@@ -491,7 +603,7 @@ where
let Some(_) = activity_poll_interval else {
return ScannerCycleWakeReason::Timer;
};
if is_lock_lost() {
if (movement.is_lock_lost)() {
return ScannerCycleWakeReason::LeaderLockLost;
}
@@ -500,7 +612,7 @@ where
let lock_lost = async {
loop {
tokio::time::sleep(SCANNER_LEADER_LOCK_POLL_INTERVAL).await;
if is_lock_lost() {
if (movement.is_lock_lost)() {
break;
}
}
@@ -519,6 +631,9 @@ where
}
match observation {
ScannerActivityObservation::Unchanged | ScannerActivityObservation::NotRequired => {}
ScannerActivityObservation::MovementChanged | ScannerActivityObservation::RemoteRestarted => {
return ScannerCycleWakeReason::ClusterActivity;
}
ScannerActivityObservation::Changed if !generations.defer_cluster_activity => {
return ScannerCycleWakeReason::ClusterActivity;
}
@@ -568,12 +683,23 @@ pub(super) fn compare_scanner_activity(
let Some(previous_activity) = previous.get(host) else {
continue;
};
if host != LOCAL_SCANNER_ACTIVITY_NODE && previous_activity.instance_id != current_activity.instance_id {
return ScannerActivityObservation::RemoteRestarted;
}
if host != LOCAL_SCANNER_ACTIVITY_NODE
&& previous_activity.instance_id == current_activity.instance_id
&& previous_activity.maintenance_generation != current_activity.maintenance_generation
{
return ScannerActivityObservation::MaintenanceChanged;
}
if previous_activity.instance_id == current_activity.instance_id
&& (previous_activity.data_movement_active != current_activity.data_movement_active
|| previous_activity.movement_generation != current_activity.movement_generation
|| previous_activity.publication_blocked != current_activity.publication_blocked)
{
return ScannerActivityObservation::MovementChanged;
}
}
ScannerActivityObservation::Changed
@@ -630,12 +756,28 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
hasher.update([u8::from(activity.data_movement_active)]);
hasher.update(activity.dirty_usage_generation.to_be_bytes());
hasher.update([u8::from(activity.dirty_usage_pending)]);
hasher.update(activity.movement_generation.to_be_bytes());
hasher.update([u8::from(activity.publication_blocked)]);
}
hasher.finalize().into()
}
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
snapshot.values().all(|activity| !activity.data_movement_active)
!snapshot.is_empty()
&& snapshot.values().all(|activity| {
activity.protocol_version == SCANNER_ACTIVITY_PROTOCOL_VERSION
&& activity.movement_generation != u64::MAX
&& !activity.data_movement_active
&& !activity.publication_blocked
})
}
pub(crate) fn scanner_activity_publication_lease_targets(snapshot: &ScannerActivitySnapshot) -> Vec<(String, String, u64)> {
snapshot
.iter()
.filter(|(host, _)| host.as_str() != LOCAL_SCANNER_ACTIVITY_NODE)
.map(|(host, activity)| (host.clone(), activity.instance_id.clone(), activity.movement_generation))
.collect()
}
pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySnapshot) -> Vec<ScannerDirtyUsageAcknowledgement> {
@@ -695,11 +837,15 @@ pub(super) fn record_scanner_activity_instance(
pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool) -> Result<ScannerActivitySnapshot, String> {
let topology_digest = scanner_topology_digest(storeapi);
let data_movement_active = storeapi.scanner_data_movement_active().await;
let (data_movement_active, publication_blocked, movement_generation) = storeapi.scanner_data_movement_activity().await;
let namespace_generation = storeapi.scanner_namespace_mutation_generation();
let maintenance_generation = scanner_maintenance_generation();
let dirty_usage = scanner_dirty_usage_state();
if namespace_generation == u64::MAX || maintenance_generation == u64::MAX || dirty_usage.generation == u64::MAX {
if namespace_generation == u64::MAX
|| maintenance_generation == u64::MAX
|| dirty_usage.generation == u64::MAX
|| movement_generation == u64::MAX
{
return Err("local scanner activity generation is exhausted".to_string());
}
let local_instance_id = crate::scanner_io::scanner_activity_epoch().to_string();
@@ -715,6 +861,8 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
data_movement_active,
dirty_usage_generation: dirty_usage.generation,
dirty_usage_pending: dirty_usage.pending,
movement_generation,
publication_blocked,
},
)]);
if !distributed {
@@ -732,39 +880,57 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
if activity.namespace_generation == u64::MAX || activity.maintenance_generation == u64::MAX {
return Err(format!("scanner activity peer {host} exhausted its activity generation"));
}
let (peer_topology_digest, peer_data_movement_active, peer_dirty_usage_generation, peer_dirty_usage_pending) =
match activity.protocol_version {
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing"));
}
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
return Err(format!(
"scanner activity peer {host} cannot safely share scanner cache locks with protocol {}",
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION
));
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => (
activity
.topology_digest
.ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?,
activity
.data_movement_active
.ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?,
activity
.dirty_usage_generation
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?,
activity
.dirty_usage_pending
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?,
),
version => {
return Err(format!(
"scanner activity peer {host} uses protocol {version}, expected {}",
SCANNER_ACTIVITY_PROTOCOL_VERSION
));
}
};
if peer_dirty_usage_generation == u64::MAX {
let (
peer_topology_digest,
peer_data_movement_active,
peer_dirty_usage_generation,
peer_dirty_usage_pending,
peer_movement_generation,
peer_publication_blocked,
) = match activity.protocol_version {
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => {
return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing"));
}
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
return Err(format!(
"scanner activity peer {host} cannot safely share scanner cache locks with protocol {}",
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION
));
}
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION => {
return Err(format!(
"scanner activity peer {host} cannot verify terminal movement state with protocol {}",
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION
));
}
SCANNER_ACTIVITY_PROTOCOL_VERSION => (
activity
.topology_digest
.ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?,
activity
.data_movement_active
.ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?,
activity
.dirty_usage_generation
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?,
activity
.dirty_usage_pending
.ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?,
activity
.movement_generation
.ok_or_else(|| format!("scanner activity peer {host} omitted its movement generation"))?,
activity
.publication_blocked
.ok_or_else(|| format!("scanner activity peer {host} omitted its publication blocked state"))?,
),
version => {
return Err(format!(
"scanner activity peer {host} uses protocol {version}, expected {}",
SCANNER_ACTIVITY_PROTOCOL_VERSION
));
}
};
if peer_dirty_usage_generation == u64::MAX || peer_movement_generation == u64::MAX {
return Err(format!("scanner activity peer {host} exhausted its dirty usage generation"));
}
if peer_topology_digest != topology_digest {
@@ -783,6 +949,8 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
data_movement_active: peer_data_movement_active,
dirty_usage_generation: peer_dirty_usage_generation,
dirty_usage_pending: peer_dirty_usage_pending,
movement_generation: peer_movement_generation,
publication_blocked: peer_publication_blocked,
},
)
.is_some()
crates/scanner/src/scanner/tests.rs
+179 -2
View File
@@ -23,7 +23,7 @@ use crate::{
};
use std::collections::{HashMap, HashSet};
use std::io::Cursor;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::task::Poll;
use temp_env::{with_var, with_var_unset};
use tokio::io::AsyncReadExt;
@@ -2888,6 +2888,47 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() {
assert_eq!(store.put_counts.lock().await.get(&key), None);
}
#[tokio::test]
async fn coordinator_does_not_put_after_remote_generation_flip() {
let store = Arc::new(MemoryConfigStore::default());
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let (sender, receiver) = mpsc::channel(1);
sender
.send(complete_usage_with_bucket_count(
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
1,
))
.await
.expect("usage snapshot should enqueue");
drop(sender);
let route_store = store.clone();
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
CancellationToken::new(),
store.clone(),
receiver,
None,
Some(DataUsagePersistBaseline {
data: None,
revision: DataUsageCacheRevision::Missing,
}),
ScannerPublicationFence::new(Some(0), None, None),
move || {
let route_store = route_store.clone();
async move {
// Model the remote lease holder flipping its movement generation
// after the activity probe but before the coordinator's PUT.
route_store.publication_admission_blocked.store(true, Ordering::Release);
false
}
},
)
.await;
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert_eq!(store.put_counts.lock().await.get(&key), None);
}
#[tokio::test]
async fn test_deferred_usage_save_keeps_last_real_save_metric() {
let metrics = global_metrics();
@@ -4974,6 +5015,41 @@ async fn test_wait_for_next_scanner_cycle_stops_after_leader_lock_loss() {
assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost);
}
#[tokio::test]
async fn movement_generation_wakes_deferred_wait_without_dirty_bucket() {
let ctx = CancellationToken::new();
let movement_generation = Arc::new(AtomicU64::new(7));
let movement_changed = Arc::new(Notify::new());
let next_generation = Arc::clone(&movement_generation);
let next_changed = Arc::clone(&movement_changed);
tokio::spawn(async move {
tokio::task::yield_now().await;
next_generation.store(8, Ordering::Release);
next_changed.notify_waiters();
});
let movement = ScannerMovementWaitContext {
movement_generation_seen: Some(7),
movement_changed,
current_movement_generation: move || movement_generation.load(Ordering::Acquire),
is_lock_lost: || false,
};
let reason = wait_for_next_scanner_cycle_with_movement(
&ctx,
Duration::from_secs(60),
ScannerCycleObservedGenerations {
dirty_usage: None,
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
maintenance: crate::scanner_io::scanner_maintenance_generation(),
defer_cluster_activity: false,
},
&movement,
)
.await;
assert_eq!(reason, ScannerCycleWakeReason::MovementGeneration);
}
fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_generation: u64) -> ScannerNodeActivity {
ScannerNodeActivity {
instance_id: epoch.to_string(),
@@ -4984,6 +5060,8 @@ fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_gen
data_movement_active: false,
dirty_usage_generation: 5,
dirty_usage_pending: false,
movement_generation: 9,
publication_blocked: false,
}
}
@@ -5024,6 +5102,7 @@ fn scanner_activity_snapshot_fences_data_movement() {
let mut moving = idle.clone();
moving.get_mut("node-2").expect("node should exist").data_movement_active = true;
assert!(!scanner_activity_allows_usage_publication(&BTreeMap::new()));
assert!(scanner_activity_allows_usage_publication(&idle));
assert!(!scanner_activity_allows_usage_publication(&moving));
assert_ne!(scanner_activity_snapshot_digest(&idle), scanner_activity_snapshot_digest(&moving));
@@ -5107,7 +5186,7 @@ fn scanner_activity_observation_requires_a_complete_baseline() {
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 8, 0))]);
let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(restarted));
assert_eq!(observation, ScannerActivityObservation::Changed);
assert_eq!(observation, ScannerActivityObservation::RemoteRestarted);
assert!(error.is_none());
let (observation, error) =
@@ -5142,6 +5221,36 @@ fn remote_maintenance_change_is_distinct_from_namespace_activity() {
);
}
#[test]
fn remote_movement_generation_change_is_distinct_from_cluster_activity() {
let previous = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote", 7, 3))]);
let movement_changed = BTreeMap::from([(
"node-2".to_string(),
ScannerNodeActivity {
movement_generation: 10,
..scanner_node_activity("remote", 7, 3)
},
)]);
assert_eq!(
compare_scanner_activity(&previous, &movement_changed),
ScannerActivityObservation::MovementChanged
);
assert!(scanner_activity_observed_work(ScannerActivityObservation::MovementChanged));
}
#[test]
fn remote_restart_is_distinct_from_deferred_cluster_activity() {
let previous = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-a", 7, 3))]);
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-b", 7, 3))]);
assert_eq!(
compare_scanner_activity(&previous, &restarted),
ScannerActivityObservation::RemoteRestarted
);
assert!(scanner_activity_observed_work(ScannerActivityObservation::RemoteRestarted));
}
#[test]
fn local_maintenance_wakeup_releases_a_remote_maintenance_block() {
let blocked = scanner_activity_backoff_blocked_after_wake(false, ScannerCycleWakeReason::ClusterMaintenance);
@@ -5231,6 +5340,74 @@ async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() {
assert_eq!(seen, Some(changed));
}
#[tokio::test(start_paused = true)]
async fn superseded_retry_wait_wakes_for_remote_movement_generation() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote", 7, 3))]));
let changed = BTreeMap::from([(
"node-2".to_string(),
ScannerNodeActivity {
movement_generation: 10,
..scanner_node_activity("remote", 7, 3)
},
)]);
let reason = wait_for_next_scanner_cycle_with_activity(
&ctx,
Duration::from_secs(120),
Some(Duration::from_secs(60)),
&mut seen,
ScannerCycleObservedGenerations {
dirty_usage: None,
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
maintenance: crate::scanner_io::scanner_maintenance_generation(),
defer_cluster_activity: true,
},
|| false,
|| std::future::ready(Ok(changed.clone())),
)
.await;
assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity);
assert_eq!(seen, Some(changed));
}
#[tokio::test(start_paused = true)]
async fn superseded_retry_wait_wakes_when_remote_restart_clears_movement_state() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let ctx = CancellationToken::new();
let blocked = BTreeMap::from([(
"node-2".to_string(),
ScannerNodeActivity {
data_movement_active: true,
publication_blocked: true,
..scanner_node_activity("remote-a", 7, 3)
},
)]);
let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("remote-b", 7, 3))]);
let mut seen = Some(blocked);
let reason = wait_for_next_scanner_cycle_with_activity(
&ctx,
Duration::from_secs(120),
Some(Duration::from_secs(60)),
&mut seen,
ScannerCycleObservedGenerations {
dirty_usage: None,
runtime_config: crate::runtime_config::scanner_runtime_config_generation(),
maintenance: crate::scanner_io::scanner_maintenance_generation(),
defer_cluster_activity: true,
},
|| false,
|| std::future::ready(Ok(restarted.clone())),
)
.await;
assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity);
assert_eq!(seen, Some(restarted));
}
#[tokio::test(start_paused = true)]
async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
crates/scanner/src/scanner/usage_store.rs
+117 -14
View File
@@ -13,6 +13,7 @@
// limitations under the License.
/// Data-usage snapshot persistence: CAS store pipeline, epoch baselines, and observed-snapshot cleanup.
use super::*;
use std::collections::HashMap;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(super) enum DataUsagePersistOutcome {
@@ -29,12 +30,40 @@ pub(super) enum DataUsagePersistOutcome {
Failed,
}
fn remote_lease_expired(deadline: Option<std::time::Instant>) -> bool {
deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline)
}
#[derive(Clone, Debug)]
pub(super) struct DataUsagePersistBaseline {
pub(super) data: Option<Bytes>,
pub(super) revision: DataUsageCacheRevision,
}
/// Short-lived publication inputs captured for one usage persistence attempt.
/// Keeping the movement epoch, lease deadline, and target fence together makes
/// it explicit that they are one proof rather than independent options.
#[derive(Clone, Debug, Default)]
pub(super) struct ScannerPublicationFence {
pub(super) expected_publication_epoch: Option<u64>,
pub(super) remote_lease_deadline: Option<std::time::Instant>,
pub(super) scanner_publication_lease_fence: Option<String>,
}
impl ScannerPublicationFence {
pub(super) fn new(
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<String>,
) -> Self {
Self {
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence,
}
}
}
#[derive(Debug)]
pub(super) enum DataUsagePersistTaskResult {
Completed(DataUsagePersistOutcome),
@@ -129,7 +158,7 @@ where
receiver,
leader_epoch,
initial_baseline,
None,
ScannerPublicationFence::default(),
route_probe,
)
.await
@@ -141,16 +170,49 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
mut receiver: mpsc::Receiver<DataUsageInfo>,
receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
expected_publication_epoch: Option<u64>,
publication_fence: ScannerPublicationFence,
route_probe: F,
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
{
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
ctx,
storeapi,
receiver,
leader_epoch,
initial_baseline,
publication_fence,
route_probe,
)
.await
}
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence<
F,
Fut,
>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
mut receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
publication_fence: ScannerPublicationFence,
route_probe: F,
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
{
let ScannerPublicationFence {
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence,
} = publication_fence;
let mut outcome = DataUsagePersistOutcome::NoUpdate;
let mut next_baseline = initial_baseline;
@@ -162,6 +224,10 @@ where
if let Some(leader_epoch) = leader_epoch {
data_usage_info.scanner_epoch = Some(leader_epoch);
}
if remote_lease_expired(remote_lease_deadline) {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
break 'updates;
}
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
@@ -430,6 +496,9 @@ where
);
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
if remote_lease_expired(remote_lease_deadline) {
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
}
let done_save = Metrics::time(Metric::SaveUsage);
let save_result = {
@@ -439,12 +508,17 @@ where
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
save_config_shared_with_preconditions(
if remote_lease_expired(remote_lease_deadline) {
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
}
save_config_shared_with_preconditions_and_lease_fence(
storeapi.clone(),
target_path,
data.clone(),
sha256hex.clone(),
revision.preconditions(),
scanner_publication_lease_fence.as_deref(),
)
.await
};
@@ -538,10 +612,12 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -559,10 +635,12 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -599,10 +677,12 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -620,8 +700,14 @@ where
if backup_due {
let done_save = Metrics::time(Metric::SaveUsage);
let backup_result =
sync_data_usage_backup_from_primary_for_epoch(&ctx, storeapi.clone(), expected_publication_epoch).await;
let backup_result = sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
&ctx,
storeapi.clone(),
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
)
.await;
done_save();
if let Err(e) = backup_result {
warn!(
@@ -647,11 +733,16 @@ where
outcome
}
pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
authoritative: &DataUsageInfo,
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<&str>,
) -> bool {
if remote_lease_expired(remote_lease_deadline) {
return false;
}
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
@@ -667,10 +758,11 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
None => return false,
},
};
if expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
if remote_lease_expired(remote_lease_deadline)
|| expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
{
return false;
}
@@ -711,6 +803,9 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
if observed_data_usage_is_newer(&observed, authoritative) {
return true;
}
if remote_lease_expired(remote_lease_deadline) {
return false;
}
let result = delete_config_with_publication_admission_for_epoch(
storeapi,
@@ -720,6 +815,14 @@ pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
delete_prefix: true,
delete_prefix_object: true,
http_preconditions: Some(revision.preconditions()),
user_defined: scanner_publication_lease_fence
.map(|fence| {
HashMap::from([(
crate::storage_api::owner::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
fence.to_string(),
)])
})
.unwrap_or_default(),
..Default::default()
},
read_epoch,