fix(scanner): fence movement generation publication (#6461)

* feat(scanner): add movement generation fencing * fix(scanner): prioritize unverified cycle deferral * feat(ecstore): add scanner publication lease fence * feat(rpc): add scanner publication lease protocol * feat(scanner): hold remote leases through usage publish * test(scanner): cover publication lease fencing * fix(scanner): fence remote leases across restart and delay * feat(rpc): fence scanner publication rename writes * fix(scanner): fence observed cleanup deletes * fix(proto): qualify lease release test types * fix(scanner): pin movement notifications * fix(scanner): clean publication imports * fix(ecstore): satisfy scanner fence clippy * refactor(scanner): group wait and publication options * fix(scanner): satisfy final lint and facade guards * fix(rpc): resolve facade export conflicts * fix(ci): remove unused decommission and healing facades * fix(ci): cfg-gate test-only usage overlay import * fix(scanner): wake on remote scanner restart
2026-08-29 16:37:07 +00:00 · 2026-08-24 14:17:35 +08:00
parent 1c5c28842a
commit eec0e0e056
30 changed files with 2926 additions and 230 deletions
@@ -56,7 +56,8 @@ use std::sync::{
    Arc, OnceLock,
    atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
 };
-use tokio::sync::{OnceCell, RwLock};
+use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
+use tokio::time::Instant;
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;

@@ -64,6 +65,19 @@ const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
 const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
 const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;

+pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
+
+/// A lease is deliberately short-lived.  The coordinator treats expiry as a
+/// failed publication rather than silently continuing with a peer that may
+/// have started movement after the lease was abandoned.
+pub(crate) const SCANNER_PUBLICATION_LEASE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
+
+pub(crate) struct ScannerPublicationLeaseEntry {
+    pub(crate) expires_at: Instant,
+    pub(crate) movement_generation: u64,
+    pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
+}
+
 /// Runtime state owned by a single `ECStore` instance.
 ///
 /// This is intentionally minimal in the first migration slice; subsequent
@@ -171,6 +185,11 @@ pub struct InstanceContext {
    /// Readers are held across one publication commit; movement transitions
    /// take the writer at their durable state commit boundary.
    data_movement_operation_gate: Arc<RwLock<()>>,
+    /// Remote scanner publication leases own a read guard until explicit
+    /// release or bounded expiry.  Keeping the guard in storage-owned state
+    /// makes a remote movement transition wait on the same fence as a local
+    /// scanner commit.
+    scanner_publication_leases: Arc<Mutex<HashMap<Uuid, ScannerPublicationLeaseEntry>>>,
    /// Monotonic admission epoch paired with the operation gate. A
    /// publication admitted before a movement transition must never be
    /// mistaken for one admitted after the transition.
@@ -180,6 +199,13 @@ pub struct InstanceContext {
    /// saturating counter prevents an unchanged `u64::MAX` value from being
    /// mistaken for a fresh epoch after overflow.
    data_movement_operation_epoch_exhausted: AtomicBool,
+    /// Storage-owned generation for movement state changes.  This is separate
+    /// from the publication admission epoch so scanners can wait for a
+    /// terminal/clear transition without treating the wake as a publication
+    /// permit.
+    data_movement_generation: AtomicU64,
+    data_movement_generation_exhausted: AtomicBool,
+    data_movement_generation_notify: Arc<Notify>,
    /// Last storage-owned movement snapshot observed under the operation
    /// gate. SetDisks cache writers fail closed until ECStore refreshes it.
    scanner_publication_state: AtomicU8,
@@ -224,8 +250,12 @@ impl InstanceContext {
            bucket_metadata_sys: std::sync::Mutex::new(None),
            background_cancel_token: OnceLock::new(),
            data_movement_operation_gate: Arc::new(RwLock::new(())),
+            scanner_publication_leases: Arc::new(Mutex::new(HashMap::new())),
            data_movement_operation_epoch: AtomicU64::new(0),
            data_movement_operation_epoch_exhausted: AtomicBool::new(false),
+            data_movement_generation: AtomicU64::new(0),
+            data_movement_generation_exhausted: AtomicBool::new(false),
+            data_movement_generation_notify: Arc::new(Notify::new()),
            scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
            object_encryption_resolver: OnceLock::new(),
            tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -249,6 +279,75 @@ impl InstanceContext {
        Arc::clone(&self.data_movement_operation_gate)
    }

+    pub(crate) async fn install_scanner_publication_lease(
+        &self,
+        token: Uuid,
+        expires_at: Instant,
+        movement_generation: u64,
+        operation_guard: OwnedRwLockReadGuard<()>,
+    ) -> bool {
+        let mut leases = self.scanner_publication_leases.lock().await;
+        if leases.len() >= SCANNER_PUBLICATION_LEASE_MAX_ENTRIES {
+            return false;
+        }
+        leases.insert(
+            token,
+            ScannerPublicationLeaseEntry {
+                expires_at,
+                movement_generation,
+                _operation_guard: operation_guard,
+            },
+        );
+        true
+    }
+
+    pub(crate) async fn remove_scanner_publication_lease(&self, token: Uuid) -> bool {
+        self.scanner_publication_leases.lock().await.remove(&token).is_some()
+    }
+
+    /// Check a lease token while the caller holds the movement read guard.
+    ///
+    /// The token table is deliberately process-owned and non-persistent: a
+    /// restarted instance has no entries from the previous process, so an old
+    /// coordinator proof cannot become valid again merely because the
+    /// movement generation counter restarted at zero.
+    pub(crate) async fn scanner_publication_lease_is_active(&self, token: Uuid) -> bool {
+        let mut leases = self.scanner_publication_leases.lock().await;
+        let now = Instant::now();
+        let Some(expires_at) = leases.get(&token).map(|entry| entry.expires_at) else {
+            return false;
+        };
+        if expires_at <= now {
+            leases.remove(&token);
+            return false;
+        }
+        true
+    }
+
+    /// Return the generation bound to a live lease.  The lease entry owns the
+    /// movement read guard, so a successful lookup remains valid for the
+    /// caller's guard-protected operation; expiry is still fail-closed.
+    pub(crate) async fn scanner_publication_lease_generation(&self, token: Uuid) -> Option<u64> {
+        let mut leases = self.scanner_publication_leases.lock().await;
+        let now = Instant::now();
+        let (expires_at, movement_generation) = leases
+            .get(&token)
+            .map(|entry| (entry.expires_at, entry.movement_generation))?;
+        if expires_at <= now {
+            leases.remove(&token);
+            return None;
+        }
+        Some(movement_generation)
+    }
+
+    pub(crate) async fn expire_scanner_publication_lease(&self, token: Uuid, expires_at: Instant) {
+        let mut leases = self.scanner_publication_leases.lock().await;
+        let should_remove = leases.get(&token).is_some_and(|entry| entry.expires_at <= expires_at);
+        if should_remove {
+            leases.remove(&token);
+        }
+    }
+
    pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
        self.data_movement_operation_epoch.load(Ordering::Acquire)
    }
@@ -257,8 +356,21 @@ impl InstanceContext {
        self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
    }

+    pub(crate) fn data_movement_generation(&self) -> u64 {
+        self.data_movement_generation.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn data_movement_generation_exhausted(&self) -> bool {
+        self.data_movement_generation_exhausted.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn data_movement_generation_notify(&self) -> Arc<Notify> {
+        Arc::clone(&self.data_movement_generation_notify)
+    }
+
    pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
        !self.data_movement_operation_epoch_exhausted()
+            && !self.data_movement_generation_exhausted()
            && self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
    }

@@ -276,6 +388,7 @@ impl InstanceContext {
    pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
        self.scanner_publication_state
            .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
+        let previous = self.data_movement_operation_epoch.load(Ordering::Acquire);
        let _ = self
            .data_movement_operation_epoch
            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
@@ -283,9 +396,42 @@ impl InstanceContext {
        if result == u64::MAX {
            self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
        }
+        if result != previous {
+            let _ = self.advance_data_movement_generation();
+        }
        result
    }

+    /// Advance the movement generation after a durable movement transition.
+    /// The generation is deliberately bounded: once it reaches `u64::MAX`,
+    /// publication and generation-based waits fail closed rather than reusing
+    /// an indistinguishable saturated value.
+    pub(crate) fn advance_data_movement_generation(&self) -> Option<u64> {
+        if self.data_movement_generation_exhausted.load(Ordering::Acquire) {
+            return None;
+        }
+        let updated = self
+            .data_movement_generation
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1));
+        match updated {
+            Ok(previous) => {
+                let Some(generation) = previous.checked_add(1) else {
+                    self.data_movement_generation_exhausted.store(true, Ordering::Release);
+                    return None;
+                };
+                if generation == u64::MAX {
+                    self.data_movement_generation_exhausted.store(true, Ordering::Release);
+                }
+                self.data_movement_generation_notify.notify_waiters();
+                Some(generation)
+            }
+            Err(_) => {
+                self.data_movement_generation_exhausted.store(true, Ordering::Release);
+                None
+            }
+        }
+    }
+
    #[cfg(test)]
    pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
        self.data_movement_operation_epoch.store(epoch, Ordering::Release);
@@ -295,6 +441,13 @@ impl InstanceContext {
            .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
    }

+    #[cfg(test)]
+    pub(crate) fn set_data_movement_generation_for_test(&self, generation: u64) {
+        self.data_movement_generation.store(generation, Ordering::Release);
+        self.data_movement_generation_exhausted
+            .store(generation == u64::MAX, Ordering::Release);
+    }
+
    /// Install the application-owned object-encryption resolver once.
    pub fn set_object_encryption_resolver(
        &self,