fix(scanner): make distributed usage convergence authoritative (#5151)

* fix(scanner): make distributed usage cycles authoritative * fix(scanner): close distributed refresh races * fix(config): align scanner reload integration * fix(admin): scope config test helpers * fix(scanner): harden distributed usage convergence * fix(scanner): preserve rolling activity compatibility * fix(admin): expose non-secret optional config values * fix(scanner): acknowledge distributed dirty usage * fix(ecstore): make bucket mutations cancellation safe * fix(scanner): preserve pending dirty acknowledgements * test(obs): account for superseded scanner metric * fix(api): reject excess detached bucket mutations * test: close scanner convergence coverage gaps * fix(scanner): make path tracking cleanup one-shot --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
2026-08-21 20:06:37 +00:00 · 2026-07-25 18:45:16 +08:00
parent 0364523dad
commit a63b79004c
69 changed files with 18073 additions and 1585 deletions
@@ -21,6 +21,71 @@

 use super::super::*;

+impl SetDisks {
+    pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec<BucketInfo>, bool)> {
+        let disks = self.disk_inventory().await;
+        let write_quorum = (disks.len() / 2) + 1;
+
+        let mut futures = Vec::with_capacity(disks.len());
+        for disk in disks {
+            futures.push(async move {
+                match disk {
+                    Some(disk) => disk.list_volumes().await,
+                    None => Err(DiskError::DiskNotFound),
+                }
+            });
+        }
+
+        let results = join_all(futures).await;
+        let mut topology_complete = results.iter().all(|result| result.is_ok());
+        let mut infos = Vec::with_capacity(results.len());
+        let mut errs = Vec::with_capacity(results.len());
+        for result in results {
+            match result {
+                Ok(volumes) => {
+                    infos.push(Some(volumes));
+                    errs.push(None);
+                }
+                Err(err) => {
+                    infos.push(None);
+                    errs.push(Some(err));
+                }
+            }
+        }
+
+        if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
+            return Err(err.into());
+        }
+
+        let mut counts: HashMap<String, (usize, BucketInfo)> = HashMap::new();
+        for volumes in infos.into_iter().flatten() {
+            for volume in volumes {
+                if is_reserved_or_invalid_bucket(&volume.name, false) {
+                    continue;
+                }
+
+                let entry = counts.entry(volume.name.clone()).or_insert((
+                    0,
+                    BucketInfo {
+                        name: volume.name.clone(),
+                        created: volume.created,
+                        ..Default::default()
+                    },
+                ));
+                entry.0 += 1;
+            }
+        }
+
+        topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum);
+        let mut buckets = counts
+            .into_values()
+            .filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket))
+            .collect::<Vec<_>>();
+        buckets.sort_by(|left, right| left.name.cmp(&right.name));
+        Ok((buckets, topology_complete))
+    }
+}
+
 #[async_trait::async_trait]
 impl BucketOperations for SetDisks {
    type Error = Error;
@@ -117,65 +182,8 @@ impl BucketOperations for SetDisks {
    }

    #[tracing::instrument(skip(self))]
-    async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
-        let disks = self.disk_inventory().await;
-        let write_quorum = (disks.len() / 2) + 1;
-
-        let mut futures = Vec::with_capacity(disks.len());
-        for disk in disks {
-            futures.push(async move {
-                match disk {
-                    Some(disk) => disk.list_volumes().await,
-                    None => Err(DiskError::DiskNotFound),
-                }
-            });
-        }
-
-        let results = join_all(futures).await;
-        let mut infos = Vec::with_capacity(results.len());
-        let mut errs = Vec::with_capacity(results.len());
-        for result in results {
-            match result {
-                Ok(volumes) => {
-                    infos.push(Some(volumes));
-                    errs.push(None);
-                }
-                Err(err) => {
-                    infos.push(None);
-                    errs.push(Some(err));
-                }
-            }
-        }
-
-        if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
-            return Err(err.into());
-        }
-
-        let mut counts: HashMap<String, (usize, BucketInfo)> = HashMap::new();
-        for volumes in infos.into_iter().flatten() {
-            for volume in volumes {
-                if is_reserved_or_invalid_bucket(&volume.name, false) {
-                    continue;
-                }
-
-                let entry = counts.entry(volume.name.clone()).or_insert((
-                    0,
-                    BucketInfo {
-                        name: volume.name.clone(),
-                        created: volume.created,
-                        ..Default::default()
-                    },
-                ));
-                entry.0 += 1;
-            }
-        }
-
-        let mut buckets = counts
-            .into_values()
-            .filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket))
-            .collect::<Vec<_>>();
-        buckets.sort_by(|left, right| left.name.cmp(&right.name));
-        Ok(buckets)
+    async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
+        Ok(self.list_bucket_for_scanner(opts).await?.0)
    }

    #[tracing::instrument(skip(self))]