fix(scanner): make distributed usage convergence authoritative (#5151)

* fix(scanner): make distributed usage cycles authoritative

* fix(scanner): close distributed refresh races

* fix(config): align scanner reload integration

* fix(admin): scope config test helpers

* fix(scanner): harden distributed usage convergence

* fix(scanner): preserve rolling activity compatibility

* fix(admin): expose non-secret optional config values

* fix(scanner): acknowledge distributed dirty usage

* fix(ecstore): make bucket mutations cancellation safe

* fix(scanner): preserve pending dirty acknowledgements

* test(obs): account for superseded scanner metric

* fix(api): reject excess detached bucket mutations

* test: close scanner convergence coverage gaps

* fix(scanner): make path tracking cleanup one-shot

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -3862,6 +3862,24 @@ impl SetDisks {
         Ok(removed)
     }
 
+    fn write_precondition_lookup_error(
+        error: StorageError,
+        http_preconditions: &HTTPPreconditions,
+        bucket: &str,
+        object: &str,
+    ) -> Option<StorageError> {
+        match error {
+            StorageError::VersionNotFound(_, _, _) | StorageError::ObjectNotFound(_, _) => {
+                if http_preconditions.if_match_value().is_some() {
+                    Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()))
+                } else {
+                    None
+                }
+            }
+            error => Some(error),
+        }
+    }
+
     pub(in crate::set_disk) async fn check_write_precondition(
         &self,
         bucket: &str,
@@ -3892,19 +3910,8 @@ impl SetDisks {
                 }
             }
 
-            Err(StorageError::VersionNotFound(_, _, _))
-            | Err(StorageError::ObjectNotFound(_, _))
-            | Err(StorageError::ErasureReadQuorum) => {
-                // When the object is not found,
-                // - if If-Match is set, we should return 404 NotFound
-                // - if If-None-Match is set, we should be able to proceed with the request
-                if http_preconditions.if_match_value().is_some() {
-                    return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()));
-                }
-            }
-
-            Err(e) => {
-                return Some(e);
+            Err(error) => {
+                return Self::write_precondition_lookup_error(error, &http_preconditions, bucket, object);
             }
         }
 
@@ -4407,6 +4414,41 @@ mod tests {
     use tempfile::TempDir;
     use tokio::io::AsyncReadExt;
 
+    #[test]
+    fn write_precondition_lookup_errors_fail_closed_unless_absence_is_known() {
+        let create_only = HTTPPreconditions {
+            if_none_match: Some("*".to_string()),
+            ..Default::default()
+        };
+        let replace_only = HTTPPreconditions {
+            if_match: Some("etag".to_string()),
+            ..Default::default()
+        };
+
+        assert!(matches!(
+            SetDisks::write_precondition_lookup_error(StorageError::ErasureReadQuorum, &create_only, "bucket", "object",),
+            Some(StorageError::ErasureReadQuorum)
+        ));
+        assert!(
+            SetDisks::write_precondition_lookup_error(
+                StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
+                &create_only,
+                "bucket",
+                "object",
+            )
+            .is_none()
+        );
+        assert!(matches!(
+            SetDisks::write_precondition_lookup_error(
+                StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
+                &replace_only,
+                "bucket",
+                "object",
+            ),
+            Some(StorageError::ObjectNotFound(_, _))
+        ));
+    }
+
     fn metadata_test_fileinfo(object: &str) -> FileInfo {
         let mut fi = FileInfo::new(object, 2, 2);
         fi.volume = "bucket".to_string();

crates/ecstore/src/set_disk/mod.rs

@@ -9275,6 +9275,48 @@ mod tests {
         ));
     }
 
+    #[tokio::test]
+    async fn set_level_if_none_match_fails_closed_without_read_quorum() {
+        let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
+        let bucket = "bucket-write-precondition-quorum";
+        let object = "existing-object.txt";
+
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created before disk loss");
+        let mut reader = PutObjReader::from_vec(b"existing object body".to_vec());
+        set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("object should be written before disk loss");
+        {
+            let mut disks = set_disks.disks.write().await;
+            disks[1..].fill(None);
+        }
+
+        let create_only = ObjectOptions {
+            http_preconditions: Some(HTTPPreconditions {
+                if_none_match: Some("*".to_string()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        let result = set_disks.check_write_precondition(bucket, object, &create_only).await;
+        assert!(
+            matches!(result, Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))),
+            "expected read-quorum failure, got {result:?}"
+        );
+    }
+
     #[tokio::test]
     async fn set_level_versioned_delete_marker_hides_object_without_corrupting_version_metadata() {
         let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;

crates/ecstore/src/set_disk/ops/bucket.rs

@@ -21,6 +21,71 @@
 
 use super::super::*;
 
+impl SetDisks {
+    pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec<BucketInfo>, bool)> {
+        let disks = self.disk_inventory().await;
+        let write_quorum = (disks.len() / 2) + 1;
+
+        let mut futures = Vec::with_capacity(disks.len());
+        for disk in disks {
+            futures.push(async move {
+                match disk {
+                    Some(disk) => disk.list_volumes().await,
+                    None => Err(DiskError::DiskNotFound),
+                }
+            });
+        }
+
+        let results = join_all(futures).await;
+        let mut topology_complete = results.iter().all(|result| result.is_ok());
+        let mut infos = Vec::with_capacity(results.len());
+        let mut errs = Vec::with_capacity(results.len());
+        for result in results {
+            match result {
+                Ok(volumes) => {
+                    infos.push(Some(volumes));
+                    errs.push(None);
+                }
+                Err(err) => {
+                    infos.push(None);
+                    errs.push(Some(err));
+                }
+            }
+        }
+
+        if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
+            return Err(err.into());
+        }
+
+        let mut counts: HashMap<String, (usize, BucketInfo)> = HashMap::new();
+        for volumes in infos.into_iter().flatten() {
+            for volume in volumes {
+                if is_reserved_or_invalid_bucket(&volume.name, false) {
+                    continue;
+                }
+
+                let entry = counts.entry(volume.name.clone()).or_insert((
+                    0,
+                    BucketInfo {
+                        name: volume.name.clone(),
+                        created: volume.created,
+                        ..Default::default()
+                    },
+                ));
+                entry.0 += 1;
+            }
+        }
+
+        topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum);
+        let mut buckets = counts
+            .into_values()
+            .filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket))
+            .collect::<Vec<_>>();
+        buckets.sort_by(|left, right| left.name.cmp(&right.name));
+        Ok((buckets, topology_complete))
+    }
+}
+
 #[async_trait::async_trait]
 impl BucketOperations for SetDisks {
     type Error = Error;
@@ -117,65 +182,8 @@ impl BucketOperations for SetDisks {
     }
 
     #[tracing::instrument(skip(self))]
-    async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
-        let disks = self.disk_inventory().await;
-        let write_quorum = (disks.len() / 2) + 1;
-
-        let mut futures = Vec::with_capacity(disks.len());
-        for disk in disks {
-            futures.push(async move {
-                match disk {
-                    Some(disk) => disk.list_volumes().await,
-                    None => Err(DiskError::DiskNotFound),
-                }
-            });
-        }
-
-        let results = join_all(futures).await;
-        let mut infos = Vec::with_capacity(results.len());
-        let mut errs = Vec::with_capacity(results.len());
-        for result in results {
-            match result {
-                Ok(volumes) => {
-                    infos.push(Some(volumes));
-                    errs.push(None);
-                }
-                Err(err) => {
-                    infos.push(None);
-                    errs.push(Some(err));
-                }
-            }
-        }
-
-        if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
-            return Err(err.into());
-        }
-
-        let mut counts: HashMap<String, (usize, BucketInfo)> = HashMap::new();
-        for volumes in infos.into_iter().flatten() {
-            for volume in volumes {
-                if is_reserved_or_invalid_bucket(&volume.name, false) {
-                    continue;
-                }
-
-                let entry = counts.entry(volume.name.clone()).or_insert((
-                    0,
-                    BucketInfo {
-                        name: volume.name.clone(),
-                        created: volume.created,
-                        ..Default::default()
-                    },
-                ));
-                entry.0 += 1;
-            }
-        }
-
-        let mut buckets = counts
-            .into_values()
-            .filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket))
-            .collect::<Vec<_>>();
-        buckets.sort_by(|left, right| left.name.cmp(&right.name));
-        Ok(buckets)
+    async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
+        Ok(self.list_bucket_for_scanner(opts).await?.0)
     }
 
     #[tracing::instrument(skip(self))]