fix(scanner): fence timed out scan cycles

Cargo.lock

@@ -9257,7 +9257,6 @@ dependencies = [
  "url",
  "urlencoding",
  "uuid",
- "x509-parser",
  "zeroize",
  "zip",
  "zstd",

Cargo.toml

@@ -204,7 +204,6 @@ rsa = { version = "=0.10.0-rc.18" }
 rustls = { default-features = false, version = "0.23.43" }
 rustls-native-certs = "0.8"
 rustls-pki-types = "1.15.1"
-x509-parser = "0.18.1"
 sha1 = "0.11.0"
 sha2 = "0.11.0"
 subtle = "2.6"

crates/common/src/metrics.rs

@@ -901,6 +901,10 @@ pub struct Metrics {
     scanner_cycle_max_duration_millis: AtomicU64,
     scanner_cycle_max_objects: AtomicU64,
     scanner_cycle_max_directories: AtomicU64,
+    scanner_cycle_timeout_total: AtomicU64,
+    scanner_cycle_recovery_required_total: AtomicU64,
+    scanner_cycle_last_progress_age_seconds: AtomicU64,
+    scanner_leader_lease_without_progress: AtomicBool,
     scanner_bitrot_cycle_enabled: AtomicBool,
     scanner_bitrot_cycle_millis: AtomicU64,
     scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
@@ -1370,6 +1374,14 @@ pub struct ScannerMetricsReport {
     #[serde(default)]
     pub cycle_max_directories: u64,
     #[serde(default)]
+    pub cycle_timeout_total: u64,
+    #[serde(default)]
+    pub cycle_recovery_required_total: u64,
+    #[serde(default)]
+    pub cycle_last_progress_age: u64,
+    #[serde(default)]
+    pub leader_lease_without_progress: bool,
+    #[serde(default)]
     pub bitrot_cycle_enabled: bool,
     #[serde(default)]
     pub bitrot_cycle_seconds: f64,
@@ -1430,6 +1442,9 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total
 const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
 const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
 const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
+const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
+const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
+const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";
 
 fn scan_cycle_result_label(result: u8) -> &'static str {
     match result {
@@ -1913,6 +1928,10 @@ impl Metrics {
             scanner_cycle_max_duration_millis: AtomicU64::new(0),
             scanner_cycle_max_objects: AtomicU64::new(0),
             scanner_cycle_max_directories: AtomicU64::new(0),
+            scanner_cycle_timeout_total: AtomicU64::new(0),
+            scanner_cycle_recovery_required_total: AtomicU64::new(0),
+            scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
+            scanner_leader_lease_without_progress: AtomicBool::new(false),
             scanner_bitrot_cycle_enabled: AtomicBool::new(false),
             scanner_bitrot_cycle_millis: AtomicU64::new(0),
             scanner_checkpoint: Mutex::new(None),
@@ -2412,12 +2431,29 @@ impl Metrics {
             .store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
         self.scanner_cycle_max_directories
             .store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
+        self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
+        self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
+        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
+        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
         self.scanner_bitrot_cycle_enabled
             .store(bitrot_cycle.is_some(), Ordering::Relaxed);
         self.scanner_bitrot_cycle_millis
             .store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
     }
 
+    pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
+        self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
+        if recovery_required {
+            self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
+        }
+        self.scanner_cycle_last_progress_age_seconds
+            .store(progress_age.as_secs(), Ordering::Relaxed);
+        self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
+        metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
+        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
+        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
+    }
+
     pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
         if let Some(concurrency_limit) = concurrency_limit {
             self.scanner_set_scan_concurrency_limit
@@ -3265,6 +3301,10 @@ impl Metrics {
         m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
         m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
         m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
+        m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
+        m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
+        m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
+        m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
         m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
         m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
         m.scan_checkpoint = match self.scanner_checkpoint.lock() {
@@ -4926,4 +4966,20 @@ mod tests {
         assert!(!report.bitrot_cycle_enabled);
         assert_eq!(report.bitrot_cycle_seconds, 0.0);
     }
+
+    #[tokio::test]
+    async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
+        let metrics = Metrics::new();
+        metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
+        let timed_out = metrics.report().await;
+        assert_eq!(timed_out.cycle_timeout_total, 1);
+        assert_eq!(timed_out.cycle_last_progress_age, 17);
+        assert!(timed_out.leader_lease_without_progress);
+
+        metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
+        let current = metrics.report().await;
+        assert_eq!(current.cycle_timeout_total, 1);
+        assert_eq!(current.cycle_last_progress_age, 0);
+        assert!(!current.leader_lease_without_progress);
+    }
 }

crates/config/README.md

@@ -84,6 +84,12 @@ Current guidance:
 - `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
 
+Scanner cycle budget controls:
+
+- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
+- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
+- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted.
+
 ## Mmap read environment aliases
 
 - `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical)

crates/config/src/constants/scanner.rs

@@ -143,9 +143,12 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
 /// Default scanner speed preset.
 pub const DEFAULT_SCANNER_SPEED: &str = "default";
 
-/// Default scanner cycle runtime budget.
-/// `0` keeps the existing unbounded per-cycle behavior.
-pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0;
+/// Default scanner cycle runtime budget when no override is configured.
+///
+/// An explicit `0` remains the compatibility escape hatch for an unbounded
+/// cycle. Keeping the unset default finite prevents a stalled scanner I/O
+/// operation from holding the leader lease forever.
+pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60;
 
 /// Default scanner per-cycle object budget.
 /// `0` keeps the existing unbounded per-cycle behavior.

crates/ecstore/src/api/mod.rs

@@ -317,6 +317,8 @@ pub mod config {
 }
 
 pub mod data_usage {
+    #[cfg(feature = "test-util")]
+    pub use crate::data_usage::seed_bucket_usage_memory_for_test;
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
         init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
@@ -328,8 +330,6 @@ pub mod data_usage {
         remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
         store_data_usage_in_backend,
     };
-    #[cfg(feature = "test-util")]
-    pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
 }
 
 pub mod disk {

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -866,7 +866,7 @@ impl BucketTargetSys {
             return Some(cli);
         }
 
-        // TODO(backlog): spawn an async task to proactively reload the replication target
+        // TODO: spawn a task to reload the target
         if self.is_reloading_target(bucket, arn).await {
             return None;
         }

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info(
         .map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
     let mut rstate = oi.replication_state();
     rstate.replicate_decision_str = dsc.to_string();
-    let asz = oi.get_actual_size_or_physical();
+    let asz = oi.get_actual_size().unwrap_or_default();
     let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
     let checksum = if ssec { oi.checksum.clone() } else { None };
 

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -1412,7 +1412,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
     };
     let mut replication_state = oi.replication_state();
     replication_state.replicate_decision_str = dsc.to_string();
-    let actual_size = oi.get_actual_size_or_physical();
+    let actual_size = oi.get_actual_size().unwrap_or_default();
 
     Ok(ReplicateObjectInfo {
         name: oi.name.clone(),

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
             .map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
         version_id: object_info.version_id.map(|version_id| version_id.to_string()),
         etag: object_info.etag.as_deref(),
-        actual_size: object_info.get_actual_size_or_physical(),
+        actual_size: object_info.get_actual_size().unwrap_or_default(),
         delete_marker: object_info.delete_marker,
         content_type: object_info.content_type.as_deref(),
         content_encoding: object_info.content_encoding.as_deref(),
@@ -542,20 +542,6 @@ mod tests {
         assert!(replication_target_head_is_newer_null_version(&source, &target));
     }
 
-    #[test]
-    fn replication_source_uses_physical_size_for_unknown_compressed_object() {
-        let mut metadata = HashMap::new();
-        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
-        let source = ObjectInfo {
-            size: 128,
-            actual_size: -1,
-            user_defined: Arc::new(metadata),
-            ..Default::default()
-        };
-
-        assert_eq!(replication_source_object(&source).actual_size, 128);
-    }
-
     #[test]
     fn replication_target_head_content_matches_compare_etag_only() {
         let source = ObjectInfo {

crates/ecstore/src/cluster/rpc/peer_s3_client.rs

@@ -454,7 +454,7 @@ impl S3PeerSys {
                 }
             }
             topology_complete &= bucket_map.values().all(|count| *count >= quorum);
-            // TODO(backlog): integrate MRF backlog stats into scanner bucket listing
+            // TODO: MRF
         }
 
         let mut buckets: Vec<BucketInfo> = result_map.into_values().collect();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -2406,7 +2406,7 @@ impl DiskAPI for RemoteDisk {
             return errors;
         }
 
-        // TODO(backlog): replace string errors with typed `StorageError` variants
+        // TODO: use Error not string
 
         let result = self
             .execute_with_timeout(

crates/ecstore/src/core/sets.rs

@@ -21,7 +21,7 @@ use crate::storage_api_contracts::{
     bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
     list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
     multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
-    object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
+    object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
     range::HTTPRangeSpec,
 };
 use crate::{
@@ -249,7 +249,7 @@ impl Sets {
 
         self.connect_disks().await;
 
-        // TODO(backlog): make monitor_and_connect interval configurable instead of hardcoded 15s
+        // TODO: config interval
         let mut interval = tokio::time::interval(Duration::from_secs(15));
         loop {
             tokio::select! {
@@ -414,66 +414,6 @@ fn apply_delete_objects_results(
     }
 }
 
-fn apply_delete_accounting_results(
-    accounting: &mut [Option<DeleteAccounting>],
-    set_objects: &[DelObj],
-    set_accounting: &[Option<DeleteAccounting>],
-) {
-    for (obj, value) in set_objects.iter().zip(set_accounting.iter()) {
-        accounting[obj.orig_idx] = value.clone();
-    }
-}
-
-impl Sets {
-    pub(crate) async fn delete_objects_with_accounting(
-        &self,
-        bucket: &str,
-        objects: Vec<ObjectToDelete>,
-        opts: ObjectOptions,
-    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
-        let mut del_objects = vec![DeletedObject::default(); objects.len()];
-        let mut del_errs = vec![None; objects.len()];
-        let mut accounting = vec![None; objects.len()];
-        let mut set_obj_map = HashMap::new();
-
-        for (i, obj) in objects.iter().enumerate() {
-            let idx = self.get_hashed_set_index(obj.object_name.as_str());
-            set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj {
-                orig_idx: i,
-                obj: obj.clone(),
-            });
-        }
-
-        let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
-        let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
-        let mut futures = FuturesUnordered::new();
-        let bucket = bucket.to_owned();
-
-        for (set_index, set_objects) in set_obj_map {
-            let disks = self.get_disks(set_index);
-            let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::<Vec<_>>();
-            let bucket = bucket.clone();
-            let opts = opts.clone();
-            let semaphore = semaphore.clone();
-            futures.push(async move {
-                let _permit = semaphore
-                    .acquire_owned()
-                    .await
-                    .expect("delete_objects semaphore should remain open");
-                let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await;
-                (set_objects, deleted, errors, accounting)
-            });
-        }
-
-        while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await {
-            apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors);
-            apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting);
-        }
-
-        (del_objects, del_errs, accounting)
-    }
-}
-
 #[async_trait::async_trait]
 impl crate::storage_api_contracts::object::ObjectIO for Sets {
     type Error = Error;
@@ -715,8 +655,65 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
         objects: Vec<ObjectToDelete>,
         opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
-        let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
-        (deleted, errors)
+        // Default return value
+        let mut del_objects = vec![DeletedObject::default(); objects.len()];
+
+        let mut del_errs = Vec::with_capacity(objects.len());
+        for _ in 0..objects.len() {
+            del_errs.push(None)
+        }
+
+        let mut set_obj_map = HashMap::new();
+
+        // hash key
+        for (i, obj) in objects.iter().enumerate() {
+            let idx = self.get_hashed_set_index(obj.object_name.as_str());
+
+            if !set_obj_map.contains_key(&idx) {
+                set_obj_map.insert(
+                    idx,
+                    vec![DelObj {
+                        // set_idx: idx,
+                        orig_idx: i,
+                        obj: obj.clone(),
+                    }],
+                );
+            } else if let Some(val) = set_obj_map.get_mut(&idx) {
+                val.push(DelObj {
+                    // set_idx: idx,
+                    orig_idx: i,
+                    obj: obj.clone(),
+                });
+            }
+        }
+
+        let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
+        let mut futures = FuturesUnordered::new();
+        let bucket = bucket.to_string();
+
+        for (k, v) in set_obj_map {
+            let disks = self.get_disks(k);
+            let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
+            let bucket = bucket.clone();
+            let opts = opts.clone();
+            let semaphore = semaphore.clone();
+
+            futures.push(async move {
+                let _permit = semaphore
+                    .acquire_owned()
+                    .await
+                    .expect("delete_objects semaphore should remain open");
+                let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await;
+                (v, dobjects, errs)
+            });
+        }
+
+        while let Some((v, dobjects, errs)) = futures.next().await {
+            apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs);
+        }
+
+        (del_objects, del_errs)
     }
 
     #[tracing::instrument(skip(self))]

crates/ecstore/src/data_usage/mod.rs

@@ -1391,37 +1391,7 @@ impl BucketUsageAccumulator {
 }
 
 pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
-    // A compressed object may carry -1 while the transformed size is unknown
-    // (legacy streaming sentinel). In that case the persisted physical size
-    // is still a valid accounting floor; every other negative value is corrupt.
-    // An explicit negative `actual-size` metadata value is corrupt, however:
-    // the sentinel is only valid in the in-memory/object-part field written by
-    // the legacy streaming path, not as a persisted declared size.
-    let compressed = object.is_compressed();
-    if object.actual_size < -1 || (object.actual_size == -1 && !compressed) {
-        return Err(Error::PartMissingOrCorrupt);
-    }
-    if object
-        .parts
-        .iter()
-        .any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
-    {
-        return Err(Error::PartMissingOrCorrupt);
-    }
-    let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
-        .filter(|value| !value.is_empty());
-    if declared_actual_size
-        .as_deref()
-        .and_then(|value| value.parse::<i64>().ok())
-        .is_some_and(|size| size < 0)
-    {
-        return Err(Error::PartMissingOrCorrupt);
-    }
-    let logical_size = match object.get_actual_size().map_err(Error::other)? {
-        size if size == -1 && compressed && declared_actual_size.is_none() => None,
-        size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?),
-        _ => return Err(Error::PartMissingOrCorrupt),
-    };
+    let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
     let persisted_part_size = if object.parts.is_empty() {
         u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
     } else {
@@ -1429,8 +1399,12 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
             // Compressed streaming objects persist -1 when the transformed
             // part size is unknown. The physical part size remains a valid
             // quota floor; reject only non-negative values that overflow.
-            let actual_size = if part.actual_size == -1 {
-                0
+            let actual_size = if part.actual_size < 0 {
+                if object.is_compressed() {
+                    0
+                } else {
+                    return Err(Error::PartMissingOrCorrupt);
+                }
             } else {
                 u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
             };
@@ -1438,7 +1412,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
             total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
         })?
     };
-    Ok(logical_size.unwrap_or(0).max(persisted_part_size))
+    Ok(logical_size.max(persisted_part_size))
 }
 
 type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
@@ -3346,80 +3320,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() {
-        let mut metadata = HashMap::new();
-        rustfs_utils::http::insert_str(
-            &mut metadata,
-            rustfs_utils::http::SUFFIX_COMPRESSION,
-            "klauspost/compress/s2".to_string(),
-        );
-        let object = ObjectInfo {
-            size: 400,
-            actual_size: -1,
-            user_defined: Arc::new(metadata),
-            ..Default::default()
-        };
-
-        assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400);
-    }
-
-    #[test]
-    fn quota_object_size_rejects_compressed_part_sum_overflow() {
-        let mut metadata = HashMap::new();
-        rustfs_utils::http::insert_str(
-            &mut metadata,
-            rustfs_utils::http::SUFFIX_COMPRESSION,
-            "klauspost/compress/s2".to_string(),
-        );
-        let object = ObjectInfo {
-            size: 1,
-            user_defined: Arc::new(metadata),
-            parts: Arc::new(vec![
-                rustfs_filemeta::ObjectPartInfo {
-                    actual_size: i64::MAX,
-                    ..Default::default()
-                },
-                rustfs_filemeta::ObjectPartInfo {
-                    actual_size: 1,
-                    ..Default::default()
-                },
-            ]),
-            ..Default::default()
-        };
-
-        assert!(matches!(quota_object_size(&object), Err(Error::Io(_))));
-    }
-
-    #[test]
-    fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() {
-        let mut metadata = HashMap::new();
-        rustfs_utils::http::insert_str(
-            &mut metadata,
-            rustfs_utils::http::SUFFIX_COMPRESSION,
-            "klauspost/compress/s2".to_string(),
-        );
-        let corrupt_object = ObjectInfo {
-            size: 400,
-            actual_size: -2,
-            user_defined: Arc::new(metadata.clone()),
-            ..Default::default()
-        };
-        assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt)));
-
-        let corrupt_part = ObjectInfo {
-            size: 400,
-            user_defined: Arc::new(metadata),
-            parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
-                size: 400,
-                actual_size: -2,
-                ..Default::default()
-            }]),
-            ..Default::default()
-        };
-        assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt)));
-    }
-
     #[tokio::test]
     #[serial]
     async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {

crates/ecstore/src/disk/local.rs

@@ -5215,8 +5215,8 @@ impl LocalDisk {
 
         let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default());
 
-        // TODO(backlog): add O_DIRECT I/O support for performance-critical paths
-        // TODO(backlog): populate DiskInfo in constructor
+        // TODO: DIRECT support
+        // TODD: DiskInfo
         let mut disk = Self {
             root: root.clone(),
             publication_root,
@@ -5751,7 +5751,7 @@ impl LocalDisk {
 
         // return Ok(());
 
-        // TODO(backlog): make disk space checks and trash cleanup event-driven instead of poll-based
+        // TODO: async notifications for disk space checks and trash cleanup
 
         let trash_path = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
         // if let Some(parent) = trash_path.parent() {
@@ -5997,7 +5997,7 @@ impl LocalDisk {
 
     #[hotpath::measure(impl_type = "LocalDisk")]
     async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
-        // TODO(backlog): add configurable timeout for read_all_data operations
+        // TODO: timeout support
         let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
         Ok(data)
     }
@@ -6674,7 +6674,7 @@ impl LocalDisk {
             return Ok(());
         }
 
-        // TODO(backlog): add directory listing lock to prevent concurrent enumeration
+        // TODO: add lock
 
         let stall = opts.stall_timeout_duration();
 
@@ -8796,7 +8796,7 @@ impl DiskAPI for LocalDisk {
         Ok(entries)
     }
 
-    // TODO(backlog): support io.writer cancellation and early termination in walk_dir
+    // FIXME: TODO: io.writer TODO cancel
     #[tracing::instrument(level = "trace", skip_all)]
     async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
         self.wait_for_startup_cleanup().await;
@@ -9880,7 +9880,7 @@ impl DiskAPI for LocalDisk {
                 );
                 return Err(e);
             }
-            // TODO(backlog): add post-setup disk health verification
+            // TODO: health check
         }
         Ok(())
     }

crates/ecstore/src/layout/endpoints.rs

@@ -249,7 +249,7 @@ impl PoolEndpointList {
             endpoint.set_set_index(0);
             endpoint.set_disk_index(0);
 
-            // TODO(backlog): check for cross-device mounts in single-drive setup
+            // TODO Check for cross device mounts if any.
 
             return Ok(Self {
                 inner: vec![Endpoints::from(vec![endpoint])],
@@ -264,7 +264,7 @@ impl PoolEndpointList {
                 // Convert args to endpoints
                 let mut eps = Endpoints::try_from(set_layout.as_slice())?;
 
-                // TODO(backlog): check for cross-device mounts in multi-pool setup
+                // TODO Check for cross device mounts if any.
 
                 for (disk_idx, ep) in eps.as_mut().iter_mut().enumerate() {
                     ep.set_pool_index(pool_idx);

crates/ecstore/src/object_api/types.rs

@@ -689,9 +689,6 @@ impl ObjectInfo {
     }
 
     pub fn get_actual_size(&self) -> std::io::Result<i64> {
-        if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) {
-            return Err(std::io::Error::other("invalid negative actual size"));
-        }
         if self.actual_size > 0 {
             return Ok(self.actual_size);
         }
@@ -703,25 +700,10 @@ impl ObjectInfo {
                 let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
                 return Ok(size);
             }
-            if self.actual_size == -1 && self.parts.is_empty() {
-                return Ok(-1);
-            }
-            let mut actual_size = 0_i64;
-            let mut unknown = false;
-            for part in self.parts.iter() {
-                match part.actual_size {
-                    -1 => unknown = true,
-                    size if size >= 0 => {
-                        actual_size = actual_size
-                            .checked_add(size)
-                            .ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?;
-                    }
-                    _ => return Err(std::io::Error::other("invalid negative compressed part size")),
-                }
-            }
-            if unknown {
-                return Ok(-1);
-            }
+            let mut actual_size = 0;
+            self.parts.iter().for_each(|part| {
+                actual_size += part.actual_size;
+            });
             if actual_size == 0 && actual_size != self.size {
                 return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size)));
             }
@@ -736,18 +718,6 @@ impl ObjectInfo {
         Ok(self.size)
     }
 
-    /// Returns a non-negative size for client and replication boundaries.
-    ///
-    /// Compressed legacy metadata can retain the internal `-1` unknown-size
-    /// sentinel. Those boundaries cannot emit a negative length, so they use
-    /// the persisted physical size while quota accounting keeps the sentinel
-    /// distinction in [`crate::data_usage::quota_object_size`].
-    pub fn get_actual_size_or_physical(&self) -> i64 {
-        self.get_actual_size()
-            .map(|size| if size >= 0 { size } else { self.size.max(0) })
-            .unwrap_or_else(|_| self.size.max(0))
-    }
-
     pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
         let mut version_id = fi.version_id;
 
@@ -1121,7 +1091,7 @@ impl ObjectInfo {
                     }
                 };
 
-                // TODO(backlog): handle VersionPurgeStatus in object listing
+                // TODO:VersionPurgeStatus
                 let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
                 objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned));
 

crates/ecstore/src/services/metrics_realtime.rs

@@ -256,6 +256,10 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
         cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
         cycle_max_objects: metrics.cycle_max_objects,
         cycle_max_directories: metrics.cycle_max_directories,
+        cycle_timeout_total: metrics.cycle_timeout_total,
+        cycle_recovery_required_total: metrics.cycle_recovery_required_total,
+        cycle_last_progress_age: metrics.cycle_last_progress_age,
+        leader_lease_without_progress: metrics.leader_lease_without_progress,
         bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
         bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
         scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport {
@@ -611,6 +615,10 @@ mod test {
             current_started: chrono_to_jiff_timestamp(current_started),
             last_cycle_partial_source: "usage".to_string(),
             last_cycle_partial_source_code: 1,
+            cycle_timeout_total: 3,
+            cycle_recovery_required_total: 2,
+            cycle_last_progress_age: 17,
+            leader_lease_without_progress: true,
             partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
                 source: "usage".to_string(),
                 cycles: 2,
@@ -622,6 +630,10 @@ mod test {
         assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
         assert_eq!(scanner.last_cycle_partial_source, "usage");
         assert_eq!(scanner.last_cycle_partial_source_code, 1);
+        assert_eq!(scanner.cycle_timeout_total, 3);
+        assert_eq!(scanner.cycle_recovery_required_total, 2);
+        assert_eq!(scanner.cycle_last_progress_age, 17);
+        assert!(scanner.leader_lease_without_progress);
         let usage = scanner
             .partial_cycles_by_source
             .iter()

crates/ecstore/src/set_disk/mod.rs

@@ -97,7 +97,7 @@ use crate::storage_api_contracts::{
         CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo,
     },
     namespace::NamespaceLocking as _,
-    object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
+    object::{DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
     range::HTTPRangeSpec,
 };
 use crate::store::utils::is_reserved_or_invalid_bucket;

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -1575,7 +1575,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         let parts_metadata = vec![fi.clone(); disks.len()];
 
         if !user_defined.contains_key("content-type") {
-            // TODO(backlog): detect content-type from part data when header is missing
+            // TODO: get content-type
         }
 
         if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
@@ -1971,7 +1971,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
             }
 
-            // TODO(backlog): integrate encryption verification during complete multipart
+            // TODO: crypto
 
             if (i < uploaded_parts.len() - 1)
                 && !(opts.data_movement && ext_part.actual_size < 0)

crates/ecstore/src/set_disk/ops/object.rs

@@ -45,7 +45,6 @@ use crate::bucket::replication::{
     DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType,
     replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta,
 };
-use crate::data_usage::quota_object_size;
 use crate::diagnostics::get::GetObjectFailureReason;
 use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
 use crate::error::is_err_invalid_upload_id;
@@ -5656,18 +5655,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         objects: Vec<ObjectToDelete>,
         opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
-        let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
-        (deleted, errors)
-    }
-
-    async fn delete_objects_with_accounting(
-        &self,
-        bucket: &str,
-        objects: Vec<ObjectToDelete>,
-        opts: ObjectOptions,
-    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
-        let mut accounting = vec![None; objects.len()];
         let delete_config_snapshot = opts
             .delete_replication_config_snapshot
             .clone()
@@ -5757,7 +5745,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 *item = Some(Error::other(message.clone()));
                             }
                         }
-                        return (del_objects, del_errs, accounting);
+                        return (del_objects, del_errs);
                     }
                 },
             }
@@ -5804,22 +5792,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             let source_missing = gerr
                 .as_ref()
                 .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err));
-            // Resolve accounting from the generation selected under this
-            // object's write lock. A request-layer pre-stat is only an
-            // optimization and cannot identify a concurrent overwrite.
-            let (accounting_size, accounting_version_id, removed_current_object) = if source_missing
-                || dobj.synthetic_version_id
-                || set_disk_delete_creates_delete_marker(&check_opts)
-                || goi.delete_marker
-            {
-                (None, None, false)
-            } else {
-                (
-                    quota_object_size(&goi).ok(),
-                    goi.version_id.filter(|version_id| !version_id.is_nil()),
-                    (dobj.version_id.is_none() || is_explicit_null_version(dobj.version_id)) && !dobj.synthetic_version_id,
-                )
-            };
             // Normalize both sides before comparing. `goi.version_id` is the
             // client-facing identity, where `from_file_info` synthesizes
             // `Some(Uuid::nil())` for a null version on a versioned or
@@ -5948,12 +5920,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     },
                     replication_state: vr.replication_state_internal.clone(),
                     ..Default::default()
-                };
-                accounting[i] = Some(DeleteAccounting {
-                    size: accounting_size,
-                    version_id: accounting_version_id,
-                    removed_current_object,
-                });
+                }
             }
 
             // Only add to vers_map if we hold the lock
@@ -5999,7 +5966,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     });
                 }
             }
-            return (del_objects, del_errs, accounting);
+            return (del_objects, del_errs);
         }
 
         let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len());
@@ -6194,7 +6161,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         join_all(rollback_futures).await;
 
-        // TODO(backlog): support partial object deletion for multi-part objects
+        // TODO: add_partial
 
         if let Some(api) = opts.tier_delete_journal_api.as_ref() {
             for (idx, je) in persisted_journal_entries {
@@ -6237,16 +6204,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
-        // An accounting identity is actionable only when the delete result is
-        // successful. Never let a failed commit (including a partial quorum
-        // failure) reach the request-layer fast delta path.
-        for (index, err) in del_errs.iter().enumerate() {
-            if err.is_some() {
-                accounting[index] = None;
-            }
-        }
-
-        (del_objects, del_errs, accounting)
+        (del_objects, del_errs)
     }
 
     #[tracing::instrument(skip(self))]
@@ -6413,7 +6371,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
-        // TODO(backlog): integrate lifecycle evaluation before object deletion
+        // TODO: Lifecycle
 
         let mut version_found = true;
         // delete_object_version below derives its own majority quorum from the
@@ -6507,7 +6465,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 mark_deleted: mark_delete,
                 mod_time: Some(mod_time),
                 replication_state_internal: opts.delete_replication.as_ref().map(replication_state_to_filemeta),
-                ..Default::default() // TODO(backlog): populate transition state on delete markers
+                ..Default::default() // TODO: Transition
             };
 
             fi.set_tier_free_version_id(&find_vid.to_string());
@@ -6575,12 +6533,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended);
         obj_info.size = goi.size;
-        // Keep the committed source metadata on the internal delete result so
-        // the request layer can derive canonical accounting for this exact
-        // generation. Delete responses do not expose these fields.
-        obj_info.actual_size = goi.actual_size;
-        obj_info.user_defined = Arc::clone(&goi.user_defined);
-        obj_info.parts = Arc::clone(&goi.parts);
         obj_info.user_tags = Arc::clone(&goi.user_tags);
         self.invalidate_get_object_metadata_cache(bucket, object).await;
         Ok(obj_info)
@@ -7872,113 +7824,6 @@ mod replication_quota_safety_tests {
         assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
     }
 
-    #[tokio::test]
-    async fn delete_returns_canonical_compressed_accounting_size() {
-        let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "compressed-delete-accounting";
-        for disk in &disks {
-            disk.make_volume(bucket).await.expect("bucket volume should be created");
-        }
-
-        let mut user_defined = HashMap::new();
-        insert_str(
-            &mut user_defined,
-            rustfs_utils::http::SUFFIX_COMPRESSION,
-            "klauspost/compress/s2".to_string(),
-        );
-        insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
-        let mut reader = PutObjReader::new(
-            HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
-                .expect("compressed fixture reader should be valid"),
-        );
-        set_disks
-            .put_object(
-                bucket,
-                "object",
-                &mut reader,
-                &ObjectOptions {
-                    user_defined,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("compressed object should be written");
-
-        let (deleted, errors, accounting) = set_disks
-            .delete_objects_with_accounting(
-                bucket,
-                vec![ObjectToDelete {
-                    object_name: "object".to_string(),
-                    ..Default::default()
-                }],
-                ObjectOptions {
-                    object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
-                        ObjectLockConfigState::ConfirmedAbsent,
-                    ))),
-                    ..Default::default()
-                },
-            )
-            .await;
-
-        assert!(errors[0].is_none(), "compressed delete should succeed: {:?}", errors[0]);
-        assert!(deleted[0].found, "the committed object must be reported as found");
-        assert_eq!(accounting[0].as_ref().and_then(|value| value.size), Some(1000));
-        assert!(accounting[0].as_ref().is_some_and(|value| value.version_id.is_none()));
-        assert!(accounting[0].as_ref().is_some_and(|value| value.removed_current_object));
-    }
-
-    #[tokio::test]
-    async fn suspended_delete_marker_does_not_return_body_accounting() {
-        let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "suspended-delete-accounting";
-        for disk in &disks {
-            disk.make_volume(bucket).await.expect("bucket volume should be created");
-        }
-
-        let mut user_defined = HashMap::new();
-        insert_str(
-            &mut user_defined,
-            rustfs_utils::http::SUFFIX_COMPRESSION,
-            "klauspost/compress/s2".to_string(),
-        );
-        insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
-        let mut reader = PutObjReader::new(
-            HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
-                .expect("compressed fixture reader should be valid"),
-        );
-        let suspended_opts = ObjectOptions {
-            version_suspended: true,
-            delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
-                s3s::dto::VersioningConfiguration {
-                    status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
-                    ..Default::default()
-                },
-                None,
-            ))),
-            user_defined,
-            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
-            ..Default::default()
-        };
-        set_disks
-            .put_object(bucket, "object", &mut reader, &suspended_opts)
-            .await
-            .expect("compressed object should be written");
-
-        let (deleted, errors, accounting) = set_disks
-            .delete_objects_with_accounting(
-                bucket,
-                vec![ObjectToDelete {
-                    object_name: "object".to_string(),
-                    ..Default::default()
-                }],
-                suspended_opts,
-            )
-            .await;
-        assert!(errors[0].is_none(), "suspended delete should create a marker: {:?}", errors[0]);
-        assert!(deleted[0].delete_marker);
-        assert!(accounting[0].is_none(), "a delete marker must not carry body accounting");
-    }
-
     #[tokio::test]
     async fn direct_put_cannot_persist_a_tiny_logical_size() {
         let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;

crates/ecstore/src/storage_api_contracts/mod.rs

@@ -62,8 +62,8 @@ pub(crate) mod object {
     use super::{Debug, Error, FileInfo, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
     use crate::storage_api_contracts::range::HTTPRangeSpec;
     pub(crate) use rustfs_storage_api::{
-        DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions,
-        ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
+        DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions, ObjectOperations,
+        ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
     };
 
     pub(crate) trait EcstoreObjectIO:

crates/ecstore/src/store/bucket.rs

@@ -601,7 +601,7 @@ impl ECStore {
 
     #[instrument(skip(self))]
     pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
-        // TODO(backlog): support cached bucket listing via opts.cached
+        // TODO: opts.cached
 
         let mut buckets = self.peer_sys.list_bucket(opts).await?;
 

crates/ecstore/src/store/list_objects.rs

@@ -4673,7 +4673,7 @@ async fn gather_results(
             entry.name = entry.name.replace("\\", "/");
         }
 
-        // TODO(backlog): integrate rx.recv() for incremental listing results
+        // TODO: rx.recv()
 
         if let Some(marker) = &opts.marker
             && ((!opts.include_marker && &entry.name <= marker) || (opts.include_marker && &entry.name < marker))
@@ -4703,7 +4703,7 @@ async fn gather_results(
             continue;
         }
 
-        // TODO(backlog): integrate lifecycle evaluation during object listing
+        // TODO: Lifecycle
 
         entries.push(Some(entry));
         candidate_entries += 1;

crates/ecstore/src/store/multipart.rs

@@ -332,7 +332,7 @@ impl ECStore {
         let expected_incarnation_id = opts.expected_bucket_incarnation_id;
 
         if request.prefix.is_empty() {
-            // TODO(backlog): return cached multipart listing when prefix is empty
+            // TODO: return from cache
         }
 
         if self.single_pool() {
@@ -610,7 +610,7 @@ impl ECStore {
         let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
         let opts = &opts;
 
-        // TODO(backlog): defer DeleteUploadID to background for faster abort response
+        // TODO: defer DeleteUploadID
 
         if self.single_pool() {
             return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;

crates/ecstore/src/store/object.rs

@@ -41,7 +41,7 @@ use crate::set_disk::{
 };
 use crate::storage_api_contracts::{
     namespace::NamespaceLocking as _,
-    object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _},
+    object::{ObjectIO as _, ObjectOperations as _},
 };
 use parking_lot::Mutex as ParkingMutex;
 use rustfs_io_metrics::{
@@ -1216,14 +1216,6 @@ fn return_batch_delete_lock_error(objects: &[ObjectToDelete], err: Error) -> (Ve
     (del_objects, del_errs)
 }
 
-fn return_batch_delete_lock_error_with_accounting(
-    objects: &[ObjectToDelete],
-    err: Error,
-) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
-    let (deleted, errors) = return_batch_delete_lock_error(objects, err);
-    (deleted, errors, vec![None; objects.len()])
-}
-
 fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> {
     let mut object_names: Vec<&str> = objects.iter().map(|object| object.object_name.as_str()).collect();
     object_names.sort_unstable();
@@ -2320,22 +2312,6 @@ impl ECStore {
         result
     }
 
-    pub async fn delete_objects_with_tier_delete_journal_and_accounting(
-        self: &Arc<Self>,
-        bucket: &str,
-        objects: Vec<ObjectToDelete>,
-        opts: ObjectOptions,
-    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
-        let result = self
-            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, Some(Arc::clone(self)))
-            .await;
-        let success_count = result.1.iter().filter(|err| err.is_none()).count();
-        if success_count > 0 {
-            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
-        }
-        result
-    }
-
     #[instrument(skip(self))]
     pub(super) async fn handle_delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
         self.handle_delete_object_with_journal(bucket, object, opts, None).await
@@ -2713,19 +2689,6 @@ impl ECStore {
         opts: ObjectOptions,
         tier_journal_api: Option<Arc<ECStore>>,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
-        let (deleted, errors, _) = self
-            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, tier_journal_api)
-            .await;
-        (deleted, errors)
-    }
-
-    pub(super) async fn handle_delete_objects_with_journal_and_accounting(
-        &self,
-        bucket: &str,
-        objects: Vec<ObjectToDelete>,
-        opts: ObjectOptions,
-        tier_journal_api: Option<Arc<ECStore>>,
-    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
         // encode object name
         let objects: Vec<ObjectToDelete> = objects
             .iter()
@@ -2738,7 +2701,6 @@ impl ECStore {
 
         // Default return value
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
-        let mut accounting = vec![None; objects.len()];
 
         let mut del_errs = Vec::with_capacity(objects.len());
         for _ in 0..objects.len() {
@@ -2752,7 +2714,7 @@ impl ECStore {
         } else {
             match self.acquire_bucket_lifecycle_read_lock(bucket).await {
                 Ok(guard) => Some(guard),
-                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
+                Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
             }
         };
         if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
@@ -2764,21 +2726,21 @@ impl ECStore {
                 Err(err) => {
                     let message = err.to_string();
                     let errors = (0..objects.len()).map(|_| Some(Error::other(message.clone()))).collect();
-                    return (del_objects, errors, accounting);
+                    return (del_objects, errors);
                 }
             }
         }
         if !is_meta_bucketname(bucket)
             && let Err(err) = get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
         {
-            return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err);
+            return return_batch_delete_lock_error(objects.as_slice(), err);
         }
         let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
             None
         } else {
             Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
                 Ok(guard) => guard,
-                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
+                Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
             })
         };
         if let Some(guard) = _object_lock_metadata_guard.as_ref() {
@@ -2788,7 +2750,7 @@ impl ECStore {
             let (state, incarnation_id, config_revision) =
                 match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
                     Ok(snapshot) => snapshot,
-                    Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
+                    Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
                 };
             opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
                 self.id,
@@ -2804,10 +2766,7 @@ impl ECStore {
         if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
             && expected != current
         {
-            return return_batch_delete_lock_error_with_accounting(
-                objects.as_slice(),
-                StorageError::BucketNotFound(bucket.to_string()),
-            );
+            return return_batch_delete_lock_error(objects.as_slice(), StorageError::BucketNotFound(bucket.to_string()));
         }
         #[cfg(test)]
         if current_bucket_incarnation_id.is_some() {
@@ -2815,7 +2774,7 @@ impl ECStore {
         }
         let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
             Ok(guards) => guards,
-            Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
+            Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
         };
 
         let mut futures = Vec::with_capacity(self.pools.len());
@@ -2824,24 +2783,22 @@ impl ECStore {
             if self.is_pool_rebalancing(pool.pool_idx).await {
                 continue;
             }
-            futures.push(pool.delete_objects_with_accounting(bucket, objects.clone(), opts.clone()));
+            futures.push(pool.delete_objects(bucket, objects.clone(), opts.clone()));
         }
 
         let results = join_all(futures).await;
 
         for idx in 0..del_objects.len() {
-            for (dels, errs, pool_accounting) in results.iter() {
+            for (dels, errs) in results.iter() {
                 if errs[idx].is_none() && dels[idx].found {
                     del_errs[idx] = None;
                     del_objects[idx] = dels[idx].clone();
-                    accounting[idx] = pool_accounting[idx].clone();
                     break;
                 }
 
                 if del_errs[idx].is_none() {
                     del_errs[idx] = errs[idx].clone();
                     del_objects[idx] = dels[idx].clone();
-                    accounting[idx] = pool_accounting[idx].clone();
                 }
             }
         }
@@ -2850,7 +2807,7 @@ impl ECStore {
             v.object_name = decode_dir_object(&v.object_name);
         });
 
-        (del_objects, del_errs, accounting)
+        (del_objects, del_errs)
 
         // let mut futures = Vec::with_capacity(objects.len());
 

crates/ecstore/src/store/rebalance.rs

@@ -385,7 +385,7 @@ impl ECStore {
     }
 
     pub(super) async fn is_suspended(&self, idx: usize) -> bool {
-        // TODO(backlog): acquire pool metadata lock for consistent suspension check
+        // TODO: LOCK
 
         let pool_meta = self.pool_meta.read().await;
 

crates/madmin/src/metrics.rs

@@ -689,6 +689,14 @@ pub struct ScannerMetrics {
     pub cycle_max_objects: u64,
     #[serde(rename = "cycle_max_directories", default)]
     pub cycle_max_directories: u64,
+    #[serde(rename = "cycle_timeout_total", default)]
+    pub cycle_timeout_total: u64,
+    #[serde(rename = "cycle_recovery_required_total", default)]
+    pub cycle_recovery_required_total: u64,
+    #[serde(rename = "cycle_last_progress_age", default)]
+    pub cycle_last_progress_age: u64,
+    #[serde(rename = "leader_lease_without_progress", default)]
+    pub leader_lease_without_progress: bool,
     #[serde(rename = "bitrot_cycle_enabled", default)]
     pub bitrot_cycle_enabled: bool,
     #[serde(rename = "bitrot_cycle_seconds", default)]
@@ -764,6 +772,8 @@ impl ScannerMetrics {
             self.cycle_max_duration_seconds = other.cycle_max_duration_seconds;
             self.cycle_max_objects = other.cycle_max_objects;
             self.cycle_max_directories = other.cycle_max_directories;
+            self.cycle_last_progress_age = other.cycle_last_progress_age;
+            self.leader_lease_without_progress = other.leader_lease_without_progress;
             self.bitrot_cycle_enabled = other.bitrot_cycle_enabled;
             self.bitrot_cycle_seconds = other.bitrot_cycle_seconds;
         }
@@ -857,6 +867,12 @@ impl ScannerMetrics {
             .saturating_add(other.last_cycle_replication_checks);
         self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves);
         self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles);
+        self.cycle_timeout_total = self.cycle_timeout_total.saturating_add(other.cycle_timeout_total);
+        self.cycle_recovery_required_total = self
+            .cycle_recovery_required_total
+            .saturating_add(other.cycle_recovery_required_total);
+        self.cycle_last_progress_age = self.cycle_last_progress_age.max(other.cycle_last_progress_age);
+        self.leader_lease_without_progress |= other.leader_lease_without_progress;
         self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles);
         self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown);
         self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime);

crates/scanner/src/runtime_config.rs

@@ -125,7 +125,10 @@ impl Default for ScannerRuntimeConfig {
             cycle_interval_source: ScannerRuntimeConfigSource::Default,
             bitrot_cycle: Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS)),
             bitrot_cycle_source: ScannerRuntimeConfigSource::Default,
-            cycle_budget: ScannerCycleBudgetConfig::default(),
+            cycle_budget: ScannerCycleBudgetConfig {
+                max_duration: Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)),
+                ..Default::default()
+            },
             cycle_max_duration_source: ScannerRuntimeConfigSource::Default,
             cycle_max_objects_source: ScannerRuntimeConfigSource::Default,
             cycle_max_directories_source: ScannerRuntimeConfigSource::Default,
@@ -436,19 +439,46 @@ fn lookup_max_wait(
     Ok((speed.max_sleep(), speed_source))
 }
 
-fn lookup_optional_seconds(
-    kvs: Option<&KVS>,
-    key: &'static str,
-    env_key: &'static str,
-    default: u64,
-) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
-    if let Some(secs) = rustfs_utils::get_env_opt_u64(env_key) {
-        return Ok((Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Env));
+fn lookup_cycle_duration(kvs: Option<&KVS>) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
+    match rustfs_utils::get_env_parse_outcome::<u64>(ENV_SCANNER_CYCLE_MAX_DURATION_SECS) {
+        rustfs_utils::EnvParseOutcome::Parsed(secs) => {
+            return cycle_duration_from_secs(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, secs)
+                .map(|duration| (duration, ScannerRuntimeConfigSource::Env));
+        }
+        rustfs_utils::EnvParseOutcome::Invalid => {
+            // Do not include the raw environment value in the typed error:
+            // deployments occasionally put sensitive material in inherited
+            // environment snapshots. The key still identifies the control.
+            return Err(invalid_value(
+                ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
+                "<invalid>",
+                "expected unsigned integer seconds",
+            ));
+        }
+        rustfs_utils::EnvParseOutcome::Absent => {}
     }
-    if let Some(value) = config_value(kvs, key, default) {
-        return parse_config_u64(key, value).map(|secs| (Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Config));
+
+    if let Some(value) = config_value(kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) {
+        let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?;
+        return cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs)
+            .map(|duration| (duration, ScannerRuntimeConfigSource::Config));
     }
-    Ok((None, ScannerRuntimeConfigSource::Default))
+
+    Ok((
+        Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)),
+        ScannerRuntimeConfigSource::Default,
+    ))
+}
+
+fn cycle_duration_from_secs(key: &'static str, secs: u64) -> Result<Option<Duration>, ScannerRuntimeConfigError> {
+    if secs == 0 {
+        return Ok(None);
+    }
+    let duration = Duration::from_secs(secs);
+    if std::time::Instant::now().checked_add(duration).is_none() {
+        return Err(invalid_value(key, "<overflow>", "duration exceeds the timer range"));
+    }
+    Ok(Some(duration))
 }
 
 fn lookup_start_delay(kvs: Option<&KVS>) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
@@ -553,12 +583,7 @@ pub(crate) fn lookup_scanner_runtime_config(
         (speed.cycle_interval(), speed_source)
     };
 
-    let (cycle_max_duration, cycle_max_duration_source) = lookup_optional_seconds(
-        scanner_kvs,
-        SCANNER_CYCLE_MAX_DURATION,
-        ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
-        DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS,
-    )?;
+    let (cycle_max_duration, cycle_max_duration_source) = lookup_cycle_duration(scanner_kvs)?;
     let (cycle_max_objects, cycle_max_objects_source) = lookup_count_budget(
         scanner_kvs,
         SCANNER_CYCLE_MAX_OBJECTS,
@@ -863,10 +888,10 @@ mod tests {
     use rustfs_config::server_config::{Config as ServerConfig, KVS};
     use rustfs_config::{
         DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS,
-        ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED,
-        HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
-        SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
-        SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
+        ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY,
+        ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE,
+        SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION,
+        SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
     };
     use serial_test::serial;
     use std::collections::HashMap;
@@ -943,6 +968,43 @@ mod tests {
         });
     }
 
+    #[test]
+    #[serial]
+    fn scanner_unset_budget_uses_safe_default_but_explicit_zero_is_unbounded() {
+        let config = server_config_with_scanner(&[]);
+        with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
+            let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
+            assert_eq!(resolved.cycle_budget.max_duration, Some(Duration::from_secs(1800)));
+            assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Default);
+        });
+
+        let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "0")]);
+        with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
+            let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
+            assert_eq!(resolved.cycle_budget.max_duration, None);
+            assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Config);
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn cycle_budget_invalid_or_overflow_config_is_rejected() {
+        with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("invalid"), || {
+            let error = lookup_scanner_runtime_config(None).expect_err("invalid duration env must be rejected");
+            assert!(error.to_string().contains(ENV_SCANNER_CYCLE_MAX_DURATION_SECS));
+            assert!(error.to_string().contains("<invalid>"));
+            assert!(!error.to_string().contains(": invalid ("));
+        });
+        with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551616"), || {
+            assert!(lookup_scanner_runtime_config(None).is_err());
+        });
+        with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551615"), || {
+            assert!(lookup_scanner_runtime_config(None).is_err());
+        });
+        let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "not-a-duration")]);
+        assert!(lookup_scanner_runtime_config(Some(&config)).is_err());
+    }
+
     #[test]
     #[serial]
     fn scanner_runtime_config_normalizes_persisted_default_speed() {

crates/scanner/src/scanner.rs

@@ -52,6 +52,7 @@ use rustfs_config::{
 };
 use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
 use rustfs_data_usage::observed_data_usage_is_newer;
+use rustfs_lock::NamespaceLockGuard;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest as _, Sha256};
 #[cfg(test)]
@@ -983,20 +984,116 @@ fn data_usage_persist_timeout() -> Duration {
     DataUsageCache::persistence_timeout()
 }
 
+#[cfg(not(test))]
+const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_secs(30);
+#[cfg(test)]
+const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_millis(50);
+
+async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
+    ctx: &CancellationToken,
+    storeapi: Arc<Store>,
+    cycle_info: &mut CurrentCycle,
+    cycle_revision: &mut DataUsageCacheRevision,
+    leader_epoch: &mut u64,
+    lock_lost: LockLost,
+) -> bool
+where
+    Store: ScannerObjectIO,
+    LockLost: Future<Output = ()>,
+{
+    let fence_ctx = ctx.child_token();
+    let claim = claim_scanner_leadership(&fence_ctx, storeapi, cycle_info, cycle_revision, leader_epoch);
+    tokio::pin!(claim);
+    tokio::pin!(lock_lost);
+    tokio::select! {
+        biased;
+        _ = &mut lock_lost => {
+            fence_ctx.cancel();
+            false
+        }
+        result = tokio::time::timeout(SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT, &mut claim) => {
+            result.unwrap_or(false) && !fence_ctx.is_cancelled()
+        }
+    }
+}
+
+struct ScannerCycleDeadlineState<'a> {
+    cycle_info: &'a mut CurrentCycle,
+    cycle_revision: &'a mut DataUsageCacheRevision,
+    leader_epoch: &'a mut u64,
+    cycle_budget: &'a ScannerCycleBudget,
+}
+
+fn cycle_timeout_requires_recovery(worker_stopped: bool, cycle_state_persisted: bool, generation_fenced: bool) -> bool {
+    !worker_stopped || !cycle_state_persisted || !generation_fenced
+}
+
+async fn handle_scanner_cycle_deadline<Store>(
+    ctx: &CancellationToken,
+    storeapi: Arc<Store>,
+    state: ScannerCycleDeadlineState<'_>,
+    worker_stopped: bool,
+    guard: &mut NamespaceLockGuard,
+) where
+    Store: ScannerObjectIO,
+{
+    let fenced = fence_scanner_epoch_after_cycle_timeout(
+        ctx,
+        storeapi,
+        state.cycle_info,
+        state.cycle_revision,
+        state.leader_epoch,
+        guard.lock_lost_notified(),
+    )
+    .await;
+    let cycle_state_persisted = state.cycle_budget.cycle_state_persisted();
+    let recovery_required = cycle_timeout_requires_recovery(worker_stopped, cycle_state_persisted, fenced);
+    warn!(
+        target: "rustfs::scanner",
+        event = EVENT_SCANNER_CYCLE_STATE,
+        component = LOG_COMPONENT_SCANNER,
+        subsystem = LOG_SUBSYSTEM_RUNTIME,
+        state = "cycle_timeout",
+        worker_stopped,
+        cycle_state_persisted,
+        generation_fenced = fenced,
+        recovery_required,
+        "Scanner cycle deadline expired; durable cursor/generation fencing completed when possible"
+    );
+    global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age());
+    // Stop renewing before releasing the lease. A new leader can then claim the
+    // higher persisted generation instead of inheriting the expired worker.
+    guard.release();
+    global_metrics().set_cycle(None).await;
+}
+
 async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
     cycle_info.current = 0;
     global_metrics().clear_current_scan_mode();
     cycle_metrics_guard.finish(cycle_info.clone()).await;
 }
 
-#[instrument(skip_all)]
-#[hotpath::measure]
+#[cfg(test)]
 async fn run_data_scanner_cycle(
     ctx: &CancellationToken,
     storeapi: &Arc<ECStore>,
     cycle_info: &mut CurrentCycle,
     cycle_revision: &mut DataUsageCacheRevision,
     leader_epoch: u64,
+) -> ScannerCycleOutcome {
+    let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config());
+    run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget).await
+}
+
+#[instrument(skip_all)]
+#[hotpath::measure]
+async fn run_data_scanner_cycle_with_budget(
+    ctx: &CancellationToken,
+    storeapi: &Arc<ECStore>,
+    cycle_info: &mut CurrentCycle,
+    cycle_revision: &mut DataUsageCacheRevision,
+    leader_epoch: u64,
+    cycle_budget: Arc<ScannerCycleBudget>,
 ) -> ScannerCycleOutcome {
     let _activity_guard = ScannerActivityGuard::new();
     if let Err(err) = refresh_scanner_runtime_config_from_global() {
@@ -1012,7 +1109,11 @@ async fn run_data_scanner_cycle(
     }
     let configured_cycle_interval = scanner_cycle_interval();
     let configured_bitrot_cycle = scanner_bitrot_cycle();
-    let cycle_budget_config = scanner_cycle_budget_config();
+    let cycle_budget_config = ScannerCycleBudgetConfig {
+        max_duration: cycle_budget.max_duration(),
+        max_objects: cycle_budget.max_objects(),
+        max_directories: cycle_budget.max_directories(),
+    };
     let usage_persist_timeout = data_usage_persist_timeout();
     global_metrics().record_scanner_cycle_config(
         configured_cycle_interval,
@@ -1083,7 +1184,6 @@ async fn run_data_scanner_cycle(
     let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
 
     let done_cycle = Metrics::time(Metric::ScanCycle);
-    let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
     let scan_result = storeapi
         .clone()
         .nsscanner_with_status(
@@ -1223,7 +1323,7 @@ async fn run_data_scanner_cycle(
                 "Scanner cycle is recovering to a newer durable cache generation"
             );
             emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
-            return if persist_required_scanner_cycle_floor(
+            let persisted = persist_required_scanner_cycle_floor(
                 ctx,
                 storeapi.clone(),
                 cycle_info,
@@ -1232,8 +1332,9 @@ async fn run_data_scanner_cycle(
                 required_cycle,
                 &mut cycle_metrics_guard,
             )
-            .await
-            {
+            .await;
+            return if persisted {
+                cycle_budget.mark_cycle_state_persisted();
                 ScannerCycleOutcome::Partial
             } else {
                 ScannerCycleOutcome::Failed
@@ -1291,7 +1392,7 @@ async fn run_data_scanner_cycle(
             scan_cycle_partial_reason(budget_reason),
             scan_cycle_partial_source(budget_reason),
         );
-        return if finalize_partial_scan_cycle(
+        let persisted = finalize_partial_scan_cycle(
             ctx,
             storeapi.clone(),
             cycle_info,
@@ -1299,8 +1400,9 @@ async fn run_data_scanner_cycle(
             leader_epoch,
             &mut cycle_metrics_guard,
         )
-        .await
-        {
+        .await;
+        return if persisted {
+            cycle_budget.mark_cycle_state_persisted();
             ScannerCycleOutcome::Partial
         } else {
             ScannerCycleOutcome::Failed
@@ -1375,7 +1477,7 @@ async fn run_data_scanner_cycle(
                 );
             }
             emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
-            return if finalize_partial_scan_cycle(
+            let persisted = finalize_partial_scan_cycle(
                 ctx,
                 storeapi.clone(),
                 cycle_info,
@@ -1383,8 +1485,9 @@ async fn run_data_scanner_cycle(
                 leader_epoch,
                 &mut cycle_metrics_guard,
             )
-            .await
-            {
+            .await;
+            return if persisted {
+                cycle_budget.mark_cycle_state_persisted();
                 ScannerCycleOutcome::Partial
             } else {
                 ScannerCycleOutcome::Failed
@@ -1425,6 +1528,7 @@ async fn run_data_scanner_cycle(
             )
             .await
             {
+                cycle_budget.mark_cycle_state_persisted();
                 emit_scan_cycle_superseded(cycle_start.elapsed());
                 return ScannerCycleOutcome::Superseded;
             }
@@ -1457,6 +1561,7 @@ async fn run_data_scanner_cycle(
         emit_scan_cycle_complete(false, cycle_start.elapsed());
         return ScannerCycleOutcome::Failed;
     }
+    cycle_budget.mark_cycle_state_persisted();
 
     done_cycle();
     emit_scan_cycle_complete(true, cycle_start.elapsed());
@@ -1521,7 +1626,7 @@ async fn run_data_scanner_with_maintenance_state(
 ) -> Result<(), ScannerError> {
     reset_scanner_cycle_schedule();
     // Acquire leader lock (write lock) to ensure only one scanner runs
-    let guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
+    let mut guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
         Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
             Ok(guard) => {
                 record_scanner_leader_lock_state("acquired");
@@ -1704,13 +1809,49 @@ async fn run_data_scanner_with_maintenance_state(
             return Ok(());
         }
         let cycle_ctx = ctx.child_token();
-        let initial_outcome = await_scanner_cycle_with_lock_fence(
+        let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
+        let initial_outcome = match await_scanner_cycle_with_budget_fence(
             &cycle_ctx,
-            run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch),
+            &cycle_budget,
+            run_data_scanner_cycle_with_budget(
+                &cycle_ctx,
+                &storeapi,
+                &mut cycle_info,
+                &mut cycle_revision,
+                leader_epoch,
+                cycle_budget.clone(),
+            ),
             guard.lock_lost_notified(),
         )
         .await
-        .unwrap_or(ScannerCycleOutcome::Failed);
+        {
+            ScannerCycleWaitOutcome::Completed(outcome) => outcome,
+            ScannerCycleWaitOutcome::LockLost => {
+                record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await;
+                global_metrics().set_cycle(None).await;
+                return Ok(());
+            }
+            ScannerCycleWaitOutcome::Cancelled => {
+                global_metrics().set_cycle(None).await;
+                return Ok(());
+            }
+            ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
+                handle_scanner_cycle_deadline(
+                    &ctx,
+                    storeapi.clone(),
+                    ScannerCycleDeadlineState {
+                        cycle_info: &mut cycle_info,
+                        cycle_revision: &mut cycle_revision,
+                        leader_epoch: &mut leader_epoch,
+                        cycle_budget: &cycle_budget,
+                    },
+                    worker_stopped,
+                    &mut guard,
+                )
+                .await;
+                return Ok(());
+            }
+        };
         superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded);
         deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_)));
         dirty_usage_generation_seen = dirty_generation_before_cycle;
@@ -1916,13 +2057,49 @@ async fn run_data_scanner_with_maintenance_state(
         }
         let dirty_generation_before_cycle = dirty_usage_generation();
         let cycle_ctx = ctx.child_token();
-        let outcome = await_scanner_cycle_with_lock_fence(
+        let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
+        let outcome = match await_scanner_cycle_with_budget_fence(
             &cycle_ctx,
-            run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch),
+            &cycle_budget,
+            run_data_scanner_cycle_with_budget(
+                &cycle_ctx,
+                &storeapi,
+                &mut cycle_info,
+                &mut cycle_revision,
+                leader_epoch,
+                cycle_budget.clone(),
+            ),
             guard.lock_lost_notified(),
         )
         .await
-        .unwrap_or(ScannerCycleOutcome::Failed);
+        {
+            ScannerCycleWaitOutcome::Completed(outcome) => outcome,
+            ScannerCycleWaitOutcome::LockLost => {
+                record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await;
+                global_metrics().set_cycle(None).await;
+                return Ok(());
+            }
+            ScannerCycleWaitOutcome::Cancelled => {
+                global_metrics().set_cycle(None).await;
+                return Ok(());
+            }
+            ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
+                handle_scanner_cycle_deadline(
+                    &ctx,
+                    storeapi.clone(),
+                    ScannerCycleDeadlineState {
+                        cycle_info: &mut cycle_info,
+                        cycle_revision: &mut cycle_revision,
+                        leader_epoch: &mut leader_epoch,
+                        cycle_budget: &cycle_budget,
+                    },
+                    worker_stopped,
+                    &mut guard,
+                )
+                .await;
+                return Ok(());
+            }
+        };
         superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded);
         deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_)));
         dirty_usage_generation_seen = dirty_generation_before_cycle;

crates/scanner/src/scanner/cycle_state.rs

@@ -496,3 +496,63 @@ where
         output = &mut cycle => Some(output),
     }
 }
+
+#[derive(Debug, PartialEq, Eq)]
+pub(super) enum ScannerCycleWaitOutcome<T> {
+    Completed(T),
+    LockLost,
+    Cancelled,
+    Deadline { worker_stopped: bool },
+}
+
+pub(super) async fn await_scanner_cycle_with_budget_fence<Cycle, LockLost>(
+    cycle_ctx: &CancellationToken,
+    budget: &ScannerCycleBudget,
+    cycle: Cycle,
+    lock_lost: LockLost,
+) -> ScannerCycleWaitOutcome<Cycle::Output>
+where
+    Cycle: Future,
+    LockLost: Future<Output = ()>,
+{
+    tokio::pin!(cycle);
+    tokio::pin!(lock_lost);
+    let deadline = async {
+        if let Some(deadline) = budget.deadline() {
+            tokio::time::sleep_until(deadline).await;
+        } else {
+            std::future::pending::<()>().await;
+        }
+    };
+    tokio::pin!(deadline);
+    tokio::select! {
+        biased;
+        _ = &mut lock_lost => {
+            cycle_ctx.cancel();
+            let _ = tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await;
+            ScannerCycleWaitOutcome::LockLost
+        }
+        _ = &mut deadline => {
+            budget.cancel_for_runtime();
+            // Let the budget cancellation reach the scanner first so it can
+            // persist a partial cursor. Only an uncooperative worker gets the
+            // parent cancellation, and it is dropped after the bounded window;
+            // the caller fences its epoch next.
+            let worker_stopped = if tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle)
+                .await
+                .is_ok()
+            {
+                true
+            } else {
+                cycle_ctx.cancel();
+                false
+            };
+            ScannerCycleWaitOutcome::Deadline { worker_stopped }
+        }
+        _ = cycle_ctx.cancelled() => {
+            let _ = tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await;
+            ScannerCycleWaitOutcome::Cancelled
+        }
+        output = &mut cycle => ScannerCycleWaitOutcome::Completed(output),
+    }
+}

crates/scanner/src/scanner/tests.rs

@@ -26,6 +26,7 @@ use std::task::Poll;
 use temp_env::{with_var, with_var_unset};
 use tokio::io::AsyncReadExt;
 use tokio::sync::Mutex;
+use tokio::time::{Duration, advance};
 
 const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
 
@@ -118,6 +119,180 @@ async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() {
     assert!(cycle_ctx.is_cancelled());
 }
 
+#[tokio::test(start_paused = true)]
+#[serial]
+async fn cycle_budget_fences_late_writer_after_timeout() {
+    let cycle_ctx = CancellationToken::new();
+    let budget = ScannerCycleBudget::new(
+        &cycle_ctx,
+        ScannerCycleBudgetConfig {
+            max_duration: Some(Duration::from_secs(5)),
+            ..Default::default()
+        },
+    );
+    let outcome = {
+        let cycle = std::future::pending::<()>();
+        let lock_lost = std::future::pending::<()>();
+        let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, cycle, lock_lost);
+        tokio::pin!(waiter);
+        tokio::task::yield_now().await;
+        advance(Duration::from_secs(5)).await;
+        tokio::task::yield_now().await;
+        advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await;
+        waiter.await
+    };
+    assert_eq!(outcome, ScannerCycleWaitOutcome::Deadline { worker_stopped: false });
+    assert!(cycle_ctx.is_cancelled());
+    assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Runtime));
+
+    // A newer leadership epoch is the durable fence that rejects a late
+    // writer after the timed-out future has been dropped.
+    let store = Arc::new(MemoryConfigStore::default());
+    let mut revision = DataUsageCacheRevision::Missing;
+    let mut cycle = CurrentCycle {
+        current: 0,
+        next: 12,
+        ..Default::default()
+    };
+    let persist_ctx = CancellationToken::new();
+    assert!(persist_scanner_cycle_state(&persist_ctx, store.clone(), &mut cycle, &mut revision, 1).await);
+    let newer = encode_scanner_cycle_state(&cycle, 2).expect("new epoch fence should encode");
+    let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    store.interleaving_puts.lock().await.insert(key, (2, newer));
+    let mut late_cycle = CurrentCycle { next: 13, ..cycle };
+    assert!(!persist_scanner_cycle_state(&persist_ctx, store, &mut late_cycle, &mut revision, 1).await);
+}
+
+#[tokio::test(start_paused = true)]
+async fn cycle_budget_parent_cancellation_is_not_reported_as_timeout() {
+    let cycle_ctx = CancellationToken::new();
+    let budget = ScannerCycleBudget::new(
+        &cycle_ctx,
+        ScannerCycleBudgetConfig {
+            max_duration: Some(Duration::from_secs(5)),
+            ..Default::default()
+        },
+    );
+    let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, std::future::pending::<()>(), std::future::pending());
+    tokio::pin!(waiter);
+    tokio::task::yield_now().await;
+    cycle_ctx.cancel();
+    tokio::task::yield_now().await;
+    advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await;
+    assert_eq!(waiter.await, ScannerCycleWaitOutcome::Cancelled);
+}
+
+#[tokio::test(start_paused = true)]
+async fn cycle_budget_deadline_wins_same_tick_as_parent_cancellation() {
+    let cycle_ctx = CancellationToken::new();
+    let budget = ScannerCycleBudget::new(
+        &cycle_ctx,
+        ScannerCycleBudgetConfig {
+            max_duration: Some(Duration::from_secs(5)),
+            ..Default::default()
+        },
+    );
+    let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, std::future::pending::<()>(), std::future::pending());
+    tokio::pin!(waiter);
+    tokio::task::yield_now().await;
+    advance(Duration::from_secs(5)).await;
+    cycle_ctx.cancel();
+    tokio::task::yield_now().await;
+    advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await;
+
+    assert_eq!(waiter.await, ScannerCycleWaitOutcome::Deadline { worker_stopped: false });
+    assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Runtime));
+}
+
+#[tokio::test]
+async fn cycle_budget_persist_cursor_failure_is_recovery_required() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
+    store.fail_put_number.lock().await.insert(key, 1);
+
+    let ctx = CancellationToken::new();
+    let mut revision = DataUsageCacheRevision::Missing;
+    let mut cycle = CurrentCycle {
+        current: 12,
+        next: 12,
+        ..Default::default()
+    };
+    let mut leader_epoch = 1;
+    let fenced = fence_scanner_epoch_after_cycle_timeout(
+        &ctx,
+        store,
+        &mut cycle,
+        &mut revision,
+        &mut leader_epoch,
+        std::future::pending(),
+    )
+    .await;
+    assert!(!fenced, "a failed cursor/generation write must require recovery");
+    let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
+    assert!(cycle_timeout_requires_recovery(true, budget.cycle_state_persisted(), fenced));
+
+    let metrics = Metrics::new();
+    metrics.record_scanner_cycle_timeout(!fenced, Duration::from_secs(17));
+    let report = metrics.report().await;
+    assert_eq!(report.cycle_timeout_total, 1);
+    assert_eq!(report.cycle_recovery_required_total, 1);
+    assert_eq!(report.cycle_last_progress_age, 17);
+    assert!(report.leader_lease_without_progress);
+}
+
+#[tokio::test]
+#[serial]
+async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let lock = store
+        .new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
+        .await
+        .expect("scanner leader lock should be created");
+    let mut guard = lock
+        .get_write_lock(Duration::from_secs(1))
+        .await
+        .expect("scanner leader lock should be acquired");
+
+    let ctx = CancellationToken::new();
+    let mut cycle_info = CurrentCycle {
+        current: 12,
+        next: 12,
+        ..Default::default()
+    };
+    let mut cycle_revision = DataUsageCacheRevision::Missing;
+    let mut leader_epoch = 1;
+    let budget = ScannerCycleBudget::new(
+        &ctx,
+        ScannerCycleBudgetConfig {
+            max_duration: Some(Duration::from_secs(60)),
+            ..Default::default()
+        },
+    );
+    budget.mark_cycle_state_persisted();
+
+    handle_scanner_cycle_deadline(
+        &ctx,
+        store.clone(),
+        ScannerCycleDeadlineState {
+            cycle_info: &mut cycle_info,
+            cycle_revision: &mut cycle_revision,
+            leader_epoch: &mut leader_epoch,
+            cycle_budget: &budget,
+        },
+        true,
+        &mut guard,
+    )
+    .await;
+
+    assert!(guard.is_released());
+    let persisted = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
+        .await
+        .expect("deadline handler should persist a fenced cursor");
+    let (_, persisted_epoch) = decode_scanner_cycle_state(&persisted).expect("fenced cursor should decode");
+    assert_eq!(persisted_epoch, 2);
+    global_metrics().set_cycle(None).await;
+}
+
 struct ScannerDefaultSpeedGuard;
 
 impl ScannerDefaultSpeedGuard {
@@ -416,14 +591,6 @@ fn test_scanner_cycle_max_duration_uses_env() {
     });
 }
 
-#[test]
-#[serial]
-fn test_scanner_cycle_max_duration_default_is_disabled() {
-    with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
-        assert_eq!(scanner_cycle_max_duration(), None);
-    });
-}
-
 #[tokio::test]
 async fn test_scanner_cycle_budget_cancels_after_duration() {
     let parent = CancellationToken::new();
@@ -1356,7 +1523,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() {
 }
 
 #[tokio::test]
-async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() {
+async fn cycle_budget_lease_takeover_rejects_old_generation() {
     let store = Arc::new(MemoryConfigStore::default());
     let ctx = CancellationToken::new();
     let mut revision = DataUsageCacheRevision::Missing;
@@ -1401,12 +1568,17 @@ async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() {
         .await
     );
 
-    let state = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
+    let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH)
         .await
         .expect("replacement leadership claim should persist");
     let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("replacement cycle state should decode");
     assert_eq!(claimed_cycle.next, 14);
     assert_eq!(claimed_epoch, 2);
+
+    let mut stale_cycle = CurrentCycle { next: 15, ..cycle };
+    let mut stale_revision = DataUsageCacheRevision::Etag("memory-2".to_string());
+    let stale_ctx = CancellationToken::new();
+    assert!(!persist_scanner_cycle_state(&stale_ctx, store, &mut stale_cycle, &mut stale_revision, 1,).await);
 }
 
 #[tokio::test]

crates/scanner/src/scanner_budget.rs

@@ -14,17 +14,16 @@
 
 use std::sync::{
     Arc,
-    atomic::{AtomicU8, AtomicU64, Ordering},
+    atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
 };
-use std::time::Instant;
-
-use tokio::time::Duration;
+use tokio::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
 
 const BUDGET_REASON_NONE: u8 = 0;
 const BUDGET_REASON_RUNTIME: u8 = 1;
 const BUDGET_REASON_OBJECTS: u8 = 2;
 const BUDGET_REASON_DIRECTORIES: u8 = 3;
+const PROGRESS_CLOCK_SAMPLE_INTERVAL: u64 = 128;
 
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 pub(crate) struct ScannerCycleBudgetConfig {
@@ -63,29 +62,51 @@ pub struct ScannerCycleBudget {
     token: CancellationToken,
     reason: Arc<AtomicU8>,
     started_at: Instant,
+    deadline: Option<Instant>,
     max_duration: Option<Duration>,
     max_objects: Option<u64>,
     max_directories: Option<u64>,
     track_progress: bool,
+    track_unbounded_counts: bool,
     objects_scanned: AtomicU64,
     directories_started: AtomicU64,
     entries_visited: AtomicU64,
+    last_progress_millis: AtomicU64,
+    cycle_state_persisted: AtomicBool,
 }
 
 impl ScannerCycleBudget {
+    #[cfg(test)]
     pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
-        Self::new_inner(parent, config, false)
+        Self::new_inner(parent, config, false, false)
     }
 
     pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
-        Self::new_inner(parent, config, true)
+        Self::new_inner(parent, config, true, true)
     }
 
-    fn new_inner(parent: &CancellationToken, config: ScannerCycleBudgetConfig, track_progress: bool) -> Arc<Self> {
+    pub(crate) fn new_with_runtime_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
+        let track_progress = config.max_duration.is_some();
+        Self::new_inner(parent, config, track_progress, false)
+    }
+
+    fn new_inner(
+        parent: &CancellationToken,
+        config: ScannerCycleBudgetConfig,
+        track_progress: bool,
+        track_unbounded_counts: bool,
+    ) -> Arc<Self> {
         let token = parent.child_token();
         let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE));
+        let started_at = Instant::now();
+        let deadline = config.max_duration.map(|duration| match started_at.checked_add(duration) {
+            Some(deadline) => deadline,
+            // Runtime config rejects this range, but keep programmatic callers
+            // fail-closed instead of panicking or silently disabling the wall clock.
+            None => started_at,
+        });
 
-        if let Some(duration) = config.max_duration {
+        if let Some(deadline) = deadline {
             let parent = parent.clone();
             let token_wait = token.clone();
             let token_cancel = token.clone();
@@ -94,7 +115,7 @@ impl ScannerCycleBudget {
                 tokio::select! {
                     _ = parent.cancelled() => {}
                     _ = token_wait.cancelled() => {}
-                    _ = tokio::time::sleep(duration) => {
+                    _ = tokio::time::sleep_until(deadline) => {
                         Self::cancel_for_reason(&reason, &token_cancel, ScannerCycleBudgetReason::Runtime);
                     }
                 }
@@ -104,14 +125,18 @@ impl ScannerCycleBudget {
         Arc::new(Self {
             token,
             reason,
-            started_at: Instant::now(),
+            started_at,
+            deadline,
             max_duration: config.max_duration,
             max_objects: config.max_objects,
             max_directories: config.max_directories,
             track_progress,
+            track_unbounded_counts,
             objects_scanned: AtomicU64::new(0),
             directories_started: AtomicU64::new(0),
             entries_visited: AtomicU64::new(0),
+            last_progress_millis: AtomicU64::new(0),
+            cycle_state_persisted: AtomicBool::new(false),
         })
     }
 
@@ -131,6 +156,14 @@ impl ScannerCycleBudget {
         self.max_duration
     }
 
+    pub(crate) fn deadline(&self) -> Option<Instant> {
+        self.deadline
+    }
+
+    pub(crate) fn cancel_for_runtime(&self) {
+        self.cancel_for(ScannerCycleBudgetReason::Runtime);
+    }
+
     pub(crate) fn max_objects(&self) -> Option<u64> {
         self.max_objects
     }
@@ -173,15 +206,43 @@ impl ScannerCycleBudget {
         self.entries_visited.load(Ordering::Relaxed)
     }
 
+    pub(crate) fn mark_cycle_state_persisted(&self) {
+        self.cycle_state_persisted.store(true, Ordering::Release);
+    }
+
+    pub(crate) fn cycle_state_persisted(&self) -> bool {
+        self.cycle_state_persisted.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn progress_age(&self) -> Duration {
+        let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
+        let last_progress = self.last_progress_millis.load(Ordering::Relaxed);
+        Duration::from_millis(elapsed_millis.saturating_sub(last_progress))
+    }
+
+    fn record_progress_sample(&self, event: u64) {
+        // Clock reads are sampled at batch/count boundaries; the scanner's
+        // per-object path does not add a second progress atomic.
+        if event == 0 || (event != 1 && !event.is_multiple_of(PROGRESS_CLOCK_SAMPLE_INTERVAL)) {
+            return;
+        }
+        let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
+        self.last_progress_millis.store(elapsed_millis, Ordering::Relaxed);
+    }
+
     pub(crate) fn record_entries_visited(&self, entries_visited: u64) {
         if self.track_progress {
-            saturating_fetch_add(&self.entries_visited, entries_visited);
+            let entries = saturating_fetch_add(&self.entries_visited, entries_visited);
+            self.record_progress_sample(entries);
         }
     }
 
     pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) {
         if self.track_progress || self.max_objects.is_some() {
             let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned);
+            if self.track_progress {
+                self.record_progress_sample(objects);
+            }
             if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
                 self.cancel_for(ScannerCycleBudgetReason::Objects);
             }
@@ -189,6 +250,9 @@ impl ScannerCycleBudget {
 
         if self.track_progress || self.max_directories.is_some() {
             let directories = saturating_fetch_add(&self.directories_started, directories_started);
+            if self.track_progress {
+                self.record_progress_sample(directories);
+            }
             if self
                 .max_directories
                 .is_some_and(|max_directories| directories > max_directories)
@@ -207,11 +271,14 @@ impl ScannerCycleBudget {
     }
 
     pub(crate) fn try_start_directory(&self) -> bool {
-        if !self.track_progress && self.max_directories.is_none() {
+        if self.max_directories.is_none() && !self.track_unbounded_counts {
             return true;
         }
 
         let directories = saturating_fetch_add(&self.directories_started, 1);
+        if self.track_progress {
+            self.record_progress_sample(directories);
+        }
         if self
             .max_directories
             .is_some_and(|max_directories| directories > max_directories)
@@ -224,11 +291,14 @@ impl ScannerCycleBudget {
     }
 
     pub(crate) fn record_object_scanned(&self) {
-        if !self.track_progress && self.max_objects.is_none() {
+        if self.max_objects.is_none() && !self.track_unbounded_counts {
             return;
         }
 
         let objects = saturating_fetch_add(&self.objects_scanned, 1);
+        if self.track_progress {
+            self.record_progress_sample(objects);
+        }
         if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
             self.cancel_for(ScannerCycleBudgetReason::Objects);
         }
@@ -461,4 +531,29 @@ mod tests {
         assert!(object_limited.requires_serial_progress_accounting());
         assert!(directory_limited.requires_serial_progress_accounting());
     }
+
+    #[tokio::test(start_paused = true)]
+    async fn progress_age_uses_virtual_time_and_sampled_progress() {
+        let parent = CancellationToken::new();
+        let budget = ScannerCycleBudget::new_with_runtime_progress_tracking(
+            &parent,
+            ScannerCycleBudgetConfig {
+                max_duration: Some(Duration::from_secs(60)),
+                ..Default::default()
+            },
+        );
+
+        tokio::time::advance(Duration::from_secs(5)).await;
+        assert_eq!(budget.progress_age(), Duration::from_secs(5));
+        budget.record_entries_visited(1);
+        assert_eq!(budget.progress_age(), Duration::ZERO);
+
+        tokio::time::advance(Duration::from_secs(2)).await;
+        for _ in 0..126 {
+            budget.record_entries_visited(1);
+        }
+        assert_eq!(budget.progress_age(), Duration::from_secs(2));
+        budget.record_entries_visited(1);
+        assert_eq!(budget.progress_age(), Duration::ZERO);
+    }
 }

crates/storage-api/src/lib.rs

@@ -76,7 +76,6 @@ pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOption
 pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
 pub use error::{StorageErrorCode, StorageResult};
 pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
-pub use object::DeleteAccounting;
 pub use object::ObjectLockDeleteOptions;
 pub use object::{DeletedObject, ObjectToDelete};
 pub use object::{ExpirationOptions, TransitionedObject};

crates/storage-api/src/object.rs

@@ -218,17 +218,6 @@ pub struct DeletedObject {
     pub force_delete_generation: Option<i64>,
 }
 
-/// Accounting identity returned by the internal commit-time delete path.
-///
-/// This is carried separately from [`DeletedObject`] so adding quota details
-/// does not change the source shape of the public S3 delete result contract.
-#[derive(Debug, Default, Clone, PartialEq, Eq)]
-pub struct DeleteAccounting {
-    pub size: Option<u64>,
-    pub version_id: Option<Uuid>,
-    pub removed_current_object: bool,
-}
-
 impl DeletedObject {
     pub fn version_purge_status(&self) -> VersionPurgeStatusType {
         self.replication_state
@@ -352,19 +341,6 @@ pub trait ObjectOperations: Send + Sync + fmt::Debug {
         objects: Vec<Self::ObjectToDelete>,
         opts: Self::ObjectOptions,
     ) -> (Vec<Self::DeletedObject>, Vec<Option<Self::Error>>);
-    /// Delete objects and optionally return commit-time accounting identities.
-    /// The default preserves the ordinary delete contract for implementations
-    /// that do not expose storage-level accounting details.
-    async fn delete_objects_with_accounting(
-        &self,
-        bucket: &str,
-        objects: Vec<Self::ObjectToDelete>,
-        opts: Self::ObjectOptions,
-    ) -> (Vec<Self::DeletedObject>, Vec<Option<Self::Error>>, Vec<Option<DeleteAccounting>>) {
-        let object_count = objects.len();
-        let (deleted, errors) = self.delete_objects(bucket, objects, opts).await;
-        (deleted, errors, vec![None; object_count])
-    }
     async fn put_object_metadata(
         &self,
         bucket: &str,

crates/utils/src/envs.rs

@@ -268,7 +268,7 @@ where
         .parse::<T>()
         .map_err(|_| {
             log_once(&format!("env_invalid_value:{used_key}"), || {
-                format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::<T>())
+                format!("Invalid {} value for {used_key}. Treating as unset.", type_name::<T>())
             });
         })
         .ok()
@@ -570,7 +570,7 @@ where
         Ok(parsed) => EnvParseOutcome::Parsed(parsed),
         Err(_) => {
             log_once(&format!("env_invalid_value:{used_key}"), || {
-                format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::<T>())
+                format!("Invalid {} value for {used_key}. Treating as unset.", type_name::<T>())
             });
             EnvParseOutcome::Invalid
         }

docs/operations/scanner-runtime-controls.md

@@ -52,7 +52,7 @@ The `/v3/scanner/status` response reports each effective runtime value with a
 | `scanner.max_wait` | `RUSTFS_SCANNER_MAX_WAIT_SECS` | seconds | preset-derived | Caps one scanner sleep. |
 | `scanner.cycle` | `RUSTFS_SCANNER_CYCLE` | seconds | preset-derived | Sets the interval between scanner cycles. |
 | `scanner.start_delay` | `RUSTFS_SCANNER_START_DELAY_SECS` | seconds | unset | Sets startup delay and, for compatibility, the cycle interval when `scanner.cycle` is unset. |
-| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `0` | Caps one cycle's runtime. `0` disables this budget. |
+| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `1800` | Caps one cycle's runtime. An explicit `0` disables this budget. |
 | `scanner.cycle_max_objects` | `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` | objects | `0` | Caps objects processed by one cycle. `0` disables this budget. |
 | `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` | Caps directories entered by one cycle. `0` disables this budget. |
 | `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. |
@@ -70,6 +70,21 @@ sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`,
 `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis
 needs a precise override.
 
+When the cycle duration control is unset, RustFS uses a finite 1800-second
+(30-minute) default, matching the scanner benchmark guidance. An explicit `0`
+preserves the compatibility behavior of an unbounded cycle; object and
+directory budgets likewise remain unbounded when explicitly set to `0`. Invalid
+or overflowing duration environment values are configuration errors rather than
+silent fallback values.
+
+When a finite deadline expires, RustFS cancels cooperative scanner work and
+waits only for the existing bounded shutdown window. A non-yielding I/O future
+is dropped after that window. RustFS then attempts a higher leadership epoch so
+late cycle, usage, cache, and remote writes from the old generation fail closed.
+If the worker cannot stop cooperatively, the cycle state was not confirmed
+durable, or that epoch fence cannot be durably persisted, the scanner reports
+`recovery-required`; it does not claim an uncooperative cursor was saved.
+
 An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle
 cadence: dirty-usage notifications do not bypass that configured interval.
 The default adaptive policy continues to use dirty-usage notifications to wake
@@ -144,6 +159,10 @@ metrics.maintenance_control.primary_control
 metrics.source_work
 metrics.replication_repair
 metrics.scan_checkpoint
+metrics.cycle_timeout_total
+metrics.cycle_last_progress_age
+metrics.leader_lease_without_progress
+metrics.cycle_recovery_required_total
 ```
 
 ## Reading Pacing Pressure

protocol/agent/v1/fixtures/auth/MANIFEST.sha256

@@ -1,6 +1,5 @@
 3d602080f7ca4c32ba9e37ad1a32665c78560726b30aeee08fd9e95eb2f36194  accept-vectors.json
 d3c19946288717088145592e0e8d6f2fa684443ba2f73d4c7bc49c415d6dd051  certificate-profile.json
-299a2ae34a8ca74bcf31deeb53a08f9eff279efa09d3f5358a0cf10866fe1a5d  error-codes.json
+060485263c51003274c056a0e04bec1b7d76157cf599ba79eebe040bc7cee71b  error-codes.json
 43fe297ffb512b1b9f4af62f1832f3aa3905157893bfdc3dcc6d56f5a98aaef6  reject-vectors.json
-0cf26a7332fa6e3f57390e081f2cceead3236f6ca57f7038e3b55c2582af1733  rotation-proof.json
-7c07100460fa23fca482466df9ca22f91c7f26b36087a7207a001f7d987f527e  surface-separation.json
+b946175b094f4a8d75091b652fbe3d4327c9c795f28e02c96e1ab90a429e418d  surface-separation.json

protocol/agent/v1/fixtures/auth/error-codes.json

@@ -2,7 +2,7 @@
   "protocolVersion": "v1",
   "fixtureSet": "auth",
   "fixture": "error-codes",
-  "description": "Frozen ErrorInfo reasons for agent authentication, negotiation, and credential-rotation authorization. Clients branch on status and reason, never on message.",
+  "description": "Frozen ErrorInfo reasons for agent authentication and negotiation. Clients branch on status and reason, never on message.",
   "domain": "rustfs.connect",
   "detailType": "type.googleapis.com/google.rpc.ErrorInfo",
   "disclosureRules": [
@@ -81,24 +81,6 @@
       "httpStatus": 401,
       "status": "UNAUTHENTICATED",
       "meaning": "A browser session cookie was presented to an authenticated agent operation. The agent surface never accepts it."
-    },
-    {
-      "reason": "ROTATION_CREDENTIAL_NOT_CURRENT",
-      "httpStatus": 409,
-      "status": "ABORTED",
-      "meaning": "The authenticated certificate is valid for ordinary agent operations but is not the device's current ACTIVE credential and therefore cannot authorize another rotation."
-    },
-    {
-      "reason": "ROTATION_REQUEST_CONFLICT",
-      "httpStatus": 409,
-      "status": "ABORTED",
-      "meaning": "The requestId already belongs to a rotation with different transcript inputs and cannot be reused."
-    },
-    {
-      "reason": "ROTATION_PROOF_INVALID",
-      "httpStatus": 401,
-      "status": "UNAUTHENTICATED",
-      "meaning": "The credential proof does not verify over the canonical rotation transcript under the presented certificate key."
     }
   ]
 }

protocol/agent/v1/fixtures/auth/rotation-proof.json

@@ -1,306 +0,0 @@
-{
-  "protocolVersion": "v1",
-  "fixtureSet": "auth",
-  "fixture": "rotation-proof",
-  "description": "Frozen proof-of-possession transcript for rotating an online device credential. The current certificate key authorizes one new certificate request for one device and one idempotent request.",
-  "operation": {
-    "method": "POST",
-    "path": "/agent/clusterDevices/{device}:rotateCredential",
-    "operationId": "rotateClusterDeviceCredential",
-    "authenticatedSurface": "/agent/*",
-    "currentCredentialRequiredForNewRotation": true,
-    "outgoingOverlapCredentialMayAuthenticateOrdinaryOperations": true,
-    "outgoingOverlapCredentialMayRotate": false,
-    "outgoingOverlapCredentialMayReplayItsCompletedRotation": true
-  },
-  "replayPolicy": {
-    "recordState": "COMPLETED",
-    "bindingFields": [
-      "currentCertificateFingerprint",
-      "clusterDeviceName",
-      "requestId",
-      "certificateRequestSha256"
-    ],
-    "exactMatch": "returnStoredResult",
-    "mismatchReason": "ROTATION_REQUEST_CONFLICT",
-    "retentionLowerBound": "outgoingCredential.validUntil",
-    "sideEffects": "No issuer call, credential write, or overlap extension."
-  },
-  "completedReplayRecord": {
-    "state": "COMPLETED",
-    "currentCertificateFingerprint": "1bc816e2d285c52ee3a0aa06dc7aec17e788626043e795259a67f2436f970d09",
-    "clusterDeviceName": "organizations/0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70/clusters/0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81/clusterDevices/0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92",
-    "requestId": "7c5d6e2a-8b91-4f03-a4d5-6e7f8091a2b3",
-    "certificateRequestSha256": "kVS-bXYxD6F22cZNy4Vnpgb5gFZbr4EGP5GvIz7uOaw",
-    "resultReference": "credential-rotation-result-01"
-  },
-  "transcript": {
-    "domain": "RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1",
-    "domainTerminator": "0x0a",
-    "fieldSeparator": "0x3a",
-    "fieldTerminator": "0x0a",
-    "fieldCount": 4,
-    "encoding": "US-ASCII",
-    "normalisationPermitted": false,
-    "fieldOrder": [
-      "currentCertificateFingerprint",
-      "clusterDeviceName",
-      "requestId",
-      "certificateRequestSha256"
-    ],
-    "fields": [
-      {
-        "name": "currentCertificateFingerprint",
-        "position": 1,
-        "source": "the exact certificate accepted by trusted ingress and resolved by Connect",
-        "pattern": "^[0-9a-f]{64}$",
-        "binds": "the one currently active credential authorizing the rotation",
-        "absenceWouldAllow": "A proof captured from one current certificate to authorize another credential after the device rotated."
-      },
-      {
-        "name": "clusterDeviceName",
-        "position": 2,
-        "source": "the authenticated identity after it is matched to the target resource",
-        "pattern": "^organizations/[0-9a-f-]{36}/clusters/[0-9a-f-]{36}/clusterDevices/[0-9a-f-]{36}$",
-        "binds": "the organization, cluster, and device being rotated",
-        "absenceWouldAllow": "A proof produced by one device to be presented against another device resource."
-      },
-      {
-        "name": "requestId",
-        "position": 3,
-        "source": "the request body",
-        "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$",
-        "binds": "the single idempotent rotation attempt",
-        "absenceWouldAllow": "A captured proof to be replayed as a fresh attempt instead of the same idempotent request."
-      },
-      {
-        "name": "certificateRequestSha256",
-        "position": 4,
-        "source": "recomputed over the exact PKCS#10 DER octets in certificateRequest",
-        "pattern": "^[A-Za-z0-9_-]{43}$",
-        "binds": "the next device public key and certificate request",
-        "absenceWouldAllow": "A valid proof to be combined with an attacker certificate request."
-      }
-    ]
-  },
-  "signature": {
-    "algorithmField": "proof.algorithm",
-    "algorithmEnumeration": ["ES256"],
-    "curve": "P-256",
-    "signatureEncoding": "fixed-width-r-s",
-    "signatureLengthBytes": 64,
-    "signatureTransferEncoding": "base64url-unpadded",
-    "signatureValuePattern": "^[A-Za-z0-9_-]{86}$",
-    "lowSRequired": true,
-    "verifyingKeySource": "the SubjectPublicKeyInfo of the exact presented certificate accepted by trusted ingress",
-    "newKeyPossession": "the PKCS#10 self-signature is verified separately under the public key inside certificateRequest",
-    "sharedEncodingContract": "protocol/agent/v1/registration-proof.md#the-signature"
-  },
-  "verificationOrder": [
-    {
-      "stage": "protocolVersion",
-      "rule": "protocolVersion must be the supported major version.",
-      "reason": "UNSUPPORTED_PROTOCOL"
-    },
-    {
-      "stage": "encoding",
-      "rule": "proof.algorithm and proof.value obey the shared ES256 fixed-width low-S contract.",
-      "reasons": ["UNSUPPORTED_ALGORITHM", "SIGNATURE_MALFORMED", "SIGNATURE_NOT_CANONICAL"]
-    },
-    {
-      "stage": "certificateRequest",
-      "rule": "certificateRequest is one self-signed PKCS#10 request whose public key is P-256.",
-      "reasons": ["CERTIFICATE_REQUEST_MALFORMED", "DEVICE_KEY_UNSUPPORTED"]
-    },
-    {
-      "stage": "proof",
-      "rule": "Connect rebuilds the transcript and verifies proof.value under the presented certificate public key.",
-      "reason": "ROTATION_PROOF_INVALID"
-    },
-    {
-      "stage": "replay",
-      "rule": "A completed record bound to the presented credential, device, requestId, and CSR digest returns its stored result without issuing again. A requestId bound to different transcript inputs is refused.",
-      "reason": "ROTATION_REQUEST_CONFLICT"
-    },
-    {
-      "stage": "credential",
-      "rule": "After replay lookup misses, the authenticated certificate must be the device current ACTIVE credential, not a credential in the outgoing overlap.",
-      "reason": "ROTATION_CREDENTIAL_NOT_CURRENT"
-    }
-  ],
-  "reasonSources": {
-    "auth": ["UNSUPPORTED_PROTOCOL", "ROTATION_CREDENTIAL_NOT_CURRENT", "ROTATION_REQUEST_CONFLICT", "ROTATION_PROOF_INVALID"],
-    "sharedFromRegistration": [
-      "UNSUPPORTED_ALGORITHM",
-      "SIGNATURE_MALFORMED",
-      "SIGNATURE_NOT_CANONICAL",
-      "CERTIFICATE_REQUEST_MALFORMED",
-      "DEVICE_KEY_UNSUPPORTED"
-    ]
-  },
-  "example": {
-    "inputs": {
-      "currentCertificateFingerprint": "1bc816e2d285c52ee3a0aa06dc7aec17e788626043e795259a67f2436f970d09",
-      "clusterDeviceName": "organizations/0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70/clusters/0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81/clusterDevices/0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92",
-      "requestId": "7c5d6e2a-8b91-4f03-a4d5-6e7f8091a2b3",
-      "certificateRequestSha256": "kVS-bXYxD6F22cZNy4Vnpgb5gFZbr4EGP5GvIz7uOaw"
-    },
-    "artifacts": {
-      "currentPublicKeySpki": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAENEpjuFZTqp0Hxh/OWV3TGkluNjCo15dk+4CozuR6aT9Vaxhkb2M9nhaVGfk8+aSiSIiKFSCsYonKl8jh743Qow==",
-      "currentPublicKeyFingerprint": "28608e223c75ed89e72041f12afae4fc1cd3f1d3bcca38cbb62d1a41687610c2",
-      "certificateRequest": "MIIBHjCBxAIBADBiMRswGQYDVQQDDBJpZ25vcmVkLWJ5LWNvbm5lY3QxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQWPosvIKr5V3CpTUdLfMQxDx31B3SglLKyqRg/oH3J+PhUnqf7pDWW1sTkP2aRIDjAwRn0DpLrz405CcvGHvYEoAAwCgYIKoZIzj0EAwIDSQAwRgIhAMJzXo/CK4E9BfjOxP35he9LLlqENhK7HTzZQTuIgLX2AiEAwOZHibk5HEijTWcJ/UT117nssfJesWZVOWwz/KTIpi8=",
-      "otherCertificateRequest": "MIIBHTCBxAIBADBiMRswGQYDVQQDDBJpZ25vcmVkLWJ5LWNvbm5lY3QxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATi0KiZGcbPJixDiQN/aIVky7xqaYhjmwAn1aozrC2eanaDENYQ3D5zw5qoZNIp8YUk/CETZrba3C5KYR5ocdlLoAAwCgYIKoZIzj0EAwIDSAAwRQIhAL0ZtvWyTSu18RF5J4ZVIuOGjJpJwSdP+87CVxNKJJduAiAahIruc3FLtO3RI7B8Ome8IsVDUpSAQilThjdOeDMsYg=="
-    },
-    "canonicalTranscript": "RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1\n64:1bc816e2d285c52ee3a0aa06dc7aec17e788626043e795259a67f2436f970d09\n148:organizations/0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70/clusters/0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81/clusterDevices/0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92\n36:7c5d6e2a-8b91-4f03-a4d5-6e7f8091a2b3\n43:kVS-bXYxD6F22cZNy4Vnpgb5gFZbr4EGP5GvIz7uOaw\n",
-    "canonicalTranscriptLengthBytes": 346,
-    "canonicalTranscriptSha256": "e5f0e9cd0d5d420bc7e51217512de76e1cf4dceb1d67804b6518c4fb5d9fe434"
-  },
-  "acceptVectors": [
-    {
-      "name": "current credential authorizes one new certificate request",
-      "requestId": "7c5d6e2a-8b91-4f03-a4d5-6e7f8091a2b3",
-      "proof": "vgz7Hm-xZ5JQ7B3oqUEKXB4tQxEzWZ988iPxAklaIZIsd4nSTUv5YriqES5yKGHvKOgWPyWjvrgI0j0T-fYEsQ",
-      "expected": {"accepted": true, "reason": null, "stage": "credential", "replayed": false}
-    },
-    {
-      "name": "the same certificate request under a new requestId needs its own proof",
-      "requestId": "82e1f3a4-9b05-4c67-8d90-1e2f3a4b5c6d",
-      "proof": "s-rkF9w41sqgdQNkOlhjcLYlrNHN_HL5xh4C3k_1a9ZQu7Uq0LoET_x7i4Rw3xQ75ec6B6tfqZ3XsHTS0FfN7Q",
-      "expected": {"accepted": true, "reason": null, "stage": "credential", "replayed": false}
-    },
-    {
-      "name": "outgoing credential replays its completed byte-equivalent rotation",
-      "requestId": "7c5d6e2a-8b91-4f03-a4d5-6e7f8091a2b3",
-      "proof": "vgz7Hm-xZ5JQ7B3oqUEKXB4tQxEzWZ988iPxAklaIZIsd4nSTUv5YriqES5yKGHvKOgWPyWjvrgI0j0T-fYEsQ",
-      "replayRecord": "completedExample",
-      "mutation": {"currentCredential": false},
-      "expected": {"accepted": true, "reason": null, "stage": "replay", "replayedResultReference": "credential-rotation-result-01"}
-    }
-  ],
-  "rejectVectors": [
-    {
-      "name": "outgoing overlap credential attempts another rotation",
-      "stage": "credential",
-      "requestId": "82e1f3a4-9b05-4c67-8d90-1e2f3a4b5c6d",
-      "proof": "s-rkF9w41sqgdQNkOlhjcLYlrNHN_HL5xh4C3k_1a9ZQu7Uq0LoET_x7i4Rw3xQ75ec6B6tfqZ3XsHTS0FfN7Q",
-      "mutation": {"currentCredential": false},
-      "expected": {"accepted": false, "reason": "ROTATION_CREDENTIAL_NOT_CURRENT"}
-    },
-    {
-      "name": "rotation declares protocol v2",
-      "stage": "protocolVersion",
-      "mutation": {"protocolVersion": "v2"},
-      "expected": {"accepted": false, "reason": "UNSUPPORTED_PROTOCOL"}
-    },
-    {
-      "name": "proof declares ES384",
-      "stage": "encoding",
-      "mutation": {"proofAlgorithm": "ES384"},
-      "expected": {"accepted": false, "reason": "UNSUPPORTED_ALGORITHM"}
-    },
-    {
-      "name": "DER encoded proof",
-      "stage": "encoding",
-      "mutation": {"proofValue": "MEUCIQC-DPseb7FnklDsHeipQQpcHi1DETNZn3zyI_ECSVohkgIgLHeJ0k1L-WK4qhEucihh7yjoFj8lo764CNI9E_n2BLE"},
-      "expected": {"accepted": false, "reason": "SIGNATURE_MALFORMED"}
-    },
-    {
-      "name": "padded base64url proof",
-      "stage": "encoding",
-      "mutation": {"proofValue": "vgz7Hm-xZ5JQ7B3oqUEKXB4tQxEzWZ988iPxAklaIZIsd4nSTUv5YriqES5yKGHvKOgWPyWjvrgI0j0T-fYEsQ=="},
-      "expected": {"accepted": false, "reason": "SIGNATURE_MALFORMED"}
-    },
-    {
-      "name": "protocol negotiation precedes proof encoding and stale credential state",
-      "stage": "protocolVersion",
-      "mutation": {"currentCredential": false, "protocolVersion": "v2", "proofValue": "not-a-signature"},
-      "expected": {"accepted": false, "reason": "UNSUPPORTED_PROTOCOL"}
-    },
-    {
-      "name": "declared algorithm precedes signature bytes",
-      "stage": "encoding",
-      "mutation": {"proofAlgorithm": "ES384", "proofValue": "not-a-signature"},
-      "expected": {"accepted": false, "reason": "UNSUPPORTED_ALGORITHM"}
-    },
-    {
-      "name": "malleated high-S proof",
-      "stage": "encoding",
-      "mutation": {"proofValue": "vgz7Hm-xZ5JQ7B3oqUEKXB4tQxEzWZ988iPxAklaIZLTiHYssrQGnkdV7tGN154Qk_7kboFz38zq542vAm0goA"},
-      "expected": {"accepted": false, "reason": "SIGNATURE_NOT_CANONICAL"}
-    },
-    {
-      "name": "proof over the rotation fields uses the registration domain",
-      "stage": "proof",
-      "mutation": {"proofValue": "as27GKXEKXtym-BU8NUl0BYhJkEUooZWPadrxwOWa40pAaN6p4VxKCCFggZl4ZQsl5CUtaMoMaxM_HzT1c1qUA"},
-      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
-    },
-    {
-      "name": "proof verification precedes stale credential state",
-      "stage": "proof",
-      "mutation": {"currentCredential": false, "proofValue": "as27GKXEKXtym-BU8NUl0BYhJkEUooZWPadrxwOWa40pAaN6p4VxKCCFggZl4ZQsl5CUtaMoMaxM_HzT1c1qUA"},
-      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
-    },
-    {
-      "name": "new certificate request key signs instead of the current credential key",
-      "stage": "proof",
-      "mutation": {"proofValue": "QEN8wMQQk5HjriDw2aeAxIfsRvrWgIjGR3KlpK81Y34kaTAacO-6_bOMAjQ0hMMIk-YYN6MDCw0Via_Ri0X0BA"},
-      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
-    },
-    {
-      "name": "proof is moved to another current certificate",
-      "stage": "proof",
-      "mutation": {"currentCertificateFingerprint": "2bc816e2d285c52ee3a0aa06dc7aec17e788626043e795259a67f2436f970d09"},
-      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
-    },
-    {
-      "name": "proof is moved to another device",
-      "stage": "proof",
-      "mutation": {"clusterDeviceName": "organizations/0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70/clusters/0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81/clusterDevices/0198f4b0-3c00-7e30-8f41-4a5b6c7d8e93"},
-      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
-    },
-    {
-      "name": "proof is replayed under another requestId",
-      "stage": "proof",
-      "mutation": {"requestId": "82e1f3a4-9b05-4c67-8d90-1e2f3a4b5c6d"},
-      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
-    },
-    {
-      "name": "proof is combined with another certificate request",
-      "stage": "proof",
-      "mutation": {"certificateRequest": "otherCertificateRequest"},
-      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
-    },
-    {
-      "name": "completed replay belongs to another certificate fingerprint",
-      "stage": "replay",
-      "replayRecord": "completedExample",
-      "replayRecordMutation": {"currentCertificateFingerprint": "2bc816e2d285c52ee3a0aa06dc7aec17e788626043e795259a67f2436f970d09"},
-      "mutation": {"currentCredential": false},
-      "expected": {"accepted": false, "reason": "ROTATION_REQUEST_CONFLICT"}
-    },
-    {
-      "name": "completed replay belongs to another device",
-      "stage": "replay",
-      "replayRecord": "completedExample",
-      "replayRecordMutation": {"clusterDeviceName": "organizations/0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70/clusters/0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81/clusterDevices/0198f4b0-3c00-7e30-8f41-4a5b6c7d8e93"},
-      "mutation": {"currentCredential": false},
-      "expected": {"accepted": false, "reason": "ROTATION_REQUEST_CONFLICT"}
-    },
-    {
-      "name": "completed replay belongs to another requestId",
-      "stage": "replay",
-      "replayRecord": "completedExample",
-      "replayRecordMutation": {"requestId": "82e1f3a4-9b05-4c67-8d90-1e2f3a4b5c6d"},
-      "mutation": {"currentCredential": false},
-      "expected": {"accepted": false, "reason": "ROTATION_REQUEST_CONFLICT"}
-    },
-    {
-      "name": "completed replay belongs to another CSR digest",
-      "stage": "replay",
-      "replayRecord": "completedExample",
-      "replayRecordMutation": {"certificateRequestSha256": "PXxfNGWtIbrqVocag6lOLDDR41AjpHXJ7UcAWkbTSAs"},
-      "mutation": {"currentCredential": false},
-      "expected": {"accepted": false, "reason": "ROTATION_REQUEST_CONFLICT"}
-    }
-  ]
-}

protocol/agent/v1/fixtures/auth/surface-separation.json

@@ -49,7 +49,7 @@
       "schemeType": "mutualTLS",
       "forbiddenSecuritySchemes": ["sessionCookie"],
       "defaultSecurity": ["agentMutualTls"],
-      "publicOperations": ["getProtocolStatus", "exchangeRegistrationToken"]
+      "publicOperations": ["getProtocolStatus"]
     },
     {
       "document": "openapi/control.json",

protocol/agent/v1/fixtures/fixture-sets.json

@@ -11,7 +11,7 @@
     {
       "name": "auth",
       "status": "populated",
-      "purpose": "Client certificate profile, RFC 9440 header profile, authentication and credential-rotation proof vectors, surface separation, and the frozen error reason registry."
+      "purpose": "Client certificate profile, RFC 9440 header profile, authentication accept and reject vectors, surface separation, and the frozen error reason registry."
     },
     {
       "name": "version",

rustfs/Cargo.toml

@@ -288,7 +288,6 @@ zstd.workspace = true
 # Cryptography and Security
 rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types = { workspace = true }
-x509-parser = { workspace = true }
 subtle = { workspace = true }
 jiff = { workspace = true, features = ["serde"] }
 time = { workspace = true, features = ["parsing", "formatting", "serde", "macros"] }

rustfs/src/app/bucket_usecase.rs

@@ -1021,7 +1021,7 @@ fn build_list_objects_v2_metadata_output(
                 object: Object {
                     key: Some(encode_list_objects_v2_value(&object.name, encoding_type)),
                     last_modified: object.mod_time.map(Timestamp::from),
-                    size: Some(object.get_actual_size_or_physical()),
+                    size: Some(object.get_actual_size().unwrap_or_default()),
                     e_tag: object.etag.clone().map(|etag| to_s3s_etag(&etag)),
                     storage_class: Some(ObjectStorageClass::from(
                         object

rustfs/src/app/object_usecase.rs

@@ -3969,55 +3969,6 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool {
     opts.version_id.is_none() && opts.versioned && !opts.version_suspended
 }
 
-fn delete_removes_current_object(opts: &ObjectOptions) -> bool {
-    delete_request_targets_current(
-        opts.version_id
-            .as_deref()
-            .and_then(|version_id| Uuid::parse_str(version_id).ok()),
-    )
-}
-
-fn delete_request_targets_current(version_id: Option<Uuid>) -> bool {
-    version_id.is_none() || version_id.is_some_and(|version_id| version_id.is_nil())
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum DeleteMemoryUpdate {
-    DeleteMarker,
-    Object { size: u64, removed_current_object: bool },
-}
-
-fn delete_memory_update(
-    creates_delete_marker: bool,
-    committed_delete_marker: bool,
-    requested_current: bool,
-    accounting_size: Option<u64>,
-    removed_current_object: bool,
-) -> Option<DeleteMemoryUpdate> {
-    if creates_delete_marker || (committed_delete_marker && requested_current) {
-        return Some(DeleteMemoryUpdate::DeleteMarker);
-    }
-
-    (!committed_delete_marker)
-        .then_some(accounting_size)
-        .flatten()
-        .map(|size| DeleteMemoryUpdate::Object {
-            size,
-            removed_current_object,
-        })
-}
-
-async fn apply_delete_memory_update(bucket: &str, update: Option<DeleteMemoryUpdate>) {
-    match update {
-        Some(DeleteMemoryUpdate::DeleteMarker) => record_bucket_delete_marker_memory(bucket).await,
-        Some(DeleteMemoryUpdate::Object {
-            size,
-            removed_current_object,
-        }) => record_bucket_object_delete_memory(bucket, size, removed_current_object).await,
-        None => {}
-    }
-}
-
 /// `DeleteObjects` is idempotent. A raw filesystem `NotFound` can cross the
 /// distributed delete path instead of its usual typed missing-object error.
 fn is_delete_objects_not_found(error: &EcstoreError) -> bool {
@@ -8458,6 +8409,8 @@ impl DefaultObjectUsecase {
             object: ObjectToDelete,
             versioned: bool,
             version_suspended: bool,
+            size: i64,
+            existing: Option<ObjectInfo>,
         }
 
         // Phase 2 (bounded concurrency, backlog#929 / HP-8): collect the
@@ -8475,23 +8428,32 @@ impl DefaultObjectUsecase {
                     skip_stat,
                 } = prepared;
                 let synthetic_version_id = object.version_id.is_none() && is_dir_object(&object.object_name);
-                if !skip_stat {
+                let (goi, source_missing) = if skip_stat {
+                    (ObjectInfo::default(), false)
+                } else {
                     match store_ref.get_object_info(bucket_ref, &object.object_name, &opts).await {
-                        Ok(_) => {}
-                        Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
+                        Ok(res) => (res, false),
+                        Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
+                            (ObjectInfo::default(), true)
+                        }
                         Err(err) => return Err(ApiError::from(err)),
                     }
-                }
+                };
+
+                let size = goi.size;
 
                 if synthetic_version_id {
                     object.version_id = Some(Uuid::nil());
                 }
 
+                let existing = (!skip_stat && !source_missing).then_some(goi);
                 Ok::<_, ApiError>(AdmittedDelete {
                     idx,
                     object,
                     versioned: opts.versioned,
                     version_suspended: opts.version_suspended,
+                    size,
+                    existing,
                 })
             }))
             .buffered(DELETE_OBJECTS_PRE_STAT_CONCURRENCY)
@@ -8502,11 +8464,15 @@ impl DefaultObjectUsecase {
         // per-key success/failure reporting is unchanged.
         let mut object_to_delete = Vec::new();
         let mut object_to_delete_idx = Vec::new();
+        let mut object_sizes = Vec::new();
+        let mut existing_object_infos = Vec::new();
         let mut object_versioning = Vec::new();
         for admitted in admitted_deletes {
+            object_sizes.push(admitted.size);
             object_to_delete_idx.push(admitted.idx);
             object_versioning.push((admitted.versioned, admitted.version_suspended));
             object_to_delete.push(admitted.object);
+            existing_object_infos.push(admitted.existing);
         }
         let cache_adapter = self.object_data_cache();
         let cache_keys_before_delete = object_to_delete
@@ -8523,8 +8489,8 @@ impl DefaultObjectUsecase {
             ..Default::default()
         };
         apply_bucket_generation_guard(&req, &bucket, &mut storage_delete_opts)?;
-        let (dobjs, errs, accounting) = store
-            .delete_objects_with_tier_delete_journal_and_accounting(&bucket, object_to_delete.clone(), storage_delete_opts)
+        let (dobjs, errs) = store
+            .delete_objects_with_tier_delete_journal(&bucket, object_to_delete.clone(), storage_delete_opts)
             .await;
 
         let _manager = get_concurrency_manager();
@@ -8549,16 +8515,17 @@ impl DefaultObjectUsecase {
                     delete_results[didx].delete_object = Some(deleted_object.clone());
                     let (versioned, version_suspended) = object_versioning[i];
                     let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended;
-                    let committed_delete_marker = dobjs[i].delete_marker;
-                    let delete_accounting = accounting.get(i).and_then(Option::as_ref);
-                    let update = delete_memory_update(
-                        creates_delete_marker,
-                        committed_delete_marker,
-                        delete_request_targets_current(object_to_delete[i].version_id),
-                        delete_accounting.and_then(|value| value.size),
-                        delete_accounting.is_some_and(|value| value.removed_current_object),
-                    );
-                    apply_delete_memory_update(&bucket, update).await;
+                    if creates_delete_marker {
+                        record_bucket_delete_marker_memory(&bucket).await;
+                    } else {
+                        let size = object_sizes[i].max(0) as u64;
+                        record_bucket_object_delete_memory(
+                            &bucket,
+                            size,
+                            existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
+                        )
+                        .await;
+                    }
                 }
                 Err(error) => {
                     delete_results[didx].error = Some(error);
@@ -8836,24 +8803,12 @@ impl DefaultObjectUsecase {
             let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await;
         }
 
-        // Fast in-memory update for immediate quota and admin usage consistency.
-        // Prefix/force deletes and synthetic directory entries do not carry one
-        // committed object identity; leave their cache delta to reconciliation.
-        let update = if force_delete || obj_info.name.is_empty() || synthetic_version_id {
-            None
+        // Fast in-memory update for immediate quota and admin usage consistency
+        if delete_creates_delete_marker(&opts) {
+            record_bucket_delete_marker_memory(&bucket).await;
         } else {
-            // The storage commit returns this object's metadata while its
-            // generation lock is held. Never fall back to a pre-delete stat:
-            // an overwrite can commit between that stat and this delete.
-            delete_memory_update(
-                delete_creates_delete_marker(&opts),
-                obj_info.delete_marker,
-                opts.version_id.is_none(),
-                quota_object_size(&obj_info).ok(),
-                delete_removes_current_object(&opts),
-            )
-        };
-        apply_delete_memory_update(&bucket, update).await;
+            record_bucket_object_delete_memory(&bucket, obj_info.size.max(0) as u64, opts.version_id.is_none()).await;
+        }
 
         if obj_info.name.is_empty() {
             if let Some((operation_id, target_arns, generation)) = force_delete_intent {
@@ -17906,158 +17861,6 @@ mod tests {
         assert!(!can_skip_delete_objects_pre_stat(false, &delete_marker_creating_opts(), false));
     }
 
-    #[test]
-    fn delete_accounting_recognizes_explicit_null_as_current_object() {
-        let opts = ObjectOptions {
-            version_id: Some(Uuid::nil().to_string()),
-            version_suspended: true,
-            ..Default::default()
-        };
-        assert!(delete_removes_current_object(&opts));
-        assert!(delete_request_targets_current(Some(Uuid::nil())));
-        assert!(!delete_request_targets_current(Some(Uuid::new_v4())));
-        assert!(!delete_removes_current_object(&ObjectOptions {
-            version_id: Some(Uuid::new_v4().to_string()),
-            ..Default::default()
-        }));
-    }
-
-    #[test]
-    fn compressed_object_delete_restores_usage_baseline() {
-        let mut metadata = HashMap::new();
-        insert_str(&mut metadata, SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string());
-        let object = ObjectInfo {
-            size: 400,
-            actual_size: 1000,
-            user_defined: Arc::new(metadata),
-            ..Default::default()
-        };
-        let accounting_size = quota_object_size(&object).expect("logical compressed size should be canonical");
-
-        assert_eq!(
-            delete_memory_update(false, false, true, Some(accounting_size), true),
-            Some(DeleteMemoryUpdate::Object {
-                size: 1000,
-                removed_current_object: true,
-            })
-        );
-    }
-
-    #[test]
-    fn invalid_accounting_metadata_is_reconciled_without_overflow() {
-        assert_eq!(delete_memory_update(false, false, true, None, true), None);
-        assert_eq!(
-            delete_memory_update(false, true, true, None, true),
-            Some(DeleteMemoryUpdate::DeleteMarker)
-        );
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn compressed_delete_requests_restore_usage_baseline() {
-        use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
-
-        let store = crate::app::gating_test_env::shared_gating_ecstore().await;
-        if current_app_context().is_none() {
-            crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
-        }
-        let bucket = format!("compressed-delete-request-{}", Uuid::new_v4().simple());
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create compressed delete request bucket");
-
-        // Seed the process-local usage with the canonical logical bytes. The
-        // direct storage PUT below intentionally does not apply an app-layer
-        // usage delta; the two real DELETE requests must remove exactly this
-        // amount through their request-layer wiring.
-        crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 2_000).await;
-
-        for object in ["single", "batch"] {
-            let mut metadata = HashMap::new();
-            insert_str(&mut metadata, SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string());
-            insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, "1000".to_string());
-            let reader = HashReader::from_stream(std::io::Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
-                .expect("compressed fixture reader should be valid");
-            let mut reader = PutObjReader::new(reader);
-            store
-                .put_object(
-                    &bucket,
-                    object,
-                    &mut reader,
-                    &ObjectOptions {
-                        user_defined: metadata,
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("compressed fixture object should be written");
-        }
-
-        let mut single_req = build_request(
-            DeleteObjectInput::builder()
-                .bucket(bucket.clone())
-                .key("single".to_string())
-                .build()
-                .expect("single delete input should build"),
-            Method::DELETE,
-        );
-        single_req.extensions.insert(crate::storage::access::ReqInfo {
-            cred: Some(rustfs_credentials::Credentials::default()),
-            is_owner: true,
-            ..Default::default()
-        });
-        DefaultObjectUsecase::from_global()
-            .execute_delete_object(single_req)
-            .await
-            .expect("single compressed delete should succeed");
-        assert_eq!(
-            crate::app::storage_api::test::data_usage::get_bucket_usage_memory(&bucket).await,
-            Some(1_000),
-            "single delete must subtract the logical accounting size"
-        );
-
-        let mut batch_req = build_request(
-            DeleteObjectsInput::builder()
-                .bucket(bucket.clone())
-                .delete(Delete {
-                    objects: vec![ObjectIdentifier {
-                        key: "batch".to_string(),
-                        ..Default::default()
-                    }],
-                    quiet: None,
-                })
-                .build()
-                .expect("batch delete input should build"),
-            Method::POST,
-        );
-        batch_req.extensions.insert(crate::storage::access::ReqInfo {
-            cred: Some(rustfs_credentials::Credentials::default()),
-            is_owner: true,
-            ..Default::default()
-        });
-        DefaultObjectUsecase::from_global()
-            .execute_delete_objects(batch_req)
-            .await
-            .expect("batch compressed delete should succeed");
-        assert_eq!(
-            crate::app::storage_api::test::data_usage::get_bucket_usage_memory(&bucket).await,
-            Some(0),
-            "batch delete must subtract the committed logical accounting size"
-        );
-
-        store
-            .delete_bucket(
-                &bucket,
-                &DeleteBucketOptions {
-                    force: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("clean up compressed delete request bucket");
-    }
-
     #[tokio::test]
     async fn execute_get_object_attributes_returns_internal_error_when_store_uninitialized() {
         let input = GetObjectAttributesInput::builder()

rustfs/src/app/storage_api.rs

@@ -72,11 +72,6 @@ pub(crate) mod data_usage {
         compute_bucket_usage, live_bucket_usage_computations, seed_bucket_usage_memory_for_test, store_data_usage_in_backend,
     };
 
-    #[cfg(test)]
-    pub(crate) async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
-        crate::storage::storage_api::ecstore_data_usage::get_bucket_usage_memory(bucket).await
-    }
-
     pub(crate) async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) {
         crate::storage::storage_api::ecstore_data_usage::record_bucket_object_delete_memory(
             bucket,
@@ -1238,10 +1233,7 @@ pub(crate) mod test {
 
     pub(crate) use super::access::ReqInfo;
     pub(crate) use super::options::VERSIONING_CONFIG_LOOKUPS;
-    pub(crate) use super::{bucket, ecfs, object_utils, runtime};
-    pub(crate) mod data_usage {
-        pub(crate) use super::super::data_usage::*;
-    }
+    pub(crate) use super::{bucket, data_usage, ecfs, object_utils, runtime};
     pub(crate) use crate::storage::storage_api::test_consumer::{get_global_bucket_metadata_sys, set_bucket_metadata};
     pub(crate) use crate::storage::storage_api::{
         ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader,

rustfs/src/connect/client.rs

@@ -1,566 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::time::Duration;
-
-use base64::Engine as _;
-use reqwest::{Client, StatusCode, Url};
-use rustls::RootCertStore;
-use rustls::pki_types::{CertificateDer, pem::PemObject as _};
-use serde::Deserialize;
-use uuid::Uuid;
-use zeroize::Zeroizing;
-
-use super::credential_store::{
-    CompletedRegistration, CredentialStore, CredentialStoreError, DeviceCredential, PendingRegistration, PendingRotation,
-};
-use super::identity::{IdentityError, RegistrationTranscript};
-use super::identity_store::{IdentityStore, StoreError};
-use super::registration::{
-    CredentialResponse, CredentialValidationError, ExpectedDevice, RegistrationRequest, RegistrationToken, RotationRequest,
-    certificate_fingerprint, certificate_request_matches, public_key_fingerprint, validate_credential,
-    validate_stored_credential,
-};
-
-const MAX_ATTEMPTS: usize = 3;
-const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
-const ROTATION_THRESHOLD_SECONDS: i64 = 8 * 60 * 60;
-
-pub struct ConnectConfig<'a> {
-    pub endpoint: &'a str,
-    pub root_ca_pem: &'a [u8],
-    pub timeout: Duration,
-}
-
-pub struct ConnectClient {
-    endpoint: Url,
-    roots: RootCertStore,
-    root_certificates: Vec<CertificateDer<'static>>,
-    client: Client,
-    timeout: Duration,
-}
-
-impl ConnectClient {
-    pub fn from_optional_config(config: Option<ConnectConfig<'_>>) -> Result<Option<Self>, ClientError> {
-        config.map(Self::new).transpose()
-    }
-
-    pub fn new(config: ConnectConfig<'_>) -> Result<Self, ClientError> {
-        let mut endpoint = Url::parse(config.endpoint).map_err(|_| ClientError::Endpoint)?;
-        if endpoint.scheme() != "https"
-            || endpoint.cannot_be_a_base()
-            || !endpoint.username().is_empty()
-            || endpoint.password().is_some()
-            || endpoint.query().is_some()
-            || endpoint.fragment().is_some()
-        {
-            return Err(ClientError::Endpoint);
-        }
-        if !endpoint.path().ends_with('/') {
-            let path = format!("{}/", endpoint.path());
-            endpoint.set_path(&path);
-        }
-
-        let root_certificates = CertificateDer::pem_slice_iter(config.root_ca_pem)
-            .collect::<Result<Vec<_>, _>>()
-            .map_err(|_| ClientError::RootCertificate)?;
-        if root_certificates.is_empty() {
-            return Err(ClientError::RootCertificate);
-        }
-        let mut roots = RootCertStore::empty();
-        let (accepted, rejected) = roots.add_parsable_certificates(root_certificates.clone());
-        if accepted != root_certificates.len() || rejected != 0 {
-            return Err(ClientError::RootCertificate);
-        }
-
-        let client = build_client(&root_certificates, config.timeout, None)?;
-        Ok(Self {
-            endpoint,
-            roots,
-            root_certificates,
-            client,
-            timeout: config.timeout,
-        })
-    }
-
-    pub async fn register(
-        &self,
-        identity_store: &IdentityStore,
-        credential_store: &CredentialStore,
-        token: &RegistrationToken,
-    ) -> Result<DeviceCredential, ClientError> {
-        let _lock = credential_store.lock().await?;
-        if let Some((credential, _)) = self.load_valid_credential(identity_store, credential_store)? {
-            ensure_credential_time(&credential, unix_now())?;
-            return Ok(credential);
-        }
-
-        let identity = identity_store.load_or_create()?;
-        let candidate = PendingRegistration {
-            token_uid: token.registration_token_uid.clone(),
-            request_id: Uuid::new_v4().to_string(),
-            certificate_request: identity.certificate_request_base64()?,
-            previous_credential_fingerprint: None,
-            next_public_key_sha256: None,
-        };
-        let pending = credential_store.claim_pending_registration(&candidate)?;
-        if pending.token_uid != token.registration_token_uid
-            || pending.previous_credential_fingerprint.is_some()
-            || pending.next_public_key_sha256.is_some()
-            || !is_request_id(&pending.request_id)
-            || !certificate_request_matches(&pending.certificate_request, &identity)?
-        {
-            return Err(ClientError::PendingRegistration);
-        }
-
-        let credential = match self.exchange_registration(token, &pending, &identity).await {
-            Ok(credential) => credential,
-            Err(error @ (ClientError::AccessRevoked { .. } | ClientError::Rejected { .. })) => {
-                credential_store.clear_pending_registration()?;
-                return Err(error);
-            }
-            Err(error) => return Err(error),
-        };
-        credential_store.save(&credential)?;
-        credential_store.clear_pending_registration()?;
-        Ok(credential)
-    }
-
-    pub async fn reenroll(
-        &self,
-        identity_store: &IdentityStore,
-        credential_store: &CredentialStore,
-        token: &RegistrationToken,
-    ) -> Result<DeviceCredential, ClientError> {
-        let _lock = credential_store.lock().await?;
-        let (credential, _) = self
-            .load_valid_credential(identity_store, credential_store)?
-            .ok_or(ClientError::NotRegistered)?;
-        let fingerprint = certificate_fingerprint(&credential.certificate)?;
-        if credential_store.load_completed_registration()?.is_some_and(|completed| {
-            completed.token_uid == token.registration_token_uid && completed.credential_fingerprint == fingerprint
-        }) {
-            return Ok(credential);
-        }
-        credential_store.clear_pending_rotation()?;
-        let next = identity_store.load_or_create_next()?;
-        let next_fingerprint = public_key_fingerprint(&next);
-        let candidate = PendingRegistration {
-            token_uid: token.registration_token_uid.clone(),
-            request_id: Uuid::new_v4().to_string(),
-            certificate_request: next.certificate_request_base64()?,
-            previous_credential_fingerprint: Some(fingerprint.clone()),
-            next_public_key_sha256: Some(next_fingerprint.clone()),
-        };
-        let pending = credential_store.claim_pending_registration(&candidate)?;
-        if pending.token_uid != token.registration_token_uid
-            || pending.previous_credential_fingerprint.as_deref() != Some(&fingerprint)
-            || pending.next_public_key_sha256.as_deref() != Some(&next_fingerprint)
-            || !is_request_id(&pending.request_id)
-            || !certificate_request_matches(&pending.certificate_request, &next)?
-        {
-            return Err(ClientError::PendingRegistration);
-        }
-        let enrolled = match self.exchange_registration(token, &pending, &next).await {
-            Ok(credential) => credential,
-            Err(error @ (ClientError::AccessRevoked { .. } | ClientError::Rejected { .. })) => {
-                credential_store.clear_pending_registration()?;
-                identity_store.clear_next()?;
-                return Err(error);
-            }
-            Err(error) => return Err(error),
-        };
-        credential_store.save(&enrolled)?;
-        identity_store.commit_next(&next)?;
-        credential_store.save_completed_registration(&CompletedRegistration {
-            token_uid: token.registration_token_uid.clone(),
-            credential_fingerprint: certificate_fingerprint(&enrolled.certificate)?,
-        })?;
-        credential_store.clear_pending_registration()?;
-        Ok(enrolled)
-    }
-
-    async fn exchange_registration(
-        &self,
-        token: &RegistrationToken,
-        pending: &PendingRegistration,
-        identity: &super::identity::DeviceIdentity,
-    ) -> Result<DeviceCredential, ClientError> {
-        let csr_der = base64::engine::general_purpose::STANDARD
-            .decode(&pending.certificate_request)
-            .map_err(|_| ClientError::PendingRegistration)?;
-        let transcript = RegistrationTranscript::build(
-            &token.registration_token_uid,
-            &token.organization_uid,
-            &token.cluster_uid,
-            &pending.request_id,
-            &token.challenge_nonce,
-            token.expires_unix,
-            &csr_der,
-        )?;
-        let proof = identity.sign_registration(&transcript);
-        let body = RegistrationRequest::new(token, &pending.request_id, &pending.certificate_request, &proof);
-        let url = self.url("./registrationTokens:exchange")?;
-        let response = self
-            .send(StatusCode::CREATED, || self.client.post(url.clone()).json(&body))
-            .await?;
-        let cluster = format!("organizations/{}/clusters/{}", token.organization_uid, token.cluster_uid);
-        let credential = validate_credential(
-            response,
-            identity,
-            &self.roots,
-            &self.root_certificates,
-            ExpectedDevice::Registration { cluster: &cluster },
-        )?;
-        Ok(credential)
-    }
-
-    pub async fn rotate_if_due(
-        &self,
-        identity_store: &IdentityStore,
-        credential_store: &CredentialStore,
-        now_unix: i64,
-    ) -> Result<Option<DeviceCredential>, ClientError> {
-        let _lock = credential_store.lock().await?;
-        let (credential, identity) = self
-            .load_valid_credential(identity_store, credential_store)?
-            .ok_or(ClientError::NotRegistered)?;
-        if credential_store.load_pending_registration()?.is_some() {
-            return Err(ClientError::PendingRegistration);
-        }
-        ensure_credential_time(&credential, now_unix)?;
-        if credential.not_after_unix - now_unix > ROTATION_THRESHOLD_SECONDS {
-            return Ok(None);
-        }
-        let fingerprint = certificate_fingerprint(&credential.certificate)?;
-        let next = identity_store.load_or_create_next()?;
-        let candidate = PendingRotation {
-            credential_fingerprint: fingerprint.clone(),
-            device_name: credential.name.clone(),
-            request_id: Uuid::new_v4().to_string(),
-            certificate_request: next.certificate_request_base64()?,
-            next_public_key_sha256: public_key_fingerprint(&next),
-        };
-        let pending = credential_store.claim_pending_rotation(&candidate)?;
-        if pending.credential_fingerprint != fingerprint
-            || pending.device_name != credential.name
-            || !is_request_id(&pending.request_id)
-            || pending.next_public_key_sha256 != public_key_fingerprint(&next)
-            || !certificate_request_matches(&pending.certificate_request, &next)?
-        {
-            return Err(ClientError::PendingRotation);
-        }
-        let body = RotationRequest::new(
-            &identity,
-            &fingerprint,
-            &credential.name,
-            &pending.request_id,
-            &pending.certificate_request,
-        )?;
-        let private_key = identity.to_pkcs8_pem()?;
-        let mut identity_pem = Zeroizing::new(Vec::with_capacity(credential.certificate_chain.len() + private_key.len() + 1));
-        identity_pem.extend_from_slice(credential.certificate_chain.as_bytes());
-        identity_pem.push(b'\n');
-        identity_pem.extend_from_slice(private_key.as_bytes());
-        let tls_identity = reqwest::Identity::from_pem(&identity_pem).map_err(|_| ClientError::IdentityCertificate)?;
-        let client = build_client(&self.root_certificates, self.timeout, Some(tls_identity))?;
-        let path = format!("clusterDevices/{}:rotateCredential", credential.uid);
-        let url = self.url(&path)?;
-        let response = self.send(StatusCode::OK, || client.post(url.clone()).json(&body)).await?;
-        let rotated = validate_credential(
-            response,
-            &next,
-            &self.roots,
-            &self.root_certificates,
-            ExpectedDevice::Rotation { name: &credential.name },
-        )?;
-        if rotated.name != credential.name || rotated.uid != credential.uid {
-            return Err(ClientError::Credential(CredentialValidationError::Identity));
-        }
-        credential_store.save(&rotated)?;
-        identity_store.commit_next(&next)?;
-        credential_store.clear_pending_rotation()?;
-        Ok(Some(rotated))
-    }
-
-    fn load_valid_credential(
-        &self,
-        identity_store: &IdentityStore,
-        credential_store: &CredentialStore,
-    ) -> Result<Option<(DeviceCredential, super::identity::DeviceIdentity)>, ClientError> {
-        let Some(credential) = credential_store.load()? else {
-            return Ok(None);
-        };
-        let current = identity_store.load()?.ok_or(ClientError::IdentityMissing)?;
-        if let Some(pending) = credential_store.load_pending_registration()? {
-            let Some(previous) = pending.previous_credential_fingerprint.as_deref() else {
-                if pending.next_public_key_sha256.is_some()
-                    || !is_request_id(&pending.request_id)
-                    || !certificate_request_matches(&pending.certificate_request, &current)?
-                {
-                    return Err(ClientError::PendingRegistration);
-                }
-                validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
-                credential_store.clear_pending_registration()?;
-                return Ok(Some((credential, current)));
-            };
-            let next_fingerprint = pending
-                .next_public_key_sha256
-                .as_deref()
-                .ok_or(ClientError::PendingRegistration)?;
-            if !is_request_id(&pending.request_id) {
-                return Err(ClientError::PendingRegistration);
-            }
-            let fingerprint = certificate_fingerprint(&credential.certificate)?;
-            if fingerprint == previous {
-                validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
-                let next = identity_store.load_next()?.ok_or(ClientError::PendingRegistration)?;
-                if public_key_fingerprint(&next) != next_fingerprint
-                    || !certificate_request_matches(&pending.certificate_request, &next)?
-                {
-                    return Err(ClientError::PendingRegistration);
-                }
-                return Ok(Some((credential, current)));
-            }
-            if public_key_fingerprint(&current) == next_fingerprint {
-                if !certificate_request_matches(&pending.certificate_request, &current)? {
-                    return Err(ClientError::PendingRegistration);
-                }
-                validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
-            } else {
-                let next = identity_store.load_next()?.ok_or(ClientError::PendingRegistration)?;
-                if public_key_fingerprint(&next) != next_fingerprint
-                    || !certificate_request_matches(&pending.certificate_request, &next)?
-                {
-                    return Err(ClientError::PendingRegistration);
-                }
-                validate_stored_credential(&credential, &next, &self.roots, &self.root_certificates)?;
-                identity_store.commit_next(&next)?;
-            }
-            credential_store.save_completed_registration(&CompletedRegistration {
-                token_uid: pending.token_uid,
-                credential_fingerprint: fingerprint,
-            })?;
-            credential_store.clear_pending_registration()?;
-            let current = identity_store.load()?.ok_or(ClientError::IdentityMissing)?;
-            return Ok(Some((credential, current)));
-        }
-        let Some(pending) = credential_store.load_pending_rotation()? else {
-            validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
-            return Ok(Some((credential, current)));
-        };
-        let fingerprint = certificate_fingerprint(&credential.certificate)?;
-        if pending.device_name != credential.name || !is_request_id(&pending.request_id) {
-            return Err(ClientError::PendingRotation);
-        }
-        if fingerprint == pending.credential_fingerprint {
-            validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
-            let next = identity_store.load_next()?.ok_or(ClientError::PendingRotation)?;
-            if pending.next_public_key_sha256 != public_key_fingerprint(&next)
-                || !certificate_request_matches(&pending.certificate_request, &next)?
-            {
-                return Err(ClientError::PendingRotation);
-            }
-            return Ok(Some((credential, current)));
-        }
-
-        if public_key_fingerprint(&current) == pending.next_public_key_sha256 {
-            if !certificate_request_matches(&pending.certificate_request, &current)? {
-                return Err(ClientError::PendingRotation);
-            }
-            validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
-        } else {
-            let next = identity_store.load_next()?.ok_or(ClientError::PendingRotation)?;
-            if public_key_fingerprint(&next) != pending.next_public_key_sha256
-                || !certificate_request_matches(&pending.certificate_request, &next)?
-            {
-                return Err(ClientError::PendingRotation);
-            }
-            validate_stored_credential(&credential, &next, &self.roots, &self.root_certificates)?;
-            identity_store.commit_next(&next)?;
-        }
-        credential_store.clear_pending_rotation()?;
-        let current = identity_store.load()?.ok_or(ClientError::IdentityMissing)?;
-        Ok(Some((credential, current)))
-    }
-
-    async fn send<F>(&self, success: StatusCode, mut request: F) -> Result<CredentialResponse, ClientError>
-    where
-        F: FnMut() -> reqwest::RequestBuilder,
-    {
-        let mut last_status = None;
-        for attempt in 0..MAX_ATTEMPTS {
-            match request().send().await {
-                Ok(response) if response.status() == success => return decode_response(response).await,
-                Ok(response) if matches!(response.status(), StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) => {
-                    let status = response.status();
-                    let reason = decode_reason(response).await;
-                    return Err(ClientError::AccessRevoked { status, reason });
-                }
-                Ok(response) if matches!(response.status(), StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS) => {
-                    last_status = Some(response.status());
-                }
-                Ok(response) if response.status().is_client_error() => {
-                    let status = response.status();
-                    let reason = decode_reason(response).await;
-                    return Err(ClientError::Rejected { status, reason });
-                }
-                Ok(response) if response.status().is_server_error() => {
-                    last_status = Some(response.status());
-                }
-                Ok(response) => {
-                    let status = response.status();
-                    let reason = decode_reason(response).await;
-                    return Err(ClientError::Rejected { status, reason });
-                }
-                Err(error) if !error.is_timeout() && !error.is_connect() => return Err(ClientError::Transport(error)),
-                Err(_) => {}
-            }
-
-            if attempt + 1 < MAX_ATTEMPTS {
-                tokio::time::sleep(Duration::from_millis(50 * (attempt as u64 + 1))).await;
-            }
-        }
-        Err(ClientError::Unavailable { status: last_status })
-    }
-
-    fn url(&self, path: &str) -> Result<Url, ClientError> {
-        self.endpoint.join(path).map_err(|_| ClientError::Endpoint)
-    }
-}
-
-fn is_request_id(value: &str) -> bool {
-    Uuid::parse_str(value).is_ok_and(|uuid| uuid.get_version() == Some(uuid::Version::Random) && uuid.to_string() == value)
-}
-
-fn unix_now() -> i64 {
-    std::time::SystemTime::now()
-        .duration_since(std::time::UNIX_EPOCH)
-        .map_or(0, |duration| duration.as_secs() as i64)
-}
-
-fn ensure_credential_time(credential: &DeviceCredential, now_unix: i64) -> Result<(), ClientError> {
-    if credential.not_before_unix > now_unix {
-        return Err(ClientError::CredentialNotYetValid);
-    }
-    if credential.not_after_unix <= now_unix {
-        return Err(ClientError::CredentialExpired);
-    }
-    Ok(())
-}
-
-fn build_client(
-    roots: &[CertificateDer<'static>],
-    timeout: Duration,
-    identity: Option<reqwest::Identity>,
-) -> Result<Client, ClientError> {
-    let certificates = roots
-        .iter()
-        .map(|root| reqwest::Certificate::from_der(root.as_ref()))
-        .collect::<Result<Vec<_>, _>>()?;
-    let mut builder = Client::builder()
-        .https_only(true)
-        .redirect(reqwest::redirect::Policy::none())
-        .timeout(timeout)
-        .tls_certs_only(certificates);
-    if let Some(identity) = identity {
-        builder = builder.identity(identity);
-    }
-    builder.build().map_err(ClientError::Transport)
-}
-
-async fn decode_response(mut response: reqwest::Response) -> Result<CredentialResponse, ClientError> {
-    let body = read_body(&mut response).await?;
-    serde_json::from_slice(&body).map_err(|_| ClientError::Response)
-}
-
-async fn read_body(response: &mut reqwest::Response) -> Result<Vec<u8>, ClientError> {
-    let mut body = Vec::new();
-    while let Some(chunk) = response.chunk().await.map_err(ClientError::Transport)? {
-        if body.len() + chunk.len() > MAX_RESPONSE_BYTES {
-            return Err(ClientError::ResponseTooLarge);
-        }
-        body.extend_from_slice(&chunk);
-    }
-    Ok(body)
-}
-
-#[derive(Deserialize)]
-struct ErrorEnvelope {
-    #[serde(default)]
-    details: Vec<ErrorDetail>,
-}
-
-#[derive(Deserialize)]
-struct ErrorDetail {
-    #[serde(default)]
-    reason: String,
-}
-
-async fn decode_reason(mut response: reqwest::Response) -> Option<String> {
-    let body = read_body(&mut response).await.ok()?;
-    serde_json::from_slice::<ErrorEnvelope>(&body)
-        .ok()?
-        .details
-        .into_iter()
-        .find_map(|detail| (!detail.reason.is_empty()).then_some(detail.reason))
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum ClientError {
-    #[error("Connect endpoint must be an HTTPS base URL without credentials, query, or fragment")]
-    Endpoint,
-    #[error("Connect root CA configuration is invalid")]
-    RootCertificate,
-    #[error(
-        "Connect registration has a pending attempt for a different token; restore the original protected token configuration"
-    )]
-    PendingRegistration,
-    #[error(
-        "Connect credential rotation has an unfinished attempt for a different current certificate; inspect the local credential store"
-    )]
-    PendingRotation,
-    #[error("RustFS is not registered with Connect")]
-    NotRegistered,
-    #[error("the Connect device private key is missing; restore device.key before using the stored certificate")]
-    IdentityMissing,
-    #[error("the Connect device certificate has expired; call ConnectClient::reenroll with a fresh registration token")]
-    CredentialExpired,
-    #[error("the Connect device certificate is not yet valid; fix local clock skew or call ConnectClient::reenroll")]
-    CredentialNotYetValid,
-    #[error("the stored Connect certificate and device private key cannot form a TLS identity")]
-    IdentityCertificate,
-    #[error(
-        "Connect rejected the device credential with HTTP {status}; reason={reason:?}; call ConnectClient::reenroll with a fresh registration token if revoked"
-    )]
-    AccessRevoked { status: StatusCode, reason: Option<String> },
-    #[error("Connect rejected the request with HTTP {status}; reason={reason:?}")]
-    Rejected { status: StatusCode, reason: Option<String> },
-    #[error("Connect remained unavailable after bounded retries; last_status={status:?}")]
-    Unavailable { status: Option<StatusCode> },
-    #[error("Connect response exceeded the 1 MiB credential-response limit")]
-    ResponseTooLarge,
-    #[error("Connect returned an invalid credential response")]
-    Response,
-    #[error(transparent)]
-    Transport(#[from] reqwest::Error),
-    #[error(transparent)]
-    Identity(#[from] IdentityError),
-    #[error(transparent)]
-    IdentityStore(#[from] StoreError),
-    #[error(transparent)]
-    CredentialStore(#[from] CredentialStoreError),
-    #[error(transparent)]
-    Credential(#[from] CredentialValidationError),
-}

rustfs/src/connect/credential_store.rs

@@ -1,370 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::fs;
-use std::io::{self, Write as _};
-use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicU64, Ordering};
-
-use serde::{Deserialize, Serialize};
-
-const CREDENTIAL_FILE: &str = "device.crt.json";
-const REGISTRATION_COMPLETED_FILE: &str = "registration.completed.json";
-const REGISTRATION_PENDING_FILE: &str = "registration.pending.json";
-const ROTATION_PENDING_FILE: &str = "rotation.pending.json";
-const LOCK_FILE: &str = ".state.lock";
-
-#[cfg(unix)]
-const FILE_MODE: u32 = 0o600;
-
-static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0);
-
-#[derive(Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct DeviceCredential {
-    pub name: String,
-    pub uid: String,
-    pub protocol_version: String,
-    pub key_id: String,
-    pub certificate_serial: String,
-    pub certificate: String,
-    pub certificate_chain: String,
-    pub not_before_unix: i64,
-    pub not_after_unix: i64,
-}
-
-impl std::fmt::Debug for DeviceCredential {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("DeviceCredential")
-            .field("name", &self.name)
-            .field("key_id", &self.key_id)
-            .field("certificate_serial", &self.certificate_serial)
-            .field("not_after_unix", &self.not_after_unix)
-            .finish_non_exhaustive()
-    }
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct PendingRegistration {
-    pub token_uid: String,
-    pub request_id: String,
-    pub certificate_request: String,
-    #[serde(default)]
-    pub previous_credential_fingerprint: Option<String>,
-    #[serde(default)]
-    pub next_public_key_sha256: Option<String>,
-}
-
-#[derive(Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct CompletedRegistration {
-    pub token_uid: String,
-    pub credential_fingerprint: String,
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct PendingRotation {
-    pub credential_fingerprint: String,
-    pub device_name: String,
-    pub request_id: String,
-    pub certificate_request: String,
-    pub next_public_key_sha256: String,
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum CredentialStoreError {
-    #[error("connect credential store I/O failed at {path}: {source}")]
-    Io {
-        path: PathBuf,
-        #[source]
-        source: io::Error,
-    },
-
-    #[error("connect credential data at {path} is invalid: {source}")]
-    Invalid {
-        path: PathBuf,
-        #[source]
-        source: serde_json::Error,
-    },
-
-    #[cfg(unix)]
-    #[error("connect credential file at {path} has mode {mode:o}, expected {expected:o}")]
-    Permissions { path: PathBuf, mode: u32, expected: u32 },
-}
-
-#[derive(Clone, Debug)]
-pub struct CredentialStore {
-    directory: PathBuf,
-}
-
-pub(crate) struct CredentialLock {
-    _file: fs::File,
-}
-
-impl CredentialStore {
-    pub fn new(directory: impl Into<PathBuf>) -> Self {
-        Self {
-            directory: directory.into(),
-        }
-    }
-
-    pub(crate) fn load(&self) -> Result<Option<DeviceCredential>, CredentialStoreError> {
-        self.read(CREDENTIAL_FILE)
-    }
-
-    pub(crate) async fn lock(&self) -> Result<CredentialLock, CredentialStoreError> {
-        let directory = self.directory.clone();
-        tokio::task::spawn_blocking(move || {
-            fs::create_dir_all(&directory).map_err(|source| CredentialStoreError::Io {
-                path: directory.clone(),
-                source,
-            })?;
-            let path = directory.join(LOCK_FILE);
-            let mut options = fs::OpenOptions::new();
-            options.create(true).truncate(false).read(true).write(true);
-            #[cfg(unix)]
-            {
-                use std::os::unix::fs::OpenOptionsExt as _;
-                options.mode(FILE_MODE);
-            }
-            let file = options.open(&path).map_err(|source| CredentialStoreError::Io {
-                path: path.clone(),
-                source,
-            })?;
-            check_mode(&path)?;
-            file.lock().map_err(|source| CredentialStoreError::Io { path, source })?;
-            Ok(CredentialLock { _file: file })
-        })
-        .await
-        .map_err(|source| CredentialStoreError::Io {
-            path: self.directory.join(LOCK_FILE),
-            source: io::Error::other(source),
-        })?
-    }
-
-    pub(crate) fn save(&self, credential: &DeviceCredential) -> Result<(), CredentialStoreError> {
-        self.write(CREDENTIAL_FILE, credential)
-    }
-
-    pub(crate) fn claim_pending_registration(
-        &self,
-        pending: &PendingRegistration,
-    ) -> Result<PendingRegistration, CredentialStoreError> {
-        self.claim(REGISTRATION_PENDING_FILE, pending)
-    }
-
-    pub(crate) fn load_pending_registration(&self) -> Result<Option<PendingRegistration>, CredentialStoreError> {
-        self.read(REGISTRATION_PENDING_FILE)
-    }
-
-    pub(crate) fn clear_pending_registration(&self) -> Result<(), CredentialStoreError> {
-        self.remove(REGISTRATION_PENDING_FILE)
-    }
-
-    pub(crate) fn load_completed_registration(&self) -> Result<Option<CompletedRegistration>, CredentialStoreError> {
-        self.read(REGISTRATION_COMPLETED_FILE)
-    }
-
-    pub(crate) fn save_completed_registration(&self, completed: &CompletedRegistration) -> Result<(), CredentialStoreError> {
-        self.write(REGISTRATION_COMPLETED_FILE, completed)
-    }
-
-    pub(crate) fn load_pending_rotation(&self) -> Result<Option<PendingRotation>, CredentialStoreError> {
-        self.read(ROTATION_PENDING_FILE)
-    }
-
-    pub(crate) fn claim_pending_rotation(&self, pending: &PendingRotation) -> Result<PendingRotation, CredentialStoreError> {
-        self.claim(ROTATION_PENDING_FILE, pending)
-    }
-
-    pub(crate) fn clear_pending_rotation(&self) -> Result<(), CredentialStoreError> {
-        self.remove(ROTATION_PENDING_FILE)
-    }
-
-    fn read<T: for<'de> Deserialize<'de>>(&self, file: &str) -> Result<Option<T>, CredentialStoreError> {
-        let path = self.directory.join(file);
-        let bytes = match fs::read(&path) {
-            Ok(bytes) => bytes,
-            Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None),
-            Err(source) => return Err(CredentialStoreError::Io { path, source }),
-        };
-
-        check_mode(&path)?;
-        serde_json::from_slice(&bytes)
-            .map(Some)
-            .map_err(|source| CredentialStoreError::Invalid { path, source })
-    }
-
-    fn write<T: Serialize>(&self, file: &str, value: &T) -> Result<(), CredentialStoreError> {
-        let bytes = serde_json::to_vec(value).map_err(|source| CredentialStoreError::Invalid {
-            path: self.directory.join(file),
-            source,
-        })?;
-
-        fs::create_dir_all(&self.directory).map_err(|source| CredentialStoreError::Io {
-            path: self.directory.clone(),
-            source,
-        })?;
-
-        let final_path = self.directory.join(file);
-        let temp_path = self.stage(file, &bytes)?;
-        let result = fs::rename(&temp_path, &final_path)
-            .map_err(|source| CredentialStoreError::Io {
-                path: final_path,
-                source,
-            })
-            .and_then(|()| {
-                fsync_dir(&self.directory).map_err(|source| CredentialStoreError::Io {
-                    path: self.directory.clone(),
-                    source,
-                })
-            });
-
-        if result.is_err() {
-            let _ = fs::remove_file(&temp_path);
-        }
-        result
-    }
-
-    fn claim<T>(&self, file: &str, value: &T) -> Result<T, CredentialStoreError>
-    where
-        T: Clone + Serialize + for<'de> Deserialize<'de>,
-    {
-        if let Some(existing) = self.read(file)? {
-            return Ok(existing);
-        }
-        let bytes = serde_json::to_vec(value).map_err(|source| CredentialStoreError::Invalid {
-            path: self.directory.join(file),
-            source,
-        })?;
-        fs::create_dir_all(&self.directory).map_err(|source| CredentialStoreError::Io {
-            path: self.directory.clone(),
-            source,
-        })?;
-        let final_path = self.directory.join(file);
-        let temp_path = self.stage(file, &bytes)?;
-        let published = fs::hard_link(&temp_path, &final_path);
-        let _ = fs::remove_file(&temp_path);
-        match published {
-            Ok(()) => {
-                fsync_dir(&self.directory).map_err(|source| CredentialStoreError::Io {
-                    path: self.directory.clone(),
-                    source,
-                })?;
-                Ok(value.clone())
-            }
-            Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {
-                fsync_dir(&self.directory).map_err(|source| CredentialStoreError::Io {
-                    path: self.directory.clone(),
-                    source,
-                })?;
-                self.read(file)?.ok_or_else(|| CredentialStoreError::Io {
-                    path: final_path,
-                    source: io::Error::new(io::ErrorKind::NotFound, "pending state vanished after publication"),
-                })
-            }
-            Err(source) => Err(CredentialStoreError::Io {
-                path: final_path,
-                source,
-            }),
-        }
-    }
-
-    fn stage(&self, file: &str, bytes: &[u8]) -> Result<PathBuf, CredentialStoreError> {
-        loop {
-            let path = self.directory.join(format!(
-                ".{file}.{}.{}.tmp",
-                std::process::id(),
-                STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed)
-            ));
-            let mut options = fs::OpenOptions::new();
-            options.write(true).create_new(true);
-            #[cfg(unix)]
-            {
-                use std::os::unix::fs::OpenOptionsExt as _;
-                options.mode(FILE_MODE);
-            }
-            let mut staging = match options.open(&path) {
-                Ok(staging) => staging,
-                Err(source) if source.kind() == io::ErrorKind::AlreadyExists => continue,
-                Err(source) => return Err(CredentialStoreError::Io { path, source }),
-            };
-            let result = staging
-                .write_all(bytes)
-                .and_then(|()| staging.sync_all())
-                .map_err(|source| CredentialStoreError::Io {
-                    path: path.clone(),
-                    source,
-                })
-                .and_then(|()| check_mode(&path));
-            if let Err(error) = result {
-                let _ = fs::remove_file(&path);
-                return Err(error);
-            }
-            return Ok(path);
-        }
-    }
-
-    fn remove(&self, file: &str) -> Result<(), CredentialStoreError> {
-        let path = self.directory.join(file);
-        match fs::remove_file(&path) {
-            Ok(()) => fsync_dir(&self.directory).map_err(|source| CredentialStoreError::Io {
-                path: self.directory.clone(),
-                source,
-            }),
-            Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()),
-            Err(source) => Err(CredentialStoreError::Io { path, source }),
-        }
-    }
-}
-
-#[cfg(unix)]
-fn check_mode(path: &Path) -> Result<(), CredentialStoreError> {
-    use std::os::unix::fs::PermissionsExt as _;
-
-    let mode = fs::metadata(path)
-        .map_err(|source| CredentialStoreError::Io {
-            path: path.to_path_buf(),
-            source,
-        })?
-        .permissions()
-        .mode()
-        & 0o7777;
-    if mode != FILE_MODE {
-        return Err(CredentialStoreError::Permissions {
-            path: path.to_path_buf(),
-            mode,
-            expected: FILE_MODE,
-        });
-    }
-    Ok(())
-}
-
-#[cfg(not(unix))]
-fn check_mode(_path: &Path) -> Result<(), CredentialStoreError> {
-    Ok(())
-}
-
-fn fsync_dir(directory: &Path) -> io::Result<()> {
-    #[cfg(unix)]
-    fs::File::open(directory)?.sync_all()?;
-    #[cfg(not(unix))]
-    let _ = directory;
-    Ok(())
-}

rustfs/src/connect/identity.rs

@@ -24,7 +24,7 @@ use base64::Engine as _;
 use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
 use p256::ecdsa::signature::Signer as _;
 use p256::ecdsa::{Signature, SigningKey};
-use p256::pkcs8::{DecodePrivateKey as _, EncodePrivateKey as _, LineEnding};
+use p256::pkcs8::{DecodePrivateKey as _, EncodePrivateKey as _};
 use sha2::{Digest as _, Sha256};
 use zeroize::Zeroizing;
 
@@ -213,12 +213,6 @@ impl DeviceIdentity {
             .map_err(|error| IdentityError::MalformedKey(error.to_string()))
     }
 
-    pub(crate) fn to_pkcs8_pem(&self) -> Result<Zeroizing<String>, IdentityError> {
-        self.signing_key
-            .to_pkcs8_pem(LineEnding::LF)
-            .map_err(|error| IdentityError::MalformedKey(error.to_string()))
-    }
-
     /// Build the PKCS#10 certificate request Connect consumes.
     ///
     /// Connect reads the request for its SubjectPublicKeyInfo and its

rustfs/src/connect/identity_store.rs

@@ -32,7 +32,6 @@ use super::identity::{DeviceIdentity, IdentityError};
 
 /// Name of the key file inside the store directory.
 const KEY_FILE: &str = "device.key";
-const NEXT_KEY_FILE: &str = "device.key.next";
 
 /// Owner read/write only. The key is the device's whole identity.
 #[cfg(unix)]
@@ -95,15 +94,7 @@ impl IdentityStore {
     /// been enrolled. Reading never creates anything, so an unconfigured
     /// server can ask without acquiring an identity as a side effect.
     pub fn load(&self) -> Result<Option<DeviceIdentity>, StoreError> {
-        self.load_file(KEY_FILE)
-    }
-
-    pub(crate) fn load_next(&self) -> Result<Option<DeviceIdentity>, StoreError> {
-        self.load_file(NEXT_KEY_FILE)
-    }
-
-    fn load_file(&self, file: &str) -> Result<Option<DeviceIdentity>, StoreError> {
-        let path = self.directory.join(file);
+        let path = self.key_path();
 
         let der = match fs::read(&path) {
             Ok(der) => Zeroizing::new(der),
@@ -151,15 +142,11 @@ impl IdentityStore {
         let candidate = DeviceIdentity::generate();
         let der = candidate.to_pkcs8_der()?;
 
-        match self.publish(KEY_FILE, &der) {
+        match self.publish(&der) {
             Ok(()) => Ok(candidate),
             // Another process published first. Its key is the identity; ours
             // was never written anywhere and simply goes out of scope.
             Err(StoreError::Io { source, .. }) if source.kind() == io::ErrorKind::AlreadyExists => {
-                fsync_dir(&self.directory).map_err(|source| StoreError::Io {
-                    path: self.directory.clone(),
-                    source,
-                })?;
                 self.load()?.ok_or_else(|| StoreError::Io {
                     path: self.key_path(),
                     source: io::Error::new(
@@ -172,106 +159,35 @@ impl IdentityStore {
         }
     }
 
-    pub(crate) fn load_or_create_next(&self) -> Result<DeviceIdentity, StoreError> {
-        if let Some(identity) = self.load_next()? {
-            return Ok(identity);
-        }
-        fs::create_dir_all(&self.directory).map_err(|source| StoreError::Io {
-            path: self.directory.clone(),
-            source,
-        })?;
-        let candidate = DeviceIdentity::generate();
-        let der = candidate.to_pkcs8_der()?;
-        match self.publish(NEXT_KEY_FILE, &der) {
-            Ok(()) => Ok(candidate),
-            Err(StoreError::Io { source, .. }) if source.kind() == io::ErrorKind::AlreadyExists => {
-                fsync_dir(&self.directory).map_err(|source| StoreError::Io {
-                    path: self.directory.clone(),
-                    source,
-                })?;
-                self.load_next()?.ok_or_else(|| StoreError::Io {
-                    path: self.directory.join(NEXT_KEY_FILE),
-                    source: io::Error::new(io::ErrorKind::NotFound, "next device key vanished after publication"),
-                })
-            }
-            Err(error) => Err(error),
-        }
-    }
-
-    pub(crate) fn commit_next(&self, expected: &DeviceIdentity) -> Result<(), StoreError> {
-        let next_path = self.directory.join(NEXT_KEY_FILE);
-        match fs::rename(&next_path, self.key_path()) {
-            Ok(()) => fsync_dir(&self.directory).map_err(|source| StoreError::Io {
-                path: self.directory.clone(),
-                source,
-            }),
-            Err(source) if source.kind() == io::ErrorKind::NotFound => {
-                let current = self.load()?.ok_or_else(|| StoreError::Io {
-                    path: self.key_path(),
-                    source,
-                })?;
-                if current.public_key_der() == expected.public_key_der() {
-                    fsync_dir(&self.directory).map_err(|source| StoreError::Io {
-                        path: self.directory.clone(),
-                        source,
-                    })
-                } else {
-                    Err(StoreError::Io {
-                        path: next_path,
-                        source: io::Error::new(io::ErrorKind::NotFound, "next device key is missing"),
-                    })
-                }
-            }
-            Err(source) => Err(StoreError::Io { path: next_path, source }),
-        }
-    }
-
-    pub(crate) fn clear_next(&self) -> Result<(), StoreError> {
-        let path = self.directory.join(NEXT_KEY_FILE);
-        match fs::remove_file(&path) {
-            Ok(()) => fsync_dir(&self.directory).map_err(|source| StoreError::Io {
-                path: self.directory.clone(),
-                source,
-            }),
-            Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()),
-            Err(source) => Err(StoreError::Io { path, source }),
-        }
-    }
-
     /// Write, seal, fsync, then link into place and fsync the directory. The
     /// key is durable before it is reachable, and it is reachable only once.
-    fn publish(&self, file: &str, der: &[u8]) -> Result<(), StoreError> {
+    fn publish(&self, der: &[u8]) -> Result<(), StoreError> {
         use std::io::Write as _;
 
-        let final_path = self.directory.join(file);
+        let final_path = self.key_path();
+        let temp_path = self.directory.join(format!(
+            "{KEY_FILE}.{}.{}.tmp",
+            std::process::id(),
+            STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed)
+        ));
 
         let io_at = |path: &Path| {
             let path = path.to_path_buf();
             move |source| StoreError::Io { path, source }
         };
 
-        let (temp_path, mut staging) = loop {
-            let temp_path = self.directory.join(format!(
-                ".{file}.{}.{}.tmp",
-                std::process::id(),
-                STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed)
-            ));
-            let mut options = fs::OpenOptions::new();
-            options.write(true).create_new(true);
-            #[cfg(unix)]
-            {
-                use std::os::unix::fs::OpenOptionsExt as _;
-                options.mode(KEY_MODE);
-            }
-            match options.open(&temp_path) {
-                Ok(staging) => break (temp_path, staging),
-                Err(source) if source.kind() == io::ErrorKind::AlreadyExists => continue,
-                Err(source) => return Err(io_at(&temp_path)(source)),
-            }
-        };
+        let mut options = fs::OpenOptions::new();
+        options.write(true).create(true).truncate(true);
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::OpenOptionsExt as _;
+            options.mode(KEY_MODE);
+        }
+
+        let mut file = options.open(&temp_path).map_err(io_at(&temp_path))?;
 
         let result = (|| -> Result<(), StoreError> {
-            staging.write_all(der).map_err(io_at(&temp_path))?;
+            file.write_all(der).map_err(io_at(&temp_path))?;
 
             // The umask can only narrow the creation mode, so set and verify
             // the exact mode before the bytes become durable.
@@ -279,10 +195,9 @@ impl IdentityStore {
             {
                 use std::os::unix::fs::PermissionsExt as _;
 
-                staging
-                    .set_permissions(fs::Permissions::from_mode(KEY_MODE))
+                file.set_permissions(fs::Permissions::from_mode(KEY_MODE))
                     .map_err(io_at(&temp_path))?;
-                let mode = staging.metadata().map_err(io_at(&temp_path))?.permissions().mode() & 0o7777;
+                let mode = file.metadata().map_err(io_at(&temp_path))?.permissions().mode() & 0o7777;
                 if mode != KEY_MODE {
                     return Err(StoreError::Permissions {
                         path: temp_path.clone(),
@@ -292,11 +207,11 @@ impl IdentityStore {
                 }
             }
 
-            staging.sync_all().map_err(io_at(&temp_path))?;
+            file.sync_all().map_err(io_at(&temp_path))?;
             Ok(())
         })();
 
-        drop(staging);
+        drop(file);
 
         if let Err(error) = result {
             let _ = fs::remove_file(&temp_path);

rustfs/src/connect/mod.rs

@@ -25,16 +25,10 @@
 //! not been enrolled into a Connect control plane never calls into it, so an
 //! unconfigured server generates no key and holds no identity.
 
-pub mod client;
-pub mod credential_store;
 pub mod identity;
 pub mod identity_store;
 pub mod offline;
-pub mod registration;
 
-pub use client::{ClientError, ConnectClient, ConnectConfig};
-pub use credential_store::{CredentialStore, DeviceCredential};
 pub use identity::{DeviceIdentity, IdentityError, RegistrationProof, RegistrationTranscript};
 pub use identity_store::{IdentityStore, StoreError};
 pub use offline::{EnrollmentError, OfflineEnrollment, OfflineKeyStore, VerifiedChallenge};
-pub use registration::{RegistrationToken, TokenError};

rustfs/src/connect/registration.rs

@@ -1,527 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::io::Read;
-use std::sync::Arc;
-
-use base64::Engine as _;
-use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD;
-use p256::ecdsa::signature::Signer as _;
-use p256::ecdsa::{Signature, SigningKey};
-use p256::pkcs8::DecodePrivateKey as _;
-use rustls::RootCertStore;
-use rustls::pki_types::{CertificateDer, UnixTime, pem::PemObject as _};
-use rustls::server::WebPkiClientVerifier;
-use serde::{Deserialize, Serialize};
-use sha2::{Digest as _, Sha256};
-use time::OffsetDateTime;
-use time::format_description::well_known::Rfc3339;
-use uuid::{Uuid, Version};
-use x509_parser::extensions::GeneralName;
-use x509_parser::oid_registry::OID_SIG_ECDSA_WITH_SHA256;
-use x509_parser::prelude::{FromDer as _, X509Certificate, X509CertificationRequest};
-use zeroize::Zeroizing;
-
-use super::credential_store::DeviceCredential;
-use super::identity::{DeviceIdentity, RegistrationProof};
-
-pub const PROTOCOL_VERSION: &str = "v1";
-
-const CERTIFICATE_LIFETIME_SECONDS: i64 = 86_400;
-const ROTATION_DOMAIN: &[u8] = b"RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1";
-const MAX_TOKEN_BYTES: u64 = 16 * 1024;
-
-#[derive(Deserialize)]
-#[serde(deny_unknown_fields, rename_all = "camelCase")]
-struct RegistrationTokenDocument {
-    registration_token_uid: String,
-    registration_token_secret: String,
-    organization_uid: String,
-    cluster_uid: String,
-    challenge_nonce: String,
-    expires_unix: i64,
-}
-
-pub struct RegistrationToken {
-    pub registration_token_uid: String,
-    registration_token_secret: Zeroizing<String>,
-    pub organization_uid: String,
-    pub cluster_uid: String,
-    pub challenge_nonce: String,
-    pub expires_unix: i64,
-}
-
-impl std::fmt::Debug for RegistrationToken {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("RegistrationToken")
-            .field("registration_token_uid", &self.registration_token_uid)
-            .field("expires_unix", &self.expires_unix)
-            .finish_non_exhaustive()
-    }
-}
-
-impl RegistrationToken {
-    pub fn from_reader(reader: impl Read) -> Result<Self, TokenError> {
-        let mut bytes = Zeroizing::new(Vec::new());
-        reader
-            .take(MAX_TOKEN_BYTES + 1)
-            .read_to_end(&mut bytes)
-            .map_err(TokenError::Read)?;
-        if bytes.len() as u64 > MAX_TOKEN_BYTES {
-            return Err(TokenError::TooLarge);
-        }
-        let document: RegistrationTokenDocument = serde_json::from_slice(&bytes).map_err(TokenError::Invalid)?;
-        let decoded = BASE64_URL_NO_PAD
-            .decode(&document.registration_token_secret)
-            .map(Zeroizing::new)
-            .map_err(|_| TokenError::SecretShape)?;
-        if decoded.len() != 32 || BASE64_URL_NO_PAD.encode(&decoded) != document.registration_token_secret {
-            return Err(TokenError::SecretShape);
-        }
-        if !is_uuid_v7(&document.registration_token_uid)
-            || !is_uuid_v7(&document.organization_uid)
-            || !is_uuid_v7(&document.cluster_uid)
-            || document.expires_unix < 0
-            || document.challenge_nonce.len() != 64
-            || !document
-                .challenge_nonce
-                .bytes()
-                .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
-        {
-            return Err(TokenError::Shape);
-        }
-
-        Ok(Self {
-            registration_token_uid: document.registration_token_uid,
-            registration_token_secret: Zeroizing::new(document.registration_token_secret),
-            organization_uid: document.organization_uid,
-            cluster_uid: document.cluster_uid,
-            challenge_nonce: document.challenge_nonce,
-            expires_unix: document.expires_unix,
-        })
-    }
-
-    pub(crate) fn secret(&self) -> &str {
-        &self.registration_token_secret
-    }
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum TokenError {
-    #[error("failed to read the Connect registration token")]
-    Read(#[source] std::io::Error),
-    #[error("Connect registration token configuration is invalid")]
-    Invalid(#[source] serde_json::Error),
-    #[error("Connect registration token secret must be 32-byte unpadded base64url")]
-    SecretShape,
-    #[error("Connect registration token configuration exceeds 16 KiB")]
-    TooLarge,
-    #[error("Connect registration token fields do not match the protocol schema")]
-    Shape,
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct RegistrationRequest<'a> {
-    protocol_version: &'static str,
-    request_id: &'a str,
-    registration_token_uid: &'a str,
-    registration_token_secret: &'a str,
-    certificate_request: &'a str,
-    proof: ProofRef<'a>,
-}
-
-#[derive(Serialize)]
-struct ProofRef<'a> {
-    algorithm: &'a str,
-    value: &'a str,
-}
-
-impl<'a> RegistrationRequest<'a> {
-    pub(crate) fn new(
-        token: &'a RegistrationToken,
-        request_id: &'a str,
-        certificate_request: &'a str,
-        proof: &'a RegistrationProof,
-    ) -> Self {
-        Self {
-            protocol_version: PROTOCOL_VERSION,
-            request_id,
-            registration_token_uid: &token.registration_token_uid,
-            registration_token_secret: token.secret(),
-            certificate_request,
-            proof: ProofRef {
-                algorithm: &proof.algorithm,
-                value: &proof.value,
-            },
-        }
-    }
-}
-
-#[derive(Serialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct RotationRequest<'a> {
-    protocol_version: &'static str,
-    request_id: &'a str,
-    certificate_request: &'a str,
-    proof: ProofOwned,
-}
-
-#[derive(Serialize)]
-struct ProofOwned {
-    algorithm: String,
-    value: String,
-}
-
-impl<'a> RotationRequest<'a> {
-    pub(crate) fn new(
-        identity: &DeviceIdentity,
-        credential_fingerprint: &str,
-        device_name: &str,
-        request_id: &'a str,
-        certificate_request: &'a str,
-    ) -> Result<Self, CredentialValidationError> {
-        let csr_der = base64::engine::general_purpose::STANDARD
-            .decode(certificate_request)
-            .map_err(|_| CredentialValidationError::CertificateRequest)?;
-        let csr_digest = BASE64_URL_NO_PAD.encode(Sha256::digest(&csr_der));
-        let transcript = rotation_transcript(credential_fingerprint, device_name, request_id, &csr_digest)?;
-        let key = identity
-            .to_pkcs8_der()
-            .map_err(|_| CredentialValidationError::CertificateRequest)?;
-        let signing_key = SigningKey::from_pkcs8_der(&key).map_err(|_| CredentialValidationError::CertificateRequest)?;
-        let signature: Signature = signing_key.sign(&transcript);
-        let canonical = signature.normalize_s().unwrap_or(signature);
-
-        Ok(Self {
-            protocol_version: PROTOCOL_VERSION,
-            request_id,
-            certificate_request,
-            proof: ProofOwned {
-                algorithm: "ES256".to_string(),
-                value: BASE64_URL_NO_PAD.encode(canonical.to_bytes()),
-            },
-        })
-    }
-}
-
-fn rotation_transcript(
-    credential_fingerprint: &str,
-    device_name: &str,
-    request_id: &str,
-    csr_digest: &str,
-) -> Result<Vec<u8>, CredentialValidationError> {
-    let fields = [credential_fingerprint, device_name, request_id, csr_digest];
-    if fields
-        .iter()
-        .any(|field| !field.is_ascii() || field.as_bytes().contains(&b'\n'))
-    {
-        return Err(CredentialValidationError::RotationTranscript);
-    }
-
-    let mut transcript = Vec::with_capacity(346);
-    transcript.extend_from_slice(ROTATION_DOMAIN);
-    transcript.push(b'\n');
-    for field in fields {
-        transcript.extend_from_slice(field.len().to_string().as_bytes());
-        transcript.push(b':');
-        transcript.extend_from_slice(field.as_bytes());
-        transcript.push(b'\n');
-    }
-    Ok(transcript)
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct CredentialResponse {
-    pub name: String,
-    #[serde(default)]
-    pub uid: String,
-    #[serde(default)]
-    pub cluster: String,
-    pub protocol_version: String,
-    pub key_id: String,
-    pub certificate_serial: String,
-    pub certificate: String,
-    pub certificate_chain: String,
-    pub not_before: String,
-    pub not_after: String,
-}
-
-pub(crate) enum ExpectedDevice<'a> {
-    Registration { cluster: &'a str },
-    Rotation { name: &'a str },
-    Stored,
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum CredentialValidationError {
-    #[error("Connect returned malformed certificate material")]
-    Certificate,
-    #[error("Connect returned a certificate chain that is not trusted")]
-    Chain,
-    #[error("Connect returned a certificate for the wrong device identity")]
-    Identity,
-    #[error("Connect returned a certificate for a different device key")]
-    Key,
-    #[error("Connect returned an invalid certificate validity window")]
-    Validity,
-    #[error("the device certificate request could not be prepared")]
-    CertificateRequest,
-    #[error("the credential rotation transcript contains an invalid field")]
-    RotationTranscript,
-}
-
-pub(crate) fn validate_credential(
-    response: CredentialResponse,
-    identity: &DeviceIdentity,
-    roots: &RootCertStore,
-    root_certificates: &[CertificateDer<'static>],
-    expected: ExpectedDevice<'_>,
-) -> Result<DeviceCredential, CredentialValidationError> {
-    validate_credential_at(response, identity, roots, root_certificates, expected, true)
-}
-
-pub(crate) fn validate_stored_credential(
-    credential: &DeviceCredential,
-    identity: &DeviceIdentity,
-    roots: &RootCertStore,
-    root_certificates: &[CertificateDer<'static>],
-) -> Result<(), CredentialValidationError> {
-    let not_before = OffsetDateTime::from_unix_timestamp(credential.not_before_unix)
-        .map_err(|_| CredentialValidationError::Validity)?
-        .format(&Rfc3339)
-        .map_err(|_| CredentialValidationError::Validity)?;
-    let not_after = OffsetDateTime::from_unix_timestamp(credential.not_after_unix)
-        .map_err(|_| CredentialValidationError::Validity)?
-        .format(&Rfc3339)
-        .map_err(|_| CredentialValidationError::Validity)?;
-    let response = CredentialResponse {
-        name: credential.name.clone(),
-        uid: credential.uid.clone(),
-        cluster: String::new(),
-        protocol_version: credential.protocol_version.clone(),
-        key_id: credential.key_id.clone(),
-        certificate_serial: credential.certificate_serial.clone(),
-        certificate: credential.certificate.clone(),
-        certificate_chain: credential.certificate_chain.clone(),
-        not_before,
-        not_after,
-    };
-    validate_credential_at(response, identity, roots, root_certificates, ExpectedDevice::Stored, false).map(|_| ())
-}
-
-fn validate_credential_at(
-    response: CredentialResponse,
-    identity: &DeviceIdentity,
-    roots: &RootCertStore,
-    root_certificates: &[CertificateDer<'static>],
-    expected: ExpectedDevice<'_>,
-    verify_now: bool,
-) -> Result<DeviceCredential, CredentialValidationError> {
-    if response.protocol_version != PROTOCOL_VERSION {
-        return Err(CredentialValidationError::Identity);
-    }
-
-    let leaves = CertificateDer::pem_slice_iter(response.certificate.as_bytes())
-        .collect::<Result<Vec<_>, _>>()
-        .map_err(|_| CredentialValidationError::Certificate)?;
-    if leaves.len() != 1 {
-        return Err(CredentialValidationError::Certificate);
-    }
-    let chain = CertificateDer::pem_slice_iter(response.certificate_chain.as_bytes())
-        .collect::<Result<Vec<_>, _>>()
-        .map_err(|_| CredentialValidationError::Certificate)?;
-    if chain.is_empty()
-        || chain[0].as_ref() != leaves[0].as_ref()
-        || chain
-            .iter()
-            .skip(1)
-            .any(|certificate| root_certificates.iter().any(|root| root.as_ref() == certificate.as_ref()))
-    {
-        return Err(CredentialValidationError::Chain);
-    }
-
-    let (remaining, certificate) =
-        X509Certificate::from_der(leaves[0].as_ref()).map_err(|_| CredentialValidationError::Certificate)?;
-    if !remaining.is_empty() {
-        return Err(CredentialValidationError::Certificate);
-    }
-
-    let uid = match expected {
-        ExpectedDevice::Registration { cluster } => {
-            if response.uid.is_empty()
-                || response.cluster != cluster
-                || response.name != format!("{cluster}/clusterDevices/{}", response.uid)
-            {
-                return Err(CredentialValidationError::Identity);
-            }
-            response.uid.clone()
-        }
-        ExpectedDevice::Rotation { name } => {
-            if response.name != name || !response.uid.is_empty() || !response.cluster.is_empty() {
-                return Err(CredentialValidationError::Identity);
-            }
-            name.rsplit_once("/clusterDevices/")
-                .map(|(_, uid)| uid.to_string())
-                .ok_or(CredentialValidationError::Identity)?
-        }
-        ExpectedDevice::Stored => {
-            let (cluster, name_uid) = response
-                .name
-                .rsplit_once("/clusterDevices/")
-                .ok_or(CredentialValidationError::Identity)?;
-            if response.uid != name_uid
-                || !response.cluster.is_empty()
-                || !valid_cluster_name(cluster)
-                || response.name.matches("/clusterDevices/").count() != 1
-            {
-                return Err(CredentialValidationError::Identity);
-            }
-            response.uid.clone()
-        }
-    };
-    let parsed_uid = Uuid::parse_str(&uid).map_err(|_| CredentialValidationError::Identity)?;
-    if parsed_uid.get_version() != Some(Version::SortRand) || parsed_uid.to_string() != uid {
-        return Err(CredentialValidationError::Identity);
-    }
-    let expected_uri = format!("urn:rustfs:connect:device:{uid}");
-    let common_names = certificate
-        .subject()
-        .iter_common_name()
-        .map(|name| name.as_str())
-        .collect::<Result<Vec<_>, _>>()
-        .map_err(|_| CredentialValidationError::Identity)?;
-    let san = certificate
-        .subject_alternative_name()
-        .map_err(|_| CredentialValidationError::Identity)?
-        .ok_or(CredentialValidationError::Identity)?;
-    let san_matches = matches!(san.value.general_names.as_slice(), [GeneralName::URI(uri)] if *uri == expected_uri);
-
-    if common_names.as_slice() != [uid.as_str()]
-        || !san_matches
-        || certificate.subject().iter().count() != 1
-        || certificate.subject().iter_attributes().count() != 1
-    {
-        return Err(CredentialValidationError::Identity);
-    }
-    if certificate.public_key().raw != identity.public_key_der() {
-        return Err(CredentialValidationError::Key);
-    }
-
-    let not_before = OffsetDateTime::parse(&response.not_before, &Rfc3339)
-        .map_err(|_| CredentialValidationError::Validity)?
-        .unix_timestamp();
-    let not_after = OffsetDateTime::parse(&response.not_after, &Rfc3339)
-        .map_err(|_| CredentialValidationError::Validity)?
-        .unix_timestamp();
-    let verify_unix = if verify_now {
-        UnixTime::now()
-    } else {
-        let midpoint = certificate.validity().not_before.timestamp()
-            + (certificate.validity().not_after.timestamp() - certificate.validity().not_before.timestamp()) / 2;
-        UnixTime::since_unix_epoch(std::time::Duration::from_secs(
-            midpoint.try_into().map_err(|_| CredentialValidationError::Validity)?,
-        ))
-    };
-    let verifier = WebPkiClientVerifier::builder(Arc::new(roots.clone()))
-        .build()
-        .map_err(|_| CredentialValidationError::Chain)?;
-    verifier
-        .verify_client_cert(&leaves[0], &chain[1..], verify_unix)
-        .map_err(|_| CredentialValidationError::Chain)?;
-    if not_before != certificate.validity().not_before.timestamp()
-        || not_after != certificate.validity().not_after.timestamp()
-        || not_after - not_before != CERTIFICATE_LIFETIME_SECONDS
-        || certificate.signature_algorithm.algorithm != OID_SIG_ECDSA_WITH_SHA256
-        || response.certificate_serial != canonical_serial(certificate.raw_serial())?
-        || response.key_id != format!("x509-{}", response.certificate_serial)
-    {
-        return Err(CredentialValidationError::Validity);
-    }
-
-    Ok(DeviceCredential {
-        name: response.name,
-        uid,
-        protocol_version: response.protocol_version,
-        key_id: response.key_id,
-        certificate_serial: response.certificate_serial,
-        certificate: response.certificate,
-        certificate_chain: response.certificate_chain,
-        not_before_unix: not_before,
-        not_after_unix: not_after,
-    })
-}
-
-pub(crate) fn public_key_fingerprint(identity: &DeviceIdentity) -> String {
-    hex_lower(&Sha256::digest(identity.public_key_der()))
-}
-
-pub(crate) fn certificate_request_matches(encoded: &str, identity: &DeviceIdentity) -> Result<bool, CredentialValidationError> {
-    let der = base64::engine::general_purpose::STANDARD
-        .decode(encoded)
-        .map_err(|_| CredentialValidationError::CertificateRequest)?;
-    let (remaining, request) =
-        X509CertificationRequest::from_der(&der).map_err(|_| CredentialValidationError::CertificateRequest)?;
-    Ok(remaining.is_empty() && request.certification_request_info.subject_pki.raw == identity.public_key_der())
-}
-
-fn canonical_serial(raw: &[u8]) -> Result<String, CredentialValidationError> {
-    let magnitude = match raw {
-        [0, first, rest @ ..] if first & 0x80 != 0 => {
-            if rest.len() + 1 > 16 {
-                return Err(CredentialValidationError::Validity);
-            }
-            &raw[1..]
-        }
-        [0] => raw,
-        [0, ..] | [] => return Err(CredentialValidationError::Validity),
-        [first, ..] if first & 0x80 != 0 => return Err(CredentialValidationError::Validity),
-        _ if raw.len() > 16 => return Err(CredentialValidationError::Validity),
-        _ => raw,
-    };
-    let mut padded = [0u8; 16];
-    padded[16 - magnitude.len()..].copy_from_slice(magnitude);
-    Ok(hex_lower(&padded))
-}
-
-fn is_uuid_v7(value: &str) -> bool {
-    Uuid::parse_str(value).is_ok_and(|uuid| uuid.get_version() == Some(Version::SortRand) && uuid.to_string() == value)
-}
-
-fn valid_cluster_name(name: &str) -> bool {
-    let Some((organization, cluster)) = name
-        .strip_prefix("organizations/")
-        .and_then(|rest| rest.split_once("/clusters/"))
-    else {
-        return false;
-    };
-    !cluster.contains('/') && is_uuid_v7(organization) && is_uuid_v7(cluster)
-}
-
-pub(crate) fn certificate_fingerprint(certificate_pem: &str) -> Result<String, CredentialValidationError> {
-    let certificate = CertificateDer::pem_slice_iter(certificate_pem.as_bytes())
-        .next()
-        .ok_or(CredentialValidationError::Certificate)?
-        .map_err(|_| CredentialValidationError::Certificate)?;
-    Ok(hex_lower(&Sha256::digest(certificate.as_ref())))
-}
-
-fn hex_lower(bytes: &[u8]) -> String {
-    bytes.iter().fold(String::with_capacity(bytes.len() * 2), |mut output, byte| {
-        use std::fmt::Write as _;
-        let _ = write!(output, "{byte:02x}");
-        output
-    })
-}

rustfs/src/storage/s3_api/bucket.rs

@@ -296,10 +296,7 @@ pub(crate) fn build_list_objects_v2_output(
             let mut obj = Object {
                 key: Some(key),
                 last_modified: v.mod_time.map(Timestamp::from),
-                // Compressed legacy objects may retain an unknown (-1)
-                // logical-size sentinel; never expose that internal value in
-                // an S3 response.
-                size: Some(v.get_actual_size_or_physical()),
+                size: Some(v.get_actual_size().unwrap_or_default()),
                 e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)),
                 storage_class: v.storage_class.clone().map(ObjectStorageClass::from),
                 ..Default::default()
@@ -659,45 +656,6 @@ mod tests {
         assert_eq!(output.common_prefixes.as_ref().map(std::vec::Vec::len), Some(2));
     }
 
-    #[test]
-    fn list_objects_never_exposes_compressed_unknown_size_sentinel() {
-        let mut metadata = std::collections::HashMap::new();
-        rustfs_utils::http::insert_str(
-            &mut metadata,
-            rustfs_utils::http::SUFFIX_COMPRESSION,
-            "klauspost/compress/s2".to_string(),
-        );
-        let output = build_list_objects_v2_output(
-            ListObjectsV2Info {
-                objects: vec![ObjectInfo {
-                    name: "legacy-compressed".to_string(),
-                    size: 128,
-                    actual_size: -1,
-                    user_defined: std::sync::Arc::new(metadata),
-                    ..Default::default()
-                }],
-                ..Default::default()
-            },
-            false,
-            1000,
-            "bucket".to_string(),
-            String::new(),
-            None,
-            None,
-            None,
-            None,
-        );
-
-        assert_eq!(
-            output
-                .contents
-                .as_ref()
-                .and_then(|objects| objects.first())
-                .and_then(|object| object.size),
-            Some(128)
-        );
-    }
-
     #[test]
     fn list_responses_report_standard_for_legacy_label_only_file_metadata() {
         let version_id = Uuid::parse_str("11111111-2222-3333-4444-555555555555").expect("fixture version ID should be valid");

rustfs/src/storage/storage_api.rs

@@ -429,8 +429,6 @@ pub(crate) mod ecstore_config {
 }
 
 pub(crate) mod ecstore_data_usage {
-    #[cfg(test)]
-    pub(crate) use rustfs_ecstore::api::data_usage::get_bucket_usage_memory;
     pub(crate) use rustfs_ecstore::api::data_usage::{
         apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached,
         load_data_usage_from_backend, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,

rustfs/tests/connect_registration.rs

@@ -1,911 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::VecDeque;
-use std::fs;
-use std::sync::{Arc, Mutex};
-use std::time::Duration;
-
-use base64::Engine as _;
-use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
-use bytes::Bytes;
-use http_body_util::{BodyExt as _, Full};
-use hyper::service::service_fn;
-use hyper::{Request, Response, StatusCode};
-use hyper_util::rt::TokioIo;
-use p256::ecdsa::signature::Verifier as _;
-use p256::ecdsa::{Signature, VerifyingKey};
-use p256::pkcs8::DecodePublicKey as _;
-use rcgen::{
-    BasicConstraints, CertificateParams, DistinguishedName, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair,
-    KeyUsagePurpose, SanType, SerialNumber,
-};
-use rustfs::connect::{ClientError, ConnectClient, ConnectConfig, CredentialStore, IdentityStore, RegistrationToken, TokenError};
-use rustls::RootCertStore;
-use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, pem::PemObject as _};
-use rustls::server::WebPkiClientVerifier;
-use serde_json::{Value, json};
-use sha2::{Digest as _, Sha256};
-use time::OffsetDateTime;
-use time::format_description::well_known::Rfc3339;
-use tokio::net::TcpListener;
-use tokio_rustls::TlsAcceptor;
-
-const ORGANIZATION_UID: &str = "0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70";
-const CLUSTER_UID: &str = "0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81";
-const DEVICE_UID: &str = "0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92";
-const TOKEN_UID: &str = "0198f4b0-6f00-7b60-9271-7d8e9fa0b1c5";
-const FRESH_TOKEN_UID: &str = "0198f4b0-7f00-7c70-a381-8e9fa0b1c2d6";
-const SECOND_TOKEN_UID: &str = "0198f4b0-8f00-7d80-b491-9fa0b1c2d3e7";
-
-struct TestPki {
-    root_params: CertificateParams,
-    root_key: KeyPair,
-    root_der: CertificateDer<'static>,
-    root_pem: String,
-    server_der: CertificateDer<'static>,
-    server_key: PrivatePkcs8KeyDer<'static>,
-}
-
-impl TestPki {
-    fn new() -> Self {
-        let now = OffsetDateTime::now_utc();
-        let root_key = KeyPair::generate().expect("generate root key");
-        let mut root_params = CertificateParams::default();
-        root_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
-        root_params.not_before = now - time::Duration::days(1);
-        root_params.not_after = now + time::Duration::days(30);
-        root_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::DigitalSignature];
-        root_params.distinguished_name.push(DnType::CommonName, "Connect test root");
-        let root = root_params.self_signed(&root_key).expect("sign root");
-
-        let server_key = KeyPair::generate().expect("generate server key");
-        let mut server_params = CertificateParams::default();
-        server_params.not_before = now - time::Duration::hours(1);
-        server_params.not_after = now + time::Duration::days(2);
-        server_params
-            .subject_alt_names
-            .push(SanType::DnsName("localhost".try_into().expect("valid DNS name")));
-        server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
-        let issuer = Issuer::from_params(&root_params, &root_key);
-        let server = server_params
-            .signed_by(&server_key, &issuer)
-            .expect("sign server certificate");
-
-        Self {
-            root_params,
-            root_key,
-            root_der: root.der().clone(),
-            root_pem: root.pem(),
-            server_der: server.der().clone(),
-            server_key: PrivatePkcs8KeyDer::from(server_key.serialize_der()),
-        }
-    }
-
-    fn credential(&self, identity: &rustfs::connect::DeviceIdentity, uri: &str, serial_byte: u8) -> Value {
-        let now = OffsetDateTime::now_utc().replace_nanosecond(0).expect("whole second");
-        self.credential_window(identity, uri, serial_byte, now, now + time::Duration::days(1))
-    }
-
-    fn credential_window(
-        &self,
-        identity: &rustfs::connect::DeviceIdentity,
-        uri: &str,
-        serial_byte: u8,
-        not_before: OffsetDateTime,
-        not_after: OffsetDateTime,
-    ) -> Value {
-        let mut params = CertificateParams::default();
-        params.not_before = not_before;
-        params.not_after = not_after;
-        params.serial_number = Some(SerialNumber::from(vec![serial_byte; 16]));
-        params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
-        params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];
-        params.distinguished_name = DistinguishedName::new();
-        params.distinguished_name.push(DnType::CommonName, DEVICE_UID);
-        params
-            .subject_alt_names
-            .push(SanType::URI(uri.try_into().expect("valid URI SAN")));
-        let private_key = identity.to_pkcs8_der().expect("serialize device key");
-        let private_key = PrivatePkcs8KeyDer::from(private_key.to_vec());
-        let device_key =
-            KeyPair::from_pkcs8_der_and_sign_algo(&private_key, &rcgen::PKCS_ECDSA_P256_SHA256).expect("parse device key");
-        let issuer = Issuer::from_params(&self.root_params, &self.root_key);
-        let certificate = params.signed_by(&device_key, &issuer).expect("sign device certificate");
-        let serial = format!("{serial_byte:02x}").repeat(16);
-        let cluster = format!("organizations/{ORGANIZATION_UID}/clusters/{CLUSTER_UID}");
-
-        json!({
-            "name": format!("{cluster}/clusterDevices/{DEVICE_UID}"),
-            "uid": DEVICE_UID,
-            "cluster": cluster,
-            "protocolVersion": "v1",
-            "keyId": format!("x509-{serial}"),
-            "certificateSerial": serial,
-            "certificate": certificate.pem(),
-            "certificateChain": certificate.pem(),
-            "notBefore": not_before.format(&Rfc3339).expect("format notBefore"),
-            "notAfter": not_after.format(&Rfc3339).expect("format notAfter"),
-        })
-    }
-
-    fn server_config(&self, require_client: bool) -> rustls::ServerConfig {
-        let mut roots = RootCertStore::empty();
-        roots.add(self.root_der.clone()).expect("add client root");
-        let verifier = WebPkiClientVerifier::builder(Arc::new(roots));
-        let verifier = if require_client {
-            verifier.build()
-        } else {
-            verifier.allow_unauthenticated().build()
-        }
-        .expect("build client verifier");
-        rustls::ServerConfig::builder()
-            .with_client_cert_verifier(verifier)
-            .with_single_cert(vec![self.server_der.clone()], PrivateKeyDer::Pkcs8(self.server_key.clone_key()))
-            .expect("build server TLS")
-    }
-}
-
-#[derive(Clone)]
-enum Reply {
-    Json(StatusCode, Value),
-    DelayedClose(Duration),
-    VerifiedRotation {
-        response: Value,
-        current_public_key: Vec<u8>,
-        current_certificate_fingerprint: String,
-        device_name: String,
-    },
-}
-
-struct TestServer {
-    endpoint: String,
-    seen: Arc<Mutex<Vec<Value>>>,
-    task: tokio::task::JoinHandle<()>,
-}
-
-impl Drop for TestServer {
-    fn drop(&mut self) {
-        self.task.abort();
-    }
-}
-
-async fn server(pki: &TestPki, replies: Vec<Reply>) -> TestServer {
-    server_with_client_auth(pki, replies, false).await
-}
-
-async fn server_with_client_auth(pki: &TestPki, replies: Vec<Reply>, require_client: bool) -> TestServer {
-    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
-    let address = listener.local_addr().expect("server address");
-    let acceptor = TlsAcceptor::from(Arc::new(pki.server_config(require_client)));
-    let replies = Arc::new(Mutex::new(VecDeque::from(replies)));
-    let seen = Arc::new(Mutex::new(Vec::new()));
-    let captured = seen.clone();
-    let task = tokio::spawn(async move {
-        loop {
-            let Ok((stream, _)) = listener.accept().await else {
-                return;
-            };
-            let acceptor = acceptor.clone();
-            let replies = replies.clone();
-            let seen = captured.clone();
-            tokio::spawn(async move {
-                let Ok(stream) = acceptor.accept(stream).await else {
-                    return;
-                };
-                let service = service_fn(move |request: Request<hyper::body::Incoming>| {
-                    let replies = replies.clone();
-                    let seen = seen.clone();
-                    async move {
-                        let body = request.into_body().collect().await.expect("read request body").to_bytes();
-                        let value: Value = serde_json::from_slice(&body).expect("request JSON");
-                        seen.lock().expect("seen lock").push(value.clone());
-                        let reply = replies.lock().expect("reply lock").pop_front().expect("planned reply");
-                        match reply {
-                            Reply::Json(status, value) => Ok::<_, hyper::Error>(
-                                Response::builder()
-                                    .status(status)
-                                    .header("content-type", "application/json")
-                                    .body(Full::new(Bytes::from(serde_json::to_vec(&value).expect("reply JSON"))))
-                                    .expect("response"),
-                            ),
-                            Reply::DelayedClose(delay) => {
-                                tokio::time::sleep(delay).await;
-                                Ok(Response::builder()
-                                    .status(StatusCode::SERVICE_UNAVAILABLE)
-                                    .body(Full::new(Bytes::new()))
-                                    .expect("response"))
-                            }
-                            Reply::VerifiedRotation {
-                                response,
-                                current_public_key,
-                                current_certificate_fingerprint,
-                                device_name,
-                            } => {
-                                verify_rotation_request(
-                                    &value,
-                                    &current_public_key,
-                                    &current_certificate_fingerprint,
-                                    &device_name,
-                                );
-                                Ok(Response::builder()
-                                    .status(StatusCode::OK)
-                                    .header("content-type", "application/json")
-                                    .body(Full::new(Bytes::from(serde_json::to_vec(&response).expect("reply JSON"))))
-                                    .expect("response"))
-                            }
-                        }
-                    }
-                });
-                let _ = hyper::server::conn::http1::Builder::new()
-                    .serve_connection(TokioIo::new(stream), service)
-                    .await;
-            });
-        }
-    });
-    TestServer {
-        endpoint: format!("https://localhost:{}/agent/", address.port()),
-        seen,
-        task,
-    }
-}
-
-fn verify_rotation_request(request: &Value, current_public_key: &[u8], fingerprint: &str, device_name: &str) {
-    assert_eq!(request["protocolVersion"], "v1");
-    assert_eq!(request["proof"]["algorithm"], "ES256");
-    let csr = BASE64_STANDARD
-        .decode(request["certificateRequest"].as_str().expect("certificateRequest"))
-        .expect("CSR base64");
-    let csr_digest = BASE64_URL_NO_PAD.encode(Sha256::digest(&csr));
-    let request_id = request["requestId"].as_str().expect("requestId");
-    let transcript = rebuilt_rotation_transcript(
-        b"RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1",
-        [fingerprint, device_name, request_id, &csr_digest],
-    );
-    let encoded = request["proof"]["value"].as_str().expect("proof value");
-    assert_eq!(encoded.len(), 86);
-    let raw = BASE64_URL_NO_PAD.decode(encoded).expect("proof base64url");
-    let signature = Signature::from_slice(&raw).expect("fixed-width signature");
-    assert!(signature.normalize_s().is_none(), "rotation proof must be low-S");
-    let verifying = VerifyingKey::from_public_key_der(current_public_key).expect("current public key");
-    verifying.verify(&transcript, &signature).expect("rotation proof verifies");
-
-    let wrong_domain = rebuilt_rotation_transcript(
-        b"RUSTFS-CONNECT-CREDENTIAL-ROTATION-V2",
-        [fingerprint, device_name, request_id, &csr_digest],
-    );
-    assert!(verifying.verify(&wrong_domain, &signature).is_err());
-    let wrong_order = rebuilt_rotation_transcript(
-        b"RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1",
-        [device_name, fingerprint, request_id, &csr_digest],
-    );
-    assert!(verifying.verify(&wrong_order, &signature).is_err());
-}
-
-fn rebuilt_rotation_transcript(domain: &[u8], fields: [&str; 4]) -> Vec<u8> {
-    let mut transcript = Vec::new();
-    transcript.extend_from_slice(domain);
-    transcript.push(b'\n');
-    for field in fields {
-        transcript.extend_from_slice(field.len().to_string().as_bytes());
-        transcript.push(b':');
-        transcript.extend_from_slice(field.as_bytes());
-        transcript.push(b'\n');
-    }
-    transcript
-}
-
-fn certificate_fingerprint(pem: &str) -> String {
-    let certificate = CertificateDer::pem_slice_iter(pem.as_bytes())
-        .next()
-        .expect("leaf certificate")
-        .expect("certificate PEM");
-    Sha256::digest(certificate.as_ref())
-        .iter()
-        .map(|byte| format!("{byte:02x}"))
-        .collect()
-}
-
-fn token_document() -> Value {
-    json!({
-        "registrationTokenUid": TOKEN_UID,
-        "registrationTokenSecret": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
-        "organizationUid": ORGANIZATION_UID,
-        "clusterUid": CLUSTER_UID,
-        "challengeNonce": "a3f1c07d9b2e4856af0c1d3b5e7f9012c4a6b8d0e2f4061738495a6b7c8d9e0f",
-        "expiresUnix": OffsetDateTime::now_utc().unix_timestamp() + 3600,
-    })
-}
-
-fn token() -> RegistrationToken {
-    token_with_uid(TOKEN_UID)
-}
-
-fn token_with_uid(uid: &str) -> RegistrationToken {
-    let document = token_document();
-    let mut document = document;
-    document["registrationTokenUid"] = json!(uid);
-    RegistrationToken::from_reader(serde_json::to_vec(&document).expect("token JSON").as_slice()).expect("token parses")
-}
-
-fn stores(temp: &tempfile::TempDir) -> (IdentityStore, CredentialStore) {
-    (
-        IdentityStore::new(temp.path().join("identity")),
-        CredentialStore::new(temp.path().join("credential")),
-    )
-}
-
-fn client(server: &TestServer, pki: &TestPki, timeout: Duration) -> ConnectClient {
-    ConnectClient::new(ConnectConfig {
-        endpoint: &server.endpoint,
-        root_ca_pem: pki.root_pem.as_bytes(),
-        timeout,
-    })
-    .expect("build Connect client")
-}
-
-fn rotation_response(pki: &TestPki, identity: &rustfs::connect::DeviceIdentity, serial: u8) -> (Value, Value) {
-    let stored = pki.credential(identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), serial);
-    let mut wire = stored.clone();
-    wire.as_object_mut().expect("response object").remove("uid");
-    wire.as_object_mut().expect("response object").remove("cluster");
-    (wire, stored)
-}
-
-fn write_stored_credential(path: &std::path::Path, response: &Value) {
-    let not_before = OffsetDateTime::parse(response["notBefore"].as_str().expect("notBefore"), &Rfc3339)
-        .expect("parse notBefore")
-        .unix_timestamp();
-    let not_after = OffsetDateTime::parse(response["notAfter"].as_str().expect("notAfter"), &Rfc3339)
-        .expect("parse notAfter")
-        .unix_timestamp();
-    let stored = json!({
-        "name": response["name"],
-        "uid": DEVICE_UID,
-        "protocolVersion": response["protocolVersion"],
-        "keyId": response["keyId"],
-        "certificateSerial": response["certificateSerial"],
-        "certificate": response["certificate"],
-        "certificateChain": response["certificateChain"],
-        "notBeforeUnix": not_before,
-        "notAfterUnix": not_after,
-    });
-    fs::write(path, serde_json::to_vec(&stored).expect("stored credential JSON")).expect("write credential");
-    set_owner_only(path);
-}
-
-#[cfg(unix)]
-fn set_owner_only(path: &std::path::Path) {
-    use std::os::unix::fs::PermissionsExt as _;
-    fs::set_permissions(path, fs::Permissions::from_mode(0o600)).expect("set owner-only mode");
-}
-
-#[cfg(not(unix))]
-fn set_owner_only(_path: &std::path::Path) {}
-
-#[tokio::test]
-async fn registration_reuses_request_and_csr_after_timeout_and_restart() {
-    let temp = tempfile::tempdir().expect("temp dir");
-    let (identity_store, credential_store) = stores(&temp);
-    let identity = identity_store.load_or_create().expect("create identity");
-    let pki = TestPki::new();
-    let response = pki.credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 0x80);
-    let server = server(
-        &pki,
-        vec![
-            Reply::DelayedClose(Duration::from_millis(200)),
-            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
-            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
-            Reply::Json(StatusCode::CREATED, response),
-        ],
-    )
-    .await;
-    let first = client(&server, &pki, Duration::from_millis(80));
-    assert!(matches!(
-        first.register(&identity_store, &credential_store, &token()).await,
-        Err(ClientError::Unavailable { .. })
-    ));
-
-    let restarted = client(&server, &pki, Duration::from_secs(2));
-    let credential = restarted
-        .register(&identity_store, &credential_store, &token())
-        .await
-        .expect("restart replays completed exchange");
-    assert_eq!(credential.uid, DEVICE_UID);
-    assert_eq!(credential.certificate_serial, "80".repeat(16));
-
-    let seen = server.seen.lock().expect("seen lock");
-    assert_eq!(seen.len(), 4);
-    for request in &seen[1..] {
-        assert_eq!(request["requestId"], seen[0]["requestId"]);
-        assert_eq!(request["certificateRequest"], seen[0]["certificateRequest"]);
-    }
-}
-
-#[tokio::test]
-async fn registration_rejects_untrusted_or_misbound_credentials() {
-    for case in ["san", "chain", "key", "key_id", "cluster", "name"] {
-        let temp = tempfile::tempdir().expect("temp dir");
-        let (identity_store, credential_store) = stores(&temp);
-        let identity = identity_store.load_or_create().expect("create identity");
-        let pki = TestPki::new();
-        let mut response = match case {
-            "san" => pki.credential(&identity, "urn:rustfs:connect:device:0198f4b0-3c00-7e30-8f41-4a5b6c7d8e93", 2),
-            "chain" => TestPki::new().credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 2),
-            "key" => pki.credential(
-                &rustfs::connect::DeviceIdentity::generate(),
-                &format!("urn:rustfs:connect:device:{DEVICE_UID}"),
-                2,
-            ),
-            _ => pki.credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 2),
-        };
-        match case {
-            "key_id" => response["keyId"] = json!("x509-deadbeef"),
-            "cluster" => response["cluster"] = json!(format!("organizations/{ORGANIZATION_UID}/clusters/other")),
-            "name" => {
-                response["name"] = json!(format!("organizations/{ORGANIZATION_UID}/clusters/other/clusterDevices/{DEVICE_UID}"))
-            }
-            _ => {}
-        }
-        let server = server(&pki, vec![Reply::Json(StatusCode::CREATED, response)]).await;
-        let error = client(&server, &pki, Duration::from_secs(2))
-            .register(&identity_store, &credential_store, &token())
-            .await
-            .expect_err("invalid returned identity must fail closed");
-        assert!(matches!(error, ClientError::Credential(_)));
-        assert!(!temp.path().join("credential/device.crt.json").exists());
-    }
-}
-
-#[tokio::test]
-async fn stored_credential_is_revalidated_before_reuse() {
-    let temp = tempfile::tempdir().expect("temp dir");
-    let (identity_store, credential_store) = stores(&temp);
-    let identity = identity_store.load_or_create().expect("create identity");
-    let pki = TestPki::new();
-    let issued = pki.credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 4);
-    let server = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
-    let client = client(&server, &pki, Duration::from_secs(2));
-    client
-        .register(&identity_store, &credential_store, &token())
-        .await
-        .expect("register");
-
-    let path = temp.path().join("credential/device.crt.json");
-    let mut stored: Value = serde_json::from_slice(&fs::read(&path).expect("read credential")).expect("credential JSON");
-    stored["certificateSerial"] = json!("00".repeat(16));
-    fs::write(&path, serde_json::to_vec(&stored).expect("credential JSON")).expect("tamper credential");
-    let error = client
-        .register(&identity_store, &credential_store, &token())
-        .await
-        .expect_err("tampered stored credential must fail closed");
-    assert!(matches!(error, ClientError::Credential(_)));
-}
-
-#[tokio::test]
-async fn register_rejects_expired_and_not_yet_valid_stored_credentials() {
-    let temp = tempfile::tempdir().expect("temp dir");
-    let (identity_store, credential_store) = stores(&temp);
-    let identity = identity_store.load_or_create().expect("create identity");
-    let pki = TestPki::new();
-    let issued = pki.credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 9);
-    let registration = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
-    let client = client(&registration, &pki, Duration::from_secs(2));
-    client
-        .register(&identity_store, &credential_store, &token())
-        .await
-        .expect("register");
-
-    let now = OffsetDateTime::now_utc().replace_nanosecond(0).expect("whole second");
-    let path = temp.path().join("credential/device.crt.json");
-    let expired = pki.credential_window(
-        &identity,
-        &format!("urn:rustfs:connect:device:{DEVICE_UID}"),
-        10,
-        now - time::Duration::days(2),
-        now - time::Duration::days(1),
-    );
-    write_stored_credential(&path, &expired);
-    assert!(matches!(
-        client.register(&identity_store, &credential_store, &token()).await,
-        Err(ClientError::CredentialExpired)
-    ));
-
-    let future = pki.credential_window(
-        &identity,
-        &format!("urn:rustfs:connect:device:{DEVICE_UID}"),
-        11,
-        now + time::Duration::hours(1),
-        now + time::Duration::hours(25),
-    );
-    write_stored_credential(&path, &future);
-    assert!(matches!(
-        client.register(&identity_store, &credential_store, &token()).await,
-        Err(ClientError::CredentialNotYetValid)
-    ));
-}
-
-#[tokio::test]
-async fn concurrent_rotation_retries_converge_and_promote_the_next_key() {
-    let temp = tempfile::tempdir().expect("temp dir");
-    let (identity_store, credential_store) = stores(&temp);
-    let current = identity_store.load_or_create().expect("create identity");
-    let pki = TestPki::new();
-    let mut issued = pki.credential(&current, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 5);
-    issued["certificateChain"] = json!(
-        issued["certificateChain"]
-            .as_str()
-            .expect("certificate chain")
-            .trim_end_matches('\n')
-    );
-    let registration = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
-    let registered = client(&registration, &pki, Duration::from_secs(2))
-        .register(&identity_store, &credential_store, &token())
-        .await
-        .expect("register");
-
-    let retries = server(
-        &pki,
-        vec![
-            Reply::DelayedClose(Duration::from_millis(200)),
-            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
-            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
-            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
-            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
-            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
-        ],
-    )
-    .await;
-    let retry_client = client(&retries, &pki, Duration::from_millis(80));
-    let due = registered.not_after_unix - 8 * 60 * 60;
-    let (first, second) = tokio::join!(
-        retry_client.rotate_if_due(&identity_store, &credential_store, due),
-        retry_client.rotate_if_due(&identity_store, &credential_store, due)
-    );
-    assert!(matches!(first, Err(ClientError::Unavailable { .. })));
-    assert!(matches!(second, Err(ClientError::Unavailable { .. })));
-    let (request_id, certificate_request) = {
-        let seen = retries.seen.lock().expect("seen lock");
-        assert!(seen.len() >= 3, "bounded retries must reach the server");
-        for request in &seen[1..] {
-            assert_eq!(request["requestId"], seen[0]["requestId"]);
-            assert_eq!(request["certificateRequest"], seen[0]["certificateRequest"]);
-        }
-        (seen[0]["requestId"].clone(), seen[0]["certificateRequest"].clone())
-    };
-
-    let next_der = fs::read(temp.path().join("identity/device.key.next")).expect("read staged next key");
-    let next = rustfs::connect::DeviceIdentity::from_pkcs8_der(&next_der).expect("parse next key");
-    assert_ne!(current.public_key_der(), next.public_key_der());
-    assert_eq!(
-        identity_store
-            .load()
-            .expect("load current key")
-            .expect("current key")
-            .public_key_der(),
-        current.public_key_der()
-    );
-    #[cfg(unix)]
-    {
-        use std::os::unix::fs::PermissionsExt as _;
-        let mode = fs::metadata(temp.path().join("identity/device.key.next"))
-            .expect("next key metadata")
-            .permissions()
-            .mode()
-            & 0o7777;
-        assert_eq!(mode, 0o600);
-    }
-    let (rotated, _) = rotation_response(&pki, &next, 6);
-    let success = server_with_client_auth(
-        &pki,
-        vec![Reply::VerifiedRotation {
-            response: rotated,
-            current_public_key: current.public_key_der(),
-            current_certificate_fingerprint: certificate_fingerprint(&registered.certificate),
-            device_name: registered.name.clone(),
-        }],
-        true,
-    )
-    .await;
-    let success_client = client(&success, &pki, Duration::from_secs(2));
-    let (due_result, current_result) = tokio::join!(
-        success_client.rotate_if_due(&identity_store, &credential_store, due),
-        success_client.rotate_if_due(&identity_store, &credential_store, OffsetDateTime::now_utc().unix_timestamp())
-    );
-    let credential = due_result
-        .expect("retry rotation")
-        .or(current_result.expect("concurrent current-state check"))
-        .expect("exactly one rotation is due");
-    assert_eq!(credential.certificate_serial, "06".repeat(16));
-    let success_seen = success.seen.lock().expect("seen lock");
-    assert_eq!(success_seen.len(), 1, "the post-commit actor must not publish stale state");
-    assert_eq!(success_seen[0]["requestId"], request_id);
-    assert_eq!(success_seen[0]["certificateRequest"], certificate_request);
-    drop(success_seen);
-    assert_eq!(
-        identity_store
-            .load()
-            .expect("load key")
-            .expect("current key")
-            .public_key_der(),
-        next.public_key_der()
-    );
-    assert!(!temp.path().join("identity/device.key.next").exists());
-    assert!(!temp.path().join("credential/rotation.pending.json").exists());
-}
-
-#[tokio::test]
-async fn rotation_commit_recovers_after_each_durable_step() {
-    let temp = tempfile::tempdir().expect("temp dir");
-    let (identity_store, credential_store) = stores(&temp);
-    let current = identity_store.load_or_create().expect("create identity");
-    let pki = TestPki::new();
-    let issued = pki.credential(&current, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 7);
-    let registration = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
-    let registered = client(&registration, &pki, Duration::from_secs(2))
-        .register(&identity_store, &credential_store, &token())
-        .await
-        .expect("register");
-    let failed = server(&pki, vec![Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})); 3]).await;
-    let due = registered.not_after_unix - 8 * 60 * 60;
-    assert!(matches!(
-        client(&failed, &pki, Duration::from_secs(2))
-            .rotate_if_due(&identity_store, &credential_store, due)
-            .await,
-        Err(ClientError::Unavailable { .. })
-    ));
-
-    let pending_path = temp.path().join("credential/rotation.pending.json");
-    let pending = fs::read(&pending_path).expect("read pending state");
-    let next_der = fs::read(temp.path().join("identity/device.key.next")).expect("read next key");
-    let next = rustfs::connect::DeviceIdentity::from_pkcs8_der(&next_der).expect("parse next key");
-    let (_, stored) = rotation_response(&pki, &next, 8);
-    write_stored_credential(&temp.path().join("credential/device.crt.json"), &stored);
-
-    let idle = server(&pki, vec![]).await;
-    assert!(
-        client(&idle, &pki, Duration::from_secs(2))
-            .rotate_if_due(&identity_store, &credential_store, OffsetDateTime::now_utc().unix_timestamp())
-            .await
-            .expect("recover after credential save")
-            .is_none()
-    );
-    assert_eq!(
-        identity_store
-            .load()
-            .expect("load key")
-            .expect("current key")
-            .public_key_der(),
-        next.public_key_der()
-    );
-
-    fs::write(&pending_path, pending).expect("restore pending after key commit");
-    set_owner_only(&pending_path);
-    assert!(
-        client(&idle, &pki, Duration::from_secs(2))
-            .rotate_if_due(&identity_store, &credential_store, OffsetDateTime::now_utc().unix_timestamp())
-            .await
-            .expect("recover after key commit")
-            .is_none()
-    );
-    assert!(!pending_path.exists());
-}
-
-#[tokio::test]
-async fn pending_reenrollment_blocks_rotation_and_resumes_original_exchange() {
-    let temp = tempfile::tempdir().expect("temp dir");
-    let (identity_store, credential_store) = stores(&temp);
-    let current = identity_store.load_or_create().expect("create identity");
-    let pki = TestPki::new();
-    let issued = pki.credential(&current, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 13);
-    let registration = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
-    let registered = client(&registration, &pki, Duration::from_secs(2))
-        .register(&identity_store, &credential_store, &token())
-        .await
-        .expect("register");
-
-    let failed = server(&pki, vec![Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})); 3]).await;
-    assert!(matches!(
-        client(&failed, &pki, Duration::from_secs(2))
-            .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
-            .await,
-        Err(ClientError::Unavailable { .. })
-    ));
-
-    let pending_path = temp.path().join("credential/registration.pending.json");
-    let pending = fs::read(&pending_path).expect("read pending reenrollment");
-    let pending_document: Value = serde_json::from_slice(&pending).expect("pending JSON");
-    let next_der = fs::read(temp.path().join("identity/device.key.next")).expect("read next key");
-    let next = rustfs::connect::DeviceIdentity::from_pkcs8_der(&next_der).expect("parse next key");
-    let enrolled = pki.credential(&next, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 14);
-
-    let rotation = server(&pki, vec![]).await;
-    let error = client(&rotation, &pki, Duration::from_secs(2))
-        .rotate_if_due(&identity_store, &credential_store, registered.not_after_unix - 8 * 60 * 60)
-        .await
-        .expect_err("pending reenrollment blocks rotation");
-    assert!(matches!(error, ClientError::PendingRegistration));
-    assert!(rotation.seen.lock().expect("seen lock").is_empty());
-
-    let resumed = server(&pki, vec![Reply::Json(StatusCode::CREATED, enrolled)]).await;
-    let credential = client(&resumed, &pki, Duration::from_secs(2))
-        .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
-        .await
-        .expect("resume reenrollment");
-    assert_eq!(credential.certificate_serial, "0e".repeat(16));
-    let resumed_seen = resumed.seen.lock().expect("seen lock");
-    assert_eq!(resumed_seen.len(), 1);
-    assert_eq!(resumed_seen[0]["requestId"], pending_document["requestId"]);
-    assert_eq!(resumed_seen[0]["certificateRequest"], pending_document["certificateRequest"]);
-}
-
-#[tokio::test]
-async fn reenrollment_commit_recovers_after_each_durable_step() {
-    let temp = tempfile::tempdir().expect("temp dir");
-    let (identity_store, credential_store) = stores(&temp);
-    let current = identity_store.load_or_create().expect("create identity");
-    let pki = TestPki::new();
-    let issued = pki.credential(&current, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 13);
-    let registration = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
-    client(&registration, &pki, Duration::from_secs(2))
-        .register(&identity_store, &credential_store, &token())
-        .await
-        .expect("register");
-
-    let failed = server(&pki, vec![Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})); 3]).await;
-    assert!(matches!(
-        client(&failed, &pki, Duration::from_secs(2))
-            .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
-            .await,
-        Err(ClientError::Unavailable { .. })
-    ));
-
-    let pending_path = temp.path().join("credential/registration.pending.json");
-    let pending = fs::read(&pending_path).expect("read pending reenrollment");
-    let next_der = fs::read(temp.path().join("identity/device.key.next")).expect("read next key");
-    let next = rustfs::connect::DeviceIdentity::from_pkcs8_der(&next_der).expect("parse next key");
-    let enrolled = pki.credential(&next, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 14);
-    write_stored_credential(&temp.path().join("credential/device.crt.json"), &enrolled);
-
-    let idle = server(&pki, vec![]).await;
-    let idle_client = client(&idle, &pki, Duration::from_secs(2));
-    let recovered = idle_client
-        .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
-        .await
-        .expect("recover after reenrollment credential save");
-    assert_eq!(recovered.certificate_serial, "0e".repeat(16));
-    assert_eq!(
-        identity_store
-            .load()
-            .expect("load key")
-            .expect("current key")
-            .public_key_der(),
-        next.public_key_der()
-    );
-
-    fs::remove_file(temp.path().join("credential/registration.completed.json")).expect("remove completed receipt");
-    fs::write(&pending_path, pending).expect("restore pending after key commit");
-    set_owner_only(&pending_path);
-    let recovered = idle_client
-        .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
-        .await
-        .expect("recover after reenrollment key commit");
-    assert_eq!(recovered.certificate_serial, "0e".repeat(16));
-    assert!(!pending_path.exists());
-    let recovered = idle_client
-        .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
-        .await
-        .expect("completed reenrollment is idempotent after pending cleanup");
-    assert_eq!(recovered.certificate_serial, "0e".repeat(16));
-    assert!(idle.seen.lock().expect("seen lock").is_empty());
-
-    let different = server(&pki, vec![Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})); 3]).await;
-    assert!(matches!(
-        client(&different, &pki, Duration::from_secs(2))
-            .reenroll(&identity_store, &credential_store, &token_with_uid(SECOND_TOKEN_UID))
-            .await,
-        Err(ClientError::Unavailable { .. })
-    ));
-    assert_eq!(different.seen.lock().expect("seen lock").len(), 3);
-}
-
-#[test]
-fn registration_token_schema_is_strict_and_bounded() {
-    let mut document = serde_json::to_value(token_document()).expect("token document");
-    document["unexpected"] = json!(true);
-    assert!(matches!(
-        RegistrationToken::from_reader(serde_json::to_vec(&document).expect("token JSON").as_slice()),
-        Err(TokenError::Invalid(_))
-    ));
-    assert!(matches!(
-        RegistrationToken::from_reader(vec![b' '; 16 * 1024 + 1].as_slice()),
-        Err(TokenError::TooLarge)
-    ));
-    let mut malformed = token_document();
-    malformed["challengeNonce"] = json!("A".repeat(64));
-    assert!(matches!(
-        RegistrationToken::from_reader(serde_json::to_vec(&malformed).expect("token JSON").as_slice()),
-        Err(TokenError::Shape)
-    ));
-}
-
-#[tokio::test]
-async fn rotation_waits_for_threshold_and_stops_on_revocation() {
-    let temp = tempfile::tempdir().expect("temp dir");
-    let (identity_store, credential_store) = stores(&temp);
-    let identity = identity_store.load_or_create().expect("create identity");
-    let pki = TestPki::new();
-    let issued = pki.credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 3);
-    let rotation_server = server(
-        &pki,
-        vec![
-            Reply::Json(StatusCode::CREATED, issued),
-            Reply::Json(StatusCode::UNAUTHORIZED, json!({"details": [{"reason": "DEVICE_REVOKED"}]})),
-        ],
-    )
-    .await;
-    let connect = client(&rotation_server, &pki, Duration::from_secs(2));
-    let current = connect
-        .register(&identity_store, &credential_store, &token())
-        .await
-        .expect("register");
-    assert!(
-        connect
-            .rotate_if_due(&identity_store, &credential_store, current.not_before_unix)
-            .await
-            .expect("not due")
-            .is_none()
-    );
-    let error = connect
-        .rotate_if_due(&identity_store, &credential_store, current.not_after_unix - 8 * 60 * 60)
-        .await
-        .expect_err("revocation must stop rotation");
-    assert!(matches!(error, ClientError::AccessRevoked { .. }));
-    assert!(error.to_string().contains("ConnectClient::reenroll"));
-    assert_eq!(rotation_server.seen.lock().expect("seen lock").len(), 2);
-    let stored: Value =
-        serde_json::from_slice(&fs::read(temp.path().join("credential/device.crt.json")).expect("read stored credential"))
-            .expect("stored credential JSON");
-    assert_eq!(stored["certificateSerial"], current.certificate_serial);
-
-    let next_der = fs::read(temp.path().join("identity/device.key.next")).expect("read staged next key");
-    let next = rustfs::connect::DeviceIdentity::from_pkcs8_der(&next_der).expect("parse staged next key");
-    let enrolled = pki.credential(&next, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 12);
-    let reenrollment = server(&pki, vec![Reply::Json(StatusCode::CREATED, enrolled)]).await;
-    let fresh = client(&reenrollment, &pki, Duration::from_secs(2))
-        .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
-        .await
-        .expect("fresh token reenrolls revoked credential");
-    assert_eq!(fresh.certificate_serial, "0c".repeat(16));
-    assert_eq!(
-        identity_store
-            .load()
-            .expect("load identity")
-            .expect("identity")
-            .public_key_der(),
-        next.public_key_der()
-    );
-}
-
-#[test]
-fn unconfigured_connect_has_no_side_effects() {
-    let temp = tempfile::tempdir().expect("temp dir");
-    let directory = temp.path().join("connect");
-    assert!(
-        ConnectClient::from_optional_config(None)
-            .expect("unconfigured is valid")
-            .is_none()
-    );
-    assert!(!directory.exists());
-}