fix(quota): account compressed deletes by committed size

Cargo.lock

@@ -9257,6 +9257,7 @@ dependencies = [
  "url",
  "urlencoding",
  "uuid",
+ "x509-parser",
  "zeroize",
  "zip",
  "zstd",

Cargo.toml

@@ -204,6 +204,7 @@ rsa = { version = "=0.10.0-rc.18" }
 rustls = { default-features = false, version = "0.23.43" }
 rustls-native-certs = "0.8"
 rustls-pki-types = "1.15.1"
+x509-parser = "0.18.1"
 sha1 = "0.11.0"
 sha2 = "0.11.0"
 subtle = "2.6"

crates/common/src/metrics.rs

@@ -901,10 +901,6 @@ pub struct Metrics {
     scanner_cycle_max_duration_millis: AtomicU64,
     scanner_cycle_max_objects: AtomicU64,
     scanner_cycle_max_directories: AtomicU64,
-    scanner_cycle_timeout_total: AtomicU64,
-    scanner_cycle_recovery_required_total: AtomicU64,
-    scanner_cycle_last_progress_age_seconds: AtomicU64,
-    scanner_leader_lease_without_progress: AtomicBool,
     scanner_bitrot_cycle_enabled: AtomicBool,
     scanner_bitrot_cycle_millis: AtomicU64,
     scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
@@ -1374,14 +1370,6 @@ pub struct ScannerMetricsReport {
     #[serde(default)]
     pub cycle_max_directories: u64,
     #[serde(default)]
-    pub cycle_timeout_total: u64,
-    #[serde(default)]
-    pub cycle_recovery_required_total: u64,
-    #[serde(default)]
-    pub cycle_last_progress_age: u64,
-    #[serde(default)]
-    pub leader_lease_without_progress: bool,
-    #[serde(default)]
     pub bitrot_cycle_enabled: bool,
     #[serde(default)]
     pub bitrot_cycle_seconds: f64,
@@ -1442,9 +1430,6 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total
 const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
 const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
 const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
-const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
-const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
-const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";
 
 fn scan_cycle_result_label(result: u8) -> &'static str {
     match result {
@@ -1928,10 +1913,6 @@ impl Metrics {
             scanner_cycle_max_duration_millis: AtomicU64::new(0),
             scanner_cycle_max_objects: AtomicU64::new(0),
             scanner_cycle_max_directories: AtomicU64::new(0),
-            scanner_cycle_timeout_total: AtomicU64::new(0),
-            scanner_cycle_recovery_required_total: AtomicU64::new(0),
-            scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
-            scanner_leader_lease_without_progress: AtomicBool::new(false),
             scanner_bitrot_cycle_enabled: AtomicBool::new(false),
             scanner_bitrot_cycle_millis: AtomicU64::new(0),
             scanner_checkpoint: Mutex::new(None),
@@ -2431,29 +2412,12 @@ impl Metrics {
             .store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
         self.scanner_cycle_max_directories
             .store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
-        self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
-        self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
-        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
-        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
         self.scanner_bitrot_cycle_enabled
             .store(bitrot_cycle.is_some(), Ordering::Relaxed);
         self.scanner_bitrot_cycle_millis
             .store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
     }
 
-    pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
-        self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
-        if recovery_required {
-            self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
-        }
-        self.scanner_cycle_last_progress_age_seconds
-            .store(progress_age.as_secs(), Ordering::Relaxed);
-        self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
-        metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
-        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
-        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
-    }
-
     pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
         if let Some(concurrency_limit) = concurrency_limit {
             self.scanner_set_scan_concurrency_limit
@@ -3301,10 +3265,6 @@ impl Metrics {
         m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
         m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
         m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
-        m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
-        m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
-        m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
-        m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
         m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
         m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
         m.scan_checkpoint = match self.scanner_checkpoint.lock() {
@@ -4966,20 +4926,4 @@ mod tests {
         assert!(!report.bitrot_cycle_enabled);
         assert_eq!(report.bitrot_cycle_seconds, 0.0);
     }
-
-    #[tokio::test]
-    async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
-        let metrics = Metrics::new();
-        metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
-        let timed_out = metrics.report().await;
-        assert_eq!(timed_out.cycle_timeout_total, 1);
-        assert_eq!(timed_out.cycle_last_progress_age, 17);
-        assert!(timed_out.leader_lease_without_progress);
-
-        metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
-        let current = metrics.report().await;
-        assert_eq!(current.cycle_timeout_total, 1);
-        assert_eq!(current.cycle_last_progress_age, 0);
-        assert!(!current.leader_lease_without_progress);
-    }
 }

crates/config/README.md

@@ -84,12 +84,6 @@ Current guidance:
 - `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
 
-Scanner cycle budget controls:
-
-- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
-- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
-- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted.
-
 ## Mmap read environment aliases
 
 - `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical)

crates/config/src/constants/scanner.rs

@@ -143,12 +143,9 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
 /// Default scanner speed preset.
 pub const DEFAULT_SCANNER_SPEED: &str = "default";
 
-/// Default scanner cycle runtime budget when no override is configured.
-///
-/// An explicit `0` remains the compatibility escape hatch for an unbounded
-/// cycle. Keeping the unset default finite prevents a stalled scanner I/O
-/// operation from holding the leader lease forever.
-pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60;
+/// Default scanner cycle runtime budget.
+/// `0` keeps the existing unbounded per-cycle behavior.
+pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0;
 
 /// Default scanner per-cycle object budget.
 /// `0` keeps the existing unbounded per-cycle behavior.

crates/ecstore/src/api/mod.rs

@@ -317,8 +317,6 @@ pub mod config {
 }
 
 pub mod data_usage {
-    #[cfg(feature = "test-util")]
-    pub use crate::data_usage::seed_bucket_usage_memory_for_test;
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
         init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
@@ -330,6 +328,8 @@ pub mod data_usage {
         remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
         store_data_usage_in_backend,
     };
+    #[cfg(feature = "test-util")]
+    pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
 }
 
 pub mod disk {

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -866,7 +866,7 @@ impl BucketTargetSys {
             return Some(cli);
         }
 
-        // TODO: spawn a task to reload the target
+        // TODO(backlog): spawn an async task to proactively reload the replication target
         if self.is_reloading_target(bucket, arn).await {
             return None;
         }

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info(
         .map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
     let mut rstate = oi.replication_state();
     rstate.replicate_decision_str = dsc.to_string();
-    let asz = oi.get_actual_size().unwrap_or_default();
+    let asz = oi.get_actual_size_or_physical();
     let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
     let checksum = if ssec { oi.checksum.clone() } else { None };
 

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -1412,7 +1412,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
     };
     let mut replication_state = oi.replication_state();
     replication_state.replicate_decision_str = dsc.to_string();
-    let actual_size = oi.get_actual_size().unwrap_or_default();
+    let actual_size = oi.get_actual_size_or_physical();
 
     Ok(ReplicateObjectInfo {
         name: oi.name.clone(),

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
             .map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
         version_id: object_info.version_id.map(|version_id| version_id.to_string()),
         etag: object_info.etag.as_deref(),
-        actual_size: object_info.get_actual_size().unwrap_or_default(),
+        actual_size: object_info.get_actual_size_or_physical(),
         delete_marker: object_info.delete_marker,
         content_type: object_info.content_type.as_deref(),
         content_encoding: object_info.content_encoding.as_deref(),
@@ -542,6 +542,20 @@ mod tests {
         assert!(replication_target_head_is_newer_null_version(&source, &target));
     }
 
+    #[test]
+    fn replication_source_uses_physical_size_for_unknown_compressed_object() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
+        let source = ObjectInfo {
+            size: 128,
+            actual_size: -1,
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+
+        assert_eq!(replication_source_object(&source).actual_size, 128);
+    }
+
     #[test]
     fn replication_target_head_content_matches_compare_etag_only() {
         let source = ObjectInfo {

crates/ecstore/src/cluster/rpc/peer_s3_client.rs

@@ -454,7 +454,7 @@ impl S3PeerSys {
                 }
             }
             topology_complete &= bucket_map.values().all(|count| *count >= quorum);
-            // TODO: MRF
+            // TODO(backlog): integrate MRF backlog stats into scanner bucket listing
         }
 
         let mut buckets: Vec<BucketInfo> = result_map.into_values().collect();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -2406,7 +2406,7 @@ impl DiskAPI for RemoteDisk {
             return errors;
         }
 
-        // TODO: use Error not string
+        // TODO(backlog): replace string errors with typed `StorageError` variants
 
         let result = self
             .execute_with_timeout(

crates/ecstore/src/core/sets.rs

@@ -21,7 +21,7 @@ use crate::storage_api_contracts::{
     bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
     list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
     multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
-    object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
+    object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
     range::HTTPRangeSpec,
 };
 use crate::{
@@ -249,7 +249,7 @@ impl Sets {
 
         self.connect_disks().await;
 
-        // TODO: config interval
+        // TODO(backlog): make monitor_and_connect interval configurable instead of hardcoded 15s
         let mut interval = tokio::time::interval(Duration::from_secs(15));
         loop {
             tokio::select! {
@@ -414,6 +414,66 @@ fn apply_delete_objects_results(
     }
 }
 
+fn apply_delete_accounting_results(
+    accounting: &mut [Option<DeleteAccounting>],
+    set_objects: &[DelObj],
+    set_accounting: &[Option<DeleteAccounting>],
+) {
+    for (obj, value) in set_objects.iter().zip(set_accounting.iter()) {
+        accounting[obj.orig_idx] = value.clone();
+    }
+}
+
+impl Sets {
+    pub(crate) async fn delete_objects_with_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
+        let mut del_objects = vec![DeletedObject::default(); objects.len()];
+        let mut del_errs = vec![None; objects.len()];
+        let mut accounting = vec![None; objects.len()];
+        let mut set_obj_map = HashMap::new();
+
+        for (i, obj) in objects.iter().enumerate() {
+            let idx = self.get_hashed_set_index(obj.object_name.as_str());
+            set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj {
+                orig_idx: i,
+                obj: obj.clone(),
+            });
+        }
+
+        let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
+        let mut futures = FuturesUnordered::new();
+        let bucket = bucket.to_owned();
+
+        for (set_index, set_objects) in set_obj_map {
+            let disks = self.get_disks(set_index);
+            let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::<Vec<_>>();
+            let bucket = bucket.clone();
+            let opts = opts.clone();
+            let semaphore = semaphore.clone();
+            futures.push(async move {
+                let _permit = semaphore
+                    .acquire_owned()
+                    .await
+                    .expect("delete_objects semaphore should remain open");
+                let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await;
+                (set_objects, deleted, errors, accounting)
+            });
+        }
+
+        while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await {
+            apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors);
+            apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting);
+        }
+
+        (del_objects, del_errs, accounting)
+    }
+}
+
 #[async_trait::async_trait]
 impl crate::storage_api_contracts::object::ObjectIO for Sets {
     type Error = Error;
@@ -655,65 +715,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
         objects: Vec<ObjectToDelete>,
         opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
-        // Default return value
-        let mut del_objects = vec![DeletedObject::default(); objects.len()];
-
-        let mut del_errs = Vec::with_capacity(objects.len());
-        for _ in 0..objects.len() {
-            del_errs.push(None)
-        }
-
-        let mut set_obj_map = HashMap::new();
-
-        // hash key
-        for (i, obj) in objects.iter().enumerate() {
-            let idx = self.get_hashed_set_index(obj.object_name.as_str());
-
-            if !set_obj_map.contains_key(&idx) {
-                set_obj_map.insert(
-                    idx,
-                    vec![DelObj {
-                        // set_idx: idx,
-                        orig_idx: i,
-                        obj: obj.clone(),
-                    }],
-                );
-            } else if let Some(val) = set_obj_map.get_mut(&idx) {
-                val.push(DelObj {
-                    // set_idx: idx,
-                    orig_idx: i,
-                    obj: obj.clone(),
-                });
-            }
-        }
-
-        let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
-        let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
-        let mut futures = FuturesUnordered::new();
-        let bucket = bucket.to_string();
-
-        for (k, v) in set_obj_map {
-            let disks = self.get_disks(k);
-            let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
-            let bucket = bucket.clone();
-            let opts = opts.clone();
-            let semaphore = semaphore.clone();
-
-            futures.push(async move {
-                let _permit = semaphore
-                    .acquire_owned()
-                    .await
-                    .expect("delete_objects semaphore should remain open");
-                let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await;
-                (v, dobjects, errs)
-            });
-        }
-
-        while let Some((v, dobjects, errs)) = futures.next().await {
-            apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs);
-        }
-
-        (del_objects, del_errs)
+        let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
+        (deleted, errors)
     }
 
     #[tracing::instrument(skip(self))]

crates/ecstore/src/data_usage/mod.rs

@@ -1391,7 +1391,37 @@ impl BucketUsageAccumulator {
 }
 
 pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
-    let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
+    // A compressed object may carry -1 while the transformed size is unknown
+    // (legacy streaming sentinel). In that case the persisted physical size
+    // is still a valid accounting floor; every other negative value is corrupt.
+    // An explicit negative `actual-size` metadata value is corrupt, however:
+    // the sentinel is only valid in the in-memory/object-part field written by
+    // the legacy streaming path, not as a persisted declared size.
+    let compressed = object.is_compressed();
+    if object.actual_size < -1 || (object.actual_size == -1 && !compressed) {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    if object
+        .parts
+        .iter()
+        .any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
+    {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
+        .filter(|value| !value.is_empty());
+    if declared_actual_size
+        .as_deref()
+        .and_then(|value| value.parse::<i64>().ok())
+        .is_some_and(|size| size < 0)
+    {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    let logical_size = match object.get_actual_size().map_err(Error::other)? {
+        size if size == -1 && compressed && declared_actual_size.is_none() => None,
+        size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?),
+        _ => return Err(Error::PartMissingOrCorrupt),
+    };
     let persisted_part_size = if object.parts.is_empty() {
         u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
     } else {
@@ -1399,12 +1429,8 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
             // Compressed streaming objects persist -1 when the transformed
             // part size is unknown. The physical part size remains a valid
             // quota floor; reject only non-negative values that overflow.
-            let actual_size = if part.actual_size < 0 {
-                if object.is_compressed() {
-                    0
-                } else {
-                    return Err(Error::PartMissingOrCorrupt);
-                }
+            let actual_size = if part.actual_size == -1 {
+                0
             } else {
                 u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
             };
@@ -1412,7 +1438,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
             total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
         })?
     };
-    Ok(logical_size.max(persisted_part_size))
+    Ok(logical_size.unwrap_or(0).max(persisted_part_size))
 }
 
 type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
@@ -3320,6 +3346,80 @@ mod tests {
         );
     }
 
+    #[test]
+    fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let object = ObjectInfo {
+            size: 400,
+            actual_size: -1,
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+
+        assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400);
+    }
+
+    #[test]
+    fn quota_object_size_rejects_compressed_part_sum_overflow() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let object = ObjectInfo {
+            size: 1,
+            user_defined: Arc::new(metadata),
+            parts: Arc::new(vec![
+                rustfs_filemeta::ObjectPartInfo {
+                    actual_size: i64::MAX,
+                    ..Default::default()
+                },
+                rustfs_filemeta::ObjectPartInfo {
+                    actual_size: 1,
+                    ..Default::default()
+                },
+            ]),
+            ..Default::default()
+        };
+
+        assert!(matches!(quota_object_size(&object), Err(Error::Io(_))));
+    }
+
+    #[test]
+    fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let corrupt_object = ObjectInfo {
+            size: 400,
+            actual_size: -2,
+            user_defined: Arc::new(metadata.clone()),
+            ..Default::default()
+        };
+        assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt)));
+
+        let corrupt_part = ObjectInfo {
+            size: 400,
+            user_defined: Arc::new(metadata),
+            parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
+                size: 400,
+                actual_size: -2,
+                ..Default::default()
+            }]),
+            ..Default::default()
+        };
+        assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt)));
+    }
+
     #[tokio::test]
     #[serial]
     async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {

crates/ecstore/src/disk/local.rs

@@ -5215,8 +5215,8 @@ impl LocalDisk {
 
         let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default());
 
-        // TODO: DIRECT support
-        // TODD: DiskInfo
+        // TODO(backlog): add O_DIRECT I/O support for performance-critical paths
+        // TODO(backlog): populate DiskInfo in constructor
         let mut disk = Self {
             root: root.clone(),
             publication_root,
@@ -5751,7 +5751,7 @@ impl LocalDisk {
 
         // return Ok(());
 
-        // TODO: async notifications for disk space checks and trash cleanup
+        // TODO(backlog): make disk space checks and trash cleanup event-driven instead of poll-based
 
         let trash_path = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
         // if let Some(parent) = trash_path.parent() {
@@ -5997,7 +5997,7 @@ impl LocalDisk {
 
     #[hotpath::measure(impl_type = "LocalDisk")]
     async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
-        // TODO: timeout support
+        // TODO(backlog): add configurable timeout for read_all_data operations
         let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
         Ok(data)
     }
@@ -6674,7 +6674,7 @@ impl LocalDisk {
             return Ok(());
         }
 
-        // TODO: add lock
+        // TODO(backlog): add directory listing lock to prevent concurrent enumeration
 
         let stall = opts.stall_timeout_duration();
 
@@ -8796,7 +8796,7 @@ impl DiskAPI for LocalDisk {
         Ok(entries)
     }
 
-    // FIXME: TODO: io.writer TODO cancel
+    // TODO(backlog): support io.writer cancellation and early termination in walk_dir
     #[tracing::instrument(level = "trace", skip_all)]
     async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
         self.wait_for_startup_cleanup().await;
@@ -9880,7 +9880,7 @@ impl DiskAPI for LocalDisk {
                 );
                 return Err(e);
             }
-            // TODO: health check
+            // TODO(backlog): add post-setup disk health verification
         }
         Ok(())
     }

crates/ecstore/src/layout/endpoints.rs

@@ -249,7 +249,7 @@ impl PoolEndpointList {
             endpoint.set_set_index(0);
             endpoint.set_disk_index(0);
 
-            // TODO Check for cross device mounts if any.
+            // TODO(backlog): check for cross-device mounts in single-drive setup
 
             return Ok(Self {
                 inner: vec![Endpoints::from(vec![endpoint])],
@@ -264,7 +264,7 @@ impl PoolEndpointList {
                 // Convert args to endpoints
                 let mut eps = Endpoints::try_from(set_layout.as_slice())?;
 
-                // TODO Check for cross device mounts if any.
+                // TODO(backlog): check for cross-device mounts in multi-pool setup
 
                 for (disk_idx, ep) in eps.as_mut().iter_mut().enumerate() {
                     ep.set_pool_index(pool_idx);

crates/ecstore/src/object_api/types.rs

@@ -689,6 +689,9 @@ impl ObjectInfo {
     }
 
     pub fn get_actual_size(&self) -> std::io::Result<i64> {
+        if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) {
+            return Err(std::io::Error::other("invalid negative actual size"));
+        }
         if self.actual_size > 0 {
             return Ok(self.actual_size);
         }
@@ -700,10 +703,25 @@ impl ObjectInfo {
                 let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
                 return Ok(size);
             }
-            let mut actual_size = 0;
-            self.parts.iter().for_each(|part| {
-                actual_size += part.actual_size;
-            });
+            if self.actual_size == -1 && self.parts.is_empty() {
+                return Ok(-1);
+            }
+            let mut actual_size = 0_i64;
+            let mut unknown = false;
+            for part in self.parts.iter() {
+                match part.actual_size {
+                    -1 => unknown = true,
+                    size if size >= 0 => {
+                        actual_size = actual_size
+                            .checked_add(size)
+                            .ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?;
+                    }
+                    _ => return Err(std::io::Error::other("invalid negative compressed part size")),
+                }
+            }
+            if unknown {
+                return Ok(-1);
+            }
             if actual_size == 0 && actual_size != self.size {
                 return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size)));
             }
@@ -718,6 +736,18 @@ impl ObjectInfo {
         Ok(self.size)
     }
 
+    /// Returns a non-negative size for client and replication boundaries.
+    ///
+    /// Compressed legacy metadata can retain the internal `-1` unknown-size
+    /// sentinel. Those boundaries cannot emit a negative length, so they use
+    /// the persisted physical size while quota accounting keeps the sentinel
+    /// distinction in [`crate::data_usage::quota_object_size`].
+    pub fn get_actual_size_or_physical(&self) -> i64 {
+        self.get_actual_size()
+            .map(|size| if size >= 0 { size } else { self.size.max(0) })
+            .unwrap_or_else(|_| self.size.max(0))
+    }
+
     pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
         let mut version_id = fi.version_id;
 
@@ -1091,7 +1121,7 @@ impl ObjectInfo {
                     }
                 };
 
-                // TODO:VersionPurgeStatus
+                // TODO(backlog): handle VersionPurgeStatus in object listing
                 let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
                 objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned));
 

crates/ecstore/src/services/metrics_realtime.rs

@@ -256,10 +256,6 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
         cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
         cycle_max_objects: metrics.cycle_max_objects,
         cycle_max_directories: metrics.cycle_max_directories,
-        cycle_timeout_total: metrics.cycle_timeout_total,
-        cycle_recovery_required_total: metrics.cycle_recovery_required_total,
-        cycle_last_progress_age: metrics.cycle_last_progress_age,
-        leader_lease_without_progress: metrics.leader_lease_without_progress,
         bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
         bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
         scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport {
@@ -615,10 +611,6 @@ mod test {
             current_started: chrono_to_jiff_timestamp(current_started),
             last_cycle_partial_source: "usage".to_string(),
             last_cycle_partial_source_code: 1,
-            cycle_timeout_total: 3,
-            cycle_recovery_required_total: 2,
-            cycle_last_progress_age: 17,
-            leader_lease_without_progress: true,
             partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
                 source: "usage".to_string(),
                 cycles: 2,
@@ -630,10 +622,6 @@ mod test {
         assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
         assert_eq!(scanner.last_cycle_partial_source, "usage");
         assert_eq!(scanner.last_cycle_partial_source_code, 1);
-        assert_eq!(scanner.cycle_timeout_total, 3);
-        assert_eq!(scanner.cycle_recovery_required_total, 2);
-        assert_eq!(scanner.cycle_last_progress_age, 17);
-        assert!(scanner.leader_lease_without_progress);
         let usage = scanner
             .partial_cycles_by_source
             .iter()

crates/ecstore/src/set_disk/mod.rs

@@ -97,7 +97,7 @@ use crate::storage_api_contracts::{
         CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo,
     },
     namespace::NamespaceLocking as _,
-    object::{DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
+    object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
     range::HTTPRangeSpec,
 };
 use crate::store::utils::is_reserved_or_invalid_bucket;

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -1575,7 +1575,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         let parts_metadata = vec![fi.clone(); disks.len()];
 
         if !user_defined.contains_key("content-type") {
-            // TODO: get content-type
+            // TODO(backlog): detect content-type from part data when header is missing
         }
 
         if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
@@ -1971,7 +1971,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
             }
 
-            // TODO: crypto
+            // TODO(backlog): integrate encryption verification during complete multipart
 
             if (i < uploaded_parts.len() - 1)
                 && !(opts.data_movement && ext_part.actual_size < 0)

crates/ecstore/src/set_disk/ops/object.rs

@@ -45,6 +45,7 @@ use crate::bucket::replication::{
     DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType,
     replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta,
 };
+use crate::data_usage::quota_object_size;
 use crate::diagnostics::get::GetObjectFailureReason;
 use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
 use crate::error::is_err_invalid_upload_id;
@@ -5655,7 +5656,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         objects: Vec<ObjectToDelete>,
         opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
+        let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
+        (deleted, errors)
+    }
+
+    async fn delete_objects_with_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
+        let mut accounting = vec![None; objects.len()];
         let delete_config_snapshot = opts
             .delete_replication_config_snapshot
             .clone()
@@ -5745,7 +5757,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 *item = Some(Error::other(message.clone()));
                             }
                         }
-                        return (del_objects, del_errs);
+                        return (del_objects, del_errs, accounting);
                     }
                 },
             }
@@ -5792,6 +5804,22 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             let source_missing = gerr
                 .as_ref()
                 .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err));
+            // Resolve accounting from the generation selected under this
+            // object's write lock. A request-layer pre-stat is only an
+            // optimization and cannot identify a concurrent overwrite.
+            let (accounting_size, accounting_version_id, removed_current_object) = if source_missing
+                || dobj.synthetic_version_id
+                || set_disk_delete_creates_delete_marker(&check_opts)
+                || goi.delete_marker
+            {
+                (None, None, false)
+            } else {
+                (
+                    quota_object_size(&goi).ok(),
+                    goi.version_id.filter(|version_id| !version_id.is_nil()),
+                    (dobj.version_id.is_none() || is_explicit_null_version(dobj.version_id)) && !dobj.synthetic_version_id,
+                )
+            };
             // Normalize both sides before comparing. `goi.version_id` is the
             // client-facing identity, where `from_file_info` synthesizes
             // `Some(Uuid::nil())` for a null version on a versioned or
@@ -5920,7 +5948,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     },
                     replication_state: vr.replication_state_internal.clone(),
                     ..Default::default()
-                }
+                };
+                accounting[i] = Some(DeleteAccounting {
+                    size: accounting_size,
+                    version_id: accounting_version_id,
+                    removed_current_object,
+                });
             }
 
             // Only add to vers_map if we hold the lock
@@ -5966,7 +5999,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     });
                 }
             }
-            return (del_objects, del_errs);
+            return (del_objects, del_errs, accounting);
         }
 
         let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len());
@@ -6161,7 +6194,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         join_all(rollback_futures).await;
 
-        // TODO: add_partial
+        // TODO(backlog): support partial object deletion for multi-part objects
 
         if let Some(api) = opts.tier_delete_journal_api.as_ref() {
             for (idx, je) in persisted_journal_entries {
@@ -6204,7 +6237,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
-        (del_objects, del_errs)
+        // An accounting identity is actionable only when the delete result is
+        // successful. Never let a failed commit (including a partial quorum
+        // failure) reach the request-layer fast delta path.
+        for (index, err) in del_errs.iter().enumerate() {
+            if err.is_some() {
+                accounting[index] = None;
+            }
+        }
+
+        (del_objects, del_errs, accounting)
     }
 
     #[tracing::instrument(skip(self))]
@@ -6371,7 +6413,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
-        // TODO: Lifecycle
+        // TODO(backlog): integrate lifecycle evaluation before object deletion
 
         let mut version_found = true;
         // delete_object_version below derives its own majority quorum from the
@@ -6465,7 +6507,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 mark_deleted: mark_delete,
                 mod_time: Some(mod_time),
                 replication_state_internal: opts.delete_replication.as_ref().map(replication_state_to_filemeta),
-                ..Default::default() // TODO: Transition
+                ..Default::default() // TODO(backlog): populate transition state on delete markers
             };
 
             fi.set_tier_free_version_id(&find_vid.to_string());
@@ -6533,6 +6575,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended);
         obj_info.size = goi.size;
+        // Keep the committed source metadata on the internal delete result so
+        // the request layer can derive canonical accounting for this exact
+        // generation. Delete responses do not expose these fields.
+        obj_info.actual_size = goi.actual_size;
+        obj_info.user_defined = Arc::clone(&goi.user_defined);
+        obj_info.parts = Arc::clone(&goi.parts);
         obj_info.user_tags = Arc::clone(&goi.user_tags);
         self.invalidate_get_object_metadata_cache(bucket, object).await;
         Ok(obj_info)
@@ -7824,6 +7872,113 @@ mod replication_quota_safety_tests {
         assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
     }
 
+    #[tokio::test]
+    async fn delete_returns_canonical_compressed_accounting_size() {
+        let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "compressed-delete-accounting";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let mut user_defined = HashMap::new();
+        insert_str(
+            &mut user_defined,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
+        let mut reader = PutObjReader::new(
+            HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
+                .expect("compressed fixture reader should be valid"),
+        );
+        set_disks
+            .put_object(
+                bucket,
+                "object",
+                &mut reader,
+                &ObjectOptions {
+                    user_defined,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("compressed object should be written");
+
+        let (deleted, errors, accounting) = set_disks
+            .delete_objects_with_accounting(
+                bucket,
+                vec![ObjectToDelete {
+                    object_name: "object".to_string(),
+                    ..Default::default()
+                }],
+                ObjectOptions {
+                    object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
+                        ObjectLockConfigState::ConfirmedAbsent,
+                    ))),
+                    ..Default::default()
+                },
+            )
+            .await;
+
+        assert!(errors[0].is_none(), "compressed delete should succeed: {:?}", errors[0]);
+        assert!(deleted[0].found, "the committed object must be reported as found");
+        assert_eq!(accounting[0].as_ref().and_then(|value| value.size), Some(1000));
+        assert!(accounting[0].as_ref().is_some_and(|value| value.version_id.is_none()));
+        assert!(accounting[0].as_ref().is_some_and(|value| value.removed_current_object));
+    }
+
+    #[tokio::test]
+    async fn suspended_delete_marker_does_not_return_body_accounting() {
+        let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "suspended-delete-accounting";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let mut user_defined = HashMap::new();
+        insert_str(
+            &mut user_defined,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
+        let mut reader = PutObjReader::new(
+            HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
+                .expect("compressed fixture reader should be valid"),
+        );
+        let suspended_opts = ObjectOptions {
+            version_suspended: true,
+            delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
+                s3s::dto::VersioningConfiguration {
+                    status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
+                    ..Default::default()
+                },
+                None,
+            ))),
+            user_defined,
+            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
+            ..Default::default()
+        };
+        set_disks
+            .put_object(bucket, "object", &mut reader, &suspended_opts)
+            .await
+            .expect("compressed object should be written");
+
+        let (deleted, errors, accounting) = set_disks
+            .delete_objects_with_accounting(
+                bucket,
+                vec![ObjectToDelete {
+                    object_name: "object".to_string(),
+                    ..Default::default()
+                }],
+                suspended_opts,
+            )
+            .await;
+        assert!(errors[0].is_none(), "suspended delete should create a marker: {:?}", errors[0]);
+        assert!(deleted[0].delete_marker);
+        assert!(accounting[0].is_none(), "a delete marker must not carry body accounting");
+    }
+
     #[tokio::test]
     async fn direct_put_cannot_persist_a_tiny_logical_size() {
         let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;

crates/ecstore/src/storage_api_contracts/mod.rs

@@ -62,8 +62,8 @@ pub(crate) mod object {
     use super::{Debug, Error, FileInfo, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
     use crate::storage_api_contracts::range::HTTPRangeSpec;
     pub(crate) use rustfs_storage_api::{
-        DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions, ObjectOperations,
-        ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
+        DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions,
+        ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
     };
 
     pub(crate) trait EcstoreObjectIO:

crates/ecstore/src/store/bucket.rs

@@ -601,7 +601,7 @@ impl ECStore {
 
     #[instrument(skip(self))]
     pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
-        // TODO: opts.cached
+        // TODO(backlog): support cached bucket listing via opts.cached
 
         let mut buckets = self.peer_sys.list_bucket(opts).await?;
 

crates/ecstore/src/store/list_objects.rs

@@ -4673,7 +4673,7 @@ async fn gather_results(
             entry.name = entry.name.replace("\\", "/");
         }
 
-        // TODO: rx.recv()
+        // TODO(backlog): integrate rx.recv() for incremental listing results
 
         if let Some(marker) = &opts.marker
             && ((!opts.include_marker && &entry.name <= marker) || (opts.include_marker && &entry.name < marker))
@@ -4703,7 +4703,7 @@ async fn gather_results(
             continue;
         }
 
-        // TODO: Lifecycle
+        // TODO(backlog): integrate lifecycle evaluation during object listing
 
         entries.push(Some(entry));
         candidate_entries += 1;

crates/ecstore/src/store/multipart.rs

@@ -332,7 +332,7 @@ impl ECStore {
         let expected_incarnation_id = opts.expected_bucket_incarnation_id;
 
         if request.prefix.is_empty() {
-            // TODO: return from cache
+            // TODO(backlog): return cached multipart listing when prefix is empty
         }
 
         if self.single_pool() {
@@ -610,7 +610,7 @@ impl ECStore {
         let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
         let opts = &opts;
 
-        // TODO: defer DeleteUploadID
+        // TODO(backlog): defer DeleteUploadID to background for faster abort response
 
         if self.single_pool() {
             return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;

crates/ecstore/src/store/object.rs

@@ -41,7 +41,7 @@ use crate::set_disk::{
 };
 use crate::storage_api_contracts::{
     namespace::NamespaceLocking as _,
-    object::{ObjectIO as _, ObjectOperations as _},
+    object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _},
 };
 use parking_lot::Mutex as ParkingMutex;
 use rustfs_io_metrics::{
@@ -1216,6 +1216,14 @@ fn return_batch_delete_lock_error(objects: &[ObjectToDelete], err: Error) -> (Ve
     (del_objects, del_errs)
 }
 
+fn return_batch_delete_lock_error_with_accounting(
+    objects: &[ObjectToDelete],
+    err: Error,
+) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
+    let (deleted, errors) = return_batch_delete_lock_error(objects, err);
+    (deleted, errors, vec![None; objects.len()])
+}
+
 fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> {
     let mut object_names: Vec<&str> = objects.iter().map(|object| object.object_name.as_str()).collect();
     object_names.sort_unstable();
@@ -2312,6 +2320,22 @@ impl ECStore {
         result
     }
 
+    pub async fn delete_objects_with_tier_delete_journal_and_accounting(
+        self: &Arc<Self>,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
+        let result = self
+            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, Some(Arc::clone(self)))
+            .await;
+        let success_count = result.1.iter().filter(|err| err.is_none()).count();
+        if success_count > 0 {
+            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
+        }
+        result
+    }
+
     #[instrument(skip(self))]
     pub(super) async fn handle_delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
         self.handle_delete_object_with_journal(bucket, object, opts, None).await
@@ -2689,6 +2713,19 @@ impl ECStore {
         opts: ObjectOptions,
         tier_journal_api: Option<Arc<ECStore>>,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
+        let (deleted, errors, _) = self
+            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, tier_journal_api)
+            .await;
+        (deleted, errors)
+    }
+
+    pub(super) async fn handle_delete_objects_with_journal_and_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+        tier_journal_api: Option<Arc<ECStore>>,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
         // encode object name
         let objects: Vec<ObjectToDelete> = objects
             .iter()
@@ -2701,6 +2738,7 @@ impl ECStore {
 
         // Default return value
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
+        let mut accounting = vec![None; objects.len()];
 
         let mut del_errs = Vec::with_capacity(objects.len());
         for _ in 0..objects.len() {
@@ -2714,7 +2752,7 @@ impl ECStore {
         } else {
             match self.acquire_bucket_lifecycle_read_lock(bucket).await {
                 Ok(guard) => Some(guard),
-                Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
+                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
             }
         };
         if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
@@ -2726,21 +2764,21 @@ impl ECStore {
                 Err(err) => {
                     let message = err.to_string();
                     let errors = (0..objects.len()).map(|_| Some(Error::other(message.clone()))).collect();
-                    return (del_objects, errors);
+                    return (del_objects, errors, accounting);
                 }
             }
         }
         if !is_meta_bucketname(bucket)
             && let Err(err) = get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
         {
-            return return_batch_delete_lock_error(objects.as_slice(), err);
+            return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err);
         }
         let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
             None
         } else {
             Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
                 Ok(guard) => guard,
-                Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
+                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
             })
         };
         if let Some(guard) = _object_lock_metadata_guard.as_ref() {
@@ -2750,7 +2788,7 @@ impl ECStore {
             let (state, incarnation_id, config_revision) =
                 match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
                     Ok(snapshot) => snapshot,
-                    Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
+                    Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
                 };
             opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
                 self.id,
@@ -2766,7 +2804,10 @@ impl ECStore {
         if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
             && expected != current
         {
-            return return_batch_delete_lock_error(objects.as_slice(), StorageError::BucketNotFound(bucket.to_string()));
+            return return_batch_delete_lock_error_with_accounting(
+                objects.as_slice(),
+                StorageError::BucketNotFound(bucket.to_string()),
+            );
         }
         #[cfg(test)]
         if current_bucket_incarnation_id.is_some() {
@@ -2774,7 +2815,7 @@ impl ECStore {
         }
         let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
             Ok(guards) => guards,
-            Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
+            Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
         };
 
         let mut futures = Vec::with_capacity(self.pools.len());
@@ -2783,22 +2824,24 @@ impl ECStore {
             if self.is_pool_rebalancing(pool.pool_idx).await {
                 continue;
             }
-            futures.push(pool.delete_objects(bucket, objects.clone(), opts.clone()));
+            futures.push(pool.delete_objects_with_accounting(bucket, objects.clone(), opts.clone()));
         }
 
         let results = join_all(futures).await;
 
         for idx in 0..del_objects.len() {
-            for (dels, errs) in results.iter() {
+            for (dels, errs, pool_accounting) in results.iter() {
                 if errs[idx].is_none() && dels[idx].found {
                     del_errs[idx] = None;
                     del_objects[idx] = dels[idx].clone();
+                    accounting[idx] = pool_accounting[idx].clone();
                     break;
                 }
 
                 if del_errs[idx].is_none() {
                     del_errs[idx] = errs[idx].clone();
                     del_objects[idx] = dels[idx].clone();
+                    accounting[idx] = pool_accounting[idx].clone();
                 }
             }
         }
@@ -2807,7 +2850,7 @@ impl ECStore {
             v.object_name = decode_dir_object(&v.object_name);
         });
 
-        (del_objects, del_errs)
+        (del_objects, del_errs, accounting)
 
         // let mut futures = Vec::with_capacity(objects.len());
 

crates/ecstore/src/store/rebalance.rs

@@ -385,7 +385,7 @@ impl ECStore {
     }
 
     pub(super) async fn is_suspended(&self, idx: usize) -> bool {
-        // TODO: LOCK
+        // TODO(backlog): acquire pool metadata lock for consistent suspension check
 
         let pool_meta = self.pool_meta.read().await;
 

crates/madmin/src/metrics.rs

@@ -689,14 +689,6 @@ pub struct ScannerMetrics {
     pub cycle_max_objects: u64,
     #[serde(rename = "cycle_max_directories", default)]
     pub cycle_max_directories: u64,
-    #[serde(rename = "cycle_timeout_total", default)]
-    pub cycle_timeout_total: u64,
-    #[serde(rename = "cycle_recovery_required_total", default)]
-    pub cycle_recovery_required_total: u64,
-    #[serde(rename = "cycle_last_progress_age", default)]
-    pub cycle_last_progress_age: u64,
-    #[serde(rename = "leader_lease_without_progress", default)]
-    pub leader_lease_without_progress: bool,
     #[serde(rename = "bitrot_cycle_enabled", default)]
     pub bitrot_cycle_enabled: bool,
     #[serde(rename = "bitrot_cycle_seconds", default)]
@@ -772,8 +764,6 @@ impl ScannerMetrics {
             self.cycle_max_duration_seconds = other.cycle_max_duration_seconds;
             self.cycle_max_objects = other.cycle_max_objects;
             self.cycle_max_directories = other.cycle_max_directories;
-            self.cycle_last_progress_age = other.cycle_last_progress_age;
-            self.leader_lease_without_progress = other.leader_lease_without_progress;
             self.bitrot_cycle_enabled = other.bitrot_cycle_enabled;
             self.bitrot_cycle_seconds = other.bitrot_cycle_seconds;
         }
@@ -867,12 +857,6 @@ impl ScannerMetrics {
             .saturating_add(other.last_cycle_replication_checks);
         self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves);
         self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles);
-        self.cycle_timeout_total = self.cycle_timeout_total.saturating_add(other.cycle_timeout_total);
-        self.cycle_recovery_required_total = self
-            .cycle_recovery_required_total
-            .saturating_add(other.cycle_recovery_required_total);
-        self.cycle_last_progress_age = self.cycle_last_progress_age.max(other.cycle_last_progress_age);
-        self.leader_lease_without_progress |= other.leader_lease_without_progress;
         self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles);
         self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown);
         self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime);

crates/scanner/src/runtime_config.rs

@@ -125,10 +125,7 @@ impl Default for ScannerRuntimeConfig {
             cycle_interval_source: ScannerRuntimeConfigSource::Default,
             bitrot_cycle: Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS)),
             bitrot_cycle_source: ScannerRuntimeConfigSource::Default,
-            cycle_budget: ScannerCycleBudgetConfig {
-                max_duration: Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)),
-                ..Default::default()
-            },
+            cycle_budget: ScannerCycleBudgetConfig::default(),
             cycle_max_duration_source: ScannerRuntimeConfigSource::Default,
             cycle_max_objects_source: ScannerRuntimeConfigSource::Default,
             cycle_max_directories_source: ScannerRuntimeConfigSource::Default,
@@ -439,46 +436,19 @@ fn lookup_max_wait(
     Ok((speed.max_sleep(), speed_source))
 }
 
-fn lookup_cycle_duration(kvs: Option<&KVS>) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
-    match rustfs_utils::get_env_parse_outcome::<u64>(ENV_SCANNER_CYCLE_MAX_DURATION_SECS) {
-        rustfs_utils::EnvParseOutcome::Parsed(secs) => {
-            return cycle_duration_from_secs(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, secs)
-                .map(|duration| (duration, ScannerRuntimeConfigSource::Env));
-        }
-        rustfs_utils::EnvParseOutcome::Invalid => {
-            // Do not include the raw environment value in the typed error:
-            // deployments occasionally put sensitive material in inherited
-            // environment snapshots. The key still identifies the control.
-            return Err(invalid_value(
-                ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
-                "<invalid>",
-                "expected unsigned integer seconds",
-            ));
-        }
-        rustfs_utils::EnvParseOutcome::Absent => {}
+fn lookup_optional_seconds(
+    kvs: Option<&KVS>,
+    key: &'static str,
+    env_key: &'static str,
+    default: u64,
+) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
+    if let Some(secs) = rustfs_utils::get_env_opt_u64(env_key) {
+        return Ok((Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Env));
     }
-
-    if let Some(value) = config_value(kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) {
-        let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?;
-        return cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs)
-            .map(|duration| (duration, ScannerRuntimeConfigSource::Config));
+    if let Some(value) = config_value(kvs, key, default) {
+        return parse_config_u64(key, value).map(|secs| (Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Config));
     }
-
-    Ok((
-        Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)),
-        ScannerRuntimeConfigSource::Default,
-    ))
-}
-
-fn cycle_duration_from_secs(key: &'static str, secs: u64) -> Result<Option<Duration>, ScannerRuntimeConfigError> {
-    if secs == 0 {
-        return Ok(None);
-    }
-    let duration = Duration::from_secs(secs);
-    if std::time::Instant::now().checked_add(duration).is_none() {
-        return Err(invalid_value(key, "<overflow>", "duration exceeds the timer range"));
-    }
-    Ok(Some(duration))
+    Ok((None, ScannerRuntimeConfigSource::Default))
 }
 
 fn lookup_start_delay(kvs: Option<&KVS>) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
@@ -583,7 +553,12 @@ pub(crate) fn lookup_scanner_runtime_config(
         (speed.cycle_interval(), speed_source)
     };
 
-    let (cycle_max_duration, cycle_max_duration_source) = lookup_cycle_duration(scanner_kvs)?;
+    let (cycle_max_duration, cycle_max_duration_source) = lookup_optional_seconds(
+        scanner_kvs,
+        SCANNER_CYCLE_MAX_DURATION,
+        ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
+        DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS,
+    )?;
     let (cycle_max_objects, cycle_max_objects_source) = lookup_count_budget(
         scanner_kvs,
         SCANNER_CYCLE_MAX_OBJECTS,
@@ -888,10 +863,10 @@ mod tests {
     use rustfs_config::server_config::{Config as ServerConfig, KVS};
     use rustfs_config::{
         DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS,
-        ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY,
-        ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE,
-        SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION,
-        SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
+        ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED,
+        HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
+        SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
+        SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
     };
     use serial_test::serial;
     use std::collections::HashMap;
@@ -968,43 +943,6 @@ mod tests {
         });
     }
 
-    #[test]
-    #[serial]
-    fn scanner_unset_budget_uses_safe_default_but_explicit_zero_is_unbounded() {
-        let config = server_config_with_scanner(&[]);
-        with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
-            let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
-            assert_eq!(resolved.cycle_budget.max_duration, Some(Duration::from_secs(1800)));
-            assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Default);
-        });
-
-        let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "0")]);
-        with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
-            let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
-            assert_eq!(resolved.cycle_budget.max_duration, None);
-            assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Config);
-        });
-    }
-
-    #[test]
-    #[serial]
-    fn cycle_budget_invalid_or_overflow_config_is_rejected() {
-        with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("invalid"), || {
-            let error = lookup_scanner_runtime_config(None).expect_err("invalid duration env must be rejected");
-            assert!(error.to_string().contains(ENV_SCANNER_CYCLE_MAX_DURATION_SECS));
-            assert!(error.to_string().contains("<invalid>"));
-            assert!(!error.to_string().contains(": invalid ("));
-        });
-        with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551616"), || {
-            assert!(lookup_scanner_runtime_config(None).is_err());
-        });
-        with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551615"), || {
-            assert!(lookup_scanner_runtime_config(None).is_err());
-        });
-        let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "not-a-duration")]);
-        assert!(lookup_scanner_runtime_config(Some(&config)).is_err());
-    }
-
     #[test]
     #[serial]
     fn scanner_runtime_config_normalizes_persisted_default_speed() {

crates/scanner/src/scanner.rs

@@ -52,7 +52,6 @@ use rustfs_config::{
 };
 use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
 use rustfs_data_usage::observed_data_usage_is_newer;
-use rustfs_lock::NamespaceLockGuard;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest as _, Sha256};
 #[cfg(test)]
@@ -984,116 +983,20 @@ fn data_usage_persist_timeout() -> Duration {
     DataUsageCache::persistence_timeout()
 }
 
-#[cfg(not(test))]
-const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_secs(30);
-#[cfg(test)]
-const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_millis(50);
-
-async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
-    ctx: &CancellationToken,
-    storeapi: Arc<Store>,
-    cycle_info: &mut CurrentCycle,
-    cycle_revision: &mut DataUsageCacheRevision,
-    leader_epoch: &mut u64,
-    lock_lost: LockLost,
-) -> bool
-where
-    Store: ScannerObjectIO,
-    LockLost: Future<Output = ()>,
-{
-    let fence_ctx = ctx.child_token();
-    let claim = claim_scanner_leadership(&fence_ctx, storeapi, cycle_info, cycle_revision, leader_epoch);
-    tokio::pin!(claim);
-    tokio::pin!(lock_lost);
-    tokio::select! {
-        biased;
-        _ = &mut lock_lost => {
-            fence_ctx.cancel();
-            false
-        }
-        result = tokio::time::timeout(SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT, &mut claim) => {
-            result.unwrap_or(false) && !fence_ctx.is_cancelled()
-        }
-    }
-}
-
-struct ScannerCycleDeadlineState<'a> {
-    cycle_info: &'a mut CurrentCycle,
-    cycle_revision: &'a mut DataUsageCacheRevision,
-    leader_epoch: &'a mut u64,
-    cycle_budget: &'a ScannerCycleBudget,
-}
-
-fn cycle_timeout_requires_recovery(worker_stopped: bool, cycle_state_persisted: bool, generation_fenced: bool) -> bool {
-    !worker_stopped || !cycle_state_persisted || !generation_fenced
-}
-
-async fn handle_scanner_cycle_deadline<Store>(
-    ctx: &CancellationToken,
-    storeapi: Arc<Store>,
-    state: ScannerCycleDeadlineState<'_>,
-    worker_stopped: bool,
-    guard: &mut NamespaceLockGuard,
-) where
-    Store: ScannerObjectIO,
-{
-    let fenced = fence_scanner_epoch_after_cycle_timeout(
-        ctx,
-        storeapi,
-        state.cycle_info,
-        state.cycle_revision,
-        state.leader_epoch,
-        guard.lock_lost_notified(),
-    )
-    .await;
-    let cycle_state_persisted = state.cycle_budget.cycle_state_persisted();
-    let recovery_required = cycle_timeout_requires_recovery(worker_stopped, cycle_state_persisted, fenced);
-    warn!(
-        target: "rustfs::scanner",
-        event = EVENT_SCANNER_CYCLE_STATE,
-        component = LOG_COMPONENT_SCANNER,
-        subsystem = LOG_SUBSYSTEM_RUNTIME,
-        state = "cycle_timeout",
-        worker_stopped,
-        cycle_state_persisted,
-        generation_fenced = fenced,
-        recovery_required,
-        "Scanner cycle deadline expired; durable cursor/generation fencing completed when possible"
-    );
-    global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age());
-    // Stop renewing before releasing the lease. A new leader can then claim the
-    // higher persisted generation instead of inheriting the expired worker.
-    guard.release();
-    global_metrics().set_cycle(None).await;
-}
-
 async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
     cycle_info.current = 0;
     global_metrics().clear_current_scan_mode();
     cycle_metrics_guard.finish(cycle_info.clone()).await;
 }
 
-#[cfg(test)]
+#[instrument(skip_all)]
+#[hotpath::measure]
 async fn run_data_scanner_cycle(
     ctx: &CancellationToken,
     storeapi: &Arc<ECStore>,
     cycle_info: &mut CurrentCycle,
     cycle_revision: &mut DataUsageCacheRevision,
     leader_epoch: u64,
-) -> ScannerCycleOutcome {
-    let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config());
-    run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget).await
-}
-
-#[instrument(skip_all)]
-#[hotpath::measure]
-async fn run_data_scanner_cycle_with_budget(
-    ctx: &CancellationToken,
-    storeapi: &Arc<ECStore>,
-    cycle_info: &mut CurrentCycle,
-    cycle_revision: &mut DataUsageCacheRevision,
-    leader_epoch: u64,
-    cycle_budget: Arc<ScannerCycleBudget>,
 ) -> ScannerCycleOutcome {
     let _activity_guard = ScannerActivityGuard::new();
     if let Err(err) = refresh_scanner_runtime_config_from_global() {
@@ -1109,11 +1012,7 @@ async fn run_data_scanner_cycle_with_budget(
     }
     let configured_cycle_interval = scanner_cycle_interval();
     let configured_bitrot_cycle = scanner_bitrot_cycle();
-    let cycle_budget_config = ScannerCycleBudgetConfig {
-        max_duration: cycle_budget.max_duration(),
-        max_objects: cycle_budget.max_objects(),
-        max_directories: cycle_budget.max_directories(),
-    };
+    let cycle_budget_config = scanner_cycle_budget_config();
     let usage_persist_timeout = data_usage_persist_timeout();
     global_metrics().record_scanner_cycle_config(
         configured_cycle_interval,
@@ -1184,6 +1083,7 @@ async fn run_data_scanner_cycle_with_budget(
     let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
 
     let done_cycle = Metrics::time(Metric::ScanCycle);
+    let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
     let scan_result = storeapi
         .clone()
         .nsscanner_with_status(
@@ -1323,7 +1223,7 @@ async fn run_data_scanner_cycle_with_budget(
                 "Scanner cycle is recovering to a newer durable cache generation"
             );
             emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
-            let persisted = persist_required_scanner_cycle_floor(
+            return if persist_required_scanner_cycle_floor(
                 ctx,
                 storeapi.clone(),
                 cycle_info,
@@ -1332,9 +1232,8 @@ async fn run_data_scanner_cycle_with_budget(
                 required_cycle,
                 &mut cycle_metrics_guard,
             )
-            .await;
-            return if persisted {
-                cycle_budget.mark_cycle_state_persisted();
+            .await
+            {
                 ScannerCycleOutcome::Partial
             } else {
                 ScannerCycleOutcome::Failed
@@ -1392,7 +1291,7 @@ async fn run_data_scanner_cycle_with_budget(
             scan_cycle_partial_reason(budget_reason),
             scan_cycle_partial_source(budget_reason),
         );
-        let persisted = finalize_partial_scan_cycle(
+        return if finalize_partial_scan_cycle(
             ctx,
             storeapi.clone(),
             cycle_info,
@@ -1400,9 +1299,8 @@ async fn run_data_scanner_cycle_with_budget(
             leader_epoch,
             &mut cycle_metrics_guard,
         )
-        .await;
-        return if persisted {
-            cycle_budget.mark_cycle_state_persisted();
+        .await
+        {
             ScannerCycleOutcome::Partial
         } else {
             ScannerCycleOutcome::Failed
@@ -1477,7 +1375,7 @@ async fn run_data_scanner_cycle_with_budget(
                 );
             }
             emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
-            let persisted = finalize_partial_scan_cycle(
+            return if finalize_partial_scan_cycle(
                 ctx,
                 storeapi.clone(),
                 cycle_info,
@@ -1485,9 +1383,8 @@ async fn run_data_scanner_cycle_with_budget(
                 leader_epoch,
                 &mut cycle_metrics_guard,
             )
-            .await;
-            return if persisted {
-                cycle_budget.mark_cycle_state_persisted();
+            .await
+            {
                 ScannerCycleOutcome::Partial
             } else {
                 ScannerCycleOutcome::Failed
@@ -1528,7 +1425,6 @@ async fn run_data_scanner_cycle_with_budget(
             )
             .await
             {
-                cycle_budget.mark_cycle_state_persisted();
                 emit_scan_cycle_superseded(cycle_start.elapsed());
                 return ScannerCycleOutcome::Superseded;
             }
@@ -1561,7 +1457,6 @@ async fn run_data_scanner_cycle_with_budget(
         emit_scan_cycle_complete(false, cycle_start.elapsed());
         return ScannerCycleOutcome::Failed;
     }
-    cycle_budget.mark_cycle_state_persisted();
 
     done_cycle();
     emit_scan_cycle_complete(true, cycle_start.elapsed());
@@ -1626,7 +1521,7 @@ async fn run_data_scanner_with_maintenance_state(
 ) -> Result<(), ScannerError> {
     reset_scanner_cycle_schedule();
     // Acquire leader lock (write lock) to ensure only one scanner runs
-    let mut guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
+    let guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
         Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
             Ok(guard) => {
                 record_scanner_leader_lock_state("acquired");
@@ -1809,49 +1704,13 @@ async fn run_data_scanner_with_maintenance_state(
             return Ok(());
         }
         let cycle_ctx = ctx.child_token();
-        let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
-        let initial_outcome = match await_scanner_cycle_with_budget_fence(
+        let initial_outcome = await_scanner_cycle_with_lock_fence(
             &cycle_ctx,
-            &cycle_budget,
-            run_data_scanner_cycle_with_budget(
-                &cycle_ctx,
-                &storeapi,
-                &mut cycle_info,
-                &mut cycle_revision,
-                leader_epoch,
-                cycle_budget.clone(),
-            ),
+            run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch),
             guard.lock_lost_notified(),
         )
         .await
-        {
-            ScannerCycleWaitOutcome::Completed(outcome) => outcome,
-            ScannerCycleWaitOutcome::LockLost => {
-                record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await;
-                global_metrics().set_cycle(None).await;
-                return Ok(());
-            }
-            ScannerCycleWaitOutcome::Cancelled => {
-                global_metrics().set_cycle(None).await;
-                return Ok(());
-            }
-            ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
-                handle_scanner_cycle_deadline(
-                    &ctx,
-                    storeapi.clone(),
-                    ScannerCycleDeadlineState {
-                        cycle_info: &mut cycle_info,
-                        cycle_revision: &mut cycle_revision,
-                        leader_epoch: &mut leader_epoch,
-                        cycle_budget: &cycle_budget,
-                    },
-                    worker_stopped,
-                    &mut guard,
-                )
-                .await;
-                return Ok(());
-            }
-        };
+        .unwrap_or(ScannerCycleOutcome::Failed);
         superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded);
         deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_)));
         dirty_usage_generation_seen = dirty_generation_before_cycle;
@@ -2057,49 +1916,13 @@ async fn run_data_scanner_with_maintenance_state(
         }
         let dirty_generation_before_cycle = dirty_usage_generation();
         let cycle_ctx = ctx.child_token();
-        let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
-        let outcome = match await_scanner_cycle_with_budget_fence(
+        let outcome = await_scanner_cycle_with_lock_fence(
             &cycle_ctx,
-            &cycle_budget,
-            run_data_scanner_cycle_with_budget(
-                &cycle_ctx,
-                &storeapi,
-                &mut cycle_info,
-                &mut cycle_revision,
-                leader_epoch,
-                cycle_budget.clone(),
-            ),
+            run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch),
             guard.lock_lost_notified(),
         )
         .await
-        {
-            ScannerCycleWaitOutcome::Completed(outcome) => outcome,
-            ScannerCycleWaitOutcome::LockLost => {
-                record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await;
-                global_metrics().set_cycle(None).await;
-                return Ok(());
-            }
-            ScannerCycleWaitOutcome::Cancelled => {
-                global_metrics().set_cycle(None).await;
-                return Ok(());
-            }
-            ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
-                handle_scanner_cycle_deadline(
-                    &ctx,
-                    storeapi.clone(),
-                    ScannerCycleDeadlineState {
-                        cycle_info: &mut cycle_info,
-                        cycle_revision: &mut cycle_revision,
-                        leader_epoch: &mut leader_epoch,
-                        cycle_budget: &cycle_budget,
-                    },
-                    worker_stopped,
-                    &mut guard,
-                )
-                .await;
-                return Ok(());
-            }
-        };
+        .unwrap_or(ScannerCycleOutcome::Failed);
         superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded);
         deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_)));
         dirty_usage_generation_seen = dirty_generation_before_cycle;

crates/scanner/src/scanner/cycle_state.rs

@@ -496,63 +496,3 @@ where
         output = &mut cycle => Some(output),
     }
 }
-
-#[derive(Debug, PartialEq, Eq)]
-pub(super) enum ScannerCycleWaitOutcome<T> {
-    Completed(T),
-    LockLost,
-    Cancelled,
-    Deadline { worker_stopped: bool },
-}
-
-pub(super) async fn await_scanner_cycle_with_budget_fence<Cycle, LockLost>(
-    cycle_ctx: &CancellationToken,
-    budget: &ScannerCycleBudget,
-    cycle: Cycle,
-    lock_lost: LockLost,
-) -> ScannerCycleWaitOutcome<Cycle::Output>
-where
-    Cycle: Future,
-    LockLost: Future<Output = ()>,
-{
-    tokio::pin!(cycle);
-    tokio::pin!(lock_lost);
-    let deadline = async {
-        if let Some(deadline) = budget.deadline() {
-            tokio::time::sleep_until(deadline).await;
-        } else {
-            std::future::pending::<()>().await;
-        }
-    };
-    tokio::pin!(deadline);
-    tokio::select! {
-        biased;
-        _ = &mut lock_lost => {
-            cycle_ctx.cancel();
-            let _ = tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await;
-            ScannerCycleWaitOutcome::LockLost
-        }
-        _ = &mut deadline => {
-            budget.cancel_for_runtime();
-            // Let the budget cancellation reach the scanner first so it can
-            // persist a partial cursor. Only an uncooperative worker gets the
-            // parent cancellation, and it is dropped after the bounded window;
-            // the caller fences its epoch next.
-            let worker_stopped = if tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle)
-                .await
-                .is_ok()
-            {
-                true
-            } else {
-                cycle_ctx.cancel();
-                false
-            };
-            ScannerCycleWaitOutcome::Deadline { worker_stopped }
-        }
-        _ = cycle_ctx.cancelled() => {
-            let _ = tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await;
-            ScannerCycleWaitOutcome::Cancelled
-        }
-        output = &mut cycle => ScannerCycleWaitOutcome::Completed(output),
-    }
-}

crates/scanner/src/scanner/tests.rs

@@ -26,7 +26,6 @@ use std::task::Poll;
 use temp_env::{with_var, with_var_unset};
 use tokio::io::AsyncReadExt;
 use tokio::sync::Mutex;
-use tokio::time::{Duration, advance};
 
 const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
 
@@ -119,180 +118,6 @@ async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() {
     assert!(cycle_ctx.is_cancelled());
 }
 
-#[tokio::test(start_paused = true)]
-#[serial]
-async fn cycle_budget_fences_late_writer_after_timeout() {
-    let cycle_ctx = CancellationToken::new();
-    let budget = ScannerCycleBudget::new(
-        &cycle_ctx,
-        ScannerCycleBudgetConfig {
-            max_duration: Some(Duration::from_secs(5)),
-            ..Default::default()
-        },
-    );
-    let outcome = {
-        let cycle = std::future::pending::<()>();
-        let lock_lost = std::future::pending::<()>();
-        let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, cycle, lock_lost);
-        tokio::pin!(waiter);
-        tokio::task::yield_now().await;
-        advance(Duration::from_secs(5)).await;
-        tokio::task::yield_now().await;
-        advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await;
-        waiter.await
-    };
-    assert_eq!(outcome, ScannerCycleWaitOutcome::Deadline { worker_stopped: false });
-    assert!(cycle_ctx.is_cancelled());
-    assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Runtime));
-
-    // A newer leadership epoch is the durable fence that rejects a late
-    // writer after the timed-out future has been dropped.
-    let store = Arc::new(MemoryConfigStore::default());
-    let mut revision = DataUsageCacheRevision::Missing;
-    let mut cycle = CurrentCycle {
-        current: 0,
-        next: 12,
-        ..Default::default()
-    };
-    let persist_ctx = CancellationToken::new();
-    assert!(persist_scanner_cycle_state(&persist_ctx, store.clone(), &mut cycle, &mut revision, 1).await);
-    let newer = encode_scanner_cycle_state(&cycle, 2).expect("new epoch fence should encode");
-    let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
-    store.interleaving_puts.lock().await.insert(key, (2, newer));
-    let mut late_cycle = CurrentCycle { next: 13, ..cycle };
-    assert!(!persist_scanner_cycle_state(&persist_ctx, store, &mut late_cycle, &mut revision, 1).await);
-}
-
-#[tokio::test(start_paused = true)]
-async fn cycle_budget_parent_cancellation_is_not_reported_as_timeout() {
-    let cycle_ctx = CancellationToken::new();
-    let budget = ScannerCycleBudget::new(
-        &cycle_ctx,
-        ScannerCycleBudgetConfig {
-            max_duration: Some(Duration::from_secs(5)),
-            ..Default::default()
-        },
-    );
-    let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, std::future::pending::<()>(), std::future::pending());
-    tokio::pin!(waiter);
-    tokio::task::yield_now().await;
-    cycle_ctx.cancel();
-    tokio::task::yield_now().await;
-    advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await;
-    assert_eq!(waiter.await, ScannerCycleWaitOutcome::Cancelled);
-}
-
-#[tokio::test(start_paused = true)]
-async fn cycle_budget_deadline_wins_same_tick_as_parent_cancellation() {
-    let cycle_ctx = CancellationToken::new();
-    let budget = ScannerCycleBudget::new(
-        &cycle_ctx,
-        ScannerCycleBudgetConfig {
-            max_duration: Some(Duration::from_secs(5)),
-            ..Default::default()
-        },
-    );
-    let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, std::future::pending::<()>(), std::future::pending());
-    tokio::pin!(waiter);
-    tokio::task::yield_now().await;
-    advance(Duration::from_secs(5)).await;
-    cycle_ctx.cancel();
-    tokio::task::yield_now().await;
-    advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await;
-
-    assert_eq!(waiter.await, ScannerCycleWaitOutcome::Deadline { worker_stopped: false });
-    assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Runtime));
-}
-
-#[tokio::test]
-async fn cycle_budget_persist_cursor_failure_is_recovery_required() {
-    let store = Arc::new(MemoryConfigStore::default());
-    let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
-    store.fail_put_number.lock().await.insert(key, 1);
-
-    let ctx = CancellationToken::new();
-    let mut revision = DataUsageCacheRevision::Missing;
-    let mut cycle = CurrentCycle {
-        current: 12,
-        next: 12,
-        ..Default::default()
-    };
-    let mut leader_epoch = 1;
-    let fenced = fence_scanner_epoch_after_cycle_timeout(
-        &ctx,
-        store,
-        &mut cycle,
-        &mut revision,
-        &mut leader_epoch,
-        std::future::pending(),
-    )
-    .await;
-    assert!(!fenced, "a failed cursor/generation write must require recovery");
-    let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
-    assert!(cycle_timeout_requires_recovery(true, budget.cycle_state_persisted(), fenced));
-
-    let metrics = Metrics::new();
-    metrics.record_scanner_cycle_timeout(!fenced, Duration::from_secs(17));
-    let report = metrics.report().await;
-    assert_eq!(report.cycle_timeout_total, 1);
-    assert_eq!(report.cycle_recovery_required_total, 1);
-    assert_eq!(report.cycle_last_progress_age, 17);
-    assert!(report.leader_lease_without_progress);
-}
-
-#[tokio::test]
-#[serial]
-async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
-    let (_temp_dir, store) = setup_scanner_cycle_store().await;
-    let lock = store
-        .new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
-        .await
-        .expect("scanner leader lock should be created");
-    let mut guard = lock
-        .get_write_lock(Duration::from_secs(1))
-        .await
-        .expect("scanner leader lock should be acquired");
-
-    let ctx = CancellationToken::new();
-    let mut cycle_info = CurrentCycle {
-        current: 12,
-        next: 12,
-        ..Default::default()
-    };
-    let mut cycle_revision = DataUsageCacheRevision::Missing;
-    let mut leader_epoch = 1;
-    let budget = ScannerCycleBudget::new(
-        &ctx,
-        ScannerCycleBudgetConfig {
-            max_duration: Some(Duration::from_secs(60)),
-            ..Default::default()
-        },
-    );
-    budget.mark_cycle_state_persisted();
-
-    handle_scanner_cycle_deadline(
-        &ctx,
-        store.clone(),
-        ScannerCycleDeadlineState {
-            cycle_info: &mut cycle_info,
-            cycle_revision: &mut cycle_revision,
-            leader_epoch: &mut leader_epoch,
-            cycle_budget: &budget,
-        },
-        true,
-        &mut guard,
-    )
-    .await;
-
-    assert!(guard.is_released());
-    let persisted = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
-        .await
-        .expect("deadline handler should persist a fenced cursor");
-    let (_, persisted_epoch) = decode_scanner_cycle_state(&persisted).expect("fenced cursor should decode");
-    assert_eq!(persisted_epoch, 2);
-    global_metrics().set_cycle(None).await;
-}
-
 struct ScannerDefaultSpeedGuard;
 
 impl ScannerDefaultSpeedGuard {
@@ -591,6 +416,14 @@ fn test_scanner_cycle_max_duration_uses_env() {
     });
 }
 
+#[test]
+#[serial]
+fn test_scanner_cycle_max_duration_default_is_disabled() {
+    with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
+        assert_eq!(scanner_cycle_max_duration(), None);
+    });
+}
+
 #[tokio::test]
 async fn test_scanner_cycle_budget_cancels_after_duration() {
     let parent = CancellationToken::new();
@@ -1523,7 +1356,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() {
 }
 
 #[tokio::test]
-async fn cycle_budget_lease_takeover_rejects_old_generation() {
+async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() {
     let store = Arc::new(MemoryConfigStore::default());
     let ctx = CancellationToken::new();
     let mut revision = DataUsageCacheRevision::Missing;
@@ -1568,17 +1401,12 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() {
         .await
     );
 
-    let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH)
+    let state = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
         .await
         .expect("replacement leadership claim should persist");
     let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("replacement cycle state should decode");
     assert_eq!(claimed_cycle.next, 14);
     assert_eq!(claimed_epoch, 2);
-
-    let mut stale_cycle = CurrentCycle { next: 15, ..cycle };
-    let mut stale_revision = DataUsageCacheRevision::Etag("memory-2".to_string());
-    let stale_ctx = CancellationToken::new();
-    assert!(!persist_scanner_cycle_state(&stale_ctx, store, &mut stale_cycle, &mut stale_revision, 1,).await);
 }
 
 #[tokio::test]

crates/scanner/src/scanner_budget.rs

@@ -14,16 +14,17 @@
 
 use std::sync::{
     Arc,
-    atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
+    atomic::{AtomicU8, AtomicU64, Ordering},
 };
-use tokio::time::{Duration, Instant};
+use std::time::Instant;
+
+use tokio::time::Duration;
 use tokio_util::sync::CancellationToken;
 
 const BUDGET_REASON_NONE: u8 = 0;
 const BUDGET_REASON_RUNTIME: u8 = 1;
 const BUDGET_REASON_OBJECTS: u8 = 2;
 const BUDGET_REASON_DIRECTORIES: u8 = 3;
-const PROGRESS_CLOCK_SAMPLE_INTERVAL: u64 = 128;
 
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 pub(crate) struct ScannerCycleBudgetConfig {
@@ -62,51 +63,29 @@ pub struct ScannerCycleBudget {
     token: CancellationToken,
     reason: Arc<AtomicU8>,
     started_at: Instant,
-    deadline: Option<Instant>,
     max_duration: Option<Duration>,
     max_objects: Option<u64>,
     max_directories: Option<u64>,
     track_progress: bool,
-    track_unbounded_counts: bool,
     objects_scanned: AtomicU64,
     directories_started: AtomicU64,
     entries_visited: AtomicU64,
-    last_progress_millis: AtomicU64,
-    cycle_state_persisted: AtomicBool,
 }
 
 impl ScannerCycleBudget {
-    #[cfg(test)]
     pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
-        Self::new_inner(parent, config, false, false)
+        Self::new_inner(parent, config, false)
     }
 
     pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
-        Self::new_inner(parent, config, true, true)
+        Self::new_inner(parent, config, true)
     }
 
-    pub(crate) fn new_with_runtime_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
-        let track_progress = config.max_duration.is_some();
-        Self::new_inner(parent, config, track_progress, false)
-    }
-
-    fn new_inner(
-        parent: &CancellationToken,
-        config: ScannerCycleBudgetConfig,
-        track_progress: bool,
-        track_unbounded_counts: bool,
-    ) -> Arc<Self> {
+    fn new_inner(parent: &CancellationToken, config: ScannerCycleBudgetConfig, track_progress: bool) -> Arc<Self> {
         let token = parent.child_token();
         let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE));
-        let started_at = Instant::now();
-        let deadline = config.max_duration.map(|duration| match started_at.checked_add(duration) {
-            Some(deadline) => deadline,
-            // Runtime config rejects this range, but keep programmatic callers
-            // fail-closed instead of panicking or silently disabling the wall clock.
-            None => started_at,
-        });
 
-        if let Some(deadline) = deadline {
+        if let Some(duration) = config.max_duration {
             let parent = parent.clone();
             let token_wait = token.clone();
             let token_cancel = token.clone();
@@ -115,7 +94,7 @@ impl ScannerCycleBudget {
                 tokio::select! {
                     _ = parent.cancelled() => {}
                     _ = token_wait.cancelled() => {}
-                    _ = tokio::time::sleep_until(deadline) => {
+                    _ = tokio::time::sleep(duration) => {
                         Self::cancel_for_reason(&reason, &token_cancel, ScannerCycleBudgetReason::Runtime);
                     }
                 }
@@ -125,18 +104,14 @@ impl ScannerCycleBudget {
         Arc::new(Self {
             token,
             reason,
-            started_at,
-            deadline,
+            started_at: Instant::now(),
             max_duration: config.max_duration,
             max_objects: config.max_objects,
             max_directories: config.max_directories,
             track_progress,
-            track_unbounded_counts,
             objects_scanned: AtomicU64::new(0),
             directories_started: AtomicU64::new(0),
             entries_visited: AtomicU64::new(0),
-            last_progress_millis: AtomicU64::new(0),
-            cycle_state_persisted: AtomicBool::new(false),
         })
     }
 
@@ -156,14 +131,6 @@ impl ScannerCycleBudget {
         self.max_duration
     }
 
-    pub(crate) fn deadline(&self) -> Option<Instant> {
-        self.deadline
-    }
-
-    pub(crate) fn cancel_for_runtime(&self) {
-        self.cancel_for(ScannerCycleBudgetReason::Runtime);
-    }
-
     pub(crate) fn max_objects(&self) -> Option<u64> {
         self.max_objects
     }
@@ -206,43 +173,15 @@ impl ScannerCycleBudget {
         self.entries_visited.load(Ordering::Relaxed)
     }
 
-    pub(crate) fn mark_cycle_state_persisted(&self) {
-        self.cycle_state_persisted.store(true, Ordering::Release);
-    }
-
-    pub(crate) fn cycle_state_persisted(&self) -> bool {
-        self.cycle_state_persisted.load(Ordering::Acquire)
-    }
-
-    pub(crate) fn progress_age(&self) -> Duration {
-        let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
-        let last_progress = self.last_progress_millis.load(Ordering::Relaxed);
-        Duration::from_millis(elapsed_millis.saturating_sub(last_progress))
-    }
-
-    fn record_progress_sample(&self, event: u64) {
-        // Clock reads are sampled at batch/count boundaries; the scanner's
-        // per-object path does not add a second progress atomic.
-        if event == 0 || (event != 1 && !event.is_multiple_of(PROGRESS_CLOCK_SAMPLE_INTERVAL)) {
-            return;
-        }
-        let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
-        self.last_progress_millis.store(elapsed_millis, Ordering::Relaxed);
-    }
-
     pub(crate) fn record_entries_visited(&self, entries_visited: u64) {
         if self.track_progress {
-            let entries = saturating_fetch_add(&self.entries_visited, entries_visited);
-            self.record_progress_sample(entries);
+            saturating_fetch_add(&self.entries_visited, entries_visited);
         }
     }
 
     pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) {
         if self.track_progress || self.max_objects.is_some() {
             let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned);
-            if self.track_progress {
-                self.record_progress_sample(objects);
-            }
             if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
                 self.cancel_for(ScannerCycleBudgetReason::Objects);
             }
@@ -250,9 +189,6 @@ impl ScannerCycleBudget {
 
         if self.track_progress || self.max_directories.is_some() {
             let directories = saturating_fetch_add(&self.directories_started, directories_started);
-            if self.track_progress {
-                self.record_progress_sample(directories);
-            }
             if self
                 .max_directories
                 .is_some_and(|max_directories| directories > max_directories)
@@ -271,14 +207,11 @@ impl ScannerCycleBudget {
     }
 
     pub(crate) fn try_start_directory(&self) -> bool {
-        if self.max_directories.is_none() && !self.track_unbounded_counts {
+        if !self.track_progress && self.max_directories.is_none() {
             return true;
         }
 
         let directories = saturating_fetch_add(&self.directories_started, 1);
-        if self.track_progress {
-            self.record_progress_sample(directories);
-        }
         if self
             .max_directories
             .is_some_and(|max_directories| directories > max_directories)
@@ -291,14 +224,11 @@ impl ScannerCycleBudget {
     }
 
     pub(crate) fn record_object_scanned(&self) {
-        if self.max_objects.is_none() && !self.track_unbounded_counts {
+        if !self.track_progress && self.max_objects.is_none() {
             return;
         }
 
         let objects = saturating_fetch_add(&self.objects_scanned, 1);
-        if self.track_progress {
-            self.record_progress_sample(objects);
-        }
         if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
             self.cancel_for(ScannerCycleBudgetReason::Objects);
         }
@@ -531,29 +461,4 @@ mod tests {
         assert!(object_limited.requires_serial_progress_accounting());
         assert!(directory_limited.requires_serial_progress_accounting());
     }
-
-    #[tokio::test(start_paused = true)]
-    async fn progress_age_uses_virtual_time_and_sampled_progress() {
-        let parent = CancellationToken::new();
-        let budget = ScannerCycleBudget::new_with_runtime_progress_tracking(
-            &parent,
-            ScannerCycleBudgetConfig {
-                max_duration: Some(Duration::from_secs(60)),
-                ..Default::default()
-            },
-        );
-
-        tokio::time::advance(Duration::from_secs(5)).await;
-        assert_eq!(budget.progress_age(), Duration::from_secs(5));
-        budget.record_entries_visited(1);
-        assert_eq!(budget.progress_age(), Duration::ZERO);
-
-        tokio::time::advance(Duration::from_secs(2)).await;
-        for _ in 0..126 {
-            budget.record_entries_visited(1);
-        }
-        assert_eq!(budget.progress_age(), Duration::from_secs(2));
-        budget.record_entries_visited(1);
-        assert_eq!(budget.progress_age(), Duration::ZERO);
-    }
 }

crates/storage-api/src/lib.rs

@@ -76,6 +76,7 @@ pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOption
 pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
 pub use error::{StorageErrorCode, StorageResult};
 pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
+pub use object::DeleteAccounting;
 pub use object::ObjectLockDeleteOptions;
 pub use object::{DeletedObject, ObjectToDelete};
 pub use object::{ExpirationOptions, TransitionedObject};

crates/storage-api/src/object.rs

@@ -218,6 +218,17 @@ pub struct DeletedObject {
     pub force_delete_generation: Option<i64>,
 }
 
+/// Accounting identity returned by the internal commit-time delete path.
+///
+/// This is carried separately from [`DeletedObject`] so adding quota details
+/// does not change the source shape of the public S3 delete result contract.
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct DeleteAccounting {
+    pub size: Option<u64>,
+    pub version_id: Option<Uuid>,
+    pub removed_current_object: bool,
+}
+
 impl DeletedObject {
     pub fn version_purge_status(&self) -> VersionPurgeStatusType {
         self.replication_state
@@ -341,6 +352,19 @@ pub trait ObjectOperations: Send + Sync + fmt::Debug {
         objects: Vec<Self::ObjectToDelete>,
         opts: Self::ObjectOptions,
     ) -> (Vec<Self::DeletedObject>, Vec<Option<Self::Error>>);
+    /// Delete objects and optionally return commit-time accounting identities.
+    /// The default preserves the ordinary delete contract for implementations
+    /// that do not expose storage-level accounting details.
+    async fn delete_objects_with_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<Self::ObjectToDelete>,
+        opts: Self::ObjectOptions,
+    ) -> (Vec<Self::DeletedObject>, Vec<Option<Self::Error>>, Vec<Option<DeleteAccounting>>) {
+        let object_count = objects.len();
+        let (deleted, errors) = self.delete_objects(bucket, objects, opts).await;
+        (deleted, errors, vec![None; object_count])
+    }
     async fn put_object_metadata(
         &self,
         bucket: &str,

crates/utils/src/envs.rs

@@ -268,7 +268,7 @@ where
         .parse::<T>()
         .map_err(|_| {
             log_once(&format!("env_invalid_value:{used_key}"), || {
-                format!("Invalid {} value for {used_key}. Treating as unset.", type_name::<T>())
+                format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::<T>())
             });
         })
         .ok()
@@ -570,7 +570,7 @@ where
         Ok(parsed) => EnvParseOutcome::Parsed(parsed),
         Err(_) => {
             log_once(&format!("env_invalid_value:{used_key}"), || {
-                format!("Invalid {} value for {used_key}. Treating as unset.", type_name::<T>())
+                format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::<T>())
             });
             EnvParseOutcome::Invalid
         }

docs/operations/scanner-runtime-controls.md

@@ -52,7 +52,7 @@ The `/v3/scanner/status` response reports each effective runtime value with a
 | `scanner.max_wait` | `RUSTFS_SCANNER_MAX_WAIT_SECS` | seconds | preset-derived | Caps one scanner sleep. |
 | `scanner.cycle` | `RUSTFS_SCANNER_CYCLE` | seconds | preset-derived | Sets the interval between scanner cycles. |
 | `scanner.start_delay` | `RUSTFS_SCANNER_START_DELAY_SECS` | seconds | unset | Sets startup delay and, for compatibility, the cycle interval when `scanner.cycle` is unset. |
-| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `1800` | Caps one cycle's runtime. An explicit `0` disables this budget. |
+| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `0` | Caps one cycle's runtime. `0` disables this budget. |
 | `scanner.cycle_max_objects` | `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` | objects | `0` | Caps objects processed by one cycle. `0` disables this budget. |
 | `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` | Caps directories entered by one cycle. `0` disables this budget. |
 | `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. |
@@ -70,21 +70,6 @@ sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`,
 `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis
 needs a precise override.
 
-When the cycle duration control is unset, RustFS uses a finite 1800-second
-(30-minute) default, matching the scanner benchmark guidance. An explicit `0`
-preserves the compatibility behavior of an unbounded cycle; object and
-directory budgets likewise remain unbounded when explicitly set to `0`. Invalid
-or overflowing duration environment values are configuration errors rather than
-silent fallback values.
-
-When a finite deadline expires, RustFS cancels cooperative scanner work and
-waits only for the existing bounded shutdown window. A non-yielding I/O future
-is dropped after that window. RustFS then attempts a higher leadership epoch so
-late cycle, usage, cache, and remote writes from the old generation fail closed.
-If the worker cannot stop cooperatively, the cycle state was not confirmed
-durable, or that epoch fence cannot be durably persisted, the scanner reports
-`recovery-required`; it does not claim an uncooperative cursor was saved.
-
 An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle
 cadence: dirty-usage notifications do not bypass that configured interval.
 The default adaptive policy continues to use dirty-usage notifications to wake
@@ -159,10 +144,6 @@ metrics.maintenance_control.primary_control
 metrics.source_work
 metrics.replication_repair
 metrics.scan_checkpoint
-metrics.cycle_timeout_total
-metrics.cycle_last_progress_age
-metrics.leader_lease_without_progress
-metrics.cycle_recovery_required_total
 ```
 
 ## Reading Pacing Pressure

protocol/agent/v1/fixtures/auth/MANIFEST.sha256

@@ -1,5 +1,6 @@
 3d602080f7ca4c32ba9e37ad1a32665c78560726b30aeee08fd9e95eb2f36194  accept-vectors.json
 d3c19946288717088145592e0e8d6f2fa684443ba2f73d4c7bc49c415d6dd051  certificate-profile.json
-060485263c51003274c056a0e04bec1b7d76157cf599ba79eebe040bc7cee71b  error-codes.json
+299a2ae34a8ca74bcf31deeb53a08f9eff279efa09d3f5358a0cf10866fe1a5d  error-codes.json
 43fe297ffb512b1b9f4af62f1832f3aa3905157893bfdc3dcc6d56f5a98aaef6  reject-vectors.json
-b946175b094f4a8d75091b652fbe3d4327c9c795f28e02c96e1ab90a429e418d  surface-separation.json
+0cf26a7332fa6e3f57390e081f2cceead3236f6ca57f7038e3b55c2582af1733  rotation-proof.json
+7c07100460fa23fca482466df9ca22f91c7f26b36087a7207a001f7d987f527e  surface-separation.json

protocol/agent/v1/fixtures/auth/error-codes.json

@@ -2,7 +2,7 @@
   "protocolVersion": "v1",
   "fixtureSet": "auth",
   "fixture": "error-codes",
-  "description": "Frozen ErrorInfo reasons for agent authentication and negotiation. Clients branch on status and reason, never on message.",
+  "description": "Frozen ErrorInfo reasons for agent authentication, negotiation, and credential-rotation authorization. Clients branch on status and reason, never on message.",
   "domain": "rustfs.connect",
   "detailType": "type.googleapis.com/google.rpc.ErrorInfo",
   "disclosureRules": [
@@ -81,6 +81,24 @@
       "httpStatus": 401,
       "status": "UNAUTHENTICATED",
       "meaning": "A browser session cookie was presented to an authenticated agent operation. The agent surface never accepts it."
+    },
+    {
+      "reason": "ROTATION_CREDENTIAL_NOT_CURRENT",
+      "httpStatus": 409,
+      "status": "ABORTED",
+      "meaning": "The authenticated certificate is valid for ordinary agent operations but is not the device's current ACTIVE credential and therefore cannot authorize another rotation."
+    },
+    {
+      "reason": "ROTATION_REQUEST_CONFLICT",
+      "httpStatus": 409,
+      "status": "ABORTED",
+      "meaning": "The requestId already belongs to a rotation with different transcript inputs and cannot be reused."
+    },
+    {
+      "reason": "ROTATION_PROOF_INVALID",
+      "httpStatus": 401,
+      "status": "UNAUTHENTICATED",
+      "meaning": "The credential proof does not verify over the canonical rotation transcript under the presented certificate key."
     }
   ]
 }

protocol/agent/v1/fixtures/auth/rotation-proof.json

@@ -0,0 +1,306 @@
+{
+  "protocolVersion": "v1",
+  "fixtureSet": "auth",
+  "fixture": "rotation-proof",
+  "description": "Frozen proof-of-possession transcript for rotating an online device credential. The current certificate key authorizes one new certificate request for one device and one idempotent request.",
+  "operation": {
+    "method": "POST",
+    "path": "/agent/clusterDevices/{device}:rotateCredential",
+    "operationId": "rotateClusterDeviceCredential",
+    "authenticatedSurface": "/agent/*",
+    "currentCredentialRequiredForNewRotation": true,
+    "outgoingOverlapCredentialMayAuthenticateOrdinaryOperations": true,
+    "outgoingOverlapCredentialMayRotate": false,
+    "outgoingOverlapCredentialMayReplayItsCompletedRotation": true
+  },
+  "replayPolicy": {
+    "recordState": "COMPLETED",
+    "bindingFields": [
+      "currentCertificateFingerprint",
+      "clusterDeviceName",
+      "requestId",
+      "certificateRequestSha256"
+    ],
+    "exactMatch": "returnStoredResult",
+    "mismatchReason": "ROTATION_REQUEST_CONFLICT",
+    "retentionLowerBound": "outgoingCredential.validUntil",
+    "sideEffects": "No issuer call, credential write, or overlap extension."
+  },
+  "completedReplayRecord": {
+    "state": "COMPLETED",
+    "currentCertificateFingerprint": "1bc816e2d285c52ee3a0aa06dc7aec17e788626043e795259a67f2436f970d09",
+    "clusterDeviceName": "organizations/0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70/clusters/0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81/clusterDevices/0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92",
+    "requestId": "7c5d6e2a-8b91-4f03-a4d5-6e7f8091a2b3",
+    "certificateRequestSha256": "kVS-bXYxD6F22cZNy4Vnpgb5gFZbr4EGP5GvIz7uOaw",
+    "resultReference": "credential-rotation-result-01"
+  },
+  "transcript": {
+    "domain": "RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1",
+    "domainTerminator": "0x0a",
+    "fieldSeparator": "0x3a",
+    "fieldTerminator": "0x0a",
+    "fieldCount": 4,
+    "encoding": "US-ASCII",
+    "normalisationPermitted": false,
+    "fieldOrder": [
+      "currentCertificateFingerprint",
+      "clusterDeviceName",
+      "requestId",
+      "certificateRequestSha256"
+    ],
+    "fields": [
+      {
+        "name": "currentCertificateFingerprint",
+        "position": 1,
+        "source": "the exact certificate accepted by trusted ingress and resolved by Connect",
+        "pattern": "^[0-9a-f]{64}$",
+        "binds": "the one currently active credential authorizing the rotation",
+        "absenceWouldAllow": "A proof captured from one current certificate to authorize another credential after the device rotated."
+      },
+      {
+        "name": "clusterDeviceName",
+        "position": 2,
+        "source": "the authenticated identity after it is matched to the target resource",
+        "pattern": "^organizations/[0-9a-f-]{36}/clusters/[0-9a-f-]{36}/clusterDevices/[0-9a-f-]{36}$",
+        "binds": "the organization, cluster, and device being rotated",
+        "absenceWouldAllow": "A proof produced by one device to be presented against another device resource."
+      },
+      {
+        "name": "requestId",
+        "position": 3,
+        "source": "the request body",
+        "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$",
+        "binds": "the single idempotent rotation attempt",
+        "absenceWouldAllow": "A captured proof to be replayed as a fresh attempt instead of the same idempotent request."
+      },
+      {
+        "name": "certificateRequestSha256",
+        "position": 4,
+        "source": "recomputed over the exact PKCS#10 DER octets in certificateRequest",
+        "pattern": "^[A-Za-z0-9_-]{43}$",
+        "binds": "the next device public key and certificate request",
+        "absenceWouldAllow": "A valid proof to be combined with an attacker certificate request."
+      }
+    ]
+  },
+  "signature": {
+    "algorithmField": "proof.algorithm",
+    "algorithmEnumeration": ["ES256"],
+    "curve": "P-256",
+    "signatureEncoding": "fixed-width-r-s",
+    "signatureLengthBytes": 64,
+    "signatureTransferEncoding": "base64url-unpadded",
+    "signatureValuePattern": "^[A-Za-z0-9_-]{86}$",
+    "lowSRequired": true,
+    "verifyingKeySource": "the SubjectPublicKeyInfo of the exact presented certificate accepted by trusted ingress",
+    "newKeyPossession": "the PKCS#10 self-signature is verified separately under the public key inside certificateRequest",
+    "sharedEncodingContract": "protocol/agent/v1/registration-proof.md#the-signature"
+  },
+  "verificationOrder": [
+    {
+      "stage": "protocolVersion",
+      "rule": "protocolVersion must be the supported major version.",
+      "reason": "UNSUPPORTED_PROTOCOL"
+    },
+    {
+      "stage": "encoding",
+      "rule": "proof.algorithm and proof.value obey the shared ES256 fixed-width low-S contract.",
+      "reasons": ["UNSUPPORTED_ALGORITHM", "SIGNATURE_MALFORMED", "SIGNATURE_NOT_CANONICAL"]
+    },
+    {
+      "stage": "certificateRequest",
+      "rule": "certificateRequest is one self-signed PKCS#10 request whose public key is P-256.",
+      "reasons": ["CERTIFICATE_REQUEST_MALFORMED", "DEVICE_KEY_UNSUPPORTED"]
+    },
+    {
+      "stage": "proof",
+      "rule": "Connect rebuilds the transcript and verifies proof.value under the presented certificate public key.",
+      "reason": "ROTATION_PROOF_INVALID"
+    },
+    {
+      "stage": "replay",
+      "rule": "A completed record bound to the presented credential, device, requestId, and CSR digest returns its stored result without issuing again. A requestId bound to different transcript inputs is refused.",
+      "reason": "ROTATION_REQUEST_CONFLICT"
+    },
+    {
+      "stage": "credential",
+      "rule": "After replay lookup misses, the authenticated certificate must be the device current ACTIVE credential, not a credential in the outgoing overlap.",
+      "reason": "ROTATION_CREDENTIAL_NOT_CURRENT"
+    }
+  ],
+  "reasonSources": {
+    "auth": ["UNSUPPORTED_PROTOCOL", "ROTATION_CREDENTIAL_NOT_CURRENT", "ROTATION_REQUEST_CONFLICT", "ROTATION_PROOF_INVALID"],
+    "sharedFromRegistration": [
+      "UNSUPPORTED_ALGORITHM",
+      "SIGNATURE_MALFORMED",
+      "SIGNATURE_NOT_CANONICAL",
+      "CERTIFICATE_REQUEST_MALFORMED",
+      "DEVICE_KEY_UNSUPPORTED"
+    ]
+  },
+  "example": {
+    "inputs": {
+      "currentCertificateFingerprint": "1bc816e2d285c52ee3a0aa06dc7aec17e788626043e795259a67f2436f970d09",
+      "clusterDeviceName": "organizations/0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70/clusters/0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81/clusterDevices/0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92",
+      "requestId": "7c5d6e2a-8b91-4f03-a4d5-6e7f8091a2b3",
+      "certificateRequestSha256": "kVS-bXYxD6F22cZNy4Vnpgb5gFZbr4EGP5GvIz7uOaw"
+    },
+    "artifacts": {
+      "currentPublicKeySpki": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAENEpjuFZTqp0Hxh/OWV3TGkluNjCo15dk+4CozuR6aT9Vaxhkb2M9nhaVGfk8+aSiSIiKFSCsYonKl8jh743Qow==",
+      "currentPublicKeyFingerprint": "28608e223c75ed89e72041f12afae4fc1cd3f1d3bcca38cbb62d1a41687610c2",
+      "certificateRequest": "MIIBHjCBxAIBADBiMRswGQYDVQQDDBJpZ25vcmVkLWJ5LWNvbm5lY3QxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQWPosvIKr5V3CpTUdLfMQxDx31B3SglLKyqRg/oH3J+PhUnqf7pDWW1sTkP2aRIDjAwRn0DpLrz405CcvGHvYEoAAwCgYIKoZIzj0EAwIDSQAwRgIhAMJzXo/CK4E9BfjOxP35he9LLlqENhK7HTzZQTuIgLX2AiEAwOZHibk5HEijTWcJ/UT117nssfJesWZVOWwz/KTIpi8=",
+      "otherCertificateRequest": "MIIBHTCBxAIBADBiMRswGQYDVQQDDBJpZ25vcmVkLWJ5LWNvbm5lY3QxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATi0KiZGcbPJixDiQN/aIVky7xqaYhjmwAn1aozrC2eanaDENYQ3D5zw5qoZNIp8YUk/CETZrba3C5KYR5ocdlLoAAwCgYIKoZIzj0EAwIDSAAwRQIhAL0ZtvWyTSu18RF5J4ZVIuOGjJpJwSdP+87CVxNKJJduAiAahIruc3FLtO3RI7B8Ome8IsVDUpSAQilThjdOeDMsYg=="
+    },
+    "canonicalTranscript": "RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1\n64:1bc816e2d285c52ee3a0aa06dc7aec17e788626043e795259a67f2436f970d09\n148:organizations/0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70/clusters/0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81/clusterDevices/0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92\n36:7c5d6e2a-8b91-4f03-a4d5-6e7f8091a2b3\n43:kVS-bXYxD6F22cZNy4Vnpgb5gFZbr4EGP5GvIz7uOaw\n",
+    "canonicalTranscriptLengthBytes": 346,
+    "canonicalTranscriptSha256": "e5f0e9cd0d5d420bc7e51217512de76e1cf4dceb1d67804b6518c4fb5d9fe434"
+  },
+  "acceptVectors": [
+    {
+      "name": "current credential authorizes one new certificate request",
+      "requestId": "7c5d6e2a-8b91-4f03-a4d5-6e7f8091a2b3",
+      "proof": "vgz7Hm-xZ5JQ7B3oqUEKXB4tQxEzWZ988iPxAklaIZIsd4nSTUv5YriqES5yKGHvKOgWPyWjvrgI0j0T-fYEsQ",
+      "expected": {"accepted": true, "reason": null, "stage": "credential", "replayed": false}
+    },
+    {
+      "name": "the same certificate request under a new requestId needs its own proof",
+      "requestId": "82e1f3a4-9b05-4c67-8d90-1e2f3a4b5c6d",
+      "proof": "s-rkF9w41sqgdQNkOlhjcLYlrNHN_HL5xh4C3k_1a9ZQu7Uq0LoET_x7i4Rw3xQ75ec6B6tfqZ3XsHTS0FfN7Q",
+      "expected": {"accepted": true, "reason": null, "stage": "credential", "replayed": false}
+    },
+    {
+      "name": "outgoing credential replays its completed byte-equivalent rotation",
+      "requestId": "7c5d6e2a-8b91-4f03-a4d5-6e7f8091a2b3",
+      "proof": "vgz7Hm-xZ5JQ7B3oqUEKXB4tQxEzWZ988iPxAklaIZIsd4nSTUv5YriqES5yKGHvKOgWPyWjvrgI0j0T-fYEsQ",
+      "replayRecord": "completedExample",
+      "mutation": {"currentCredential": false},
+      "expected": {"accepted": true, "reason": null, "stage": "replay", "replayedResultReference": "credential-rotation-result-01"}
+    }
+  ],
+  "rejectVectors": [
+    {
+      "name": "outgoing overlap credential attempts another rotation",
+      "stage": "credential",
+      "requestId": "82e1f3a4-9b05-4c67-8d90-1e2f3a4b5c6d",
+      "proof": "s-rkF9w41sqgdQNkOlhjcLYlrNHN_HL5xh4C3k_1a9ZQu7Uq0LoET_x7i4Rw3xQ75ec6B6tfqZ3XsHTS0FfN7Q",
+      "mutation": {"currentCredential": false},
+      "expected": {"accepted": false, "reason": "ROTATION_CREDENTIAL_NOT_CURRENT"}
+    },
+    {
+      "name": "rotation declares protocol v2",
+      "stage": "protocolVersion",
+      "mutation": {"protocolVersion": "v2"},
+      "expected": {"accepted": false, "reason": "UNSUPPORTED_PROTOCOL"}
+    },
+    {
+      "name": "proof declares ES384",
+      "stage": "encoding",
+      "mutation": {"proofAlgorithm": "ES384"},
+      "expected": {"accepted": false, "reason": "UNSUPPORTED_ALGORITHM"}
+    },
+    {
+      "name": "DER encoded proof",
+      "stage": "encoding",
+      "mutation": {"proofValue": "MEUCIQC-DPseb7FnklDsHeipQQpcHi1DETNZn3zyI_ECSVohkgIgLHeJ0k1L-WK4qhEucihh7yjoFj8lo764CNI9E_n2BLE"},
+      "expected": {"accepted": false, "reason": "SIGNATURE_MALFORMED"}
+    },
+    {
+      "name": "padded base64url proof",
+      "stage": "encoding",
+      "mutation": {"proofValue": "vgz7Hm-xZ5JQ7B3oqUEKXB4tQxEzWZ988iPxAklaIZIsd4nSTUv5YriqES5yKGHvKOgWPyWjvrgI0j0T-fYEsQ=="},
+      "expected": {"accepted": false, "reason": "SIGNATURE_MALFORMED"}
+    },
+    {
+      "name": "protocol negotiation precedes proof encoding and stale credential state",
+      "stage": "protocolVersion",
+      "mutation": {"currentCredential": false, "protocolVersion": "v2", "proofValue": "not-a-signature"},
+      "expected": {"accepted": false, "reason": "UNSUPPORTED_PROTOCOL"}
+    },
+    {
+      "name": "declared algorithm precedes signature bytes",
+      "stage": "encoding",
+      "mutation": {"proofAlgorithm": "ES384", "proofValue": "not-a-signature"},
+      "expected": {"accepted": false, "reason": "UNSUPPORTED_ALGORITHM"}
+    },
+    {
+      "name": "malleated high-S proof",
+      "stage": "encoding",
+      "mutation": {"proofValue": "vgz7Hm-xZ5JQ7B3oqUEKXB4tQxEzWZ988iPxAklaIZLTiHYssrQGnkdV7tGN154Qk_7kboFz38zq542vAm0goA"},
+      "expected": {"accepted": false, "reason": "SIGNATURE_NOT_CANONICAL"}
+    },
+    {
+      "name": "proof over the rotation fields uses the registration domain",
+      "stage": "proof",
+      "mutation": {"proofValue": "as27GKXEKXtym-BU8NUl0BYhJkEUooZWPadrxwOWa40pAaN6p4VxKCCFggZl4ZQsl5CUtaMoMaxM_HzT1c1qUA"},
+      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
+    },
+    {
+      "name": "proof verification precedes stale credential state",
+      "stage": "proof",
+      "mutation": {"currentCredential": false, "proofValue": "as27GKXEKXtym-BU8NUl0BYhJkEUooZWPadrxwOWa40pAaN6p4VxKCCFggZl4ZQsl5CUtaMoMaxM_HzT1c1qUA"},
+      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
+    },
+    {
+      "name": "new certificate request key signs instead of the current credential key",
+      "stage": "proof",
+      "mutation": {"proofValue": "QEN8wMQQk5HjriDw2aeAxIfsRvrWgIjGR3KlpK81Y34kaTAacO-6_bOMAjQ0hMMIk-YYN6MDCw0Via_Ri0X0BA"},
+      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
+    },
+    {
+      "name": "proof is moved to another current certificate",
+      "stage": "proof",
+      "mutation": {"currentCertificateFingerprint": "2bc816e2d285c52ee3a0aa06dc7aec17e788626043e795259a67f2436f970d09"},
+      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
+    },
+    {
+      "name": "proof is moved to another device",
+      "stage": "proof",
+      "mutation": {"clusterDeviceName": "organizations/0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70/clusters/0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81/clusterDevices/0198f4b0-3c00-7e30-8f41-4a5b6c7d8e93"},
+      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
+    },
+    {
+      "name": "proof is replayed under another requestId",
+      "stage": "proof",
+      "mutation": {"requestId": "82e1f3a4-9b05-4c67-8d90-1e2f3a4b5c6d"},
+      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
+    },
+    {
+      "name": "proof is combined with another certificate request",
+      "stage": "proof",
+      "mutation": {"certificateRequest": "otherCertificateRequest"},
+      "expected": {"accepted": false, "reason": "ROTATION_PROOF_INVALID"}
+    },
+    {
+      "name": "completed replay belongs to another certificate fingerprint",
+      "stage": "replay",
+      "replayRecord": "completedExample",
+      "replayRecordMutation": {"currentCertificateFingerprint": "2bc816e2d285c52ee3a0aa06dc7aec17e788626043e795259a67f2436f970d09"},
+      "mutation": {"currentCredential": false},
+      "expected": {"accepted": false, "reason": "ROTATION_REQUEST_CONFLICT"}
+    },
+    {
+      "name": "completed replay belongs to another device",
+      "stage": "replay",
+      "replayRecord": "completedExample",
+      "replayRecordMutation": {"clusterDeviceName": "organizations/0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70/clusters/0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81/clusterDevices/0198f4b0-3c00-7e30-8f41-4a5b6c7d8e93"},
+      "mutation": {"currentCredential": false},
+      "expected": {"accepted": false, "reason": "ROTATION_REQUEST_CONFLICT"}
+    },
+    {
+      "name": "completed replay belongs to another requestId",
+      "stage": "replay",
+      "replayRecord": "completedExample",
+      "replayRecordMutation": {"requestId": "82e1f3a4-9b05-4c67-8d90-1e2f3a4b5c6d"},
+      "mutation": {"currentCredential": false},
+      "expected": {"accepted": false, "reason": "ROTATION_REQUEST_CONFLICT"}
+    },
+    {
+      "name": "completed replay belongs to another CSR digest",
+      "stage": "replay",
+      "replayRecord": "completedExample",
+      "replayRecordMutation": {"certificateRequestSha256": "PXxfNGWtIbrqVocag6lOLDDR41AjpHXJ7UcAWkbTSAs"},
+      "mutation": {"currentCredential": false},
+      "expected": {"accepted": false, "reason": "ROTATION_REQUEST_CONFLICT"}
+    }
+  ]
+}

protocol/agent/v1/fixtures/auth/surface-separation.json

@@ -49,7 +49,7 @@
       "schemeType": "mutualTLS",
       "forbiddenSecuritySchemes": ["sessionCookie"],
       "defaultSecurity": ["agentMutualTls"],
-      "publicOperations": ["getProtocolStatus"]
+      "publicOperations": ["getProtocolStatus", "exchangeRegistrationToken"]
     },
     {
       "document": "openapi/control.json",

protocol/agent/v1/fixtures/fixture-sets.json

@@ -11,7 +11,7 @@
     {
       "name": "auth",
       "status": "populated",
-      "purpose": "Client certificate profile, RFC 9440 header profile, authentication accept and reject vectors, surface separation, and the frozen error reason registry."
+      "purpose": "Client certificate profile, RFC 9440 header profile, authentication and credential-rotation proof vectors, surface separation, and the frozen error reason registry."
     },
     {
       "name": "version",

rustfs/Cargo.toml

@@ -288,6 +288,7 @@ zstd.workspace = true
 # Cryptography and Security
 rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types = { workspace = true }
+x509-parser = { workspace = true }
 subtle = { workspace = true }
 jiff = { workspace = true, features = ["serde"] }
 time = { workspace = true, features = ["parsing", "formatting", "serde", "macros"] }

rustfs/src/app/bucket_usecase.rs

@@ -1021,7 +1021,7 @@ fn build_list_objects_v2_metadata_output(
                 object: Object {
                     key: Some(encode_list_objects_v2_value(&object.name, encoding_type)),
                     last_modified: object.mod_time.map(Timestamp::from),
-                    size: Some(object.get_actual_size().unwrap_or_default()),
+                    size: Some(object.get_actual_size_or_physical()),
                     e_tag: object.etag.clone().map(|etag| to_s3s_etag(&etag)),
                     storage_class: Some(ObjectStorageClass::from(
                         object

rustfs/src/app/object_usecase.rs

@@ -3969,6 +3969,55 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool {
     opts.version_id.is_none() && opts.versioned && !opts.version_suspended
 }
 
+fn delete_removes_current_object(opts: &ObjectOptions) -> bool {
+    delete_request_targets_current(
+        opts.version_id
+            .as_deref()
+            .and_then(|version_id| Uuid::parse_str(version_id).ok()),
+    )
+}
+
+fn delete_request_targets_current(version_id: Option<Uuid>) -> bool {
+    version_id.is_none() || version_id.is_some_and(|version_id| version_id.is_nil())
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum DeleteMemoryUpdate {
+    DeleteMarker,
+    Object { size: u64, removed_current_object: bool },
+}
+
+fn delete_memory_update(
+    creates_delete_marker: bool,
+    committed_delete_marker: bool,
+    requested_current: bool,
+    accounting_size: Option<u64>,
+    removed_current_object: bool,
+) -> Option<DeleteMemoryUpdate> {
+    if creates_delete_marker || (committed_delete_marker && requested_current) {
+        return Some(DeleteMemoryUpdate::DeleteMarker);
+    }
+
+    (!committed_delete_marker)
+        .then_some(accounting_size)
+        .flatten()
+        .map(|size| DeleteMemoryUpdate::Object {
+            size,
+            removed_current_object,
+        })
+}
+
+async fn apply_delete_memory_update(bucket: &str, update: Option<DeleteMemoryUpdate>) {
+    match update {
+        Some(DeleteMemoryUpdate::DeleteMarker) => record_bucket_delete_marker_memory(bucket).await,
+        Some(DeleteMemoryUpdate::Object {
+            size,
+            removed_current_object,
+        }) => record_bucket_object_delete_memory(bucket, size, removed_current_object).await,
+        None => {}
+    }
+}
+
 /// `DeleteObjects` is idempotent. A raw filesystem `NotFound` can cross the
 /// distributed delete path instead of its usual typed missing-object error.
 fn is_delete_objects_not_found(error: &EcstoreError) -> bool {
@@ -8409,8 +8458,6 @@ impl DefaultObjectUsecase {
             object: ObjectToDelete,
             versioned: bool,
             version_suspended: bool,
-            size: i64,
-            existing: Option<ObjectInfo>,
         }
 
         // Phase 2 (bounded concurrency, backlog#929 / HP-8): collect the
@@ -8428,32 +8475,23 @@ impl DefaultObjectUsecase {
                     skip_stat,
                 } = prepared;
                 let synthetic_version_id = object.version_id.is_none() && is_dir_object(&object.object_name);
-                let (goi, source_missing) = if skip_stat {
-                    (ObjectInfo::default(), false)
-                } else {
+                if !skip_stat {
                     match store_ref.get_object_info(bucket_ref, &object.object_name, &opts).await {
-                        Ok(res) => (res, false),
-                        Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
-                            (ObjectInfo::default(), true)
-                        }
+                        Ok(_) => {}
+                        Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
                         Err(err) => return Err(ApiError::from(err)),
                     }
-                };
-
-                let size = goi.size;
+                }
 
                 if synthetic_version_id {
                     object.version_id = Some(Uuid::nil());
                 }
 
-                let existing = (!skip_stat && !source_missing).then_some(goi);
                 Ok::<_, ApiError>(AdmittedDelete {
                     idx,
                     object,
                     versioned: opts.versioned,
                     version_suspended: opts.version_suspended,
-                    size,
-                    existing,
                 })
             }))
             .buffered(DELETE_OBJECTS_PRE_STAT_CONCURRENCY)
@@ -8464,15 +8502,11 @@ impl DefaultObjectUsecase {
         // per-key success/failure reporting is unchanged.
         let mut object_to_delete = Vec::new();
         let mut object_to_delete_idx = Vec::new();
-        let mut object_sizes = Vec::new();
-        let mut existing_object_infos = Vec::new();
         let mut object_versioning = Vec::new();
         for admitted in admitted_deletes {
-            object_sizes.push(admitted.size);
             object_to_delete_idx.push(admitted.idx);
             object_versioning.push((admitted.versioned, admitted.version_suspended));
             object_to_delete.push(admitted.object);
-            existing_object_infos.push(admitted.existing);
         }
         let cache_adapter = self.object_data_cache();
         let cache_keys_before_delete = object_to_delete
@@ -8489,8 +8523,8 @@ impl DefaultObjectUsecase {
             ..Default::default()
         };
         apply_bucket_generation_guard(&req, &bucket, &mut storage_delete_opts)?;
-        let (dobjs, errs) = store
-            .delete_objects_with_tier_delete_journal(&bucket, object_to_delete.clone(), storage_delete_opts)
+        let (dobjs, errs, accounting) = store
+            .delete_objects_with_tier_delete_journal_and_accounting(&bucket, object_to_delete.clone(), storage_delete_opts)
             .await;
 
         let _manager = get_concurrency_manager();
@@ -8515,17 +8549,16 @@ impl DefaultObjectUsecase {
                     delete_results[didx].delete_object = Some(deleted_object.clone());
                     let (versioned, version_suspended) = object_versioning[i];
                     let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended;
-                    if creates_delete_marker {
-                        record_bucket_delete_marker_memory(&bucket).await;
-                    } else {
-                        let size = object_sizes[i].max(0) as u64;
-                        record_bucket_object_delete_memory(
-                            &bucket,
-                            size,
-                            existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
-                        )
-                        .await;
-                    }
+                    let committed_delete_marker = dobjs[i].delete_marker;
+                    let delete_accounting = accounting.get(i).and_then(Option::as_ref);
+                    let update = delete_memory_update(
+                        creates_delete_marker,
+                        committed_delete_marker,
+                        delete_request_targets_current(object_to_delete[i].version_id),
+                        delete_accounting.and_then(|value| value.size),
+                        delete_accounting.is_some_and(|value| value.removed_current_object),
+                    );
+                    apply_delete_memory_update(&bucket, update).await;
                 }
                 Err(error) => {
                     delete_results[didx].error = Some(error);
@@ -8803,12 +8836,24 @@ impl DefaultObjectUsecase {
             let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await;
         }
 
-        // Fast in-memory update for immediate quota and admin usage consistency
-        if delete_creates_delete_marker(&opts) {
-            record_bucket_delete_marker_memory(&bucket).await;
+        // Fast in-memory update for immediate quota and admin usage consistency.
+        // Prefix/force deletes and synthetic directory entries do not carry one
+        // committed object identity; leave their cache delta to reconciliation.
+        let update = if force_delete || obj_info.name.is_empty() || synthetic_version_id {
+            None
         } else {
-            record_bucket_object_delete_memory(&bucket, obj_info.size.max(0) as u64, opts.version_id.is_none()).await;
-        }
+            // The storage commit returns this object's metadata while its
+            // generation lock is held. Never fall back to a pre-delete stat:
+            // an overwrite can commit between that stat and this delete.
+            delete_memory_update(
+                delete_creates_delete_marker(&opts),
+                obj_info.delete_marker,
+                opts.version_id.is_none(),
+                quota_object_size(&obj_info).ok(),
+                delete_removes_current_object(&opts),
+            )
+        };
+        apply_delete_memory_update(&bucket, update).await;
 
         if obj_info.name.is_empty() {
             if let Some((operation_id, target_arns, generation)) = force_delete_intent {
@@ -17861,6 +17906,158 @@ mod tests {
         assert!(!can_skip_delete_objects_pre_stat(false, &delete_marker_creating_opts(), false));
     }
 
+    #[test]
+    fn delete_accounting_recognizes_explicit_null_as_current_object() {
+        let opts = ObjectOptions {
+            version_id: Some(Uuid::nil().to_string()),
+            version_suspended: true,
+            ..Default::default()
+        };
+        assert!(delete_removes_current_object(&opts));
+        assert!(delete_request_targets_current(Some(Uuid::nil())));
+        assert!(!delete_request_targets_current(Some(Uuid::new_v4())));
+        assert!(!delete_removes_current_object(&ObjectOptions {
+            version_id: Some(Uuid::new_v4().to_string()),
+            ..Default::default()
+        }));
+    }
+
+    #[test]
+    fn compressed_object_delete_restores_usage_baseline() {
+        let mut metadata = HashMap::new();
+        insert_str(&mut metadata, SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string());
+        let object = ObjectInfo {
+            size: 400,
+            actual_size: 1000,
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+        let accounting_size = quota_object_size(&object).expect("logical compressed size should be canonical");
+
+        assert_eq!(
+            delete_memory_update(false, false, true, Some(accounting_size), true),
+            Some(DeleteMemoryUpdate::Object {
+                size: 1000,
+                removed_current_object: true,
+            })
+        );
+    }
+
+    #[test]
+    fn invalid_accounting_metadata_is_reconciled_without_overflow() {
+        assert_eq!(delete_memory_update(false, false, true, None, true), None);
+        assert_eq!(
+            delete_memory_update(false, true, true, None, true),
+            Some(DeleteMemoryUpdate::DeleteMarker)
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn compressed_delete_requests_restore_usage_baseline() {
+        use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
+
+        let store = crate::app::gating_test_env::shared_gating_ecstore().await;
+        if current_app_context().is_none() {
+            crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
+        }
+        let bucket = format!("compressed-delete-request-{}", Uuid::new_v4().simple());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create compressed delete request bucket");
+
+        // Seed the process-local usage with the canonical logical bytes. The
+        // direct storage PUT below intentionally does not apply an app-layer
+        // usage delta; the two real DELETE requests must remove exactly this
+        // amount through their request-layer wiring.
+        crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 2_000).await;
+
+        for object in ["single", "batch"] {
+            let mut metadata = HashMap::new();
+            insert_str(&mut metadata, SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string());
+            insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, "1000".to_string());
+            let reader = HashReader::from_stream(std::io::Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
+                .expect("compressed fixture reader should be valid");
+            let mut reader = PutObjReader::new(reader);
+            store
+                .put_object(
+                    &bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        user_defined: metadata,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("compressed fixture object should be written");
+        }
+
+        let mut single_req = build_request(
+            DeleteObjectInput::builder()
+                .bucket(bucket.clone())
+                .key("single".to_string())
+                .build()
+                .expect("single delete input should build"),
+            Method::DELETE,
+        );
+        single_req.extensions.insert(crate::storage::access::ReqInfo {
+            cred: Some(rustfs_credentials::Credentials::default()),
+            is_owner: true,
+            ..Default::default()
+        });
+        DefaultObjectUsecase::from_global()
+            .execute_delete_object(single_req)
+            .await
+            .expect("single compressed delete should succeed");
+        assert_eq!(
+            crate::app::storage_api::test::data_usage::get_bucket_usage_memory(&bucket).await,
+            Some(1_000),
+            "single delete must subtract the logical accounting size"
+        );
+
+        let mut batch_req = build_request(
+            DeleteObjectsInput::builder()
+                .bucket(bucket.clone())
+                .delete(Delete {
+                    objects: vec![ObjectIdentifier {
+                        key: "batch".to_string(),
+                        ..Default::default()
+                    }],
+                    quiet: None,
+                })
+                .build()
+                .expect("batch delete input should build"),
+            Method::POST,
+        );
+        batch_req.extensions.insert(crate::storage::access::ReqInfo {
+            cred: Some(rustfs_credentials::Credentials::default()),
+            is_owner: true,
+            ..Default::default()
+        });
+        DefaultObjectUsecase::from_global()
+            .execute_delete_objects(batch_req)
+            .await
+            .expect("batch compressed delete should succeed");
+        assert_eq!(
+            crate::app::storage_api::test::data_usage::get_bucket_usage_memory(&bucket).await,
+            Some(0),
+            "batch delete must subtract the committed logical accounting size"
+        );
+
+        store
+            .delete_bucket(
+                &bucket,
+                &DeleteBucketOptions {
+                    force: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("clean up compressed delete request bucket");
+    }
+
     #[tokio::test]
     async fn execute_get_object_attributes_returns_internal_error_when_store_uninitialized() {
         let input = GetObjectAttributesInput::builder()

rustfs/src/app/storage_api.rs

@@ -72,6 +72,11 @@ pub(crate) mod data_usage {
         compute_bucket_usage, live_bucket_usage_computations, seed_bucket_usage_memory_for_test, store_data_usage_in_backend,
     };
 
+    #[cfg(test)]
+    pub(crate) async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
+        crate::storage::storage_api::ecstore_data_usage::get_bucket_usage_memory(bucket).await
+    }
+
     pub(crate) async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) {
         crate::storage::storage_api::ecstore_data_usage::record_bucket_object_delete_memory(
             bucket,
@@ -1233,7 +1238,10 @@ pub(crate) mod test {
 
     pub(crate) use super::access::ReqInfo;
     pub(crate) use super::options::VERSIONING_CONFIG_LOOKUPS;
-    pub(crate) use super::{bucket, data_usage, ecfs, object_utils, runtime};
+    pub(crate) use super::{bucket, ecfs, object_utils, runtime};
+    pub(crate) mod data_usage {
+        pub(crate) use super::super::data_usage::*;
+    }
     pub(crate) use crate::storage::storage_api::test_consumer::{get_global_bucket_metadata_sys, set_bucket_metadata};
     pub(crate) use crate::storage::storage_api::{
         ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader,

rustfs/src/connect/client.rs

@@ -0,0 +1,566 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+use base64::Engine as _;
+use reqwest::{Client, StatusCode, Url};
+use rustls::RootCertStore;
+use rustls::pki_types::{CertificateDer, pem::PemObject as _};
+use serde::Deserialize;
+use uuid::Uuid;
+use zeroize::Zeroizing;
+
+use super::credential_store::{
+    CompletedRegistration, CredentialStore, CredentialStoreError, DeviceCredential, PendingRegistration, PendingRotation,
+};
+use super::identity::{IdentityError, RegistrationTranscript};
+use super::identity_store::{IdentityStore, StoreError};
+use super::registration::{
+    CredentialResponse, CredentialValidationError, ExpectedDevice, RegistrationRequest, RegistrationToken, RotationRequest,
+    certificate_fingerprint, certificate_request_matches, public_key_fingerprint, validate_credential,
+    validate_stored_credential,
+};
+
+const MAX_ATTEMPTS: usize = 3;
+const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
+const ROTATION_THRESHOLD_SECONDS: i64 = 8 * 60 * 60;
+
+pub struct ConnectConfig<'a> {
+    pub endpoint: &'a str,
+    pub root_ca_pem: &'a [u8],
+    pub timeout: Duration,
+}
+
+pub struct ConnectClient {
+    endpoint: Url,
+    roots: RootCertStore,
+    root_certificates: Vec<CertificateDer<'static>>,
+    client: Client,
+    timeout: Duration,
+}
+
+impl ConnectClient {
+    pub fn from_optional_config(config: Option<ConnectConfig<'_>>) -> Result<Option<Self>, ClientError> {
+        config.map(Self::new).transpose()
+    }
+
+    pub fn new(config: ConnectConfig<'_>) -> Result<Self, ClientError> {
+        let mut endpoint = Url::parse(config.endpoint).map_err(|_| ClientError::Endpoint)?;
+        if endpoint.scheme() != "https"
+            || endpoint.cannot_be_a_base()
+            || !endpoint.username().is_empty()
+            || endpoint.password().is_some()
+            || endpoint.query().is_some()
+            || endpoint.fragment().is_some()
+        {
+            return Err(ClientError::Endpoint);
+        }
+        if !endpoint.path().ends_with('/') {
+            let path = format!("{}/", endpoint.path());
+            endpoint.set_path(&path);
+        }
+
+        let root_certificates = CertificateDer::pem_slice_iter(config.root_ca_pem)
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|_| ClientError::RootCertificate)?;
+        if root_certificates.is_empty() {
+            return Err(ClientError::RootCertificate);
+        }
+        let mut roots = RootCertStore::empty();
+        let (accepted, rejected) = roots.add_parsable_certificates(root_certificates.clone());
+        if accepted != root_certificates.len() || rejected != 0 {
+            return Err(ClientError::RootCertificate);
+        }
+
+        let client = build_client(&root_certificates, config.timeout, None)?;
+        Ok(Self {
+            endpoint,
+            roots,
+            root_certificates,
+            client,
+            timeout: config.timeout,
+        })
+    }
+
+    pub async fn register(
+        &self,
+        identity_store: &IdentityStore,
+        credential_store: &CredentialStore,
+        token: &RegistrationToken,
+    ) -> Result<DeviceCredential, ClientError> {
+        let _lock = credential_store.lock().await?;
+        if let Some((credential, _)) = self.load_valid_credential(identity_store, credential_store)? {
+            ensure_credential_time(&credential, unix_now())?;
+            return Ok(credential);
+        }
+
+        let identity = identity_store.load_or_create()?;
+        let candidate = PendingRegistration {
+            token_uid: token.registration_token_uid.clone(),
+            request_id: Uuid::new_v4().to_string(),
+            certificate_request: identity.certificate_request_base64()?,
+            previous_credential_fingerprint: None,
+            next_public_key_sha256: None,
+        };
+        let pending = credential_store.claim_pending_registration(&candidate)?;
+        if pending.token_uid != token.registration_token_uid
+            || pending.previous_credential_fingerprint.is_some()
+            || pending.next_public_key_sha256.is_some()
+            || !is_request_id(&pending.request_id)
+            || !certificate_request_matches(&pending.certificate_request, &identity)?
+        {
+            return Err(ClientError::PendingRegistration);
+        }
+
+        let credential = match self.exchange_registration(token, &pending, &identity).await {
+            Ok(credential) => credential,
+            Err(error @ (ClientError::AccessRevoked { .. } | ClientError::Rejected { .. })) => {
+                credential_store.clear_pending_registration()?;
+                return Err(error);
+            }
+            Err(error) => return Err(error),
+        };
+        credential_store.save(&credential)?;
+        credential_store.clear_pending_registration()?;
+        Ok(credential)
+    }
+
+    pub async fn reenroll(
+        &self,
+        identity_store: &IdentityStore,
+        credential_store: &CredentialStore,
+        token: &RegistrationToken,
+    ) -> Result<DeviceCredential, ClientError> {
+        let _lock = credential_store.lock().await?;
+        let (credential, _) = self
+            .load_valid_credential(identity_store, credential_store)?
+            .ok_or(ClientError::NotRegistered)?;
+        let fingerprint = certificate_fingerprint(&credential.certificate)?;
+        if credential_store.load_completed_registration()?.is_some_and(|completed| {
+            completed.token_uid == token.registration_token_uid && completed.credential_fingerprint == fingerprint
+        }) {
+            return Ok(credential);
+        }
+        credential_store.clear_pending_rotation()?;
+        let next = identity_store.load_or_create_next()?;
+        let next_fingerprint = public_key_fingerprint(&next);
+        let candidate = PendingRegistration {
+            token_uid: token.registration_token_uid.clone(),
+            request_id: Uuid::new_v4().to_string(),
+            certificate_request: next.certificate_request_base64()?,
+            previous_credential_fingerprint: Some(fingerprint.clone()),
+            next_public_key_sha256: Some(next_fingerprint.clone()),
+        };
+        let pending = credential_store.claim_pending_registration(&candidate)?;
+        if pending.token_uid != token.registration_token_uid
+            || pending.previous_credential_fingerprint.as_deref() != Some(&fingerprint)
+            || pending.next_public_key_sha256.as_deref() != Some(&next_fingerprint)
+            || !is_request_id(&pending.request_id)
+            || !certificate_request_matches(&pending.certificate_request, &next)?
+        {
+            return Err(ClientError::PendingRegistration);
+        }
+        let enrolled = match self.exchange_registration(token, &pending, &next).await {
+            Ok(credential) => credential,
+            Err(error @ (ClientError::AccessRevoked { .. } | ClientError::Rejected { .. })) => {
+                credential_store.clear_pending_registration()?;
+                identity_store.clear_next()?;
+                return Err(error);
+            }
+            Err(error) => return Err(error),
+        };
+        credential_store.save(&enrolled)?;
+        identity_store.commit_next(&next)?;
+        credential_store.save_completed_registration(&CompletedRegistration {
+            token_uid: token.registration_token_uid.clone(),
+            credential_fingerprint: certificate_fingerprint(&enrolled.certificate)?,
+        })?;
+        credential_store.clear_pending_registration()?;
+        Ok(enrolled)
+    }
+
+    async fn exchange_registration(
+        &self,
+        token: &RegistrationToken,
+        pending: &PendingRegistration,
+        identity: &super::identity::DeviceIdentity,
+    ) -> Result<DeviceCredential, ClientError> {
+        let csr_der = base64::engine::general_purpose::STANDARD
+            .decode(&pending.certificate_request)
+            .map_err(|_| ClientError::PendingRegistration)?;
+        let transcript = RegistrationTranscript::build(
+            &token.registration_token_uid,
+            &token.organization_uid,
+            &token.cluster_uid,
+            &pending.request_id,
+            &token.challenge_nonce,
+            token.expires_unix,
+            &csr_der,
+        )?;
+        let proof = identity.sign_registration(&transcript);
+        let body = RegistrationRequest::new(token, &pending.request_id, &pending.certificate_request, &proof);
+        let url = self.url("./registrationTokens:exchange")?;
+        let response = self
+            .send(StatusCode::CREATED, || self.client.post(url.clone()).json(&body))
+            .await?;
+        let cluster = format!("organizations/{}/clusters/{}", token.organization_uid, token.cluster_uid);
+        let credential = validate_credential(
+            response,
+            identity,
+            &self.roots,
+            &self.root_certificates,
+            ExpectedDevice::Registration { cluster: &cluster },
+        )?;
+        Ok(credential)
+    }
+
+    pub async fn rotate_if_due(
+        &self,
+        identity_store: &IdentityStore,
+        credential_store: &CredentialStore,
+        now_unix: i64,
+    ) -> Result<Option<DeviceCredential>, ClientError> {
+        let _lock = credential_store.lock().await?;
+        let (credential, identity) = self
+            .load_valid_credential(identity_store, credential_store)?
+            .ok_or(ClientError::NotRegistered)?;
+        if credential_store.load_pending_registration()?.is_some() {
+            return Err(ClientError::PendingRegistration);
+        }
+        ensure_credential_time(&credential, now_unix)?;
+        if credential.not_after_unix - now_unix > ROTATION_THRESHOLD_SECONDS {
+            return Ok(None);
+        }
+        let fingerprint = certificate_fingerprint(&credential.certificate)?;
+        let next = identity_store.load_or_create_next()?;
+        let candidate = PendingRotation {
+            credential_fingerprint: fingerprint.clone(),
+            device_name: credential.name.clone(),
+            request_id: Uuid::new_v4().to_string(),
+            certificate_request: next.certificate_request_base64()?,
+            next_public_key_sha256: public_key_fingerprint(&next),
+        };
+        let pending = credential_store.claim_pending_rotation(&candidate)?;
+        if pending.credential_fingerprint != fingerprint
+            || pending.device_name != credential.name
+            || !is_request_id(&pending.request_id)
+            || pending.next_public_key_sha256 != public_key_fingerprint(&next)
+            || !certificate_request_matches(&pending.certificate_request, &next)?
+        {
+            return Err(ClientError::PendingRotation);
+        }
+        let body = RotationRequest::new(
+            &identity,
+            &fingerprint,
+            &credential.name,
+            &pending.request_id,
+            &pending.certificate_request,
+        )?;
+        let private_key = identity.to_pkcs8_pem()?;
+        let mut identity_pem = Zeroizing::new(Vec::with_capacity(credential.certificate_chain.len() + private_key.len() + 1));
+        identity_pem.extend_from_slice(credential.certificate_chain.as_bytes());
+        identity_pem.push(b'\n');
+        identity_pem.extend_from_slice(private_key.as_bytes());
+        let tls_identity = reqwest::Identity::from_pem(&identity_pem).map_err(|_| ClientError::IdentityCertificate)?;
+        let client = build_client(&self.root_certificates, self.timeout, Some(tls_identity))?;
+        let path = format!("clusterDevices/{}:rotateCredential", credential.uid);
+        let url = self.url(&path)?;
+        let response = self.send(StatusCode::OK, || client.post(url.clone()).json(&body)).await?;
+        let rotated = validate_credential(
+            response,
+            &next,
+            &self.roots,
+            &self.root_certificates,
+            ExpectedDevice::Rotation { name: &credential.name },
+        )?;
+        if rotated.name != credential.name || rotated.uid != credential.uid {
+            return Err(ClientError::Credential(CredentialValidationError::Identity));
+        }
+        credential_store.save(&rotated)?;
+        identity_store.commit_next(&next)?;
+        credential_store.clear_pending_rotation()?;
+        Ok(Some(rotated))
+    }
+
+    fn load_valid_credential(
+        &self,
+        identity_store: &IdentityStore,
+        credential_store: &CredentialStore,
+    ) -> Result<Option<(DeviceCredential, super::identity::DeviceIdentity)>, ClientError> {
+        let Some(credential) = credential_store.load()? else {
+            return Ok(None);
+        };
+        let current = identity_store.load()?.ok_or(ClientError::IdentityMissing)?;
+        if let Some(pending) = credential_store.load_pending_registration()? {
+            let Some(previous) = pending.previous_credential_fingerprint.as_deref() else {
+                if pending.next_public_key_sha256.is_some()
+                    || !is_request_id(&pending.request_id)
+                    || !certificate_request_matches(&pending.certificate_request, &current)?
+                {
+                    return Err(ClientError::PendingRegistration);
+                }
+                validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
+                credential_store.clear_pending_registration()?;
+                return Ok(Some((credential, current)));
+            };
+            let next_fingerprint = pending
+                .next_public_key_sha256
+                .as_deref()
+                .ok_or(ClientError::PendingRegistration)?;
+            if !is_request_id(&pending.request_id) {
+                return Err(ClientError::PendingRegistration);
+            }
+            let fingerprint = certificate_fingerprint(&credential.certificate)?;
+            if fingerprint == previous {
+                validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
+                let next = identity_store.load_next()?.ok_or(ClientError::PendingRegistration)?;
+                if public_key_fingerprint(&next) != next_fingerprint
+                    || !certificate_request_matches(&pending.certificate_request, &next)?
+                {
+                    return Err(ClientError::PendingRegistration);
+                }
+                return Ok(Some((credential, current)));
+            }
+            if public_key_fingerprint(&current) == next_fingerprint {
+                if !certificate_request_matches(&pending.certificate_request, &current)? {
+                    return Err(ClientError::PendingRegistration);
+                }
+                validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
+            } else {
+                let next = identity_store.load_next()?.ok_or(ClientError::PendingRegistration)?;
+                if public_key_fingerprint(&next) != next_fingerprint
+                    || !certificate_request_matches(&pending.certificate_request, &next)?
+                {
+                    return Err(ClientError::PendingRegistration);
+                }
+                validate_stored_credential(&credential, &next, &self.roots, &self.root_certificates)?;
+                identity_store.commit_next(&next)?;
+            }
+            credential_store.save_completed_registration(&CompletedRegistration {
+                token_uid: pending.token_uid,
+                credential_fingerprint: fingerprint,
+            })?;
+            credential_store.clear_pending_registration()?;
+            let current = identity_store.load()?.ok_or(ClientError::IdentityMissing)?;
+            return Ok(Some((credential, current)));
+        }
+        let Some(pending) = credential_store.load_pending_rotation()? else {
+            validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
+            return Ok(Some((credential, current)));
+        };
+        let fingerprint = certificate_fingerprint(&credential.certificate)?;
+        if pending.device_name != credential.name || !is_request_id(&pending.request_id) {
+            return Err(ClientError::PendingRotation);
+        }
+        if fingerprint == pending.credential_fingerprint {
+            validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
+            let next = identity_store.load_next()?.ok_or(ClientError::PendingRotation)?;
+            if pending.next_public_key_sha256 != public_key_fingerprint(&next)
+                || !certificate_request_matches(&pending.certificate_request, &next)?
+            {
+                return Err(ClientError::PendingRotation);
+            }
+            return Ok(Some((credential, current)));
+        }
+
+        if public_key_fingerprint(&current) == pending.next_public_key_sha256 {
+            if !certificate_request_matches(&pending.certificate_request, &current)? {
+                return Err(ClientError::PendingRotation);
+            }
+            validate_stored_credential(&credential, &current, &self.roots, &self.root_certificates)?;
+        } else {
+            let next = identity_store.load_next()?.ok_or(ClientError::PendingRotation)?;
+            if public_key_fingerprint(&next) != pending.next_public_key_sha256
+                || !certificate_request_matches(&pending.certificate_request, &next)?
+            {
+                return Err(ClientError::PendingRotation);
+            }
+            validate_stored_credential(&credential, &next, &self.roots, &self.root_certificates)?;
+            identity_store.commit_next(&next)?;
+        }
+        credential_store.clear_pending_rotation()?;
+        let current = identity_store.load()?.ok_or(ClientError::IdentityMissing)?;
+        Ok(Some((credential, current)))
+    }
+
+    async fn send<F>(&self, success: StatusCode, mut request: F) -> Result<CredentialResponse, ClientError>
+    where
+        F: FnMut() -> reqwest::RequestBuilder,
+    {
+        let mut last_status = None;
+        for attempt in 0..MAX_ATTEMPTS {
+            match request().send().await {
+                Ok(response) if response.status() == success => return decode_response(response).await,
+                Ok(response) if matches!(response.status(), StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) => {
+                    let status = response.status();
+                    let reason = decode_reason(response).await;
+                    return Err(ClientError::AccessRevoked { status, reason });
+                }
+                Ok(response) if matches!(response.status(), StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS) => {
+                    last_status = Some(response.status());
+                }
+                Ok(response) if response.status().is_client_error() => {
+                    let status = response.status();
+                    let reason = decode_reason(response).await;
+                    return Err(ClientError::Rejected { status, reason });
+                }
+                Ok(response) if response.status().is_server_error() => {
+                    last_status = Some(response.status());
+                }
+                Ok(response) => {
+                    let status = response.status();
+                    let reason = decode_reason(response).await;
+                    return Err(ClientError::Rejected { status, reason });
+                }
+                Err(error) if !error.is_timeout() && !error.is_connect() => return Err(ClientError::Transport(error)),
+                Err(_) => {}
+            }
+
+            if attempt + 1 < MAX_ATTEMPTS {
+                tokio::time::sleep(Duration::from_millis(50 * (attempt as u64 + 1))).await;
+            }
+        }
+        Err(ClientError::Unavailable { status: last_status })
+    }
+
+    fn url(&self, path: &str) -> Result<Url, ClientError> {
+        self.endpoint.join(path).map_err(|_| ClientError::Endpoint)
+    }
+}
+
+fn is_request_id(value: &str) -> bool {
+    Uuid::parse_str(value).is_ok_and(|uuid| uuid.get_version() == Some(uuid::Version::Random) && uuid.to_string() == value)
+}
+
+fn unix_now() -> i64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map_or(0, |duration| duration.as_secs() as i64)
+}
+
+fn ensure_credential_time(credential: &DeviceCredential, now_unix: i64) -> Result<(), ClientError> {
+    if credential.not_before_unix > now_unix {
+        return Err(ClientError::CredentialNotYetValid);
+    }
+    if credential.not_after_unix <= now_unix {
+        return Err(ClientError::CredentialExpired);
+    }
+    Ok(())
+}
+
+fn build_client(
+    roots: &[CertificateDer<'static>],
+    timeout: Duration,
+    identity: Option<reqwest::Identity>,
+) -> Result<Client, ClientError> {
+    let certificates = roots
+        .iter()
+        .map(|root| reqwest::Certificate::from_der(root.as_ref()))
+        .collect::<Result<Vec<_>, _>>()?;
+    let mut builder = Client::builder()
+        .https_only(true)
+        .redirect(reqwest::redirect::Policy::none())
+        .timeout(timeout)
+        .tls_certs_only(certificates);
+    if let Some(identity) = identity {
+        builder = builder.identity(identity);
+    }
+    builder.build().map_err(ClientError::Transport)
+}
+
+async fn decode_response(mut response: reqwest::Response) -> Result<CredentialResponse, ClientError> {
+    let body = read_body(&mut response).await?;
+    serde_json::from_slice(&body).map_err(|_| ClientError::Response)
+}
+
+async fn read_body(response: &mut reqwest::Response) -> Result<Vec<u8>, ClientError> {
+    let mut body = Vec::new();
+    while let Some(chunk) = response.chunk().await.map_err(ClientError::Transport)? {
+        if body.len() + chunk.len() > MAX_RESPONSE_BYTES {
+            return Err(ClientError::ResponseTooLarge);
+        }
+        body.extend_from_slice(&chunk);
+    }
+    Ok(body)
+}
+
+#[derive(Deserialize)]
+struct ErrorEnvelope {
+    #[serde(default)]
+    details: Vec<ErrorDetail>,
+}
+
+#[derive(Deserialize)]
+struct ErrorDetail {
+    #[serde(default)]
+    reason: String,
+}
+
+async fn decode_reason(mut response: reqwest::Response) -> Option<String> {
+    let body = read_body(&mut response).await.ok()?;
+    serde_json::from_slice::<ErrorEnvelope>(&body)
+        .ok()?
+        .details
+        .into_iter()
+        .find_map(|detail| (!detail.reason.is_empty()).then_some(detail.reason))
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum ClientError {
+    #[error("Connect endpoint must be an HTTPS base URL without credentials, query, or fragment")]
+    Endpoint,
+    #[error("Connect root CA configuration is invalid")]
+    RootCertificate,
+    #[error(
+        "Connect registration has a pending attempt for a different token; restore the original protected token configuration"
+    )]
+    PendingRegistration,
+    #[error(
+        "Connect credential rotation has an unfinished attempt for a different current certificate; inspect the local credential store"
+    )]
+    PendingRotation,
+    #[error("RustFS is not registered with Connect")]
+    NotRegistered,
+    #[error("the Connect device private key is missing; restore device.key before using the stored certificate")]
+    IdentityMissing,
+    #[error("the Connect device certificate has expired; call ConnectClient::reenroll with a fresh registration token")]
+    CredentialExpired,
+    #[error("the Connect device certificate is not yet valid; fix local clock skew or call ConnectClient::reenroll")]
+    CredentialNotYetValid,
+    #[error("the stored Connect certificate and device private key cannot form a TLS identity")]
+    IdentityCertificate,
+    #[error(
+        "Connect rejected the device credential with HTTP {status}; reason={reason:?}; call ConnectClient::reenroll with a fresh registration token if revoked"
+    )]
+    AccessRevoked { status: StatusCode, reason: Option<String> },
+    #[error("Connect rejected the request with HTTP {status}; reason={reason:?}")]
+    Rejected { status: StatusCode, reason: Option<String> },
+    #[error("Connect remained unavailable after bounded retries; last_status={status:?}")]
+    Unavailable { status: Option<StatusCode> },
+    #[error("Connect response exceeded the 1 MiB credential-response limit")]
+    ResponseTooLarge,
+    #[error("Connect returned an invalid credential response")]
+    Response,
+    #[error(transparent)]
+    Transport(#[from] reqwest::Error),
+    #[error(transparent)]
+    Identity(#[from] IdentityError),
+    #[error(transparent)]
+    IdentityStore(#[from] StoreError),
+    #[error(transparent)]
+    CredentialStore(#[from] CredentialStoreError),
+    #[error(transparent)]
+    Credential(#[from] CredentialValidationError),
+}

rustfs/src/connect/credential_store.rs

@@ -0,0 +1,370 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fs;
+use std::io::{self, Write as _};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use serde::{Deserialize, Serialize};
+
+const CREDENTIAL_FILE: &str = "device.crt.json";
+const REGISTRATION_COMPLETED_FILE: &str = "registration.completed.json";
+const REGISTRATION_PENDING_FILE: &str = "registration.pending.json";
+const ROTATION_PENDING_FILE: &str = "rotation.pending.json";
+const LOCK_FILE: &str = ".state.lock";
+
+#[cfg(unix)]
+const FILE_MODE: u32 = 0o600;
+
+static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0);
+
+#[derive(Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DeviceCredential {
+    pub name: String,
+    pub uid: String,
+    pub protocol_version: String,
+    pub key_id: String,
+    pub certificate_serial: String,
+    pub certificate: String,
+    pub certificate_chain: String,
+    pub not_before_unix: i64,
+    pub not_after_unix: i64,
+}
+
+impl std::fmt::Debug for DeviceCredential {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DeviceCredential")
+            .field("name", &self.name)
+            .field("key_id", &self.key_id)
+            .field("certificate_serial", &self.certificate_serial)
+            .field("not_after_unix", &self.not_after_unix)
+            .finish_non_exhaustive()
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PendingRegistration {
+    pub token_uid: String,
+    pub request_id: String,
+    pub certificate_request: String,
+    #[serde(default)]
+    pub previous_credential_fingerprint: Option<String>,
+    #[serde(default)]
+    pub next_public_key_sha256: Option<String>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CompletedRegistration {
+    pub token_uid: String,
+    pub credential_fingerprint: String,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PendingRotation {
+    pub credential_fingerprint: String,
+    pub device_name: String,
+    pub request_id: String,
+    pub certificate_request: String,
+    pub next_public_key_sha256: String,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum CredentialStoreError {
+    #[error("connect credential store I/O failed at {path}: {source}")]
+    Io {
+        path: PathBuf,
+        #[source]
+        source: io::Error,
+    },
+
+    #[error("connect credential data at {path} is invalid: {source}")]
+    Invalid {
+        path: PathBuf,
+        #[source]
+        source: serde_json::Error,
+    },
+
+    #[cfg(unix)]
+    #[error("connect credential file at {path} has mode {mode:o}, expected {expected:o}")]
+    Permissions { path: PathBuf, mode: u32, expected: u32 },
+}
+
+#[derive(Clone, Debug)]
+pub struct CredentialStore {
+    directory: PathBuf,
+}
+
+pub(crate) struct CredentialLock {
+    _file: fs::File,
+}
+
+impl CredentialStore {
+    pub fn new(directory: impl Into<PathBuf>) -> Self {
+        Self {
+            directory: directory.into(),
+        }
+    }
+
+    pub(crate) fn load(&self) -> Result<Option<DeviceCredential>, CredentialStoreError> {
+        self.read(CREDENTIAL_FILE)
+    }
+
+    pub(crate) async fn lock(&self) -> Result<CredentialLock, CredentialStoreError> {
+        let directory = self.directory.clone();
+        tokio::task::spawn_blocking(move || {
+            fs::create_dir_all(&directory).map_err(|source| CredentialStoreError::Io {
+                path: directory.clone(),
+                source,
+            })?;
+            let path = directory.join(LOCK_FILE);
+            let mut options = fs::OpenOptions::new();
+            options.create(true).truncate(false).read(true).write(true);
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::OpenOptionsExt as _;
+                options.mode(FILE_MODE);
+            }
+            let file = options.open(&path).map_err(|source| CredentialStoreError::Io {
+                path: path.clone(),
+                source,
+            })?;
+            check_mode(&path)?;
+            file.lock().map_err(|source| CredentialStoreError::Io { path, source })?;
+            Ok(CredentialLock { _file: file })
+        })
+        .await
+        .map_err(|source| CredentialStoreError::Io {
+            path: self.directory.join(LOCK_FILE),
+            source: io::Error::other(source),
+        })?
+    }
+
+    pub(crate) fn save(&self, credential: &DeviceCredential) -> Result<(), CredentialStoreError> {
+        self.write(CREDENTIAL_FILE, credential)
+    }
+
+    pub(crate) fn claim_pending_registration(
+        &self,
+        pending: &PendingRegistration,
+    ) -> Result<PendingRegistration, CredentialStoreError> {
+        self.claim(REGISTRATION_PENDING_FILE, pending)
+    }
+
+    pub(crate) fn load_pending_registration(&self) -> Result<Option<PendingRegistration>, CredentialStoreError> {
+        self.read(REGISTRATION_PENDING_FILE)
+    }
+
+    pub(crate) fn clear_pending_registration(&self) -> Result<(), CredentialStoreError> {
+        self.remove(REGISTRATION_PENDING_FILE)
+    }
+
+    pub(crate) fn load_completed_registration(&self) -> Result<Option<CompletedRegistration>, CredentialStoreError> {
+        self.read(REGISTRATION_COMPLETED_FILE)
+    }
+
+    pub(crate) fn save_completed_registration(&self, completed: &CompletedRegistration) -> Result<(), CredentialStoreError> {
+        self.write(REGISTRATION_COMPLETED_FILE, completed)
+    }
+
+    pub(crate) fn load_pending_rotation(&self) -> Result<Option<PendingRotation>, CredentialStoreError> {
+        self.read(ROTATION_PENDING_FILE)
+    }
+
+    pub(crate) fn claim_pending_rotation(&self, pending: &PendingRotation) -> Result<PendingRotation, CredentialStoreError> {
+        self.claim(ROTATION_PENDING_FILE, pending)
+    }
+
+    pub(crate) fn clear_pending_rotation(&self) -> Result<(), CredentialStoreError> {
+        self.remove(ROTATION_PENDING_FILE)
+    }
+
+    fn read<T: for<'de> Deserialize<'de>>(&self, file: &str) -> Result<Option<T>, CredentialStoreError> {
+        let path = self.directory.join(file);
+        let bytes = match fs::read(&path) {
+            Ok(bytes) => bytes,
+            Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None),
+            Err(source) => return Err(CredentialStoreError::Io { path, source }),
+        };
+
+        check_mode(&path)?;
+        serde_json::from_slice(&bytes)
+            .map(Some)
+            .map_err(|source| CredentialStoreError::Invalid { path, source })
+    }
+
+    fn write<T: Serialize>(&self, file: &str, value: &T) -> Result<(), CredentialStoreError> {
+        let bytes = serde_json::to_vec(value).map_err(|source| CredentialStoreError::Invalid {
+            path: self.directory.join(file),
+            source,
+        })?;
+
+        fs::create_dir_all(&self.directory).map_err(|source| CredentialStoreError::Io {
+            path: self.directory.clone(),
+            source,
+        })?;
+
+        let final_path = self.directory.join(file);
+        let temp_path = self.stage(file, &bytes)?;
+        let result = fs::rename(&temp_path, &final_path)
+            .map_err(|source| CredentialStoreError::Io {
+                path: final_path,
+                source,
+            })
+            .and_then(|()| {
+                fsync_dir(&self.directory).map_err(|source| CredentialStoreError::Io {
+                    path: self.directory.clone(),
+                    source,
+                })
+            });
+
+        if result.is_err() {
+            let _ = fs::remove_file(&temp_path);
+        }
+        result
+    }
+
+    fn claim<T>(&self, file: &str, value: &T) -> Result<T, CredentialStoreError>
+    where
+        T: Clone + Serialize + for<'de> Deserialize<'de>,
+    {
+        if let Some(existing) = self.read(file)? {
+            return Ok(existing);
+        }
+        let bytes = serde_json::to_vec(value).map_err(|source| CredentialStoreError::Invalid {
+            path: self.directory.join(file),
+            source,
+        })?;
+        fs::create_dir_all(&self.directory).map_err(|source| CredentialStoreError::Io {
+            path: self.directory.clone(),
+            source,
+        })?;
+        let final_path = self.directory.join(file);
+        let temp_path = self.stage(file, &bytes)?;
+        let published = fs::hard_link(&temp_path, &final_path);
+        let _ = fs::remove_file(&temp_path);
+        match published {
+            Ok(()) => {
+                fsync_dir(&self.directory).map_err(|source| CredentialStoreError::Io {
+                    path: self.directory.clone(),
+                    source,
+                })?;
+                Ok(value.clone())
+            }
+            Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {
+                fsync_dir(&self.directory).map_err(|source| CredentialStoreError::Io {
+                    path: self.directory.clone(),
+                    source,
+                })?;
+                self.read(file)?.ok_or_else(|| CredentialStoreError::Io {
+                    path: final_path,
+                    source: io::Error::new(io::ErrorKind::NotFound, "pending state vanished after publication"),
+                })
+            }
+            Err(source) => Err(CredentialStoreError::Io {
+                path: final_path,
+                source,
+            }),
+        }
+    }
+
+    fn stage(&self, file: &str, bytes: &[u8]) -> Result<PathBuf, CredentialStoreError> {
+        loop {
+            let path = self.directory.join(format!(
+                ".{file}.{}.{}.tmp",
+                std::process::id(),
+                STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed)
+            ));
+            let mut options = fs::OpenOptions::new();
+            options.write(true).create_new(true);
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::OpenOptionsExt as _;
+                options.mode(FILE_MODE);
+            }
+            let mut staging = match options.open(&path) {
+                Ok(staging) => staging,
+                Err(source) if source.kind() == io::ErrorKind::AlreadyExists => continue,
+                Err(source) => return Err(CredentialStoreError::Io { path, source }),
+            };
+            let result = staging
+                .write_all(bytes)
+                .and_then(|()| staging.sync_all())
+                .map_err(|source| CredentialStoreError::Io {
+                    path: path.clone(),
+                    source,
+                })
+                .and_then(|()| check_mode(&path));
+            if let Err(error) = result {
+                let _ = fs::remove_file(&path);
+                return Err(error);
+            }
+            return Ok(path);
+        }
+    }
+
+    fn remove(&self, file: &str) -> Result<(), CredentialStoreError> {
+        let path = self.directory.join(file);
+        match fs::remove_file(&path) {
+            Ok(()) => fsync_dir(&self.directory).map_err(|source| CredentialStoreError::Io {
+                path: self.directory.clone(),
+                source,
+            }),
+            Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()),
+            Err(source) => Err(CredentialStoreError::Io { path, source }),
+        }
+    }
+}
+
+#[cfg(unix)]
+fn check_mode(path: &Path) -> Result<(), CredentialStoreError> {
+    use std::os::unix::fs::PermissionsExt as _;
+
+    let mode = fs::metadata(path)
+        .map_err(|source| CredentialStoreError::Io {
+            path: path.to_path_buf(),
+            source,
+        })?
+        .permissions()
+        .mode()
+        & 0o7777;
+    if mode != FILE_MODE {
+        return Err(CredentialStoreError::Permissions {
+            path: path.to_path_buf(),
+            mode,
+            expected: FILE_MODE,
+        });
+    }
+    Ok(())
+}
+
+#[cfg(not(unix))]
+fn check_mode(_path: &Path) -> Result<(), CredentialStoreError> {
+    Ok(())
+}
+
+fn fsync_dir(directory: &Path) -> io::Result<()> {
+    #[cfg(unix)]
+    fs::File::open(directory)?.sync_all()?;
+    #[cfg(not(unix))]
+    let _ = directory;
+    Ok(())
+}

rustfs/src/connect/identity.rs

@@ -24,7 +24,7 @@ use base64::Engine as _;
 use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
 use p256::ecdsa::signature::Signer as _;
 use p256::ecdsa::{Signature, SigningKey};
-use p256::pkcs8::{DecodePrivateKey as _, EncodePrivateKey as _};
+use p256::pkcs8::{DecodePrivateKey as _, EncodePrivateKey as _, LineEnding};
 use sha2::{Digest as _, Sha256};
 use zeroize::Zeroizing;
 
@@ -213,6 +213,12 @@ impl DeviceIdentity {
             .map_err(|error| IdentityError::MalformedKey(error.to_string()))
     }
 
+    pub(crate) fn to_pkcs8_pem(&self) -> Result<Zeroizing<String>, IdentityError> {
+        self.signing_key
+            .to_pkcs8_pem(LineEnding::LF)
+            .map_err(|error| IdentityError::MalformedKey(error.to_string()))
+    }
+
     /// Build the PKCS#10 certificate request Connect consumes.
     ///
     /// Connect reads the request for its SubjectPublicKeyInfo and its

rustfs/src/connect/identity_store.rs

@@ -32,6 +32,7 @@ use super::identity::{DeviceIdentity, IdentityError};
 
 /// Name of the key file inside the store directory.
 const KEY_FILE: &str = "device.key";
+const NEXT_KEY_FILE: &str = "device.key.next";
 
 /// Owner read/write only. The key is the device's whole identity.
 #[cfg(unix)]
@@ -94,7 +95,15 @@ impl IdentityStore {
     /// been enrolled. Reading never creates anything, so an unconfigured
     /// server can ask without acquiring an identity as a side effect.
     pub fn load(&self) -> Result<Option<DeviceIdentity>, StoreError> {
-        let path = self.key_path();
+        self.load_file(KEY_FILE)
+    }
+
+    pub(crate) fn load_next(&self) -> Result<Option<DeviceIdentity>, StoreError> {
+        self.load_file(NEXT_KEY_FILE)
+    }
+
+    fn load_file(&self, file: &str) -> Result<Option<DeviceIdentity>, StoreError> {
+        let path = self.directory.join(file);
 
         let der = match fs::read(&path) {
             Ok(der) => Zeroizing::new(der),
@@ -142,11 +151,15 @@ impl IdentityStore {
         let candidate = DeviceIdentity::generate();
         let der = candidate.to_pkcs8_der()?;
 
-        match self.publish(&der) {
+        match self.publish(KEY_FILE, &der) {
             Ok(()) => Ok(candidate),
             // Another process published first. Its key is the identity; ours
             // was never written anywhere and simply goes out of scope.
             Err(StoreError::Io { source, .. }) if source.kind() == io::ErrorKind::AlreadyExists => {
+                fsync_dir(&self.directory).map_err(|source| StoreError::Io {
+                    path: self.directory.clone(),
+                    source,
+                })?;
                 self.load()?.ok_or_else(|| StoreError::Io {
                     path: self.key_path(),
                     source: io::Error::new(
@@ -159,35 +172,106 @@ impl IdentityStore {
         }
     }
 
+    pub(crate) fn load_or_create_next(&self) -> Result<DeviceIdentity, StoreError> {
+        if let Some(identity) = self.load_next()? {
+            return Ok(identity);
+        }
+        fs::create_dir_all(&self.directory).map_err(|source| StoreError::Io {
+            path: self.directory.clone(),
+            source,
+        })?;
+        let candidate = DeviceIdentity::generate();
+        let der = candidate.to_pkcs8_der()?;
+        match self.publish(NEXT_KEY_FILE, &der) {
+            Ok(()) => Ok(candidate),
+            Err(StoreError::Io { source, .. }) if source.kind() == io::ErrorKind::AlreadyExists => {
+                fsync_dir(&self.directory).map_err(|source| StoreError::Io {
+                    path: self.directory.clone(),
+                    source,
+                })?;
+                self.load_next()?.ok_or_else(|| StoreError::Io {
+                    path: self.directory.join(NEXT_KEY_FILE),
+                    source: io::Error::new(io::ErrorKind::NotFound, "next device key vanished after publication"),
+                })
+            }
+            Err(error) => Err(error),
+        }
+    }
+
+    pub(crate) fn commit_next(&self, expected: &DeviceIdentity) -> Result<(), StoreError> {
+        let next_path = self.directory.join(NEXT_KEY_FILE);
+        match fs::rename(&next_path, self.key_path()) {
+            Ok(()) => fsync_dir(&self.directory).map_err(|source| StoreError::Io {
+                path: self.directory.clone(),
+                source,
+            }),
+            Err(source) if source.kind() == io::ErrorKind::NotFound => {
+                let current = self.load()?.ok_or_else(|| StoreError::Io {
+                    path: self.key_path(),
+                    source,
+                })?;
+                if current.public_key_der() == expected.public_key_der() {
+                    fsync_dir(&self.directory).map_err(|source| StoreError::Io {
+                        path: self.directory.clone(),
+                        source,
+                    })
+                } else {
+                    Err(StoreError::Io {
+                        path: next_path,
+                        source: io::Error::new(io::ErrorKind::NotFound, "next device key is missing"),
+                    })
+                }
+            }
+            Err(source) => Err(StoreError::Io { path: next_path, source }),
+        }
+    }
+
+    pub(crate) fn clear_next(&self) -> Result<(), StoreError> {
+        let path = self.directory.join(NEXT_KEY_FILE);
+        match fs::remove_file(&path) {
+            Ok(()) => fsync_dir(&self.directory).map_err(|source| StoreError::Io {
+                path: self.directory.clone(),
+                source,
+            }),
+            Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()),
+            Err(source) => Err(StoreError::Io { path, source }),
+        }
+    }
+
     /// Write, seal, fsync, then link into place and fsync the directory. The
     /// key is durable before it is reachable, and it is reachable only once.
-    fn publish(&self, der: &[u8]) -> Result<(), StoreError> {
+    fn publish(&self, file: &str, der: &[u8]) -> Result<(), StoreError> {
         use std::io::Write as _;
 
-        let final_path = self.key_path();
-        let temp_path = self.directory.join(format!(
-            "{KEY_FILE}.{}.{}.tmp",
-            std::process::id(),
-            STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed)
-        ));
+        let final_path = self.directory.join(file);
 
         let io_at = |path: &Path| {
             let path = path.to_path_buf();
             move |source| StoreError::Io { path, source }
         };
 
-        let mut options = fs::OpenOptions::new();
-        options.write(true).create(true).truncate(true);
-        #[cfg(unix)]
-        {
-            use std::os::unix::fs::OpenOptionsExt as _;
-            options.mode(KEY_MODE);
-        }
-
-        let mut file = options.open(&temp_path).map_err(io_at(&temp_path))?;
+        let (temp_path, mut staging) = loop {
+            let temp_path = self.directory.join(format!(
+                ".{file}.{}.{}.tmp",
+                std::process::id(),
+                STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed)
+            ));
+            let mut options = fs::OpenOptions::new();
+            options.write(true).create_new(true);
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::OpenOptionsExt as _;
+                options.mode(KEY_MODE);
+            }
+            match options.open(&temp_path) {
+                Ok(staging) => break (temp_path, staging),
+                Err(source) if source.kind() == io::ErrorKind::AlreadyExists => continue,
+                Err(source) => return Err(io_at(&temp_path)(source)),
+            }
+        };
 
         let result = (|| -> Result<(), StoreError> {
-            file.write_all(der).map_err(io_at(&temp_path))?;
+            staging.write_all(der).map_err(io_at(&temp_path))?;
 
             // The umask can only narrow the creation mode, so set and verify
             // the exact mode before the bytes become durable.
@@ -195,9 +279,10 @@ impl IdentityStore {
             {
                 use std::os::unix::fs::PermissionsExt as _;
 
-                file.set_permissions(fs::Permissions::from_mode(KEY_MODE))
+                staging
+                    .set_permissions(fs::Permissions::from_mode(KEY_MODE))
                     .map_err(io_at(&temp_path))?;
-                let mode = file.metadata().map_err(io_at(&temp_path))?.permissions().mode() & 0o7777;
+                let mode = staging.metadata().map_err(io_at(&temp_path))?.permissions().mode() & 0o7777;
                 if mode != KEY_MODE {
                     return Err(StoreError::Permissions {
                         path: temp_path.clone(),
@@ -207,11 +292,11 @@ impl IdentityStore {
                 }
             }
 
-            file.sync_all().map_err(io_at(&temp_path))?;
+            staging.sync_all().map_err(io_at(&temp_path))?;
             Ok(())
         })();
 
-        drop(file);
+        drop(staging);
 
         if let Err(error) = result {
             let _ = fs::remove_file(&temp_path);

rustfs/src/connect/mod.rs

@@ -25,10 +25,16 @@
 //! not been enrolled into a Connect control plane never calls into it, so an
 //! unconfigured server generates no key and holds no identity.
 
+pub mod client;
+pub mod credential_store;
 pub mod identity;
 pub mod identity_store;
 pub mod offline;
+pub mod registration;
 
+pub use client::{ClientError, ConnectClient, ConnectConfig};
+pub use credential_store::{CredentialStore, DeviceCredential};
 pub use identity::{DeviceIdentity, IdentityError, RegistrationProof, RegistrationTranscript};
 pub use identity_store::{IdentityStore, StoreError};
 pub use offline::{EnrollmentError, OfflineEnrollment, OfflineKeyStore, VerifiedChallenge};
+pub use registration::{RegistrationToken, TokenError};

rustfs/src/connect/registration.rs

@@ -0,0 +1,527 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::io::Read;
+use std::sync::Arc;
+
+use base64::Engine as _;
+use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD;
+use p256::ecdsa::signature::Signer as _;
+use p256::ecdsa::{Signature, SigningKey};
+use p256::pkcs8::DecodePrivateKey as _;
+use rustls::RootCertStore;
+use rustls::pki_types::{CertificateDer, UnixTime, pem::PemObject as _};
+use rustls::server::WebPkiClientVerifier;
+use serde::{Deserialize, Serialize};
+use sha2::{Digest as _, Sha256};
+use time::OffsetDateTime;
+use time::format_description::well_known::Rfc3339;
+use uuid::{Uuid, Version};
+use x509_parser::extensions::GeneralName;
+use x509_parser::oid_registry::OID_SIG_ECDSA_WITH_SHA256;
+use x509_parser::prelude::{FromDer as _, X509Certificate, X509CertificationRequest};
+use zeroize::Zeroizing;
+
+use super::credential_store::DeviceCredential;
+use super::identity::{DeviceIdentity, RegistrationProof};
+
+pub const PROTOCOL_VERSION: &str = "v1";
+
+const CERTIFICATE_LIFETIME_SECONDS: i64 = 86_400;
+const ROTATION_DOMAIN: &[u8] = b"RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1";
+const MAX_TOKEN_BYTES: u64 = 16 * 1024;
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields, rename_all = "camelCase")]
+struct RegistrationTokenDocument {
+    registration_token_uid: String,
+    registration_token_secret: String,
+    organization_uid: String,
+    cluster_uid: String,
+    challenge_nonce: String,
+    expires_unix: i64,
+}
+
+pub struct RegistrationToken {
+    pub registration_token_uid: String,
+    registration_token_secret: Zeroizing<String>,
+    pub organization_uid: String,
+    pub cluster_uid: String,
+    pub challenge_nonce: String,
+    pub expires_unix: i64,
+}
+
+impl std::fmt::Debug for RegistrationToken {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RegistrationToken")
+            .field("registration_token_uid", &self.registration_token_uid)
+            .field("expires_unix", &self.expires_unix)
+            .finish_non_exhaustive()
+    }
+}
+
+impl RegistrationToken {
+    pub fn from_reader(reader: impl Read) -> Result<Self, TokenError> {
+        let mut bytes = Zeroizing::new(Vec::new());
+        reader
+            .take(MAX_TOKEN_BYTES + 1)
+            .read_to_end(&mut bytes)
+            .map_err(TokenError::Read)?;
+        if bytes.len() as u64 > MAX_TOKEN_BYTES {
+            return Err(TokenError::TooLarge);
+        }
+        let document: RegistrationTokenDocument = serde_json::from_slice(&bytes).map_err(TokenError::Invalid)?;
+        let decoded = BASE64_URL_NO_PAD
+            .decode(&document.registration_token_secret)
+            .map(Zeroizing::new)
+            .map_err(|_| TokenError::SecretShape)?;
+        if decoded.len() != 32 || BASE64_URL_NO_PAD.encode(&decoded) != document.registration_token_secret {
+            return Err(TokenError::SecretShape);
+        }
+        if !is_uuid_v7(&document.registration_token_uid)
+            || !is_uuid_v7(&document.organization_uid)
+            || !is_uuid_v7(&document.cluster_uid)
+            || document.expires_unix < 0
+            || document.challenge_nonce.len() != 64
+            || !document
+                .challenge_nonce
+                .bytes()
+                .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+        {
+            return Err(TokenError::Shape);
+        }
+
+        Ok(Self {
+            registration_token_uid: document.registration_token_uid,
+            registration_token_secret: Zeroizing::new(document.registration_token_secret),
+            organization_uid: document.organization_uid,
+            cluster_uid: document.cluster_uid,
+            challenge_nonce: document.challenge_nonce,
+            expires_unix: document.expires_unix,
+        })
+    }
+
+    pub(crate) fn secret(&self) -> &str {
+        &self.registration_token_secret
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum TokenError {
+    #[error("failed to read the Connect registration token")]
+    Read(#[source] std::io::Error),
+    #[error("Connect registration token configuration is invalid")]
+    Invalid(#[source] serde_json::Error),
+    #[error("Connect registration token secret must be 32-byte unpadded base64url")]
+    SecretShape,
+    #[error("Connect registration token configuration exceeds 16 KiB")]
+    TooLarge,
+    #[error("Connect registration token fields do not match the protocol schema")]
+    Shape,
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct RegistrationRequest<'a> {
+    protocol_version: &'static str,
+    request_id: &'a str,
+    registration_token_uid: &'a str,
+    registration_token_secret: &'a str,
+    certificate_request: &'a str,
+    proof: ProofRef<'a>,
+}
+
+#[derive(Serialize)]
+struct ProofRef<'a> {
+    algorithm: &'a str,
+    value: &'a str,
+}
+
+impl<'a> RegistrationRequest<'a> {
+    pub(crate) fn new(
+        token: &'a RegistrationToken,
+        request_id: &'a str,
+        certificate_request: &'a str,
+        proof: &'a RegistrationProof,
+    ) -> Self {
+        Self {
+            protocol_version: PROTOCOL_VERSION,
+            request_id,
+            registration_token_uid: &token.registration_token_uid,
+            registration_token_secret: token.secret(),
+            certificate_request,
+            proof: ProofRef {
+                algorithm: &proof.algorithm,
+                value: &proof.value,
+            },
+        }
+    }
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct RotationRequest<'a> {
+    protocol_version: &'static str,
+    request_id: &'a str,
+    certificate_request: &'a str,
+    proof: ProofOwned,
+}
+
+#[derive(Serialize)]
+struct ProofOwned {
+    algorithm: String,
+    value: String,
+}
+
+impl<'a> RotationRequest<'a> {
+    pub(crate) fn new(
+        identity: &DeviceIdentity,
+        credential_fingerprint: &str,
+        device_name: &str,
+        request_id: &'a str,
+        certificate_request: &'a str,
+    ) -> Result<Self, CredentialValidationError> {
+        let csr_der = base64::engine::general_purpose::STANDARD
+            .decode(certificate_request)
+            .map_err(|_| CredentialValidationError::CertificateRequest)?;
+        let csr_digest = BASE64_URL_NO_PAD.encode(Sha256::digest(&csr_der));
+        let transcript = rotation_transcript(credential_fingerprint, device_name, request_id, &csr_digest)?;
+        let key = identity
+            .to_pkcs8_der()
+            .map_err(|_| CredentialValidationError::CertificateRequest)?;
+        let signing_key = SigningKey::from_pkcs8_der(&key).map_err(|_| CredentialValidationError::CertificateRequest)?;
+        let signature: Signature = signing_key.sign(&transcript);
+        let canonical = signature.normalize_s().unwrap_or(signature);
+
+        Ok(Self {
+            protocol_version: PROTOCOL_VERSION,
+            request_id,
+            certificate_request,
+            proof: ProofOwned {
+                algorithm: "ES256".to_string(),
+                value: BASE64_URL_NO_PAD.encode(canonical.to_bytes()),
+            },
+        })
+    }
+}
+
+fn rotation_transcript(
+    credential_fingerprint: &str,
+    device_name: &str,
+    request_id: &str,
+    csr_digest: &str,
+) -> Result<Vec<u8>, CredentialValidationError> {
+    let fields = [credential_fingerprint, device_name, request_id, csr_digest];
+    if fields
+        .iter()
+        .any(|field| !field.is_ascii() || field.as_bytes().contains(&b'\n'))
+    {
+        return Err(CredentialValidationError::RotationTranscript);
+    }
+
+    let mut transcript = Vec::with_capacity(346);
+    transcript.extend_from_slice(ROTATION_DOMAIN);
+    transcript.push(b'\n');
+    for field in fields {
+        transcript.extend_from_slice(field.len().to_string().as_bytes());
+        transcript.push(b':');
+        transcript.extend_from_slice(field.as_bytes());
+        transcript.push(b'\n');
+    }
+    Ok(transcript)
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct CredentialResponse {
+    pub name: String,
+    #[serde(default)]
+    pub uid: String,
+    #[serde(default)]
+    pub cluster: String,
+    pub protocol_version: String,
+    pub key_id: String,
+    pub certificate_serial: String,
+    pub certificate: String,
+    pub certificate_chain: String,
+    pub not_before: String,
+    pub not_after: String,
+}
+
+pub(crate) enum ExpectedDevice<'a> {
+    Registration { cluster: &'a str },
+    Rotation { name: &'a str },
+    Stored,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum CredentialValidationError {
+    #[error("Connect returned malformed certificate material")]
+    Certificate,
+    #[error("Connect returned a certificate chain that is not trusted")]
+    Chain,
+    #[error("Connect returned a certificate for the wrong device identity")]
+    Identity,
+    #[error("Connect returned a certificate for a different device key")]
+    Key,
+    #[error("Connect returned an invalid certificate validity window")]
+    Validity,
+    #[error("the device certificate request could not be prepared")]
+    CertificateRequest,
+    #[error("the credential rotation transcript contains an invalid field")]
+    RotationTranscript,
+}
+
+pub(crate) fn validate_credential(
+    response: CredentialResponse,
+    identity: &DeviceIdentity,
+    roots: &RootCertStore,
+    root_certificates: &[CertificateDer<'static>],
+    expected: ExpectedDevice<'_>,
+) -> Result<DeviceCredential, CredentialValidationError> {
+    validate_credential_at(response, identity, roots, root_certificates, expected, true)
+}
+
+pub(crate) fn validate_stored_credential(
+    credential: &DeviceCredential,
+    identity: &DeviceIdentity,
+    roots: &RootCertStore,
+    root_certificates: &[CertificateDer<'static>],
+) -> Result<(), CredentialValidationError> {
+    let not_before = OffsetDateTime::from_unix_timestamp(credential.not_before_unix)
+        .map_err(|_| CredentialValidationError::Validity)?
+        .format(&Rfc3339)
+        .map_err(|_| CredentialValidationError::Validity)?;
+    let not_after = OffsetDateTime::from_unix_timestamp(credential.not_after_unix)
+        .map_err(|_| CredentialValidationError::Validity)?
+        .format(&Rfc3339)
+        .map_err(|_| CredentialValidationError::Validity)?;
+    let response = CredentialResponse {
+        name: credential.name.clone(),
+        uid: credential.uid.clone(),
+        cluster: String::new(),
+        protocol_version: credential.protocol_version.clone(),
+        key_id: credential.key_id.clone(),
+        certificate_serial: credential.certificate_serial.clone(),
+        certificate: credential.certificate.clone(),
+        certificate_chain: credential.certificate_chain.clone(),
+        not_before,
+        not_after,
+    };
+    validate_credential_at(response, identity, roots, root_certificates, ExpectedDevice::Stored, false).map(|_| ())
+}
+
+fn validate_credential_at(
+    response: CredentialResponse,
+    identity: &DeviceIdentity,
+    roots: &RootCertStore,
+    root_certificates: &[CertificateDer<'static>],
+    expected: ExpectedDevice<'_>,
+    verify_now: bool,
+) -> Result<DeviceCredential, CredentialValidationError> {
+    if response.protocol_version != PROTOCOL_VERSION {
+        return Err(CredentialValidationError::Identity);
+    }
+
+    let leaves = CertificateDer::pem_slice_iter(response.certificate.as_bytes())
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|_| CredentialValidationError::Certificate)?;
+    if leaves.len() != 1 {
+        return Err(CredentialValidationError::Certificate);
+    }
+    let chain = CertificateDer::pem_slice_iter(response.certificate_chain.as_bytes())
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|_| CredentialValidationError::Certificate)?;
+    if chain.is_empty()
+        || chain[0].as_ref() != leaves[0].as_ref()
+        || chain
+            .iter()
+            .skip(1)
+            .any(|certificate| root_certificates.iter().any(|root| root.as_ref() == certificate.as_ref()))
+    {
+        return Err(CredentialValidationError::Chain);
+    }
+
+    let (remaining, certificate) =
+        X509Certificate::from_der(leaves[0].as_ref()).map_err(|_| CredentialValidationError::Certificate)?;
+    if !remaining.is_empty() {
+        return Err(CredentialValidationError::Certificate);
+    }
+
+    let uid = match expected {
+        ExpectedDevice::Registration { cluster } => {
+            if response.uid.is_empty()
+                || response.cluster != cluster
+                || response.name != format!("{cluster}/clusterDevices/{}", response.uid)
+            {
+                return Err(CredentialValidationError::Identity);
+            }
+            response.uid.clone()
+        }
+        ExpectedDevice::Rotation { name } => {
+            if response.name != name || !response.uid.is_empty() || !response.cluster.is_empty() {
+                return Err(CredentialValidationError::Identity);
+            }
+            name.rsplit_once("/clusterDevices/")
+                .map(|(_, uid)| uid.to_string())
+                .ok_or(CredentialValidationError::Identity)?
+        }
+        ExpectedDevice::Stored => {
+            let (cluster, name_uid) = response
+                .name
+                .rsplit_once("/clusterDevices/")
+                .ok_or(CredentialValidationError::Identity)?;
+            if response.uid != name_uid
+                || !response.cluster.is_empty()
+                || !valid_cluster_name(cluster)
+                || response.name.matches("/clusterDevices/").count() != 1
+            {
+                return Err(CredentialValidationError::Identity);
+            }
+            response.uid.clone()
+        }
+    };
+    let parsed_uid = Uuid::parse_str(&uid).map_err(|_| CredentialValidationError::Identity)?;
+    if parsed_uid.get_version() != Some(Version::SortRand) || parsed_uid.to_string() != uid {
+        return Err(CredentialValidationError::Identity);
+    }
+    let expected_uri = format!("urn:rustfs:connect:device:{uid}");
+    let common_names = certificate
+        .subject()
+        .iter_common_name()
+        .map(|name| name.as_str())
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|_| CredentialValidationError::Identity)?;
+    let san = certificate
+        .subject_alternative_name()
+        .map_err(|_| CredentialValidationError::Identity)?
+        .ok_or(CredentialValidationError::Identity)?;
+    let san_matches = matches!(san.value.general_names.as_slice(), [GeneralName::URI(uri)] if *uri == expected_uri);
+
+    if common_names.as_slice() != [uid.as_str()]
+        || !san_matches
+        || certificate.subject().iter().count() != 1
+        || certificate.subject().iter_attributes().count() != 1
+    {
+        return Err(CredentialValidationError::Identity);
+    }
+    if certificate.public_key().raw != identity.public_key_der() {
+        return Err(CredentialValidationError::Key);
+    }
+
+    let not_before = OffsetDateTime::parse(&response.not_before, &Rfc3339)
+        .map_err(|_| CredentialValidationError::Validity)?
+        .unix_timestamp();
+    let not_after = OffsetDateTime::parse(&response.not_after, &Rfc3339)
+        .map_err(|_| CredentialValidationError::Validity)?
+        .unix_timestamp();
+    let verify_unix = if verify_now {
+        UnixTime::now()
+    } else {
+        let midpoint = certificate.validity().not_before.timestamp()
+            + (certificate.validity().not_after.timestamp() - certificate.validity().not_before.timestamp()) / 2;
+        UnixTime::since_unix_epoch(std::time::Duration::from_secs(
+            midpoint.try_into().map_err(|_| CredentialValidationError::Validity)?,
+        ))
+    };
+    let verifier = WebPkiClientVerifier::builder(Arc::new(roots.clone()))
+        .build()
+        .map_err(|_| CredentialValidationError::Chain)?;
+    verifier
+        .verify_client_cert(&leaves[0], &chain[1..], verify_unix)
+        .map_err(|_| CredentialValidationError::Chain)?;
+    if not_before != certificate.validity().not_before.timestamp()
+        || not_after != certificate.validity().not_after.timestamp()
+        || not_after - not_before != CERTIFICATE_LIFETIME_SECONDS
+        || certificate.signature_algorithm.algorithm != OID_SIG_ECDSA_WITH_SHA256
+        || response.certificate_serial != canonical_serial(certificate.raw_serial())?
+        || response.key_id != format!("x509-{}", response.certificate_serial)
+    {
+        return Err(CredentialValidationError::Validity);
+    }
+
+    Ok(DeviceCredential {
+        name: response.name,
+        uid,
+        protocol_version: response.protocol_version,
+        key_id: response.key_id,
+        certificate_serial: response.certificate_serial,
+        certificate: response.certificate,
+        certificate_chain: response.certificate_chain,
+        not_before_unix: not_before,
+        not_after_unix: not_after,
+    })
+}
+
+pub(crate) fn public_key_fingerprint(identity: &DeviceIdentity) -> String {
+    hex_lower(&Sha256::digest(identity.public_key_der()))
+}
+
+pub(crate) fn certificate_request_matches(encoded: &str, identity: &DeviceIdentity) -> Result<bool, CredentialValidationError> {
+    let der = base64::engine::general_purpose::STANDARD
+        .decode(encoded)
+        .map_err(|_| CredentialValidationError::CertificateRequest)?;
+    let (remaining, request) =
+        X509CertificationRequest::from_der(&der).map_err(|_| CredentialValidationError::CertificateRequest)?;
+    Ok(remaining.is_empty() && request.certification_request_info.subject_pki.raw == identity.public_key_der())
+}
+
+fn canonical_serial(raw: &[u8]) -> Result<String, CredentialValidationError> {
+    let magnitude = match raw {
+        [0, first, rest @ ..] if first & 0x80 != 0 => {
+            if rest.len() + 1 > 16 {
+                return Err(CredentialValidationError::Validity);
+            }
+            &raw[1..]
+        }
+        [0] => raw,
+        [0, ..] | [] => return Err(CredentialValidationError::Validity),
+        [first, ..] if first & 0x80 != 0 => return Err(CredentialValidationError::Validity),
+        _ if raw.len() > 16 => return Err(CredentialValidationError::Validity),
+        _ => raw,
+    };
+    let mut padded = [0u8; 16];
+    padded[16 - magnitude.len()..].copy_from_slice(magnitude);
+    Ok(hex_lower(&padded))
+}
+
+fn is_uuid_v7(value: &str) -> bool {
+    Uuid::parse_str(value).is_ok_and(|uuid| uuid.get_version() == Some(Version::SortRand) && uuid.to_string() == value)
+}
+
+fn valid_cluster_name(name: &str) -> bool {
+    let Some((organization, cluster)) = name
+        .strip_prefix("organizations/")
+        .and_then(|rest| rest.split_once("/clusters/"))
+    else {
+        return false;
+    };
+    !cluster.contains('/') && is_uuid_v7(organization) && is_uuid_v7(cluster)
+}
+
+pub(crate) fn certificate_fingerprint(certificate_pem: &str) -> Result<String, CredentialValidationError> {
+    let certificate = CertificateDer::pem_slice_iter(certificate_pem.as_bytes())
+        .next()
+        .ok_or(CredentialValidationError::Certificate)?
+        .map_err(|_| CredentialValidationError::Certificate)?;
+    Ok(hex_lower(&Sha256::digest(certificate.as_ref())))
+}
+
+fn hex_lower(bytes: &[u8]) -> String {
+    bytes.iter().fold(String::with_capacity(bytes.len() * 2), |mut output, byte| {
+        use std::fmt::Write as _;
+        let _ = write!(output, "{byte:02x}");
+        output
+    })
+}

rustfs/src/storage/s3_api/bucket.rs

@@ -296,7 +296,10 @@ pub(crate) fn build_list_objects_v2_output(
             let mut obj = Object {
                 key: Some(key),
                 last_modified: v.mod_time.map(Timestamp::from),
-                size: Some(v.get_actual_size().unwrap_or_default()),
+                // Compressed legacy objects may retain an unknown (-1)
+                // logical-size sentinel; never expose that internal value in
+                // an S3 response.
+                size: Some(v.get_actual_size_or_physical()),
                 e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)),
                 storage_class: v.storage_class.clone().map(ObjectStorageClass::from),
                 ..Default::default()
@@ -656,6 +659,45 @@ mod tests {
         assert_eq!(output.common_prefixes.as_ref().map(std::vec::Vec::len), Some(2));
     }
 
+    #[test]
+    fn list_objects_never_exposes_compressed_unknown_size_sentinel() {
+        let mut metadata = std::collections::HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let output = build_list_objects_v2_output(
+            ListObjectsV2Info {
+                objects: vec![ObjectInfo {
+                    name: "legacy-compressed".to_string(),
+                    size: 128,
+                    actual_size: -1,
+                    user_defined: std::sync::Arc::new(metadata),
+                    ..Default::default()
+                }],
+                ..Default::default()
+            },
+            false,
+            1000,
+            "bucket".to_string(),
+            String::new(),
+            None,
+            None,
+            None,
+            None,
+        );
+
+        assert_eq!(
+            output
+                .contents
+                .as_ref()
+                .and_then(|objects| objects.first())
+                .and_then(|object| object.size),
+            Some(128)
+        );
+    }
+
     #[test]
     fn list_responses_report_standard_for_legacy_label_only_file_metadata() {
         let version_id = Uuid::parse_str("11111111-2222-3333-4444-555555555555").expect("fixture version ID should be valid");

rustfs/src/storage/storage_api.rs

@@ -429,6 +429,8 @@ pub(crate) mod ecstore_config {
 }
 
 pub(crate) mod ecstore_data_usage {
+    #[cfg(test)]
+    pub(crate) use rustfs_ecstore::api::data_usage::get_bucket_usage_memory;
     pub(crate) use rustfs_ecstore::api::data_usage::{
         apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached,
         load_data_usage_from_backend, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,

rustfs/tests/connect_registration.rs

@@ -0,0 +1,911 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::VecDeque;
+use std::fs;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use base64::Engine as _;
+use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
+use bytes::Bytes;
+use http_body_util::{BodyExt as _, Full};
+use hyper::service::service_fn;
+use hyper::{Request, Response, StatusCode};
+use hyper_util::rt::TokioIo;
+use p256::ecdsa::signature::Verifier as _;
+use p256::ecdsa::{Signature, VerifyingKey};
+use p256::pkcs8::DecodePublicKey as _;
+use rcgen::{
+    BasicConstraints, CertificateParams, DistinguishedName, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair,
+    KeyUsagePurpose, SanType, SerialNumber,
+};
+use rustfs::connect::{ClientError, ConnectClient, ConnectConfig, CredentialStore, IdentityStore, RegistrationToken, TokenError};
+use rustls::RootCertStore;
+use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, pem::PemObject as _};
+use rustls::server::WebPkiClientVerifier;
+use serde_json::{Value, json};
+use sha2::{Digest as _, Sha256};
+use time::OffsetDateTime;
+use time::format_description::well_known::Rfc3339;
+use tokio::net::TcpListener;
+use tokio_rustls::TlsAcceptor;
+
+const ORGANIZATION_UID: &str = "0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70";
+const CLUSTER_UID: &str = "0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81";
+const DEVICE_UID: &str = "0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92";
+const TOKEN_UID: &str = "0198f4b0-6f00-7b60-9271-7d8e9fa0b1c5";
+const FRESH_TOKEN_UID: &str = "0198f4b0-7f00-7c70-a381-8e9fa0b1c2d6";
+const SECOND_TOKEN_UID: &str = "0198f4b0-8f00-7d80-b491-9fa0b1c2d3e7";
+
+struct TestPki {
+    root_params: CertificateParams,
+    root_key: KeyPair,
+    root_der: CertificateDer<'static>,
+    root_pem: String,
+    server_der: CertificateDer<'static>,
+    server_key: PrivatePkcs8KeyDer<'static>,
+}
+
+impl TestPki {
+    fn new() -> Self {
+        let now = OffsetDateTime::now_utc();
+        let root_key = KeyPair::generate().expect("generate root key");
+        let mut root_params = CertificateParams::default();
+        root_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
+        root_params.not_before = now - time::Duration::days(1);
+        root_params.not_after = now + time::Duration::days(30);
+        root_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::DigitalSignature];
+        root_params.distinguished_name.push(DnType::CommonName, "Connect test root");
+        let root = root_params.self_signed(&root_key).expect("sign root");
+
+        let server_key = KeyPair::generate().expect("generate server key");
+        let mut server_params = CertificateParams::default();
+        server_params.not_before = now - time::Duration::hours(1);
+        server_params.not_after = now + time::Duration::days(2);
+        server_params
+            .subject_alt_names
+            .push(SanType::DnsName("localhost".try_into().expect("valid DNS name")));
+        server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
+        let issuer = Issuer::from_params(&root_params, &root_key);
+        let server = server_params
+            .signed_by(&server_key, &issuer)
+            .expect("sign server certificate");
+
+        Self {
+            root_params,
+            root_key,
+            root_der: root.der().clone(),
+            root_pem: root.pem(),
+            server_der: server.der().clone(),
+            server_key: PrivatePkcs8KeyDer::from(server_key.serialize_der()),
+        }
+    }
+
+    fn credential(&self, identity: &rustfs::connect::DeviceIdentity, uri: &str, serial_byte: u8) -> Value {
+        let now = OffsetDateTime::now_utc().replace_nanosecond(0).expect("whole second");
+        self.credential_window(identity, uri, serial_byte, now, now + time::Duration::days(1))
+    }
+
+    fn credential_window(
+        &self,
+        identity: &rustfs::connect::DeviceIdentity,
+        uri: &str,
+        serial_byte: u8,
+        not_before: OffsetDateTime,
+        not_after: OffsetDateTime,
+    ) -> Value {
+        let mut params = CertificateParams::default();
+        params.not_before = not_before;
+        params.not_after = not_after;
+        params.serial_number = Some(SerialNumber::from(vec![serial_byte; 16]));
+        params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
+        params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];
+        params.distinguished_name = DistinguishedName::new();
+        params.distinguished_name.push(DnType::CommonName, DEVICE_UID);
+        params
+            .subject_alt_names
+            .push(SanType::URI(uri.try_into().expect("valid URI SAN")));
+        let private_key = identity.to_pkcs8_der().expect("serialize device key");
+        let private_key = PrivatePkcs8KeyDer::from(private_key.to_vec());
+        let device_key =
+            KeyPair::from_pkcs8_der_and_sign_algo(&private_key, &rcgen::PKCS_ECDSA_P256_SHA256).expect("parse device key");
+        let issuer = Issuer::from_params(&self.root_params, &self.root_key);
+        let certificate = params.signed_by(&device_key, &issuer).expect("sign device certificate");
+        let serial = format!("{serial_byte:02x}").repeat(16);
+        let cluster = format!("organizations/{ORGANIZATION_UID}/clusters/{CLUSTER_UID}");
+
+        json!({
+            "name": format!("{cluster}/clusterDevices/{DEVICE_UID}"),
+            "uid": DEVICE_UID,
+            "cluster": cluster,
+            "protocolVersion": "v1",
+            "keyId": format!("x509-{serial}"),
+            "certificateSerial": serial,
+            "certificate": certificate.pem(),
+            "certificateChain": certificate.pem(),
+            "notBefore": not_before.format(&Rfc3339).expect("format notBefore"),
+            "notAfter": not_after.format(&Rfc3339).expect("format notAfter"),
+        })
+    }
+
+    fn server_config(&self, require_client: bool) -> rustls::ServerConfig {
+        let mut roots = RootCertStore::empty();
+        roots.add(self.root_der.clone()).expect("add client root");
+        let verifier = WebPkiClientVerifier::builder(Arc::new(roots));
+        let verifier = if require_client {
+            verifier.build()
+        } else {
+            verifier.allow_unauthenticated().build()
+        }
+        .expect("build client verifier");
+        rustls::ServerConfig::builder()
+            .with_client_cert_verifier(verifier)
+            .with_single_cert(vec![self.server_der.clone()], PrivateKeyDer::Pkcs8(self.server_key.clone_key()))
+            .expect("build server TLS")
+    }
+}
+
+#[derive(Clone)]
+enum Reply {
+    Json(StatusCode, Value),
+    DelayedClose(Duration),
+    VerifiedRotation {
+        response: Value,
+        current_public_key: Vec<u8>,
+        current_certificate_fingerprint: String,
+        device_name: String,
+    },
+}
+
+struct TestServer {
+    endpoint: String,
+    seen: Arc<Mutex<Vec<Value>>>,
+    task: tokio::task::JoinHandle<()>,
+}
+
+impl Drop for TestServer {
+    fn drop(&mut self) {
+        self.task.abort();
+    }
+}
+
+async fn server(pki: &TestPki, replies: Vec<Reply>) -> TestServer {
+    server_with_client_auth(pki, replies, false).await
+}
+
+async fn server_with_client_auth(pki: &TestPki, replies: Vec<Reply>, require_client: bool) -> TestServer {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
+    let address = listener.local_addr().expect("server address");
+    let acceptor = TlsAcceptor::from(Arc::new(pki.server_config(require_client)));
+    let replies = Arc::new(Mutex::new(VecDeque::from(replies)));
+    let seen = Arc::new(Mutex::new(Vec::new()));
+    let captured = seen.clone();
+    let task = tokio::spawn(async move {
+        loop {
+            let Ok((stream, _)) = listener.accept().await else {
+                return;
+            };
+            let acceptor = acceptor.clone();
+            let replies = replies.clone();
+            let seen = captured.clone();
+            tokio::spawn(async move {
+                let Ok(stream) = acceptor.accept(stream).await else {
+                    return;
+                };
+                let service = service_fn(move |request: Request<hyper::body::Incoming>| {
+                    let replies = replies.clone();
+                    let seen = seen.clone();
+                    async move {
+                        let body = request.into_body().collect().await.expect("read request body").to_bytes();
+                        let value: Value = serde_json::from_slice(&body).expect("request JSON");
+                        seen.lock().expect("seen lock").push(value.clone());
+                        let reply = replies.lock().expect("reply lock").pop_front().expect("planned reply");
+                        match reply {
+                            Reply::Json(status, value) => Ok::<_, hyper::Error>(
+                                Response::builder()
+                                    .status(status)
+                                    .header("content-type", "application/json")
+                                    .body(Full::new(Bytes::from(serde_json::to_vec(&value).expect("reply JSON"))))
+                                    .expect("response"),
+                            ),
+                            Reply::DelayedClose(delay) => {
+                                tokio::time::sleep(delay).await;
+                                Ok(Response::builder()
+                                    .status(StatusCode::SERVICE_UNAVAILABLE)
+                                    .body(Full::new(Bytes::new()))
+                                    .expect("response"))
+                            }
+                            Reply::VerifiedRotation {
+                                response,
+                                current_public_key,
+                                current_certificate_fingerprint,
+                                device_name,
+                            } => {
+                                verify_rotation_request(
+                                    &value,
+                                    &current_public_key,
+                                    &current_certificate_fingerprint,
+                                    &device_name,
+                                );
+                                Ok(Response::builder()
+                                    .status(StatusCode::OK)
+                                    .header("content-type", "application/json")
+                                    .body(Full::new(Bytes::from(serde_json::to_vec(&response).expect("reply JSON"))))
+                                    .expect("response"))
+                            }
+                        }
+                    }
+                });
+                let _ = hyper::server::conn::http1::Builder::new()
+                    .serve_connection(TokioIo::new(stream), service)
+                    .await;
+            });
+        }
+    });
+    TestServer {
+        endpoint: format!("https://localhost:{}/agent/", address.port()),
+        seen,
+        task,
+    }
+}
+
+fn verify_rotation_request(request: &Value, current_public_key: &[u8], fingerprint: &str, device_name: &str) {
+    assert_eq!(request["protocolVersion"], "v1");
+    assert_eq!(request["proof"]["algorithm"], "ES256");
+    let csr = BASE64_STANDARD
+        .decode(request["certificateRequest"].as_str().expect("certificateRequest"))
+        .expect("CSR base64");
+    let csr_digest = BASE64_URL_NO_PAD.encode(Sha256::digest(&csr));
+    let request_id = request["requestId"].as_str().expect("requestId");
+    let transcript = rebuilt_rotation_transcript(
+        b"RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1",
+        [fingerprint, device_name, request_id, &csr_digest],
+    );
+    let encoded = request["proof"]["value"].as_str().expect("proof value");
+    assert_eq!(encoded.len(), 86);
+    let raw = BASE64_URL_NO_PAD.decode(encoded).expect("proof base64url");
+    let signature = Signature::from_slice(&raw).expect("fixed-width signature");
+    assert!(signature.normalize_s().is_none(), "rotation proof must be low-S");
+    let verifying = VerifyingKey::from_public_key_der(current_public_key).expect("current public key");
+    verifying.verify(&transcript, &signature).expect("rotation proof verifies");
+
+    let wrong_domain = rebuilt_rotation_transcript(
+        b"RUSTFS-CONNECT-CREDENTIAL-ROTATION-V2",
+        [fingerprint, device_name, request_id, &csr_digest],
+    );
+    assert!(verifying.verify(&wrong_domain, &signature).is_err());
+    let wrong_order = rebuilt_rotation_transcript(
+        b"RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1",
+        [device_name, fingerprint, request_id, &csr_digest],
+    );
+    assert!(verifying.verify(&wrong_order, &signature).is_err());
+}
+
+fn rebuilt_rotation_transcript(domain: &[u8], fields: [&str; 4]) -> Vec<u8> {
+    let mut transcript = Vec::new();
+    transcript.extend_from_slice(domain);
+    transcript.push(b'\n');
+    for field in fields {
+        transcript.extend_from_slice(field.len().to_string().as_bytes());
+        transcript.push(b':');
+        transcript.extend_from_slice(field.as_bytes());
+        transcript.push(b'\n');
+    }
+    transcript
+}
+
+fn certificate_fingerprint(pem: &str) -> String {
+    let certificate = CertificateDer::pem_slice_iter(pem.as_bytes())
+        .next()
+        .expect("leaf certificate")
+        .expect("certificate PEM");
+    Sha256::digest(certificate.as_ref())
+        .iter()
+        .map(|byte| format!("{byte:02x}"))
+        .collect()
+}
+
+fn token_document() -> Value {
+    json!({
+        "registrationTokenUid": TOKEN_UID,
+        "registrationTokenSecret": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+        "organizationUid": ORGANIZATION_UID,
+        "clusterUid": CLUSTER_UID,
+        "challengeNonce": "a3f1c07d9b2e4856af0c1d3b5e7f9012c4a6b8d0e2f4061738495a6b7c8d9e0f",
+        "expiresUnix": OffsetDateTime::now_utc().unix_timestamp() + 3600,
+    })
+}
+
+fn token() -> RegistrationToken {
+    token_with_uid(TOKEN_UID)
+}
+
+fn token_with_uid(uid: &str) -> RegistrationToken {
+    let document = token_document();
+    let mut document = document;
+    document["registrationTokenUid"] = json!(uid);
+    RegistrationToken::from_reader(serde_json::to_vec(&document).expect("token JSON").as_slice()).expect("token parses")
+}
+
+fn stores(temp: &tempfile::TempDir) -> (IdentityStore, CredentialStore) {
+    (
+        IdentityStore::new(temp.path().join("identity")),
+        CredentialStore::new(temp.path().join("credential")),
+    )
+}
+
+fn client(server: &TestServer, pki: &TestPki, timeout: Duration) -> ConnectClient {
+    ConnectClient::new(ConnectConfig {
+        endpoint: &server.endpoint,
+        root_ca_pem: pki.root_pem.as_bytes(),
+        timeout,
+    })
+    .expect("build Connect client")
+}
+
+fn rotation_response(pki: &TestPki, identity: &rustfs::connect::DeviceIdentity, serial: u8) -> (Value, Value) {
+    let stored = pki.credential(identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), serial);
+    let mut wire = stored.clone();
+    wire.as_object_mut().expect("response object").remove("uid");
+    wire.as_object_mut().expect("response object").remove("cluster");
+    (wire, stored)
+}
+
+fn write_stored_credential(path: &std::path::Path, response: &Value) {
+    let not_before = OffsetDateTime::parse(response["notBefore"].as_str().expect("notBefore"), &Rfc3339)
+        .expect("parse notBefore")
+        .unix_timestamp();
+    let not_after = OffsetDateTime::parse(response["notAfter"].as_str().expect("notAfter"), &Rfc3339)
+        .expect("parse notAfter")
+        .unix_timestamp();
+    let stored = json!({
+        "name": response["name"],
+        "uid": DEVICE_UID,
+        "protocolVersion": response["protocolVersion"],
+        "keyId": response["keyId"],
+        "certificateSerial": response["certificateSerial"],
+        "certificate": response["certificate"],
+        "certificateChain": response["certificateChain"],
+        "notBeforeUnix": not_before,
+        "notAfterUnix": not_after,
+    });
+    fs::write(path, serde_json::to_vec(&stored).expect("stored credential JSON")).expect("write credential");
+    set_owner_only(path);
+}
+
+#[cfg(unix)]
+fn set_owner_only(path: &std::path::Path) {
+    use std::os::unix::fs::PermissionsExt as _;
+    fs::set_permissions(path, fs::Permissions::from_mode(0o600)).expect("set owner-only mode");
+}
+
+#[cfg(not(unix))]
+fn set_owner_only(_path: &std::path::Path) {}
+
+#[tokio::test]
+async fn registration_reuses_request_and_csr_after_timeout_and_restart() {
+    let temp = tempfile::tempdir().expect("temp dir");
+    let (identity_store, credential_store) = stores(&temp);
+    let identity = identity_store.load_or_create().expect("create identity");
+    let pki = TestPki::new();
+    let response = pki.credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 0x80);
+    let server = server(
+        &pki,
+        vec![
+            Reply::DelayedClose(Duration::from_millis(200)),
+            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
+            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
+            Reply::Json(StatusCode::CREATED, response),
+        ],
+    )
+    .await;
+    let first = client(&server, &pki, Duration::from_millis(80));
+    assert!(matches!(
+        first.register(&identity_store, &credential_store, &token()).await,
+        Err(ClientError::Unavailable { .. })
+    ));
+
+    let restarted = client(&server, &pki, Duration::from_secs(2));
+    let credential = restarted
+        .register(&identity_store, &credential_store, &token())
+        .await
+        .expect("restart replays completed exchange");
+    assert_eq!(credential.uid, DEVICE_UID);
+    assert_eq!(credential.certificate_serial, "80".repeat(16));
+
+    let seen = server.seen.lock().expect("seen lock");
+    assert_eq!(seen.len(), 4);
+    for request in &seen[1..] {
+        assert_eq!(request["requestId"], seen[0]["requestId"]);
+        assert_eq!(request["certificateRequest"], seen[0]["certificateRequest"]);
+    }
+}
+
+#[tokio::test]
+async fn registration_rejects_untrusted_or_misbound_credentials() {
+    for case in ["san", "chain", "key", "key_id", "cluster", "name"] {
+        let temp = tempfile::tempdir().expect("temp dir");
+        let (identity_store, credential_store) = stores(&temp);
+        let identity = identity_store.load_or_create().expect("create identity");
+        let pki = TestPki::new();
+        let mut response = match case {
+            "san" => pki.credential(&identity, "urn:rustfs:connect:device:0198f4b0-3c00-7e30-8f41-4a5b6c7d8e93", 2),
+            "chain" => TestPki::new().credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 2),
+            "key" => pki.credential(
+                &rustfs::connect::DeviceIdentity::generate(),
+                &format!("urn:rustfs:connect:device:{DEVICE_UID}"),
+                2,
+            ),
+            _ => pki.credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 2),
+        };
+        match case {
+            "key_id" => response["keyId"] = json!("x509-deadbeef"),
+            "cluster" => response["cluster"] = json!(format!("organizations/{ORGANIZATION_UID}/clusters/other")),
+            "name" => {
+                response["name"] = json!(format!("organizations/{ORGANIZATION_UID}/clusters/other/clusterDevices/{DEVICE_UID}"))
+            }
+            _ => {}
+        }
+        let server = server(&pki, vec![Reply::Json(StatusCode::CREATED, response)]).await;
+        let error = client(&server, &pki, Duration::from_secs(2))
+            .register(&identity_store, &credential_store, &token())
+            .await
+            .expect_err("invalid returned identity must fail closed");
+        assert!(matches!(error, ClientError::Credential(_)));
+        assert!(!temp.path().join("credential/device.crt.json").exists());
+    }
+}
+
+#[tokio::test]
+async fn stored_credential_is_revalidated_before_reuse() {
+    let temp = tempfile::tempdir().expect("temp dir");
+    let (identity_store, credential_store) = stores(&temp);
+    let identity = identity_store.load_or_create().expect("create identity");
+    let pki = TestPki::new();
+    let issued = pki.credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 4);
+    let server = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
+    let client = client(&server, &pki, Duration::from_secs(2));
+    client
+        .register(&identity_store, &credential_store, &token())
+        .await
+        .expect("register");
+
+    let path = temp.path().join("credential/device.crt.json");
+    let mut stored: Value = serde_json::from_slice(&fs::read(&path).expect("read credential")).expect("credential JSON");
+    stored["certificateSerial"] = json!("00".repeat(16));
+    fs::write(&path, serde_json::to_vec(&stored).expect("credential JSON")).expect("tamper credential");
+    let error = client
+        .register(&identity_store, &credential_store, &token())
+        .await
+        .expect_err("tampered stored credential must fail closed");
+    assert!(matches!(error, ClientError::Credential(_)));
+}
+
+#[tokio::test]
+async fn register_rejects_expired_and_not_yet_valid_stored_credentials() {
+    let temp = tempfile::tempdir().expect("temp dir");
+    let (identity_store, credential_store) = stores(&temp);
+    let identity = identity_store.load_or_create().expect("create identity");
+    let pki = TestPki::new();
+    let issued = pki.credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 9);
+    let registration = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
+    let client = client(&registration, &pki, Duration::from_secs(2));
+    client
+        .register(&identity_store, &credential_store, &token())
+        .await
+        .expect("register");
+
+    let now = OffsetDateTime::now_utc().replace_nanosecond(0).expect("whole second");
+    let path = temp.path().join("credential/device.crt.json");
+    let expired = pki.credential_window(
+        &identity,
+        &format!("urn:rustfs:connect:device:{DEVICE_UID}"),
+        10,
+        now - time::Duration::days(2),
+        now - time::Duration::days(1),
+    );
+    write_stored_credential(&path, &expired);
+    assert!(matches!(
+        client.register(&identity_store, &credential_store, &token()).await,
+        Err(ClientError::CredentialExpired)
+    ));
+
+    let future = pki.credential_window(
+        &identity,
+        &format!("urn:rustfs:connect:device:{DEVICE_UID}"),
+        11,
+        now + time::Duration::hours(1),
+        now + time::Duration::hours(25),
+    );
+    write_stored_credential(&path, &future);
+    assert!(matches!(
+        client.register(&identity_store, &credential_store, &token()).await,
+        Err(ClientError::CredentialNotYetValid)
+    ));
+}
+
+#[tokio::test]
+async fn concurrent_rotation_retries_converge_and_promote_the_next_key() {
+    let temp = tempfile::tempdir().expect("temp dir");
+    let (identity_store, credential_store) = stores(&temp);
+    let current = identity_store.load_or_create().expect("create identity");
+    let pki = TestPki::new();
+    let mut issued = pki.credential(&current, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 5);
+    issued["certificateChain"] = json!(
+        issued["certificateChain"]
+            .as_str()
+            .expect("certificate chain")
+            .trim_end_matches('\n')
+    );
+    let registration = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
+    let registered = client(&registration, &pki, Duration::from_secs(2))
+        .register(&identity_store, &credential_store, &token())
+        .await
+        .expect("register");
+
+    let retries = server(
+        &pki,
+        vec![
+            Reply::DelayedClose(Duration::from_millis(200)),
+            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
+            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
+            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
+            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
+            Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})),
+        ],
+    )
+    .await;
+    let retry_client = client(&retries, &pki, Duration::from_millis(80));
+    let due = registered.not_after_unix - 8 * 60 * 60;
+    let (first, second) = tokio::join!(
+        retry_client.rotate_if_due(&identity_store, &credential_store, due),
+        retry_client.rotate_if_due(&identity_store, &credential_store, due)
+    );
+    assert!(matches!(first, Err(ClientError::Unavailable { .. })));
+    assert!(matches!(second, Err(ClientError::Unavailable { .. })));
+    let (request_id, certificate_request) = {
+        let seen = retries.seen.lock().expect("seen lock");
+        assert!(seen.len() >= 3, "bounded retries must reach the server");
+        for request in &seen[1..] {
+            assert_eq!(request["requestId"], seen[0]["requestId"]);
+            assert_eq!(request["certificateRequest"], seen[0]["certificateRequest"]);
+        }
+        (seen[0]["requestId"].clone(), seen[0]["certificateRequest"].clone())
+    };
+
+    let next_der = fs::read(temp.path().join("identity/device.key.next")).expect("read staged next key");
+    let next = rustfs::connect::DeviceIdentity::from_pkcs8_der(&next_der).expect("parse next key");
+    assert_ne!(current.public_key_der(), next.public_key_der());
+    assert_eq!(
+        identity_store
+            .load()
+            .expect("load current key")
+            .expect("current key")
+            .public_key_der(),
+        current.public_key_der()
+    );
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt as _;
+        let mode = fs::metadata(temp.path().join("identity/device.key.next"))
+            .expect("next key metadata")
+            .permissions()
+            .mode()
+            & 0o7777;
+        assert_eq!(mode, 0o600);
+    }
+    let (rotated, _) = rotation_response(&pki, &next, 6);
+    let success = server_with_client_auth(
+        &pki,
+        vec![Reply::VerifiedRotation {
+            response: rotated,
+            current_public_key: current.public_key_der(),
+            current_certificate_fingerprint: certificate_fingerprint(&registered.certificate),
+            device_name: registered.name.clone(),
+        }],
+        true,
+    )
+    .await;
+    let success_client = client(&success, &pki, Duration::from_secs(2));
+    let (due_result, current_result) = tokio::join!(
+        success_client.rotate_if_due(&identity_store, &credential_store, due),
+        success_client.rotate_if_due(&identity_store, &credential_store, OffsetDateTime::now_utc().unix_timestamp())
+    );
+    let credential = due_result
+        .expect("retry rotation")
+        .or(current_result.expect("concurrent current-state check"))
+        .expect("exactly one rotation is due");
+    assert_eq!(credential.certificate_serial, "06".repeat(16));
+    let success_seen = success.seen.lock().expect("seen lock");
+    assert_eq!(success_seen.len(), 1, "the post-commit actor must not publish stale state");
+    assert_eq!(success_seen[0]["requestId"], request_id);
+    assert_eq!(success_seen[0]["certificateRequest"], certificate_request);
+    drop(success_seen);
+    assert_eq!(
+        identity_store
+            .load()
+            .expect("load key")
+            .expect("current key")
+            .public_key_der(),
+        next.public_key_der()
+    );
+    assert!(!temp.path().join("identity/device.key.next").exists());
+    assert!(!temp.path().join("credential/rotation.pending.json").exists());
+}
+
+#[tokio::test]
+async fn rotation_commit_recovers_after_each_durable_step() {
+    let temp = tempfile::tempdir().expect("temp dir");
+    let (identity_store, credential_store) = stores(&temp);
+    let current = identity_store.load_or_create().expect("create identity");
+    let pki = TestPki::new();
+    let issued = pki.credential(&current, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 7);
+    let registration = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
+    let registered = client(&registration, &pki, Duration::from_secs(2))
+        .register(&identity_store, &credential_store, &token())
+        .await
+        .expect("register");
+    let failed = server(&pki, vec![Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})); 3]).await;
+    let due = registered.not_after_unix - 8 * 60 * 60;
+    assert!(matches!(
+        client(&failed, &pki, Duration::from_secs(2))
+            .rotate_if_due(&identity_store, &credential_store, due)
+            .await,
+        Err(ClientError::Unavailable { .. })
+    ));
+
+    let pending_path = temp.path().join("credential/rotation.pending.json");
+    let pending = fs::read(&pending_path).expect("read pending state");
+    let next_der = fs::read(temp.path().join("identity/device.key.next")).expect("read next key");
+    let next = rustfs::connect::DeviceIdentity::from_pkcs8_der(&next_der).expect("parse next key");
+    let (_, stored) = rotation_response(&pki, &next, 8);
+    write_stored_credential(&temp.path().join("credential/device.crt.json"), &stored);
+
+    let idle = server(&pki, vec![]).await;
+    assert!(
+        client(&idle, &pki, Duration::from_secs(2))
+            .rotate_if_due(&identity_store, &credential_store, OffsetDateTime::now_utc().unix_timestamp())
+            .await
+            .expect("recover after credential save")
+            .is_none()
+    );
+    assert_eq!(
+        identity_store
+            .load()
+            .expect("load key")
+            .expect("current key")
+            .public_key_der(),
+        next.public_key_der()
+    );
+
+    fs::write(&pending_path, pending).expect("restore pending after key commit");
+    set_owner_only(&pending_path);
+    assert!(
+        client(&idle, &pki, Duration::from_secs(2))
+            .rotate_if_due(&identity_store, &credential_store, OffsetDateTime::now_utc().unix_timestamp())
+            .await
+            .expect("recover after key commit")
+            .is_none()
+    );
+    assert!(!pending_path.exists());
+}
+
+#[tokio::test]
+async fn pending_reenrollment_blocks_rotation_and_resumes_original_exchange() {
+    let temp = tempfile::tempdir().expect("temp dir");
+    let (identity_store, credential_store) = stores(&temp);
+    let current = identity_store.load_or_create().expect("create identity");
+    let pki = TestPki::new();
+    let issued = pki.credential(&current, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 13);
+    let registration = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
+    let registered = client(&registration, &pki, Duration::from_secs(2))
+        .register(&identity_store, &credential_store, &token())
+        .await
+        .expect("register");
+
+    let failed = server(&pki, vec![Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})); 3]).await;
+    assert!(matches!(
+        client(&failed, &pki, Duration::from_secs(2))
+            .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
+            .await,
+        Err(ClientError::Unavailable { .. })
+    ));
+
+    let pending_path = temp.path().join("credential/registration.pending.json");
+    let pending = fs::read(&pending_path).expect("read pending reenrollment");
+    let pending_document: Value = serde_json::from_slice(&pending).expect("pending JSON");
+    let next_der = fs::read(temp.path().join("identity/device.key.next")).expect("read next key");
+    let next = rustfs::connect::DeviceIdentity::from_pkcs8_der(&next_der).expect("parse next key");
+    let enrolled = pki.credential(&next, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 14);
+
+    let rotation = server(&pki, vec![]).await;
+    let error = client(&rotation, &pki, Duration::from_secs(2))
+        .rotate_if_due(&identity_store, &credential_store, registered.not_after_unix - 8 * 60 * 60)
+        .await
+        .expect_err("pending reenrollment blocks rotation");
+    assert!(matches!(error, ClientError::PendingRegistration));
+    assert!(rotation.seen.lock().expect("seen lock").is_empty());
+
+    let resumed = server(&pki, vec![Reply::Json(StatusCode::CREATED, enrolled)]).await;
+    let credential = client(&resumed, &pki, Duration::from_secs(2))
+        .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
+        .await
+        .expect("resume reenrollment");
+    assert_eq!(credential.certificate_serial, "0e".repeat(16));
+    let resumed_seen = resumed.seen.lock().expect("seen lock");
+    assert_eq!(resumed_seen.len(), 1);
+    assert_eq!(resumed_seen[0]["requestId"], pending_document["requestId"]);
+    assert_eq!(resumed_seen[0]["certificateRequest"], pending_document["certificateRequest"]);
+}
+
+#[tokio::test]
+async fn reenrollment_commit_recovers_after_each_durable_step() {
+    let temp = tempfile::tempdir().expect("temp dir");
+    let (identity_store, credential_store) = stores(&temp);
+    let current = identity_store.load_or_create().expect("create identity");
+    let pki = TestPki::new();
+    let issued = pki.credential(&current, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 13);
+    let registration = server(&pki, vec![Reply::Json(StatusCode::CREATED, issued)]).await;
+    client(&registration, &pki, Duration::from_secs(2))
+        .register(&identity_store, &credential_store, &token())
+        .await
+        .expect("register");
+
+    let failed = server(&pki, vec![Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})); 3]).await;
+    assert!(matches!(
+        client(&failed, &pki, Duration::from_secs(2))
+            .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
+            .await,
+        Err(ClientError::Unavailable { .. })
+    ));
+
+    let pending_path = temp.path().join("credential/registration.pending.json");
+    let pending = fs::read(&pending_path).expect("read pending reenrollment");
+    let next_der = fs::read(temp.path().join("identity/device.key.next")).expect("read next key");
+    let next = rustfs::connect::DeviceIdentity::from_pkcs8_der(&next_der).expect("parse next key");
+    let enrolled = pki.credential(&next, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 14);
+    write_stored_credential(&temp.path().join("credential/device.crt.json"), &enrolled);
+
+    let idle = server(&pki, vec![]).await;
+    let idle_client = client(&idle, &pki, Duration::from_secs(2));
+    let recovered = idle_client
+        .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
+        .await
+        .expect("recover after reenrollment credential save");
+    assert_eq!(recovered.certificate_serial, "0e".repeat(16));
+    assert_eq!(
+        identity_store
+            .load()
+            .expect("load key")
+            .expect("current key")
+            .public_key_der(),
+        next.public_key_der()
+    );
+
+    fs::remove_file(temp.path().join("credential/registration.completed.json")).expect("remove completed receipt");
+    fs::write(&pending_path, pending).expect("restore pending after key commit");
+    set_owner_only(&pending_path);
+    let recovered = idle_client
+        .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
+        .await
+        .expect("recover after reenrollment key commit");
+    assert_eq!(recovered.certificate_serial, "0e".repeat(16));
+    assert!(!pending_path.exists());
+    let recovered = idle_client
+        .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
+        .await
+        .expect("completed reenrollment is idempotent after pending cleanup");
+    assert_eq!(recovered.certificate_serial, "0e".repeat(16));
+    assert!(idle.seen.lock().expect("seen lock").is_empty());
+
+    let different = server(&pki, vec![Reply::Json(StatusCode::SERVICE_UNAVAILABLE, json!({})); 3]).await;
+    assert!(matches!(
+        client(&different, &pki, Duration::from_secs(2))
+            .reenroll(&identity_store, &credential_store, &token_with_uid(SECOND_TOKEN_UID))
+            .await,
+        Err(ClientError::Unavailable { .. })
+    ));
+    assert_eq!(different.seen.lock().expect("seen lock").len(), 3);
+}
+
+#[test]
+fn registration_token_schema_is_strict_and_bounded() {
+    let mut document = serde_json::to_value(token_document()).expect("token document");
+    document["unexpected"] = json!(true);
+    assert!(matches!(
+        RegistrationToken::from_reader(serde_json::to_vec(&document).expect("token JSON").as_slice()),
+        Err(TokenError::Invalid(_))
+    ));
+    assert!(matches!(
+        RegistrationToken::from_reader(vec![b' '; 16 * 1024 + 1].as_slice()),
+        Err(TokenError::TooLarge)
+    ));
+    let mut malformed = token_document();
+    malformed["challengeNonce"] = json!("A".repeat(64));
+    assert!(matches!(
+        RegistrationToken::from_reader(serde_json::to_vec(&malformed).expect("token JSON").as_slice()),
+        Err(TokenError::Shape)
+    ));
+}
+
+#[tokio::test]
+async fn rotation_waits_for_threshold_and_stops_on_revocation() {
+    let temp = tempfile::tempdir().expect("temp dir");
+    let (identity_store, credential_store) = stores(&temp);
+    let identity = identity_store.load_or_create().expect("create identity");
+    let pki = TestPki::new();
+    let issued = pki.credential(&identity, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 3);
+    let rotation_server = server(
+        &pki,
+        vec![
+            Reply::Json(StatusCode::CREATED, issued),
+            Reply::Json(StatusCode::UNAUTHORIZED, json!({"details": [{"reason": "DEVICE_REVOKED"}]})),
+        ],
+    )
+    .await;
+    let connect = client(&rotation_server, &pki, Duration::from_secs(2));
+    let current = connect
+        .register(&identity_store, &credential_store, &token())
+        .await
+        .expect("register");
+    assert!(
+        connect
+            .rotate_if_due(&identity_store, &credential_store, current.not_before_unix)
+            .await
+            .expect("not due")
+            .is_none()
+    );
+    let error = connect
+        .rotate_if_due(&identity_store, &credential_store, current.not_after_unix - 8 * 60 * 60)
+        .await
+        .expect_err("revocation must stop rotation");
+    assert!(matches!(error, ClientError::AccessRevoked { .. }));
+    assert!(error.to_string().contains("ConnectClient::reenroll"));
+    assert_eq!(rotation_server.seen.lock().expect("seen lock").len(), 2);
+    let stored: Value =
+        serde_json::from_slice(&fs::read(temp.path().join("credential/device.crt.json")).expect("read stored credential"))
+            .expect("stored credential JSON");
+    assert_eq!(stored["certificateSerial"], current.certificate_serial);
+
+    let next_der = fs::read(temp.path().join("identity/device.key.next")).expect("read staged next key");
+    let next = rustfs::connect::DeviceIdentity::from_pkcs8_der(&next_der).expect("parse staged next key");
+    let enrolled = pki.credential(&next, &format!("urn:rustfs:connect:device:{DEVICE_UID}"), 12);
+    let reenrollment = server(&pki, vec![Reply::Json(StatusCode::CREATED, enrolled)]).await;
+    let fresh = client(&reenrollment, &pki, Duration::from_secs(2))
+        .reenroll(&identity_store, &credential_store, &token_with_uid(FRESH_TOKEN_UID))
+        .await
+        .expect("fresh token reenrolls revoked credential");
+    assert_eq!(fresh.certificate_serial, "0c".repeat(16));
+    assert_eq!(
+        identity_store
+            .load()
+            .expect("load identity")
+            .expect("identity")
+            .public_key_der(),
+        next.public_key_der()
+    );
+}
+
+#[test]
+fn unconfigured_connect_has_no_side_effects() {
+    let temp = tempfile::tempdir().expect("temp dir");
+    let directory = temp.path().join("connect");
+    assert!(
+        ConnectClient::from_optional_config(None)
+            .expect("unconfigured is valid")
+            .is_none()
+    );
+    assert!(!directory.exists());
+}