Merge remote-tracking branch 'origin/main' into overtrue/fix-1917-peer-reload-generation-merge

crates/ecstore/src/api/mod.rs

@@ -317,8 +317,6 @@ pub mod config {
 }
 
 pub mod data_usage {
-    #[cfg(feature = "test-util")]
-    pub use crate::data_usage::seed_bucket_usage_memory_for_test;
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
         init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
@@ -330,6 +328,8 @@ pub mod data_usage {
         remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
         store_data_usage_in_backend,
     };
+    #[cfg(feature = "test-util")]
+    pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
 }
 
 pub mod disk {

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -3425,6 +3425,44 @@ mod tests {
         assert!(mutexes.contains_key("second"));
     }
 
+    #[tokio::test]
+    async fn update_all_targets_publishes_disable_proxy_on_target_client() {
+        // The read-proxy selector (replication_proxy::get_proxy_targets) skips
+        // targets whose TargetClient carries disable_proxy — the persisted
+        // per-target opt-out must survive client publication.
+        let sys = BucketTargetSys::default();
+        let target = |arn: &str, disable_proxy: bool| BucketTarget {
+            arn: arn.to_string(),
+            endpoint: "192.168.1.10:9000".to_string(),
+            target_bucket: "target-bucket".to_string(),
+            region: "us-east-1".to_string(),
+            disable_proxy,
+            credentials: Some(Credentials {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                session_token: None,
+                expiration: None,
+            }),
+            ..Default::default()
+        };
+        let targets = BucketTargets {
+            targets: vec![target("arn:proxied", false), target("arn:opted-out", true)],
+        };
+
+        sys.update_all_targets("bucket", Some(&targets)).await;
+
+        let proxied = sys
+            .get_remote_target_client("bucket", "arn:proxied")
+            .await
+            .expect("client should be published");
+        assert!(!proxied.disable_proxy);
+        let opted_out = sys
+            .get_remote_target_client("bucket", "arn:opted-out")
+            .await
+            .expect("client should be published");
+        assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn target_updates_serialize_client_build_through_publication_per_bucket() {
         let sys = Arc::new(BucketTargetSys::default());

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info(
         .map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
     let mut rstate = oi.replication_state();
     rstate.replicate_decision_str = dsc.to_string();
-    let asz = oi.get_actual_size().unwrap_or_default();
+    let asz = oi.get_actual_size_or_physical();
     let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
     let checksum = if ssec { oi.checksum.clone() } else { None };
 

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -1412,7 +1412,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
     };
     let mut replication_state = oi.replication_state();
     replication_state.replicate_decision_str = dsc.to_string();
-    let actual_size = oi.get_actual_size().unwrap_or_default();
+    let actual_size = oi.get_actual_size_or_physical();
 
     Ok(ReplicateObjectInfo {
         name: oi.name.clone(),

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
             .map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
         version_id: object_info.version_id.map(|version_id| version_id.to_string()),
         etag: object_info.etag.as_deref(),
-        actual_size: object_info.get_actual_size().unwrap_or_default(),
+        actual_size: object_info.get_actual_size_or_physical(),
         delete_marker: object_info.delete_marker,
         content_type: object_info.content_type.as_deref(),
         content_encoding: object_info.content_encoding.as_deref(),
@@ -542,6 +542,20 @@ mod tests {
         assert!(replication_target_head_is_newer_null_version(&source, &target));
     }
 
+    #[test]
+    fn replication_source_uses_physical_size_for_unknown_compressed_object() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
+        let source = ObjectInfo {
+            size: 128,
+            actual_size: -1,
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+
+        assert_eq!(replication_source_object(&source).actual_size, 128);
+    }
+
     #[test]
     fn replication_target_head_content_matches_compare_etag_only() {
         let source = ObjectInfo {

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -50,10 +50,10 @@ use rustfs_protos::evict_failed_connection;
 use rustfs_protos::proto_gen::node_service::RenamePartRequest;
 use rustfs_protos::proto_gen::node_service::{
     BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
-    DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
-    MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
-    ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
-    RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
+    DeleteVersionRequest, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest,
+    ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest,
+    ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
+    RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
     SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
     WriteMetadataRequest, node_service_client::NodeServiceClient,
 };
@@ -112,6 +112,28 @@ const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
 const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
 pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60);
 
+fn decode_delete_versions_errors(response: DeleteVersionsResponse, expected_len: usize) -> Vec<Option<Error>> {
+    if !response.item_errors.is_empty() {
+        if response.item_errors.len() != expected_len {
+            return vec![Some(Error::other("malformed delete_versions item errors")); expected_len];
+        }
+        return response
+            .item_errors
+            .into_iter()
+            .map(|error| (error.code != 0).then(|| error.into()))
+            .collect();
+    }
+
+    if response.errors.len() != expected_len {
+        return vec![Some(Error::other("malformed delete_versions errors")); expected_len];
+    }
+    response
+        .errors
+        .into_iter()
+        .map(|error| (!error.is_empty()).then(|| Error::other(error)))
+        .collect()
+}
+
 fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result<SnapshotLeaseToken> {
     if !response.success {
         return Err(response.error.unwrap_or_default().into());
@@ -2406,8 +2428,6 @@ impl DiskAPI for RemoteDisk {
             return errors;
         }
 
-        // TODO(backlog): replace string errors with typed `StorageError` variants
-
         let result = self
             .execute_with_timeout(
                 || async {
@@ -2439,17 +2459,7 @@ impl DiskAPI for RemoteDisk {
             }
             return errors;
         }
-        response
-            .errors
-            .iter()
-            .map(|error| {
-                if error.is_empty() {
-                    None
-                } else {
-                    Some(Error::other(error.to_string()))
-                }
-            })
-            .collect()
+        decode_delete_versions_errors(response, versions.len())
     }
 
     #[tracing::instrument(level = "trace", skip_all)]
@@ -3760,6 +3770,63 @@ mod tests {
 
     static INIT: Once = Once::new();
 
+    #[test]
+    fn delete_versions_response_preserves_typed_item_errors() {
+        let errors = decode_delete_versions_errors(
+            DeleteVersionsResponse {
+                success: true,
+                errors: vec!["file not found".to_string(), String::new()],
+                error: None,
+                item_errors: vec![
+                    rustfs_protos::proto_gen::node_service::Error {
+                        code: DiskError::FileNotFound.to_u32(),
+                        error_info: "file not found".to_string(),
+                    },
+                    rustfs_protos::proto_gen::node_service::Error::default(),
+                ],
+            },
+            2,
+        );
+
+        assert!(matches!(errors.as_slice(), [Some(DiskError::FileNotFound), None]));
+    }
+
+    #[test]
+    fn delete_versions_response_accepts_legacy_string_errors() {
+        let errors = decode_delete_versions_errors(
+            DeleteVersionsResponse {
+                success: true,
+                errors: vec!["legacy error".to_string(), String::new()],
+                error: None,
+                item_errors: Vec::new(),
+            },
+            2,
+        );
+
+        assert_eq!(errors.len(), 2);
+        assert_eq!(errors[0].as_ref().map(ToString::to_string).as_deref(), Some("io error legacy error"));
+        assert!(errors[1].is_none());
+    }
+
+    #[test]
+    fn delete_versions_response_rejects_misaligned_item_errors() {
+        let errors = decode_delete_versions_errors(
+            DeleteVersionsResponse {
+                success: true,
+                errors: vec!["file not found".to_string()],
+                error: None,
+                item_errors: vec![rustfs_protos::proto_gen::node_service::Error {
+                    code: DiskError::FileNotFound.to_u32(),
+                    error_info: "file not found".to_string(),
+                }],
+            },
+            2,
+        );
+
+        assert_eq!(errors.len(), 2);
+        assert!(errors.iter().all(Option::is_some));
+    }
+
     #[test]
     fn disk_mutation_digest_marks_rolling_compatibility() {
         let mut request = Request::new(());

crates/ecstore/src/core/pools.rs

@@ -36,7 +36,8 @@ use crate::disk::error::DiskError;
 use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result};
 use crate::error::{
-    StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
+    StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_object_not_found, is_err_operation_canceled,
+    is_err_version_not_found,
 };
 use crate::layout::endpoints::EndpointServerPools;
 use crate::object_api::{GetObjectReader, ObjectOptions};
@@ -89,6 +90,7 @@ const DECOMMISSION_STAGE_SOURCE_CLEANUP: &str = "source_cleanup";
 const DECOMMISSION_STAGE_ENTRY_FINISHED: &str = "entry_finished";
 const DECOMMISSION_PROGRESS_SAVE_INTERVAL: Duration = Duration::seconds(30);
 const DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD: usize = 1000;
+const DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF: Duration = Duration::seconds(1);
 const DECOMMISSION_BUCKET_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY";
 const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
 const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30;
@@ -638,22 +640,6 @@ fn track_decommission_current_object(meta: &mut PoolMeta, idx: usize, bucket: &s
     track_decommission_current_object_stage(meta, idx, bucket, object, "")
 }
 
-fn touch_decommission_progress(meta: &mut PoolMeta, idx: usize) -> Result<()> {
-    let pool_count = meta.pools.len();
-    ensure_valid_decommission_pool_index(pool_count, idx)?;
-
-    let Some(pool) = meta.pools.get_mut(idx) else {
-        return Err(invalid_decommission_pool_index_error(pool_count, idx));
-    };
-    let Some(info) = pool.decommission.as_mut() else {
-        return Err(decommission_metadata_not_initialized_error("touch decommission progress"));
-    };
-
-    pool.last_update = OffsetDateTime::now_utc();
-    info.mark_progress_saved();
-    Ok(())
-}
-
 fn resolve_decommission_update_after_result(result: Result<bool>) -> Result<bool> {
     result.map_err(|err| Error::other(format!("decommission metadata update failed: {err}")))
 }
@@ -773,7 +759,76 @@ async fn load_decommission_entry_exact_versions(
 }
 
 fn resolve_decommission_check_after_list_result(list_result: Result<()>, entry_error: Option<Error>) -> Result<()> {
-    if let Some(err) = entry_error { Err(err) } else { list_result }
+    match list_result {
+        Ok(()) => entry_error.map_or(Ok(()), Err),
+        Err(list_err) => resolve_decommission_listing_error(Some(list_err), entry_error).map_or(Ok(()), Err),
+    }
+}
+
+fn resolve_decommission_listing_error(listing_error: Option<Error>, entry_error: Option<Error>) -> Option<Error> {
+    match (listing_error, entry_error) {
+        (Some(listing_error), Some(entry_error)) if is_err_operation_canceled(&listing_error) => Some(entry_error),
+        (Some(listing_error), Some(entry_error)) if is_err_operation_canceled(&entry_error) => Some(listing_error),
+        (Some(listing_error), _) => Some(listing_error),
+        (None, entry_error) => entry_error,
+    }
+}
+
+fn decommission_unresolved_listing_error(
+    bucket: &str,
+    prefix: &str,
+    candidate: Option<&str>,
+    candidate_count: usize,
+    disk_error_count: usize,
+    pool_index: usize,
+    set_index: usize,
+) -> Error {
+    let location = candidate.unwrap_or(prefix);
+    Error::other(format!(
+        "decommission listing could not resolve metadata for {bucket}/{location} on pool {pool_index} set {set_index} ({candidate_count} candidate(s), {disk_error_count} disk error(s))"
+    ))
+}
+
+fn resolve_decommission_partial_listing_entry(
+    entries: MetaCacheEntries,
+    resolver: MetadataResolutionParams,
+    bucket: &str,
+    prefix: &str,
+    disk_error_count: usize,
+    pool_index: usize,
+    set_index: usize,
+) -> Result<MetaCacheEntry> {
+    let candidate_count = entries.as_ref().iter().flatten().count();
+    if let Some(entry) = entries.resolve(resolver) {
+        return Ok(entry);
+    }
+
+    let candidate = entries.as_ref().iter().flatten().map(|entry| entry.name.as_str()).next();
+    Err(decommission_unresolved_listing_error(
+        bucket,
+        prefix,
+        candidate,
+        candidate_count,
+        disk_error_count,
+        pool_index,
+        set_index,
+    ))
+}
+
+async fn record_decommission_entry_error(
+    entry_error: &Arc<tokio::sync::Mutex<Option<Error>>>,
+    rx: &CancellationToken,
+    err: Error,
+) {
+    if rx.is_cancelled() {
+        return;
+    }
+
+    let mut first_err = entry_error.lock().await;
+    if first_err.is_none() && !rx.is_cancelled() {
+        *first_err = Some(err);
+        rx.cancel();
+    }
 }
 
 fn resolve_decommission_pool_meta_reload_result(result: Result<()>, stage: &str) -> Result<()> {
@@ -1492,6 +1547,7 @@ impl TryFrom<PersistedPoolDecommissionInfo> for PoolDecommissionInfo {
             terminal_reload_attempt_at: value.terminal_reload_attempt_at,
             terminal_reload_failures: value.terminal_reload_failures,
             progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
+            progress_save_retry_after: None,
         })
     }
 }
@@ -1523,6 +1579,7 @@ impl TryFrom<LegacyPoolDecommissionInfo> for PoolDecommissionInfo {
             terminal_reload_attempt_at: None,
             terminal_reload_failures: Vec::new(),
             progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
+            progress_save_retry_after: None,
         })
     }
 }
@@ -1636,6 +1693,82 @@ impl PoolMeta {
         }
     }
 
+    fn decommission_progress_checkpoint(
+        &self,
+        idx: usize,
+        duration: Duration,
+        now: OffsetDateTime,
+    ) -> Result<Option<DecommissionProgressCheckpoint>> {
+        let pool_count = self.pools.len();
+        ensure_valid_decommission_pool_index(pool_count, idx)?;
+
+        let Some(pool) = self.pools.get(idx) else {
+            return Err(invalid_decommission_pool_index_error(pool_count, idx));
+        };
+        let Some(info) = pool.decommission.as_ref() else {
+            return Err(decommission_metadata_not_initialized_error("update decommission metadata timestamp"));
+        };
+
+        if info.progress_save_retry_after.is_some_and(|retry_after| now < retry_after) {
+            return Ok(None);
+        }
+
+        let time_threshold_reached = now.unix_timestamp() - pool.last_update.unix_timestamp() >= duration.whole_seconds();
+        let item_threshold_reached = info.items_since_last_progress_save() >= DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD;
+        if !time_threshold_reached && !item_threshold_reached {
+            return Ok(None);
+        }
+
+        Ok(Some(DecommissionProgressCheckpoint {
+            start_time: info.start_time,
+            queued: info.queued,
+            counted_items: info.counted_items(),
+            checkpoint_at: now,
+        }))
+    }
+
+    fn commit_decommission_progress_checkpoint(&mut self, idx: usize, checkpoint: DecommissionProgressCheckpoint) -> bool {
+        let Some(pool) = self.pools.get_mut(idx) else {
+            return false;
+        };
+        let Some(info) = pool.decommission.as_mut() else {
+            return false;
+        };
+
+        if info.start_time != checkpoint.start_time
+            || info.queued != checkpoint.queued
+            || !is_decommission_active(info.complete, info.failed, info.canceled)
+        {
+            return false;
+        }
+
+        info.progress_save_item_baseline = info.progress_save_item_baseline.max(checkpoint.counted_items);
+        info.progress_save_retry_after = None;
+        pool.last_update = pool.last_update.max(checkpoint.checkpoint_at);
+        true
+    }
+
+    fn defer_decommission_progress_checkpoint(
+        &mut self,
+        idx: usize,
+        checkpoint: DecommissionProgressCheckpoint,
+        retry_after: OffsetDateTime,
+    ) {
+        let Some(pool) = self.pools.get_mut(idx) else {
+            return;
+        };
+        let Some(info) = pool.decommission.as_mut() else {
+            return;
+        };
+
+        if info.start_time == checkpoint.start_time
+            && info.queued == checkpoint.queued
+            && is_decommission_active(info.complete, info.failed, info.canceled)
+        {
+            info.progress_save_retry_after = Some(retry_after);
+        }
+    }
+
     fn load_from_config_data(&mut self, data: Vec<u8>) -> Result<()> {
         if data.is_empty() {
             return Ok(());
@@ -1996,30 +2129,9 @@ impl PoolMeta {
     }
 
     pub fn update_after(&mut self, idx: usize, duration: Duration) -> Result<bool> {
-        let pool_count = self.pools.len();
-        ensure_valid_decommission_pool_index(pool_count, idx)?;
-
-        let (last_update, item_threshold_reached) = match self.pools.get(idx) {
-            Some(pool) if let Some(info) = pool.decommission.as_ref() => (
-                pool.last_update,
-                info.items_since_last_progress_save() >= DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
-            ),
-            Some(_) => {
-                return Err(decommission_metadata_not_initialized_error("update decommission metadata timestamp"));
-            }
-            None => return Err(invalid_decommission_pool_index_error(pool_count, idx)),
-        };
-        let now = OffsetDateTime::now_utc();
-
-        if now.unix_timestamp() - last_update.unix_timestamp() >= duration.whole_seconds() || item_threshold_reached {
-            let Some(pool) = self.pools.get_mut(idx) else {
-                return Err(invalid_decommission_pool_index_error(pool_count, idx));
-            };
-            pool.last_update = now;
-            return Ok(true);
-        }
-
-        Ok(false)
+        Ok(self
+            .decommission_progress_checkpoint(idx, duration, OffsetDateTime::now_utc())?
+            .is_some())
     }
 
     pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
@@ -2160,6 +2272,16 @@ pub struct PoolDecommissionInfo {
     pub terminal_reload_failures: Vec<String>,
     #[serde(skip)]
     pub progress_save_item_baseline: usize,
+    #[serde(skip)]
+    pub progress_save_retry_after: Option<OffsetDateTime>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct DecommissionProgressCheckpoint {
+    start_time: Option<OffsetDateTime>,
+    queued: bool,
+    counted_items: usize,
+    checkpoint_at: OffsetDateTime,
 }
 
 impl PoolDecommissionInfo {
@@ -2194,6 +2316,7 @@ impl PoolDecommissionInfo {
 
     fn mark_progress_saved(&mut self) {
         self.progress_save_item_baseline = self.counted_items();
+        self.progress_save_retry_after = None;
     }
 
     pub fn bucket_push(&mut self, bucket: &DecomBucketInfo) {
@@ -2498,6 +2621,40 @@ impl ECStore {
         snapshot.save(self.pools.clone()).await
     }
 
+    async fn save_decommission_progress_checkpoint(&self, idx: usize) -> Result<bool> {
+        // Lock order: save gate, then the short pool metadata read/write sections. Peer
+        // reloads are intentionally performed by the caller after both locks are released.
+        let _save_guard = self.pool_meta_save_gate.lock().await;
+        let (snapshot, checkpoint) = {
+            let pool_meta = self.pool_meta.read().await;
+            let Some(checkpoint) = pool_meta.decommission_progress_checkpoint(
+                idx,
+                DECOMMISSION_PROGRESS_SAVE_INTERVAL,
+                OffsetDateTime::now_utc(),
+            )?
+            else {
+                return Ok(false);
+            };
+
+            let mut snapshot = pool_meta.clone();
+            let Some(pool) = snapshot.pools.get_mut(idx) else {
+                return Err(invalid_decommission_pool_index_error(snapshot.pools.len(), idx));
+            };
+            pool.last_update = checkpoint.checkpoint_at;
+            (snapshot, checkpoint)
+        };
+
+        if let Err(err) = snapshot.save(self.pools.clone()).await {
+            let retry_after = OffsetDateTime::now_utc() + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
+            let mut pool_meta = self.pool_meta.write().await;
+            pool_meta.defer_decommission_progress_checkpoint(idx, checkpoint, retry_after);
+            return Err(err);
+        }
+
+        let mut pool_meta = self.pool_meta.write().await;
+        Ok(pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint))
+    }
+
     async fn save_current_pool_meta_for_decommission_start(
         &self,
         indices: &[usize],
@@ -2880,7 +3037,7 @@ impl ECStore {
         Ok(())
     }
 
-    async fn save_decommission_entry_progress_stage(
+    async fn track_decommission_entry_progress_stage(
         &self,
         idx: usize,
         bucket: &str,
@@ -2891,22 +3048,6 @@ impl ECStore {
             let mut pool_meta = self.pool_meta.write().await;
             track_decommission_current_object_stage(&mut pool_meta, idx, bucket, object, stage)
                 .map_err(|err| with_decommission_entry_context(stage, bucket, object, err))?;
-            touch_decommission_progress(&mut pool_meta, idx)
-                .map_err(|err| with_decommission_entry_context(stage, bucket, object, err))?;
-        }
-
-        if let Some(err) = resolve_decommission_progress_save_result(self.save_current_pool_meta().await) {
-            warn!(
-                event = EVENT_DECOMMISSION_ENTRY,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_POOLS,
-                pool_index = idx,
-                bucket = %bucket,
-                object = %object,
-                stage,
-                error = ?err,
-                "Decommission progress stage save failed"
-            );
         }
 
         Ok(())
@@ -3174,7 +3315,7 @@ impl ECStore {
                 let bucket_name = bucket.clone();
                 let object_name = rd.object_info.name.clone();
 
-                self.save_decommission_entry_progress_stage(
+                self.track_decommission_entry_progress_stage(
                     idx,
                     bucket_name.as_str(),
                     object_name.as_str(),
@@ -3268,7 +3409,7 @@ impl ECStore {
             }
             decommission_cancel_signal_result(rx.is_cancelled())?;
 
-            self.save_decommission_entry_progress_stage(
+            self.track_decommission_entry_progress_stage(
                 idx,
                 bucket.as_str(),
                 entry.name.as_str(),
@@ -3276,7 +3417,7 @@ impl ECStore {
             )
             .await?;
 
-            self.save_decommission_entry_progress_stage(
+            self.track_decommission_entry_progress_stage(
                 idx,
                 bucket.as_str(),
                 entry.name.as_str(),
@@ -3343,34 +3484,42 @@ impl ECStore {
             }
         };
 
-        self.save_decommission_entry_progress_stage(idx, bucket.as_str(), entry.name.as_str(), DECOMMISSION_STAGE_ENTRY_FINISHED)
-            .await?;
+        self.track_decommission_entry_progress_stage(
+            idx,
+            bucket.as_str(),
+            entry.name.as_str(),
+            DECOMMISSION_STAGE_ENTRY_FINISHED,
+        )
+        .await?;
 
         if should_save_progress {
-            let save_result = self.save_current_pool_meta().await;
-            if let Some(err) = resolve_decommission_progress_save_result(save_result) {
-                warn!(
-                    event = EVENT_DECOMMISSION_ENTRY,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_POOLS,
-                    pool_index = idx,
-                    bucket = %bucket,
-                    object = %entry.name,
-                    state = "progress_save_failed",
-                    error = %err,
-                    "Decommission progress save failed; continuing and will retry at the next checkpoint"
-                );
-            } else {
-                let mut pool_meta = self.pool_meta.write().await;
-                pool_meta.mark_decommission_progress_saved();
-                if let Some(notification_sys) = runtime_sources::notification_sys()
-                    && let Err(err) = resolve_decommission_entry_reload_result(
-                        notification_sys.reload_pool_meta().await,
-                        bucket.as_str(),
-                        entry.name.as_str(),
-                    )
-                {
-                    warn!("{err}");
+            match self.save_decommission_progress_checkpoint(idx).await {
+                Ok(true) => {
+                    if let Some(notification_sys) = runtime_sources::notification_sys()
+                        && let Err(err) = resolve_decommission_entry_reload_result(
+                            notification_sys.reload_pool_meta().await,
+                            bucket.as_str(),
+                            entry.name.as_str(),
+                        )
+                    {
+                        warn!("{err}");
+                    }
+                }
+                Ok(false) => {}
+                Err(err) => {
+                    if let Some(err) = resolve_decommission_progress_save_result(Err(err)) {
+                        warn!(
+                            event = EVENT_DECOMMISSION_ENTRY,
+                            component = LOG_COMPONENT_ECSTORE,
+                            subsystem = LOG_SUBSYSTEM_POOLS,
+                            pool_index = idx,
+                            bucket = %bucket,
+                            object = %entry.name,
+                            state = "progress_save_failed",
+                            error = %err,
+                            "Decommission progress save failed; continuing and will retry at the next checkpoint"
+                        );
+                    }
                 }
             }
         }
@@ -3547,6 +3696,7 @@ impl ECStore {
             let rx_clone = rx.clone();
             let bi = bi.clone();
             let set_id = set_idx;
+            let listing_entry_error = entry_error.clone();
             let worker = tokio::spawn(async move {
                 let _listing_permit = listing_permit;
                 run_decommission_listing_with_retry(
@@ -3560,7 +3710,11 @@ impl ECStore {
                         let set = set.clone();
                         let rx = rx_clone.clone();
                         let bucket = bi.clone();
-                        async move { set.list_objects_to_decommission(rx, bucket, callback).await }
+                        let entry_error = listing_entry_error.clone();
+                        async move {
+                            set.list_objects_to_decommission(rx, bucket, callback, entry_error.clone(), idx, set_id)
+                                .await
+                        }
                     },
                 )
                 .await
@@ -3590,11 +3744,7 @@ impl ECStore {
 
         wait_decommission_worker_drain(&workers, worker_limit).await?;
 
-        if let Some(err) = listing_worker_error {
-            return Err(err);
-        }
-
-        if let Some(err) = entry_error.lock().await.clone() {
+        if let Some(err) = resolve_decommission_listing_error(listing_worker_error, entry_error.lock().await.clone()) {
             return Err(err);
         }
 
@@ -4200,7 +4350,7 @@ impl ECStore {
         let buckets = self.get_buckets_to_decommission().await?;
         let pool = self.pools[idx].clone();
 
-        for set in &pool.disk_set {
+        for (set_index, set) in pool.disk_set.iter().enumerate() {
             for bucket_info in &buckets {
                 let mut lifecycle_config = None;
                 let mut object_lock_config = None;
@@ -4295,7 +4445,7 @@ impl ECStore {
                 });
 
                 let list_result = set
-                    .list_objects_to_decommission(callback_rx, bucket_info.clone(), callback)
+                    .list_objects_to_decommission(callback_rx, bucket_info.clone(), callback, entry_error.clone(), idx, set_index)
                     .await;
                 let entry_error = entry_error.lock().await.clone();
                 resolve_decommission_check_after_list_result(list_result, entry_error)?;
@@ -5030,12 +5180,15 @@ mod tests {
 pub type ListCallback = Arc<dyn Fn(MetaCacheEntry) -> BoxFuture<'static, ()> + Send + Sync + 'static>;
 
 impl SetDisks {
-    #[tracing::instrument(skip(self, rx, cb_func))]
+    #[tracing::instrument(skip(self, rx, cb_func, entry_error))]
     async fn list_objects_to_decommission(
         self: &Arc<Self>,
         rx: CancellationToken,
         bucket_info: DecomBucketInfo,
         cb_func: ListCallback,
+        entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
+        pool_index: usize,
+        set_index: usize,
     ) -> Result<()> {
         let (disks, _) = self.get_online_disks_with_healing(false).await;
         ensure_decommission_listing_disks_available(!disks.is_empty(), &bucket_info.name)?;
@@ -5050,6 +5203,12 @@ impl SetDisks {
         };
 
         let cb1 = cb_func.clone();
+        let unresolved_error = entry_error.clone();
+        let unresolved_rx = rx.clone();
+        let unresolved_bucket = bucket_info.name.clone();
+        let unresolved_prefix = bucket_info.prefix.clone();
+        let unresolved_pool_index = pool_index;
+        let unresolved_set_index = set_index;
 
         list_path_raw(
             rx,
@@ -5062,20 +5221,51 @@ impl SetDisks {
                 skip_walkdir_total_timeout: true,
                 walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
                 agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
-                partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
+                partial: Some(Box::new(move |entries: MetaCacheEntries, errs: &[Option<DiskError>]| {
                     let resolver = resolver.clone();
                     let cb_func = cb_func.clone();
-                    match entries.resolve(resolver) {
-                        Some(entry) => {
+                    let bucket = unresolved_bucket.clone();
+                    let prefix = unresolved_prefix.clone();
+                    let unresolved_error = unresolved_error.clone();
+                    let unresolved_rx = unresolved_rx.clone();
+                    let pool_index = unresolved_pool_index;
+                    let set_index = unresolved_set_index;
+                    let disk_error_count = errs.iter().flatten().count();
+                    if unresolved_rx.is_cancelled() {
+                        return Box::pin(async {});
+                    }
+
+                    match resolve_decommission_partial_listing_entry(
+                        entries,
+                        resolver,
+                        &bucket,
+                        &prefix,
+                        disk_error_count,
+                        pool_index,
+                        set_index,
+                    ) {
+                        Ok(entry) => {
                             warn!("decommission_pool: list_objects_to_decommission get {}", &entry.name);
                             Box::pin(async move {
                                 cb_func(entry).await;
                             })
                         }
-                        None => {
-                            warn!("decommission_pool: list_objects_to_decommission get none");
-                            Box::pin(async {})
-                        }
+                        Err(err) => Box::pin(async move {
+                            if unresolved_rx.is_cancelled() {
+                                return;
+                            }
+                            warn!(
+                                event = EVENT_DECOMMISSION_BUCKET,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_POOLS,
+                                bucket = %bucket,
+                                prefix = %prefix,
+                                state = "unresolved_entry",
+                                error = %err,
+                                "Decommission listing failed closed on unresolved metadata"
+                            );
+                            record_decommission_entry_error(&unresolved_error, &unresolved_rx, err).await;
+                        }),
                     }
                 })),
                 ..Default::default()
@@ -5083,6 +5273,10 @@ impl SetDisks {
         )
         .await?;
 
+        if let Some(err) = entry_error.lock().await.clone() {
+            return Err(err);
+        }
+
         Ok(())
     }
 }
@@ -5273,11 +5467,11 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi
 #[cfg(test)]
 mod pools_tests {
     use super::{
-        DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo,
-        DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo,
-        PoolStatus, apply_decommission_status_space_info, bind_decommission_cancelers, bind_missing_decommission_cancelers,
-        cancel_decommission_canceler, classify_decommission_terminal_state, count_decommission_item,
-        decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options,
+        DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF,
+        DecomBucketInfo, DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta,
+        PoolSpaceInfo, PoolStatus, apply_decommission_status_space_info, bind_decommission_cancelers,
+        bind_missing_decommission_cancelers, cancel_decommission_canceler, classify_decommission_terminal_state,
+        count_decommission_item, decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options,
         decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency,
         ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_listing_disks_available,
         ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool,
@@ -5288,11 +5482,12 @@ mod pools_tests {
         has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested,
         load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done,
         merge_pool_status_refresh, missing_decommission_worker_prefix, observe_decommission_terminal_reload_result,
-        pool_meta_has_active_decommission, require_decommission_store, resolve_decommission_bucket_done_save_result,
-        resolve_decommission_bucket_state, resolve_decommission_check_after_list_result,
-        resolve_decommission_entry_cleanup_delete_result, resolve_decommission_entry_exact_versions,
-        resolve_decommission_entry_reload_result, resolve_decommission_listing_worker_result,
-        resolve_decommission_optional_bucket_config_result, resolve_decommission_pool_meta_reload_result,
+        pool_meta_has_active_decommission, record_decommission_entry_error, require_decommission_store,
+        resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state,
+        resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result,
+        resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result, resolve_decommission_listing_error,
+        resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result,
+        resolve_decommission_partial_listing_entry, resolve_decommission_pool_meta_reload_result,
         resolve_decommission_preflight_heal_result, resolve_decommission_progress_save_result,
         resolve_decommission_spawn_failure_result, resolve_decommission_terminal_mark_after_error_result,
         resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result,
@@ -5302,16 +5497,17 @@ mod pools_tests {
         should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal,
         should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine,
         split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler,
-        touch_decommission_progress, track_decommission_current_object, track_decommission_current_object_stage,
-        validate_start_decommission_request, wait_decommission_listing_retry, wait_decommission_worker_drain,
-        with_decommission_entry_context,
+        track_decommission_current_object, track_decommission_current_object_stage, validate_start_decommission_request,
+        wait_decommission_listing_retry, wait_decommission_worker_drain, with_decommission_entry_context,
     };
     use crate::data_movement;
     use crate::disk::endpoint::Endpoint;
     use crate::error::{Error, StorageError};
     use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
     use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
-    use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntry, ObjectPartInfo};
+    use rustfs_filemeta::{
+        FileInfo, FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo,
+    };
     use rustfs_rio::Index;
     use std::sync::{
         Arc,
@@ -6379,6 +6575,65 @@ mod pools_tests {
         assert!(matches!(err, Error::SlowDown));
     }
 
+    #[test]
+    fn test_resolve_decommission_partial_listing_entry_rejects_unresolved_metadata() {
+        let err = resolve_decommission_partial_listing_entry(
+            MetaCacheEntries(vec![None]),
+            MetadataResolutionParams {
+                dir_quorum: 2,
+                obj_quorum: 2,
+                bucket: "bucket-a".to_string(),
+                ..Default::default()
+            },
+            "bucket-a",
+            "prefix/",
+            1,
+            2,
+            3,
+        )
+        .expect_err("unresolved partial listing must fail closed");
+
+        let message = err.to_string();
+        assert!(message.contains("decommission listing could not resolve metadata"));
+        assert!(message.contains("bucket-a/prefix/"));
+        assert!(message.contains("pool 2 set 3"));
+        assert!(message.contains("1 disk error(s)"));
+    }
+
+    #[tokio::test]
+    async fn test_record_decommission_entry_error_cancels_listing_and_preserves_first_error() {
+        let entry_error = Arc::new(tokio::sync::Mutex::new(None));
+        let rx = CancellationToken::new();
+
+        record_decommission_entry_error(&entry_error, &rx, Error::SlowDown).await;
+        record_decommission_entry_error(&entry_error, &rx, Error::OperationCanceled).await;
+
+        assert!(rx.is_cancelled());
+        assert!(matches!(*entry_error.lock().await, Some(Error::SlowDown)));
+    }
+
+    #[tokio::test]
+    async fn test_record_decommission_entry_error_ignores_already_canceled_listing() {
+        let entry_error = Arc::new(tokio::sync::Mutex::new(None));
+        let rx = CancellationToken::new();
+        rx.cancel();
+
+        record_decommission_entry_error(&entry_error, &rx, Error::SlowDown).await;
+
+        assert!(entry_error.lock().await.is_none());
+    }
+
+    #[test]
+    fn test_resolve_decommission_listing_error_preserves_real_listing_failure() {
+        let err = resolve_decommission_listing_error(Some(Error::SlowDown), Some(Error::OperationCanceled))
+            .expect("listing failure should be returned");
+        assert!(matches!(err, Error::SlowDown));
+
+        let err = resolve_decommission_listing_error(Some(Error::OperationCanceled), Some(Error::SlowDown))
+            .expect("entry failure should be returned");
+        assert!(matches!(err, Error::SlowDown));
+    }
+
     #[test]
     fn test_resolve_decommission_check_after_list_result_returns_list_result_without_entry_error() {
         let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), None)
@@ -6596,7 +6851,7 @@ mod pools_tests {
     }
 
     #[test]
-    fn test_touch_decommission_progress_updates_last_update_and_save_baseline() {
+    fn test_track_decommission_stage_does_not_advance_checkpoint_state() {
         let mut meta = PoolMeta {
             pools: vec![PoolStatus {
                 id: 0,
@@ -6611,11 +6866,13 @@ mod pools_tests {
             ..Default::default()
         };
 
-        touch_decommission_progress(&mut meta, 0).expect("valid decommission progress should be touched");
+        track_decommission_current_object_stage(&mut meta, 0, "bucket", "object", "migrate_object")
+            .expect("valid decommission progress should be tracked");
 
-        assert!(meta.pools[0].last_update > OffsetDateTime::UNIX_EPOCH);
+        assert_eq!(meta.pools[0].last_update, OffsetDateTime::UNIX_EPOCH);
         let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
-        assert_eq!(info.items_since_last_progress_save(), 0);
+        assert_eq!(info.items_since_last_progress_save(), 5);
+        assert_eq!(info.stage, "migrate_object");
     }
 
     #[test]
@@ -6690,6 +6947,134 @@ mod pools_tests {
         assert_eq!(info.items_since_last_progress_save(), 1);
     }
 
+    #[test]
+    fn test_pool_meta_update_after_does_not_advance_last_update_before_save() {
+        let last_update = OffsetDateTime::UNIX_EPOCH;
+        let mut meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update,
+                decommission: Some(PoolDecommissionInfo {
+                    start_time: Some(last_update),
+                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+
+        assert!(
+            meta.update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
+                .expect("item threshold should request a checkpoint")
+        );
+        assert_eq!(meta.pools[0].last_update, last_update);
+    }
+
+    #[test]
+    fn test_decommission_progress_checkpoint_commits_exact_snapshot_watermark() {
+        let start_time = OffsetDateTime::UNIX_EPOCH;
+        let checkpoint_at = start_time + Duration::seconds(30);
+        let mut meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: start_time,
+                decommission: Some(PoolDecommissionInfo {
+                    start_time: Some(start_time),
+                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+
+        let checkpoint = meta
+            .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at)
+            .expect("valid decommission state should produce a checkpoint")
+            .expect("item threshold should produce a checkpoint");
+        meta.count_item(0, 1, false);
+
+        assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
+        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
+        assert_eq!(info.progress_save_item_baseline, checkpoint.counted_items);
+        assert_eq!(info.items_since_last_progress_save(), 1);
+        assert_eq!(meta.pools[0].last_update, checkpoint_at);
+    }
+
+    #[test]
+    fn test_decommission_progress_checkpoint_backoff_does_not_advance_baseline() {
+        let start_time = OffsetDateTime::UNIX_EPOCH;
+        let checkpoint_at = start_time + Duration::seconds(30);
+        let retry_after = checkpoint_at + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
+        let mut meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: start_time,
+                decommission: Some(PoolDecommissionInfo {
+                    start_time: Some(start_time),
+                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+
+        let checkpoint = meta
+            .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at)
+            .expect("valid decommission state should produce a checkpoint")
+            .expect("item threshold should produce a checkpoint");
+        meta.defer_decommission_progress_checkpoint(0, checkpoint, retry_after);
+
+        assert!(
+            meta.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at)
+                .expect("retry backoff check should succeed")
+                .is_none()
+        );
+        assert_eq!(meta.pools[0].last_update, start_time);
+        assert_eq!(
+            meta.pools[0]
+                .decommission
+                .as_ref()
+                .expect("decommission info should exist")
+                .progress_save_item_baseline,
+            0
+        );
+    }
+
+    #[test]
+    fn test_decommission_progress_checkpoint_count_scales_with_threshold() {
+        let start_time = OffsetDateTime::UNIX_EPOCH;
+        let checkpoint_at = start_time;
+        let mut meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: start_time,
+                decommission: Some(PoolDecommissionInfo {
+                    start_time: Some(start_time),
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+        let mut checkpoint_count = 0;
+
+        for _ in 0..(DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD * 10) {
+            meta.count_item(0, 1, false);
+            if let Some(checkpoint) = meta
+                .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at)
+                .expect("valid decommission state should produce a checkpoint")
+            {
+                checkpoint_count += 1;
+                assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
+            }
+        }
+
+        assert_eq!(checkpoint_count, 10);
+    }
+
     #[test]
     fn test_ensure_decommission_not_rebalancing_rejects_running_rebalance() {
         let err = ensure_decommission_not_rebalancing(true).expect_err("rebalance running should be rejected");

crates/ecstore/src/core/sets.rs

@@ -21,7 +21,7 @@ use crate::storage_api_contracts::{
     bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
     list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
     multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
-    object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
+    object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
     range::HTTPRangeSpec,
 };
 use crate::{
@@ -414,6 +414,66 @@ fn apply_delete_objects_results(
     }
 }
 
+fn apply_delete_accounting_results(
+    accounting: &mut [Option<DeleteAccounting>],
+    set_objects: &[DelObj],
+    set_accounting: &[Option<DeleteAccounting>],
+) {
+    for (obj, value) in set_objects.iter().zip(set_accounting.iter()) {
+        accounting[obj.orig_idx] = value.clone();
+    }
+}
+
+impl Sets {
+    pub(crate) async fn delete_objects_with_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
+        let mut del_objects = vec![DeletedObject::default(); objects.len()];
+        let mut del_errs = vec![None; objects.len()];
+        let mut accounting = vec![None; objects.len()];
+        let mut set_obj_map = HashMap::new();
+
+        for (i, obj) in objects.iter().enumerate() {
+            let idx = self.get_hashed_set_index(obj.object_name.as_str());
+            set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj {
+                orig_idx: i,
+                obj: obj.clone(),
+            });
+        }
+
+        let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
+        let mut futures = FuturesUnordered::new();
+        let bucket = bucket.to_owned();
+
+        for (set_index, set_objects) in set_obj_map {
+            let disks = self.get_disks(set_index);
+            let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::<Vec<_>>();
+            let bucket = bucket.clone();
+            let opts = opts.clone();
+            let semaphore = semaphore.clone();
+            futures.push(async move {
+                let _permit = semaphore
+                    .acquire_owned()
+                    .await
+                    .expect("delete_objects semaphore should remain open");
+                let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await;
+                (set_objects, deleted, errors, accounting)
+            });
+        }
+
+        while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await {
+            apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors);
+            apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting);
+        }
+
+        (del_objects, del_errs, accounting)
+    }
+}
+
 #[async_trait::async_trait]
 impl crate::storage_api_contracts::object::ObjectIO for Sets {
     type Error = Error;
@@ -655,65 +715,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
         objects: Vec<ObjectToDelete>,
         opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
-        // Default return value
-        let mut del_objects = vec![DeletedObject::default(); objects.len()];
-
-        let mut del_errs = Vec::with_capacity(objects.len());
-        for _ in 0..objects.len() {
-            del_errs.push(None)
-        }
-
-        let mut set_obj_map = HashMap::new();
-
-        // hash key
-        for (i, obj) in objects.iter().enumerate() {
-            let idx = self.get_hashed_set_index(obj.object_name.as_str());
-
-            if !set_obj_map.contains_key(&idx) {
-                set_obj_map.insert(
-                    idx,
-                    vec![DelObj {
-                        // set_idx: idx,
-                        orig_idx: i,
-                        obj: obj.clone(),
-                    }],
-                );
-            } else if let Some(val) = set_obj_map.get_mut(&idx) {
-                val.push(DelObj {
-                    // set_idx: idx,
-                    orig_idx: i,
-                    obj: obj.clone(),
-                });
-            }
-        }
-
-        let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
-        let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
-        let mut futures = FuturesUnordered::new();
-        let bucket = bucket.to_string();
-
-        for (k, v) in set_obj_map {
-            let disks = self.get_disks(k);
-            let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
-            let bucket = bucket.clone();
-            let opts = opts.clone();
-            let semaphore = semaphore.clone();
-
-            futures.push(async move {
-                let _permit = semaphore
-                    .acquire_owned()
-                    .await
-                    .expect("delete_objects semaphore should remain open");
-                let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await;
-                (v, dobjects, errs)
-            });
-        }
-
-        while let Some((v, dobjects, errs)) = futures.next().await {
-            apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs);
-        }
-
-        (del_objects, del_errs)
+        let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
+        (deleted, errors)
     }
 
     #[tracing::instrument(skip(self))]

crates/ecstore/src/data_usage/mod.rs

@@ -1391,7 +1391,37 @@ impl BucketUsageAccumulator {
 }
 
 pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
-    let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
+    // A compressed object may carry -1 while the transformed size is unknown
+    // (legacy streaming sentinel). In that case the persisted physical size
+    // is still a valid accounting floor; every other negative value is corrupt.
+    // An explicit negative `actual-size` metadata value is corrupt, however:
+    // the sentinel is only valid in the in-memory/object-part field written by
+    // the legacy streaming path, not as a persisted declared size.
+    let compressed = object.is_compressed();
+    if object.actual_size < -1 || (object.actual_size == -1 && !compressed) {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    if object
+        .parts
+        .iter()
+        .any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
+    {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
+        .filter(|value| !value.is_empty());
+    if declared_actual_size
+        .as_deref()
+        .and_then(|value| value.parse::<i64>().ok())
+        .is_some_and(|size| size < 0)
+    {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    let logical_size = match object.get_actual_size().map_err(Error::other)? {
+        size if size == -1 && compressed && declared_actual_size.is_none() => None,
+        size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?),
+        _ => return Err(Error::PartMissingOrCorrupt),
+    };
     let persisted_part_size = if object.parts.is_empty() {
         u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
     } else {
@@ -1399,12 +1429,8 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
             // Compressed streaming objects persist -1 when the transformed
             // part size is unknown. The physical part size remains a valid
             // quota floor; reject only non-negative values that overflow.
-            let actual_size = if part.actual_size < 0 {
-                if object.is_compressed() {
-                    0
-                } else {
-                    return Err(Error::PartMissingOrCorrupt);
-                }
+            let actual_size = if part.actual_size == -1 {
+                0
             } else {
                 u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
             };
@@ -1412,7 +1438,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
             total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
         })?
     };
-    Ok(logical_size.max(persisted_part_size))
+    Ok(logical_size.unwrap_or(0).max(persisted_part_size))
 }
 
 type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
@@ -3320,6 +3346,80 @@ mod tests {
         );
     }
 
+    #[test]
+    fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let object = ObjectInfo {
+            size: 400,
+            actual_size: -1,
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+
+        assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400);
+    }
+
+    #[test]
+    fn quota_object_size_rejects_compressed_part_sum_overflow() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let object = ObjectInfo {
+            size: 1,
+            user_defined: Arc::new(metadata),
+            parts: Arc::new(vec![
+                rustfs_filemeta::ObjectPartInfo {
+                    actual_size: i64::MAX,
+                    ..Default::default()
+                },
+                rustfs_filemeta::ObjectPartInfo {
+                    actual_size: 1,
+                    ..Default::default()
+                },
+            ]),
+            ..Default::default()
+        };
+
+        assert!(matches!(quota_object_size(&object), Err(Error::Io(_))));
+    }
+
+    #[test]
+    fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let corrupt_object = ObjectInfo {
+            size: 400,
+            actual_size: -2,
+            user_defined: Arc::new(metadata.clone()),
+            ..Default::default()
+        };
+        assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt)));
+
+        let corrupt_part = ObjectInfo {
+            size: 400,
+            user_defined: Arc::new(metadata),
+            parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
+                size: 400,
+                actual_size: -2,
+                ..Default::default()
+            }]),
+            ..Default::default()
+        };
+        assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt)));
+    }
+
     #[tokio::test]
     #[serial]
     async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {

crates/ecstore/src/object_api/types.rs

@@ -689,6 +689,9 @@ impl ObjectInfo {
     }
 
     pub fn get_actual_size(&self) -> std::io::Result<i64> {
+        if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) {
+            return Err(std::io::Error::other("invalid negative actual size"));
+        }
         if self.actual_size > 0 {
             return Ok(self.actual_size);
         }
@@ -700,10 +703,25 @@ impl ObjectInfo {
                 let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
                 return Ok(size);
             }
-            let mut actual_size = 0;
-            self.parts.iter().for_each(|part| {
-                actual_size += part.actual_size;
-            });
+            if self.actual_size == -1 && self.parts.is_empty() {
+                return Ok(-1);
+            }
+            let mut actual_size = 0_i64;
+            let mut unknown = false;
+            for part in self.parts.iter() {
+                match part.actual_size {
+                    -1 => unknown = true,
+                    size if size >= 0 => {
+                        actual_size = actual_size
+                            .checked_add(size)
+                            .ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?;
+                    }
+                    _ => return Err(std::io::Error::other("invalid negative compressed part size")),
+                }
+            }
+            if unknown {
+                return Ok(-1);
+            }
             if actual_size == 0 && actual_size != self.size {
                 return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size)));
             }
@@ -718,6 +736,18 @@ impl ObjectInfo {
         Ok(self.size)
     }
 
+    /// Returns a non-negative size for client and replication boundaries.
+    ///
+    /// Compressed legacy metadata can retain the internal `-1` unknown-size
+    /// sentinel. Those boundaries cannot emit a negative length, so they use
+    /// the persisted physical size while quota accounting keeps the sentinel
+    /// distinction in [`crate::data_usage::quota_object_size`].
+    pub fn get_actual_size_or_physical(&self) -> i64 {
+        self.get_actual_size()
+            .map(|size| if size >= 0 { size } else { self.size.max(0) })
+            .unwrap_or_else(|_| self.size.max(0))
+    }
+
     pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
         let mut version_id = fi.version_id;
 

crates/ecstore/src/set_disk/mod.rs

@@ -97,7 +97,7 @@ use crate::storage_api_contracts::{
         CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo,
     },
     namespace::NamespaceLocking as _,
-    object::{DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
+    object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
     range::HTTPRangeSpec,
 };
 use crate::store::utils::is_reserved_or_invalid_bucket;

crates/ecstore/src/set_disk/ops/object.rs

@@ -45,6 +45,7 @@ use crate::bucket::replication::{
     DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType,
     replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta,
 };
+use crate::data_usage::quota_object_size;
 use crate::diagnostics::get::GetObjectFailureReason;
 use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
 use crate::error::is_err_invalid_upload_id;
@@ -5655,7 +5656,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         objects: Vec<ObjectToDelete>,
         opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
+        let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
+        (deleted, errors)
+    }
+
+    async fn delete_objects_with_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
+        let mut accounting = vec![None; objects.len()];
         let delete_config_snapshot = opts
             .delete_replication_config_snapshot
             .clone()
@@ -5745,7 +5757,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 *item = Some(Error::other(message.clone()));
                             }
                         }
-                        return (del_objects, del_errs);
+                        return (del_objects, del_errs, accounting);
                     }
                 },
             }
@@ -5792,6 +5804,22 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             let source_missing = gerr
                 .as_ref()
                 .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err));
+            // Resolve accounting from the generation selected under this
+            // object's write lock. A request-layer pre-stat is only an
+            // optimization and cannot identify a concurrent overwrite.
+            let (accounting_size, accounting_version_id, removed_current_object) = if source_missing
+                || dobj.synthetic_version_id
+                || set_disk_delete_creates_delete_marker(&check_opts)
+                || goi.delete_marker
+            {
+                (None, None, false)
+            } else {
+                (
+                    quota_object_size(&goi).ok(),
+                    goi.version_id.filter(|version_id| !version_id.is_nil()),
+                    (dobj.version_id.is_none() || is_explicit_null_version(dobj.version_id)) && !dobj.synthetic_version_id,
+                )
+            };
             // Normalize both sides before comparing. `goi.version_id` is the
             // client-facing identity, where `from_file_info` synthesizes
             // `Some(Uuid::nil())` for a null version on a versioned or
@@ -5920,7 +5948,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     },
                     replication_state: vr.replication_state_internal.clone(),
                     ..Default::default()
-                }
+                };
+                accounting[i] = Some(DeleteAccounting {
+                    size: accounting_size,
+                    version_id: accounting_version_id,
+                    removed_current_object,
+                });
             }
 
             // Only add to vers_map if we hold the lock
@@ -5966,7 +5999,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     });
                 }
             }
-            return (del_objects, del_errs);
+            return (del_objects, del_errs, accounting);
         }
 
         let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len());
@@ -6204,7 +6237,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
-        (del_objects, del_errs)
+        // An accounting identity is actionable only when the delete result is
+        // successful. Never let a failed commit (including a partial quorum
+        // failure) reach the request-layer fast delta path.
+        for (index, err) in del_errs.iter().enumerate() {
+            if err.is_some() {
+                accounting[index] = None;
+            }
+        }
+
+        (del_objects, del_errs, accounting)
     }
 
     #[tracing::instrument(skip(self))]
@@ -6533,6 +6575,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended);
         obj_info.size = goi.size;
+        // Keep the committed source metadata on the internal delete result so
+        // the request layer can derive canonical accounting for this exact
+        // generation. Delete responses do not expose these fields.
+        obj_info.actual_size = goi.actual_size;
+        obj_info.user_defined = Arc::clone(&goi.user_defined);
+        obj_info.parts = Arc::clone(&goi.parts);
         obj_info.user_tags = Arc::clone(&goi.user_tags);
         self.invalidate_get_object_metadata_cache(bucket, object).await;
         Ok(obj_info)
@@ -7824,6 +7872,113 @@ mod replication_quota_safety_tests {
         assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
     }
 
+    #[tokio::test]
+    async fn delete_returns_canonical_compressed_accounting_size() {
+        let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "compressed-delete-accounting";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let mut user_defined = HashMap::new();
+        insert_str(
+            &mut user_defined,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
+        let mut reader = PutObjReader::new(
+            HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
+                .expect("compressed fixture reader should be valid"),
+        );
+        set_disks
+            .put_object(
+                bucket,
+                "object",
+                &mut reader,
+                &ObjectOptions {
+                    user_defined,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("compressed object should be written");
+
+        let (deleted, errors, accounting) = set_disks
+            .delete_objects_with_accounting(
+                bucket,
+                vec![ObjectToDelete {
+                    object_name: "object".to_string(),
+                    ..Default::default()
+                }],
+                ObjectOptions {
+                    object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
+                        ObjectLockConfigState::ConfirmedAbsent,
+                    ))),
+                    ..Default::default()
+                },
+            )
+            .await;
+
+        assert!(errors[0].is_none(), "compressed delete should succeed: {:?}", errors[0]);
+        assert!(deleted[0].found, "the committed object must be reported as found");
+        assert_eq!(accounting[0].as_ref().and_then(|value| value.size), Some(1000));
+        assert!(accounting[0].as_ref().is_some_and(|value| value.version_id.is_none()));
+        assert!(accounting[0].as_ref().is_some_and(|value| value.removed_current_object));
+    }
+
+    #[tokio::test]
+    async fn suspended_delete_marker_does_not_return_body_accounting() {
+        let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "suspended-delete-accounting";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let mut user_defined = HashMap::new();
+        insert_str(
+            &mut user_defined,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
+        let mut reader = PutObjReader::new(
+            HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
+                .expect("compressed fixture reader should be valid"),
+        );
+        let suspended_opts = ObjectOptions {
+            version_suspended: true,
+            delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
+                s3s::dto::VersioningConfiguration {
+                    status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
+                    ..Default::default()
+                },
+                None,
+            ))),
+            user_defined,
+            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
+            ..Default::default()
+        };
+        set_disks
+            .put_object(bucket, "object", &mut reader, &suspended_opts)
+            .await
+            .expect("compressed object should be written");
+
+        let (deleted, errors, accounting) = set_disks
+            .delete_objects_with_accounting(
+                bucket,
+                vec![ObjectToDelete {
+                    object_name: "object".to_string(),
+                    ..Default::default()
+                }],
+                suspended_opts,
+            )
+            .await;
+        assert!(errors[0].is_none(), "suspended delete should create a marker: {:?}", errors[0]);
+        assert!(deleted[0].delete_marker);
+        assert!(accounting[0].is_none(), "a delete marker must not carry body accounting");
+    }
+
     #[tokio::test]
     async fn direct_put_cannot_persist_a_tiny_logical_size() {
         let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;

crates/ecstore/src/storage_api_contracts/mod.rs

@@ -62,8 +62,8 @@ pub(crate) mod object {
     use super::{Debug, Error, FileInfo, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
     use crate::storage_api_contracts::range::HTTPRangeSpec;
     pub(crate) use rustfs_storage_api::{
-        DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions, ObjectOperations,
-        ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
+        DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions,
+        ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
     };
 
     pub(crate) trait EcstoreObjectIO:

crates/ecstore/src/store/heal.rs

@@ -297,10 +297,16 @@ impl ECStore {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::bucket::metadata_sys;
     use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
-    use crate::disk::{DiskOption, format::FormatV3, new_disk};
-    use crate::layout::endpoints::{Endpoints, PoolEndpoints};
+    use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk};
+    use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
+    use crate::runtime::instance::InstanceContext;
+    use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
+    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
     use crate::store::init_format::{load_format_erasure, save_format_file};
+    use crate::store::init_local_disks_with_instance_ctx;
+    use tokio_util::sync::CancellationToken;
 
     async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
         let format = FormatV3::new(1, 1);
@@ -347,6 +353,51 @@ mod tests {
         }
     }
 
+    async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
+        let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
+        let mut pool_endpoints = Vec::new();
+        for pool_index in 0..2 {
+            let mut endpoints = Vec::new();
+            for disk_index in 0..4 {
+                let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
+                tokio::fs::create_dir_all(&disk_path)
+                    .await
+                    .expect("multi-pool heal test disk should be created");
+                let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
+                    .expect("test endpoint should parse");
+                endpoint.set_pool_index(pool_index);
+                endpoint.set_set_index(0);
+                endpoint.set_disk_index(disk_index);
+                endpoints.push(endpoint);
+            }
+            pool_endpoints.push(PoolEndpoints {
+                legacy: false,
+                set_count: 1,
+                drives_per_set: 4,
+                endpoints: Endpoints::from(endpoints),
+                cmd_line: format!("heal-owner-pool-{pool_index}"),
+                platform: "test".to_string(),
+            });
+        }
+
+        let endpoint_pools = EndpointServerPools::from(pool_endpoints);
+        let instance_ctx = Arc::new(InstanceContext::new());
+        init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
+            .await
+            .expect("multi-pool local disks should initialize");
+        let shutdown = CancellationToken::new();
+        let store = ECStore::new_with_instance_ctx(
+            "127.0.0.1:0".parse().expect("test address should parse"),
+            endpoint_pools,
+            shutdown.clone(),
+            instance_ctx,
+        )
+        .await
+        .expect("multi-pool test store should initialize");
+        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        (temp_dir, store, shutdown)
+    }
+
     #[tokio::test]
     async fn heal_object_pool_scope_selects_only_requested_pool() {
         let store = minimal_heal_store().await;
@@ -506,6 +557,229 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn unscoped_heal_object_suspended_owner_semantics() {
+        let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
+        let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
+        let active_object = "active-owner";
+        let suspended_only_object = "suspended-only";
+        let duplicate_object = "duplicate-owner";
+        let marker_object = "marker-owner";
+        let quorum_object = "quorum-owner";
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created in all pools");
+
+        let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
+        store.pools[0]
+            .put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
+            .await
+            .expect("active owner object should be written");
+        let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
+        let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
+        missing_active_disk
+            .delete(
+                &bucket,
+                active_object,
+                DeleteOptions {
+                    recursive: true,
+                    immediate: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("active owner shard should be removed for repair");
+        assert!(
+            missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
+            "the active owner fixture must start with one missing metadata copy"
+        );
+
+        let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
+        store.pools[1]
+            .put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
+            .await
+            .expect("suspended owner object should be written");
+        for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
+            let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
+            store.pools[pool_index]
+                .put_object(
+                    &bucket,
+                    duplicate_object,
+                    &mut duplicate_reader,
+                    &ObjectOptions {
+                        mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("duplicate owner object should be written");
+        }
+        let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0]
+            .clone()
+            .expect("duplicate active owner disk should be online");
+        duplicate_missing_disk
+            .delete(
+                &bucket,
+                duplicate_object,
+                DeleteOptions {
+                    recursive: true,
+                    immediate: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("duplicate active owner shard should be removed for repair");
+        let history_version = Uuid::new_v4();
+        let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
+        store.pools[0]
+            .put_object(
+                &bucket,
+                marker_object,
+                &mut history_reader,
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(history_version.to_string()),
+                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("versioned marker history should be written");
+        store.pools[0]
+            .delete_object(
+                &bucket,
+                marker_object,
+                ObjectOptions {
+                    versioned: true,
+                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("delete marker should be written");
+        let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
+        store.pools[0]
+            .put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
+            .await
+            .expect("quorum boundary object should be written");
+        {
+            let mut pool_meta = store.pool_meta.write().await;
+            let mut next = PoolMeta::new(&store.pools, &pool_meta);
+            next.pools[1].decommission = Some(PoolDecommissionInfo {
+                start_time: Some(OffsetDateTime::UNIX_EPOCH),
+                ..Default::default()
+            });
+            *pool_meta = next;
+        }
+
+        let (_, duplicate_owner) = store
+            .get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
+            .await
+            .expect("duplicate owner should resolve");
+        assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
+        let (_, active_duplicate_owner) = store
+            .get_latest_object_info_with_idx(
+                &bucket,
+                duplicate_object,
+                &ObjectOptions {
+                    skip_decommissioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("active duplicate owner should resolve");
+        assert_eq!(
+            active_duplicate_owner, 0,
+            "suspended duplicate must be excluded from active owner selection"
+        );
+        let (duplicate_result, duplicate_err) = store
+            .handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default())
+            .await
+            .expect("duplicate owner heal should complete through the production path");
+        assert_eq!(duplicate_result.object, duplicate_object);
+        assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}");
+        assert!(
+            duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(),
+            "production heal must repair the active duplicate owner rather than the suspended owner"
+        );
+        let (marker_info, marker_owner) = store
+            .get_latest_object_info_with_idx(
+                &bucket,
+                marker_object,
+                &ObjectOptions {
+                    skip_decommissioned: true,
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("latest delete marker should resolve");
+        assert_eq!(marker_owner, 0);
+        assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
+
+        let (active_result, active_err) = store
+            .handle_heal_object(&bucket, active_object, "", &HealOpts::default())
+            .await
+            .expect("unscoped active-owner heal should complete");
+        assert_eq!(active_result.object, active_object);
+        assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
+        assert!(
+            missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
+            "active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
+        );
+        assert!(
+            store.pools[1]
+                .get_object_info(&bucket, active_object, &ObjectOptions::default())
+                .await
+                .is_err(),
+            "the suspended pool must not be written for an active-owner object"
+        );
+
+        let (suspended_result, suspended_err) = store
+            .handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
+            .await
+            .expect("unscoped suspended-only heal should return a terminal result");
+        assert!(suspended_result.object.is_empty());
+        assert!(matches!(suspended_err, Some(Error::FileNotFound)));
+        assert!(
+            store.pools[1]
+                .get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
+                .await
+                .is_ok(),
+            "suspended-only data must remain untouched when unscoped heal reports absent"
+        );
+
+        let (_, explicit_err) = store
+            .handle_heal_object(
+                &bucket,
+                suspended_only_object,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("explicit suspended-owner heal should return a mapped error");
+        assert!(matches!(explicit_err, Some(Error::SlowDown)));
+
+        let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
+        let surviving_quorum_disk = original_quorum_disks[3].clone();
+        *store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
+        let (_, quorum_err) = store
+            .handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
+            .await
+            .expect("quorum boundary heal should return a mapped result");
+        *store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
+        assert!(
+            matches!(quorum_err, Some(Error::ErasureReadQuorum)),
+            "quorum-boundary heal must preserve quorum error, got {quorum_err:?}"
+        );
+        shutdown.cancel();
+    }
+
     #[tokio::test]
     async fn handle_heal_format_continues_after_a_pool_error() {
         let canonical_format = FormatV3::new(1, 3);

crates/ecstore/src/store/object.rs

@@ -41,7 +41,7 @@ use crate::set_disk::{
 };
 use crate::storage_api_contracts::{
     namespace::NamespaceLocking as _,
-    object::{ObjectIO as _, ObjectOperations as _},
+    object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _},
 };
 use parking_lot::Mutex as ParkingMutex;
 use rustfs_io_metrics::{
@@ -1216,6 +1216,14 @@ fn return_batch_delete_lock_error(objects: &[ObjectToDelete], err: Error) -> (Ve
     (del_objects, del_errs)
 }
 
+fn return_batch_delete_lock_error_with_accounting(
+    objects: &[ObjectToDelete],
+    err: Error,
+) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
+    let (deleted, errors) = return_batch_delete_lock_error(objects, err);
+    (deleted, errors, vec![None; objects.len()])
+}
+
 fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> {
     let mut object_names: Vec<&str> = objects.iter().map(|object| object.object_name.as_str()).collect();
     object_names.sort_unstable();
@@ -2312,6 +2320,22 @@ impl ECStore {
         result
     }
 
+    pub async fn delete_objects_with_tier_delete_journal_and_accounting(
+        self: &Arc<Self>,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
+        let result = self
+            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, Some(Arc::clone(self)))
+            .await;
+        let success_count = result.1.iter().filter(|err| err.is_none()).count();
+        if success_count > 0 {
+            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
+        }
+        result
+    }
+
     #[instrument(skip(self))]
     pub(super) async fn handle_delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
         self.handle_delete_object_with_journal(bucket, object, opts, None).await
@@ -2689,6 +2713,19 @@ impl ECStore {
         opts: ObjectOptions,
         tier_journal_api: Option<Arc<ECStore>>,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
+        let (deleted, errors, _) = self
+            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, tier_journal_api)
+            .await;
+        (deleted, errors)
+    }
+
+    pub(super) async fn handle_delete_objects_with_journal_and_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+        tier_journal_api: Option<Arc<ECStore>>,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
         // encode object name
         let objects: Vec<ObjectToDelete> = objects
             .iter()
@@ -2701,6 +2738,7 @@ impl ECStore {
 
         // Default return value
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
+        let mut accounting = vec![None; objects.len()];
 
         let mut del_errs = Vec::with_capacity(objects.len());
         for _ in 0..objects.len() {
@@ -2714,7 +2752,7 @@ impl ECStore {
         } else {
             match self.acquire_bucket_lifecycle_read_lock(bucket).await {
                 Ok(guard) => Some(guard),
-                Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
+                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
             }
         };
         if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
@@ -2726,21 +2764,21 @@ impl ECStore {
                 Err(err) => {
                     let message = err.to_string();
                     let errors = (0..objects.len()).map(|_| Some(Error::other(message.clone()))).collect();
-                    return (del_objects, errors);
+                    return (del_objects, errors, accounting);
                 }
             }
         }
         if !is_meta_bucketname(bucket)
             && let Err(err) = get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
         {
-            return return_batch_delete_lock_error(objects.as_slice(), err);
+            return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err);
         }
         let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
             None
         } else {
             Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
                 Ok(guard) => guard,
-                Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
+                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
             })
         };
         if let Some(guard) = _object_lock_metadata_guard.as_ref() {
@@ -2750,7 +2788,7 @@ impl ECStore {
             let (state, incarnation_id, config_revision) =
                 match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
                     Ok(snapshot) => snapshot,
-                    Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
+                    Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
                 };
             opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
                 self.id,
@@ -2766,7 +2804,10 @@ impl ECStore {
         if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
             && expected != current
         {
-            return return_batch_delete_lock_error(objects.as_slice(), StorageError::BucketNotFound(bucket.to_string()));
+            return return_batch_delete_lock_error_with_accounting(
+                objects.as_slice(),
+                StorageError::BucketNotFound(bucket.to_string()),
+            );
         }
         #[cfg(test)]
         if current_bucket_incarnation_id.is_some() {
@@ -2774,7 +2815,7 @@ impl ECStore {
         }
         let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
             Ok(guards) => guards,
-            Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
+            Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
         };
 
         let mut futures = Vec::with_capacity(self.pools.len());
@@ -2783,22 +2824,24 @@ impl ECStore {
             if self.is_pool_rebalancing(pool.pool_idx).await {
                 continue;
             }
-            futures.push(pool.delete_objects(bucket, objects.clone(), opts.clone()));
+            futures.push(pool.delete_objects_with_accounting(bucket, objects.clone(), opts.clone()));
         }
 
         let results = join_all(futures).await;
 
         for idx in 0..del_objects.len() {
-            for (dels, errs) in results.iter() {
+            for (dels, errs, pool_accounting) in results.iter() {
                 if errs[idx].is_none() && dels[idx].found {
                     del_errs[idx] = None;
                     del_objects[idx] = dels[idx].clone();
+                    accounting[idx] = pool_accounting[idx].clone();
                     break;
                 }
 
                 if del_errs[idx].is_none() {
                     del_errs[idx] = errs[idx].clone();
                     del_objects[idx] = dels[idx].clone();
+                    accounting[idx] = pool_accounting[idx].clone();
                 }
             }
         }
@@ -2807,7 +2850,7 @@ impl ECStore {
             v.object_name = decode_dir_object(&v.object_name);
         });
 
-        (del_objects, del_errs)
+        (del_objects, del_errs, accounting)
 
         // let mut futures = Vec::with_capacity(objects.len());