fix(scanner): require authoritative usage snapshots (#5333)

rustfs/src/admin/handlers/quota.rs

@@ -27,7 +27,7 @@ use hyper::{Method, StatusCode};
 use matchit::Params;
 use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
 use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
-use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
+use s3s::{Body, S3Error, S3Request, S3Response, S3Result, s3_error};
 use serde::{Deserialize, Serialize};
 use serde_json;
 use std::sync::Arc;
@@ -126,6 +126,16 @@ fn compat_bucket_quota_response(quota: &BucketQuota) -> CompatibleBucketQuotaRes
     }
 }
 
+fn map_quota_query_error(error: QuotaError, bucket: &str, operation: &str) -> S3Error {
+    match error {
+        QuotaError::ConfigNotFound { .. } => s3_error!(NoSuchBucket, "bucket not found: {}", bucket),
+        QuotaError::UsageUnavailable { .. } => {
+            s3_error!(ServiceUnavailable, "authoritative bucket usage is not available yet")
+        }
+        error => s3_error!(InternalError, "failed to {}: {}", operation, error),
+    }
+}
+
 #[derive(Debug, Serialize)]
 pub struct BucketQuotaResponse {
     pub bucket: String,
@@ -376,27 +386,34 @@ impl Operation for GetBucketQuotaHandler {
 
         let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
 
-        let (quota, current_usage) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e {
-            QuotaError::ConfigNotFound { .. } => {
-                s3_error!(NoSuchBucket, "bucket not found: {}", bucket)
+        if is_compat_get_bucket_quota_path(req.uri.path()) {
+            if !quota_checker.bucket_exists(&bucket).await {
+                return Err(s3_error!(NoSuchBucket, "bucket not found: {}", bucket));
             }
-            _ => s3_error!(InternalError, "failed to get quota: {}", e),
-        })?;
+            let quota = quota_checker
+                .get_quota_config(&bucket)
+                .await
+                .map_err(|e| s3_error!(InternalError, "failed to get quota: {}", e))?;
+            let json = serde_json::to_string(&compat_bucket_quota_response(&quota))
+                .map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
+            return Ok(S3Response::new((StatusCode::OK, Body::from(json))));
+        }
 
-        let json = if is_compat_get_bucket_quota_path(req.uri.path()) {
-            serde_json::to_string(&compat_bucket_quota_response(&quota))
-                .map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
-        } else {
-            let response = BucketQuotaResponse {
-                bucket,
-                quota: quota.quota,
-                size: current_usage.unwrap_or(0),
-                quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
-            };
+        let (quota, current_usage) = quota_checker
+            .get_quota_stats(&bucket)
+            .await
+            .map_err(|error| map_quota_query_error(error, &bucket, "get quota"))?;
 
-            serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?
+        let response = BucketQuotaResponse {
+            bucket,
+            quota: quota.quota,
+            size: current_usage.unwrap_or(0),
+            quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
         };
 
+        let json =
+            serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
+
         Ok(S3Response::new((StatusCode::OK, Body::from(json))))
     }
 }
@@ -550,12 +567,10 @@ impl Operation for GetBucketQuotaStatsHandler {
 
         let quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
 
-        let (quota, current_usage_opt) = quota_checker.get_quota_stats(&bucket).await.map_err(|e| match e {
-            QuotaError::ConfigNotFound { .. } => {
-                s3_error!(NoSuchBucket, "bucket not found: {}", bucket)
-            }
-            _ => s3_error!(InternalError, "failed to get quota stats: {}", e),
-        })?;
+        let (quota, current_usage_opt) = quota_checker
+            .get_quota_stats(&bucket)
+            .await
+            .map_err(|error| map_quota_query_error(error, &bucket, "get quota stats"))?;
 
         let current_usage = current_usage_opt.unwrap_or(0);
         let usage_percentage = quota.quota.and_then(|limit| {
@@ -659,7 +674,7 @@ impl Operation for CheckBucketQuotaHandler {
         let result = quota_checker
             .check_quota_with_usage_reporting(&bucket, operation, request.operation_size, true)
             .await
-            .map_err(|e| s3_error!(InternalError, "failed to check quota: {}", e))?;
+            .map_err(|error| map_quota_query_error(error, &bucket, "check quota"))?;
 
         let response = CheckQuotaResponse {
             bucket,
@@ -715,6 +730,19 @@ mod tests {
         assert!(json.contains("\"quotatype\":\"hard\""));
     }
 
+    #[test]
+    fn unknown_authoritative_usage_is_retryable() {
+        let error = map_quota_query_error(
+            QuotaError::UsageUnavailable {
+                bucket: "test-bucket".to_string(),
+            },
+            "test-bucket",
+            "check quota",
+        );
+
+        assert_eq!(error.code(), &s3s::S3ErrorCode::ServiceUnavailable);
+    }
+
     #[test]
     fn test_quota_operation_parsing() {
         let parse_operation = |operation: &str| match operation.to_uppercase().as_str() {

rustfs/src/app/admin_usecase.rs

@@ -19,6 +19,7 @@ use super::storage_api::admin_usecase::capacity::{
     PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free,
 };
 use super::storage_api::admin_usecase::contract::StorageAdminApi;
+use super::storage_api::admin_usecase::contract::bucket::{BucketOperations as _, BucketOptions};
 use super::storage_api::admin_usecase::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend_cached};
 use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
 use crate::app::runtime_sources::{
@@ -33,6 +34,7 @@ use crate::server::{DependencyReadiness, collect_dependency_readiness_report as
 use rustfs_data_usage::DataUsageInfo;
 use rustfs_madmin::{InfoMessage, StorageInfo};
 use s3s::S3ErrorCode;
+use std::collections::HashSet;
 use std::sync::Arc;
 use tracing::{debug, error, info, warn};
 
@@ -212,6 +214,16 @@ impl DefaultAdminUsecase {
         result.map_err(|_| Self::app_error(S3ErrorCode::InternalError, "load_data_usage_from_backend failed"))
     }
 
+    fn data_usage_snapshot_covers_namespace(info: &DataUsageInfo, buckets: impl IntoIterator<Item = String>) -> bool {
+        if !info.is_complete_bucket_usage_snapshot() {
+            return false;
+        }
+
+        let current_buckets: HashSet<String> = buckets.into_iter().filter(|bucket| !bucket.starts_with('.')).collect();
+        current_buckets.len() == info.buckets_usage.len()
+            && current_buckets.iter().all(|bucket| info.buckets_usage.contains_key(bucket))
+    }
+
     async fn refresh_rebalance_status_snapshot(store: &ECStore) -> AdminUsecaseResult<()> {
         store.refresh_rebalance_status_meta().await.map_err(|err| {
             error!("refresh rebalance metadata for pool status failed: {:?}", err);
@@ -255,6 +267,20 @@ impl DefaultAdminUsecase {
     pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
         let mut info = Self::map_data_usage_load_result(load_data_usage_from_backend_cached(store.clone()).await)?;
         apply_bucket_usage_memory_overlay(&mut info).await;
+        let buckets = store
+            .list_bucket(&BucketOptions {
+                cached: true,
+                no_metadata: true,
+                ..Default::default()
+            })
+            .await
+            .map_err(|_| Self::app_error(S3ErrorCode::InternalError, "list_bucket failed"))?;
+        if !Self::data_usage_snapshot_covers_namespace(&info, buckets.into_iter().map(|bucket| bucket.name)) {
+            return Err(Self::app_error(
+                S3ErrorCode::ServiceUnavailable,
+                "authoritative data usage snapshot is not available yet",
+            ));
+        }
 
         let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
 
@@ -645,6 +671,32 @@ mod tests {
         });
     }
 
+    #[test]
+    fn data_usage_snapshot_must_cover_the_current_bucket_namespace() {
+        let mut info = DataUsageInfo {
+            last_update: Some(std::time::SystemTime::UNIX_EPOCH),
+            usage_snapshot_complete: true,
+            buckets_count: 1,
+            ..Default::default()
+        };
+        info.buckets_usage.insert("bucket-a".to_string(), Default::default());
+
+        assert!(DefaultAdminUsecase::data_usage_snapshot_covers_namespace(
+            &info,
+            ["bucket-a".to_string(), ".rustfs.sys".to_string()]
+        ));
+        assert!(!DefaultAdminUsecase::data_usage_snapshot_covers_namespace(
+            &info,
+            ["bucket-a".to_string(), "bucket-b".to_string()]
+        ));
+
+        info.usage_snapshot_complete = false;
+        assert!(!DefaultAdminUsecase::data_usage_snapshot_covers_namespace(
+            &info,
+            ["bucket-a".to_string()]
+        ));
+    }
+
     #[tokio::test]
     async fn execute_query_storage_info_returns_internal_error_when_store_uninitialized() {
         let usecase = DefaultAdminUsecase::without_context();

rustfs/src/app/data_usage_snapshot_gating_test.rs

@@ -25,23 +25,36 @@
 //! while the response carries the seeded numbers.
 
 use super::gating_test_env::shared_gating_ecstore;
-use super::storage_api::test::StoragePutObjReader as PutObjReader;
-use super::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions};
+use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions, MakeBucketOptions};
 use super::storage_api::test::contract::object::ObjectIO as _;
 use super::storage_api::test::data_usage::{
     compute_bucket_usage, live_bucket_usage_computations, load_data_usage_from_backend_cached, record_bucket_object_write_memory,
     store_data_usage_in_backend,
 };
+use super::storage_api::test::{ECStore, StoragePutObjReader as PutObjReader};
 use crate::app::admin_usecase::DefaultAdminUsecase;
 use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
 use serial_test::serial;
 use std::collections::HashMap;
+use std::sync::Arc;
 use std::time::{Duration, SystemTime};
+use tokio::sync::mpsc;
+use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
 
 const SEEDED_BUCKET_SIZE: u64 = 123_456;
 const SEEDED_BUCKET_OBJECTS: u64 = 42;
 
+async fn next_snapshot_time(ecstore: Arc<ECStore>) -> SystemTime {
+    let now = SystemTime::now();
+    let latest = load_data_usage_from_backend_cached(ecstore)
+        .await
+        .ok()
+        .and_then(|info| info.last_update)
+        .unwrap_or(now);
+    latest.max(now) + Duration::from_nanos(1)
+}
+
 fn seeded_data_usage_info(bucket: &str, last_update: SystemTime) -> DataUsageInfo {
     let usage = BucketUsageInfo {
         size: SEEDED_BUCKET_SIZE,
@@ -58,6 +71,7 @@ fn seeded_data_usage_info(bucket: &str, last_update: SystemTime) -> DataUsageInf
         ..Default::default()
     };
     info.buckets_usage = HashMap::from([(bucket.to_string(), usage)]);
+    info.usage_snapshot_complete = true;
     info.bucket_sizes = HashMap::from([(bucket.to_string(), SEEDED_BUCKET_SIZE)]);
     info
 }
@@ -78,10 +92,18 @@ fn sized_data_usage_info(bucket: &str, size: u64, last_update: SystemTime) -> Da
         ..Default::default()
     };
     info.buckets_usage = HashMap::from([(bucket.to_string(), usage)]);
+    info.usage_snapshot_complete = true;
     info.bucket_sizes = HashMap::from([(bucket.to_string(), size)]);
     info
 }
 
+async fn persist_scanner_snapshot(ecstore: Arc<ECStore>, info: DataUsageInfo) {
+    let (sender, receiver) = mpsc::channel(1);
+    sender.send(info).await.expect("enqueue scanner snapshot");
+    drop(sender);
+    rustfs_scanner::scanner::store_data_usage_in_backend(CancellationToken::new(), ecstore, receiver).await;
+}
+
 #[tokio::test]
 #[serial]
 async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
@@ -121,10 +143,34 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
         .make_bucket(&seeded_bucket, &MakeBucketOptions::default())
         .await
         .expect("create seeded bucket");
-    let seeded_at = SystemTime::now();
-    store_data_usage_in_backend(seeded_data_usage_info(&seeded_bucket, seeded_at), ecstore.clone())
+    ecstore
+        .make_bucket(&overlay_bucket, &MakeBucketOptions::default())
         .await
-        .expect("persist seeded data usage snapshot");
+        .expect("create overlay bucket");
+    let seeded_at = next_snapshot_time(ecstore.clone()).await;
+    let mut seeded_info = seeded_data_usage_info(&seeded_bucket, seeded_at);
+    for bucket in ecstore
+        .list_bucket(&BucketOptions {
+            no_metadata: true,
+            ..Default::default()
+        })
+        .await
+        .expect("list the complete live bucket namespace")
+        .into_iter()
+        .filter(|bucket| !bucket.name.starts_with('.'))
+    {
+        seeded_info.buckets_usage.entry(bucket.name.clone()).or_default();
+        seeded_info.bucket_sizes.entry(bucket.name).or_default();
+    }
+    seeded_info.bucket_sizes.insert(live_bucket.clone(), control_usage.size);
+    seeded_info.buckets_usage.insert(live_bucket.clone(), control_usage);
+    seeded_info
+        .buckets_usage
+        .insert(overlay_bucket.clone(), BucketUsageInfo::default());
+    seeded_info.bucket_sizes.insert(overlay_bucket.clone(), 0);
+    seeded_info.buckets_count = u64::try_from(seeded_info.buckets_usage.len()).expect("test bucket count should fit into u64");
+    seeded_info.calculate_totals();
+    persist_scanner_snapshot(ecstore.clone(), seeded_info).await;
 
     record_bucket_object_write_memory(&overlay_bucket, None, 512).await;
 
@@ -182,6 +228,138 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
     assert_eq!(overlay_usage.size, 512);
 }
 
+#[tokio::test]
+#[serial]
+async fn data_usage_endpoint_treats_legacy_partial_stats_as_unknown_until_covered() {
+    let ecstore = shared_gating_ecstore().await;
+    let agent_bucket = format!("usage-legacy-agent-{}", Uuid::new_v4());
+    let small_bucket = format!("usage-legacy-small-{}", Uuid::new_v4());
+    ecstore
+        .make_bucket(&agent_bucket, &MakeBucketOptions::default())
+        .await
+        .expect("create legacy partial snapshot bucket");
+    ecstore
+        .make_bucket(&small_bucket, &MakeBucketOptions::default())
+        .await
+        .expect("create legacy zero snapshot bucket");
+    let mut reader = PutObjReader::from_vec(b"object exists".to_vec());
+    ecstore
+        .put_object(&agent_bucket, "object.bin", &mut reader, &Default::default())
+        .await
+        .expect("put object before persisting legacy partial snapshot");
+    let mut small_reader = PutObjReader::from_vec(vec![b'x'; 57_344]);
+    ecstore
+        .put_object(&small_bucket, "small.bin", &mut small_reader, &Default::default())
+        .await
+        .expect("put object before persisting legacy zero snapshot");
+
+    let persisted_at = next_snapshot_time(ecstore.clone()).await;
+    let mut legacy = DataUsageInfo {
+        last_update: Some(persisted_at),
+        buckets_count: 2,
+        ..Default::default()
+    };
+    legacy.buckets_usage.insert(
+        agent_bucket.clone(),
+        BucketUsageInfo {
+            size: 196_870_144,
+            objects_count: 1_502,
+            versions_count: 1_502,
+            ..Default::default()
+        },
+    );
+    legacy.buckets_usage.insert(small_bucket.clone(), BucketUsageInfo::default());
+    legacy.bucket_sizes.insert(agent_bucket.clone(), 196_870_144);
+    legacy.bucket_sizes.insert(small_bucket.clone(), 0);
+    store_data_usage_in_backend(legacy, ecstore.clone())
+        .await
+        .expect("persist legacy partial snapshot");
+
+    let before_endpoint = live_bucket_usage_computations();
+    let unknown = DefaultAdminUsecase::query_data_usage_info_with_store(ecstore.clone())
+        .await
+        .expect_err("an uncovered snapshot must be reported as temporarily unavailable");
+
+    assert_eq!(unknown.code, s3s::S3ErrorCode::ServiceUnavailable);
+    assert_eq!(
+        live_bucket_usage_computations(),
+        before_endpoint,
+        "uncovered snapshot handling must not restore request-time object listings"
+    );
+
+    let completed_at = persisted_at + Duration::from_nanos(1);
+    let mut completed = DataUsageInfo {
+        last_update: Some(completed_at),
+        usage_snapshot_complete: true,
+        ..Default::default()
+    };
+    for bucket in ecstore
+        .list_bucket(&BucketOptions {
+            no_metadata: true,
+            ..Default::default()
+        })
+        .await
+        .expect("list the complete live bucket namespace")
+        .into_iter()
+        .filter(|bucket| !bucket.name.starts_with('.'))
+    {
+        completed
+            .buckets_usage
+            .insert(bucket.name.clone(), BucketUsageInfo::default());
+        completed.bucket_sizes.insert(bucket.name, 0);
+    }
+    completed.buckets_usage.insert(
+        agent_bucket.clone(),
+        BucketUsageInfo {
+            size: 222_822_400,
+            objects_count: 1_700,
+            versions_count: 1_700,
+            ..Default::default()
+        },
+    );
+    completed.bucket_sizes.insert(agent_bucket.clone(), 222_822_400);
+    completed.buckets_usage.insert(
+        small_bucket.clone(),
+        BucketUsageInfo {
+            size: 57_344,
+            objects_count: 1,
+            versions_count: 1,
+            ..Default::default()
+        },
+    );
+    completed.bucket_sizes.insert(small_bucket.clone(), 57_344);
+    completed.buckets_count = u64::try_from(completed.buckets_usage.len()).expect("test bucket count should fit into u64");
+    completed.calculate_totals();
+    store_data_usage_in_backend(completed, ecstore.clone())
+        .await
+        .expect("persist complete covered snapshot");
+
+    let covered = DefaultAdminUsecase::query_data_usage_info_with_store(ecstore)
+        .await
+        .expect("query covered data usage info");
+    assert_eq!(covered.last_update, Some(completed_at));
+    assert!(covered.usage_snapshot_complete);
+    assert_eq!(
+        covered
+            .buckets_usage
+            .get(&agent_bucket)
+            .map(|usage| (usage.objects_count, usage.size)),
+        Some((1_700, 222_822_400))
+    );
+    assert_eq!(
+        covered
+            .buckets_usage
+            .get(&small_bucket)
+            .map(|usage| (usage.objects_count, usage.size)),
+        Some((1, 57_344))
+    );
+    assert_eq!(
+        live_bucket_usage_computations(),
+        before_endpoint,
+        "covered snapshot publication must not trigger request-time object listings"
+    );
+}
+
 /// Revert detector for the snapshot-cache invalidation in
 /// `save_data_usage_in_backend` (rustfs/backlog#1306): a fresh scanner save must
 /// be visible to the very next cached read, not deferred until the 30s TTL
@@ -194,11 +372,11 @@ async fn save_data_usage_invalidates_snapshot_cache() {
     let ecstore = shared_gating_ecstore().await;
     let bucket = format!("usage-invalidate-{}", Uuid::new_v4());
 
-    // Future-date both snapshots beyond the stale-guard's 5min tolerance so the
-    // persists win deterministically over whatever snapshot a sibling serial
-    // test may have left behind, regardless of test execution order.
-    let warm_at = SystemTime::now() + Duration::from_secs(600);
-    let changed_at = warm_at + Duration::from_secs(1);
+    // Advance beyond the latest persisted timestamp by the smallest possible
+    // increment so this save wins the monotonicity guard without masking a
+    // subsequent in-memory mutation as older than the snapshot.
+    let warm_at = next_snapshot_time(ecstore.clone()).await;
+    let changed_at = warm_at + Duration::from_nanos(1);
 
     const WARM_SIZE: u64 = 111_000;
     const CHANGED_SIZE: u64 = 222_000;
@@ -236,6 +414,38 @@ async fn save_data_usage_invalidates_snapshot_cache() {
     );
 }
 
+#[tokio::test]
+#[serial]
+async fn scanner_save_invalidates_snapshot_cache() {
+    let ecstore = shared_gating_ecstore().await;
+    let bucket = format!("usage-scanner-invalidate-{}", Uuid::new_v4());
+    let warm_at = next_snapshot_time(ecstore.clone()).await;
+    let changed_at = warm_at + Duration::from_nanos(1);
+
+    const WARM_SIZE: u64 = 333_000;
+    const CHANGED_SIZE: u64 = 444_000;
+
+    store_data_usage_in_backend(sized_data_usage_info(&bucket, WARM_SIZE, warm_at), ecstore.clone())
+        .await
+        .expect("persist initial snapshot");
+    let warmed = load_data_usage_from_backend_cached(ecstore.clone())
+        .await
+        .expect("warm cached snapshot");
+    assert_eq!(warmed.buckets_usage.get(&bucket).map(|usage| usage.size), Some(WARM_SIZE));
+
+    persist_scanner_snapshot(ecstore.clone(), sized_data_usage_info(&bucket, CHANGED_SIZE, changed_at)).await;
+
+    let reloaded = load_data_usage_from_backend_cached(ecstore)
+        .await
+        .expect("reload cached snapshot after scanner save");
+    assert_eq!(
+        reloaded.buckets_usage.get(&bucket).map(|usage| usage.size),
+        Some(CHANGED_SIZE),
+        "scanner persistence must invalidate the process-local snapshot cache"
+    );
+    assert_eq!(reloaded.last_update, Some(changed_at));
+}
+
 /// Wire pin for the no-snapshot response shape (rustfs/backlog#1306): a
 /// default `DataUsageInfo` must keep serializing `last_update` as `null` with
 /// empty bucket maps, so "no snapshot yet" stays distinguishable from real

rustfs/src/app/multipart_usecase.rs

@@ -58,7 +58,9 @@ use crate::app::object_data_cache::{
     ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success,
     invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation,
 };
-use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write};
+use crate::app::object_usecase::{
+    build_put_like_object_lock_metadata, map_quota_check_outcome, validate_existing_object_lock_for_write,
+};
 use crate::app::runtime_sources::{
     AppContext, current_app_context, current_object_data_cache_for_context, current_object_store_handle_for_context,
 };
@@ -516,6 +518,12 @@ impl DefaultMultipartUsecase {
             _ => None,
         };
 
+        let quota_metadata_sys = self.bucket_metadata_sys();
+        if let Some(metadata_sys) = quota_metadata_sys.as_ref() {
+            let quota_checker = QuotaChecker::new(metadata_sys.clone());
+            map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
+        }
+
         let obj_info = store
             .clone()
             .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts)
@@ -524,17 +532,18 @@ impl DefaultMultipartUsecase {
         let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await;
         record_capacity_write(Some(capacity_scope_token)).await;
 
-        // check quota after completing multipart upload
-        if let Some(metadata_sys) = self.bucket_metadata_sys() {
+        if let Some(metadata_sys) = quota_metadata_sys {
             let quota_checker = QuotaChecker::new(metadata_sys);
 
             match quota_checker
-                .check_quota(&bucket, QuotaOperation::PutObject, obj_info.size as u64)
+                .check_quota(&bucket, QuotaOperation::PutObject, obj_info.size.max(0) as u64)
                 .await
             {
                 Ok(check_result) => {
                     if !check_result.allowed {
-                        // Quota exceeded, delete the completed object
+                        // Preserve the established compensation behavior for
+                        // a known over-quota result. Unknown usage is rejected
+                        // by the preflight check before the upload is committed.
                         let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await;
                         let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await;
                         return Err(S3Error::with_message(
@@ -546,18 +555,17 @@ impl DefaultMultipartUsecase {
                             ),
                         ));
                     }
-                    // Update quota tracking after successful multipart upload
-                    if versioned {
-                        record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64)
-                            .await;
-                    } else {
-                        record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
-                    }
                 }
-                Err(e) => {
-                    warn!("Quota check failed for bucket {}: {}, allowing operation", bucket, e);
+                Err(err) => {
+                    warn!("Quota check failed for bucket {} after multipart completion: {}", bucket, err);
                 }
             }
+
+            if versioned {
+                record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
+            } else {
+                record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
+            }
         }
 
         enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;

rustfs/src/app/object_usecase.rs

@@ -501,7 +501,7 @@ fn request_body_is_aws_chunked_framed(headers: &HeaderMap) -> bool {
 /// Map a bucket-quota checker outcome onto the S3 admission result.
 ///
 /// Hard is the only supported quota type, so a checker fault (bucket-config read, config parse, or usage lookup) must fail closed rather than admit the write: allowing it would silently bypass a configured hard quota. The no-quota happy path never reaches the error arm — `QuotaChecker::check_quota` returns `Ok(allowed)` via the zero-extra-I/O fast path when no quota is configured, so failing closed here cannot penalise buckets without a quota. A fault surfaces as a retryable `ServiceUnavailable` and is counted; the client-facing message stays generic so internal config/usage details are not leaked.
-fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckResult, QuotaError>) -> S3Result<()> {
+pub(super) fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckResult, QuotaError>) -> S3Result<QuotaCheckResult> {
     match outcome {
         Ok(result) if !result.allowed => Err(S3Error::with_message(
             S3ErrorCode::InvalidRequest,
@@ -513,13 +513,17 @@ fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckResult, Quota
         )),
         Err(e) => {
             counter!("rustfs_bucket_quota_check_failed_total").increment(1);
-            warn!(bucket, error = %e, state = "checker_failed", "Bucket quota check failed closed");
+            if matches!(&e, QuotaError::UsageUnavailable { .. }) {
+                debug!(bucket, error = %e, state = "usage_pending", "Bucket quota check waiting for authoritative usage");
+            } else {
+                warn!(bucket, error = %e, state = "checker_failed", "Bucket quota check failed closed");
+            }
             Err(S3Error::with_message(
                 S3ErrorCode::ServiceUnavailable,
                 "Bucket quota check temporarily unavailable, please retry".to_string(),
             ))
         }
-        _ => Ok(()),
+        Ok(result) => Ok(result),
     }
 }
 
@@ -3263,7 +3267,7 @@ impl DefaultObjectUsecase {
             return Ok(());
         };
         let quota_checker = QuotaChecker::new(metadata_sys);
-        map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await)
+        map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(|_| ())
     }
 
     fn build_memory_bytes_blob(
@@ -7983,25 +7987,9 @@ impl DefaultObjectUsecase {
         let extract_limits = put_object_extract_limits();
         let extract_quota_snapshot = if let Some(metadata_sys) = self.bucket_metadata_sys() {
             let quota_checker = QuotaChecker::new(metadata_sys);
-            match quota_checker.get_quota_config(&bucket).await {
-                Ok(quota) => {
-                    if let Some(limit) = quota.quota {
-                        match quota_checker.get_real_time_usage(&bucket).await {
-                            Ok(current_usage) => Some((current_usage, limit)),
-                            Err(err) => {
-                                warn!(bucket, error = %err, state = "extract_usage_snapshot_failed", "Bucket quota snapshot degraded to allow");
-                                None
-                            }
-                        }
-                    } else {
-                        None
-                    }
-                }
-                Err(err) => {
-                    warn!(bucket, error = %err, state = "extract_quota_snapshot_failed", "Bucket quota snapshot degraded to allow");
-                    None
-                }
-            }
+            let check_result =
+                map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
+            check_result.current_usage.zip(check_result.quota_limit)
         } else {
             None
         };
@@ -8191,8 +8179,11 @@ impl DefaultObjectUsecase {
             let cache_adapter = self.object_data_cache();
             let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &fpath).await;
 
-            let obj_info = match store.put_object(&bucket, &fpath, &mut reader, &opts).await {
-                Ok(info) => info,
+            let (obj_info, backfilled_old_current_size) = match store
+                .put_object_with_old_current_size(&bucket, &fpath, &mut reader, &opts)
+                .await
+            {
+                Ok(result) => result,
                 Err(e) => {
                     if extract_options.ignore_errors {
                         warn!(error = %e, "Archive object write skipped due to ignore-errors");
@@ -8201,6 +8192,21 @@ impl DefaultObjectUsecase {
                     return Err(ApiError::from(e).into());
                 }
             };
+            let extract_versioned = BucketVersioningSys::prefix_enabled(&bucket, &fpath).await;
+            match previous_current_size_from_backfill(backfilled_old_current_size) {
+                Some(previous_current_size) => {
+                    if extract_versioned {
+                        record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64)
+                            .await;
+                    } else {
+                        record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
+                    }
+                }
+                None => {
+                    record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, extract_versioned)
+                        .await;
+                }
+            }
             let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await;
             if !wrote_any_entry {
                 rustfs_scanner::record_dirty_usage_bucket(&bucket);
@@ -14064,7 +14070,12 @@ mod tests {
 
     #[test]
     fn quota_admission_allows_within_limit() {
-        map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write");
+        let result = map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write");
+
+        assert_eq!(result.current_usage, Some(1024));
+        assert_eq!(result.quota_limit, Some(2048));
+        assert_eq!(result.operation_size, 512);
+        assert_eq!(result.remaining, Some(512));
     }
 
     #[test]
@@ -14085,4 +14096,16 @@ mod tests {
         .expect_err("a checker fault must fail closed");
         assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
     }
+
+    #[test]
+    fn quota_admission_fails_closed_on_unknown_authoritative_usage() {
+        let err = map_quota_check_outcome(
+            "bucket",
+            Err(QuotaError::UsageUnavailable {
+                bucket: "bucket".to_string(),
+            }),
+        )
+        .expect_err("unknown authoritative usage must not admit a quota-controlled write");
+        assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
+    }
 }

rustfs/src/app/storage_api.rs

@@ -931,6 +931,10 @@ pub(crate) mod s3_api {
 
 pub(crate) mod admin_usecase {
     pub(crate) mod contract {
+        pub(crate) mod bucket {
+            pub(crate) use super::super::super::storage_contracts::{BucketOperations, BucketOptions};
+        }
+
         pub(crate) use super::super::storage_contracts::StorageAdminApi;
     }
 

rustfs/src/error.rs

@@ -358,6 +358,7 @@ impl From<QuotaError> for ApiError {
         let code = match &err {
             QuotaError::QuotaExceeded { .. } => S3ErrorCode::InvalidRequest,
             QuotaError::ConfigNotFound { .. } => S3ErrorCode::NoSuchBucket,
+            QuotaError::UsageUnavailable { .. } => S3ErrorCode::ServiceUnavailable,
             QuotaError::InvalidConfig { .. } => S3ErrorCode::InvalidArgument,
             QuotaError::StorageError(_) => S3ErrorCode::InternalError,
         };
@@ -486,6 +487,16 @@ mod tests {
         assert_eq!(api_error.message, "The service is unavailable. Please retry.");
     }
 
+    #[test]
+    fn test_unknown_authoritative_quota_usage_maps_to_retryable_error() {
+        let api_error = ApiError::from(QuotaError::UsageUnavailable {
+            bucket: "bucket".to_string(),
+        });
+
+        assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
+        assert_eq!(api_error.message, "The service is unavailable. Please retry.");
+    }
+
     #[test]
     fn test_api_error_from_storage_error_mappings() {
         let test_cases = vec![