fix(quota): enforce durable hard quota reservations (#6058)

* fix(quota): enforce durable hard quota reservations

* fix(quota): close reservation bypasses

* fix(quota): isolate tests and box object futures

* fix(quota): close legacy and deferred settlement bypasses

* fix(app): keep object futures off caller stacks

* fix(metrics): preserve object operation labels

* fix(logging): retain GET trace guard contract

crates/ecstore/src/bucket/metadata_sys.rs

@@ -50,6 +50,72 @@ use uuid::Uuid;
 
 const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
 
+#[cfg(any(test, feature = "test-util"))]
+struct ConfigWriteLockProbeState {
+    bucket: String,
+    arrived: tokio::sync::Notify,
+}
+
+#[cfg(any(test, feature = "test-util"))]
+static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
+
+#[cfg(any(test, feature = "test-util"))]
+pub struct ConfigWriteLockProbe {
+    state: Arc<ConfigWriteLockProbeState>,
+}
+
+#[cfg(any(test, feature = "test-util"))]
+impl ConfigWriteLockProbe {
+    pub fn install(bucket: &str) -> Self {
+        let state = Arc::new(ConfigWriteLockProbeState {
+            bucket: bucket.to_string(),
+            arrived: tokio::sync::Notify::new(),
+        });
+        let mut probes = CONFIG_WRITE_LOCK_PROBES
+            .get_or_init(|| StdMutex::new(Vec::new()))
+            .lock()
+            .expect("config write lock probe mutex should not poison");
+        assert!(
+            !probes.iter().any(|current| current.bucket == state.bucket),
+            "config write lock probe must be unique for a bucket"
+        );
+        probes.push(Arc::clone(&state));
+        drop(probes);
+        Self { state }
+    }
+
+    pub async fn wait_until_attempted(&self) {
+        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("bucket config update should attempt the transaction lock");
+    }
+}
+
+#[cfg(any(test, feature = "test-util"))]
+impl Drop for ConfigWriteLockProbe {
+    fn drop(&mut self) {
+        let mut probes = CONFIG_WRITE_LOCK_PROBES
+            .get_or_init(|| StdMutex::new(Vec::new()))
+            .lock()
+            .expect("config write lock probe mutex should not poison");
+        probes.retain(|state| !Arc::ptr_eq(state, &self.state));
+    }
+}
+
+#[cfg(any(test, feature = "test-util"))]
+fn notify_config_write_lock_attempt(bucket: &str) {
+    let probe = CONFIG_WRITE_LOCK_PROBES
+        .get_or_init(|| StdMutex::new(Vec::new()))
+        .lock()
+        .expect("config write lock probe mutex should not poison")
+        .iter()
+        .find(|probe| probe.bucket == bucket)
+        .cloned();
+    if let Some(probe) = probe {
+        probe.arrived.notify_one();
+    }
+}
+
 #[derive(Clone, Copy)]
 enum MetadataLoadMode {
     Initial,
@@ -590,6 +656,31 @@ pub async fn update_under_transaction_lock(
     update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
 }
 
+pub async fn update_quota_if_incarnation(
+    bucket: &str,
+    data: Vec<u8>,
+    expected_incarnation_id: Uuid,
+    proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
+) -> Result<OffsetDateTime> {
+    let sys = get_bucket_metadata_sys()?;
+    let guard = Box::pin(acquire_config_write_guard_for_incarnation(
+        sys.clone(),
+        bucket,
+        Some(expected_incarnation_id),
+    ))
+    .await?;
+    if !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof) {
+        return Err(Error::NamespaceLockQuorumUnavailable {
+            mode: "quota_capability",
+            bucket: bucket.to_string(),
+            object: rustfs_config::QUOTA_CONFIG_FILE.to_string(),
+            required: 1,
+            achieved: 0,
+        });
+    }
+    update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await
+}
+
 pub async fn update_bucket_targets_under_transaction_lock(
     guard: &BucketMetadataMutationGuard,
     bucket: &str,
@@ -734,7 +825,26 @@ async fn acquire_transaction_lock_with_sys(
     let lock = api
         .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
         .await?;
-    Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
+    let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout());
+    #[cfg(any(test, feature = "test-util"))]
+    {
+        tokio::pin!(acquire);
+        let mut notified = false;
+        let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
+            std::task::Poll::Pending => {
+                if !notified {
+                    notify_config_write_lock_attempt(bucket);
+                    notified = true;
+                }
+                std::task::Poll::Pending
+            }
+            std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
+        })
+        .await?;
+        Ok(guard)
+    }
+    #[cfg(not(any(test, feature = "test-util")))]
+    Ok(acquire.await?)
 }
 
 /// The lock resource name is deliberately still the `bucket-targets` one it
@@ -889,6 +999,37 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
     }
 }
 
+/// Re-read the quota configuration and bucket incarnation from the same
+/// authoritative metadata blob while the caller holds the bucket metadata
+/// transaction read lock.
+pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+) -> Result<(Option<BucketQuota>, Uuid, OffsetDateTime)> {
+    let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
+
+    match bucket_meta_sys
+        .read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
+        .await?
+    {
+        BucketMetadataAuthority::Authoritative(metadata)
+            if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
+        {
+            Ok((
+                metadata.quota_config.clone(),
+                metadata.bucket_incarnation_id,
+                metadata.quota_config_updated_at,
+            ))
+        }
+        BucketMetadataAuthority::Authoritative(_) => {
+            Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
+        }
+        BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
+        BucketMetadataAuthority::Fabricated => Err(Error::other(format!("bucket quota metadata is not authoritative: {bucket}"))),
+    }
+}
+
 pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;

crates/ecstore/src/bucket/quota/checker.rs

@@ -52,6 +52,7 @@ impl QuotaChecker {
     ) -> Result<QuotaCheckResult, QuotaError> {
         let start_time = Instant::now();
         let quota_config = self.get_quota_config(bucket).await?;
+        let uses_durable_reservations = quota_config.uses_durable_reservations();
 
         // If no quota limit is set, allow operation
         let quota_limit = match quota_config.quota {
@@ -67,6 +68,7 @@ impl QuotaChecker {
                     quota_limit: None,
                     operation_size,
                     remaining: None,
+                    uses_durable_reservations,
                 });
             }
             Some(q) => q,
@@ -74,14 +76,17 @@ impl QuotaChecker {
 
         let current_usage = self.get_real_time_usage(bucket).await?;
 
+        let admission_size = if uses_durable_reservations { 0 } else { operation_size };
         let expected_usage = match operation {
-            QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => current_usage + operation_size,
+            QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
+                current_usage.saturating_add(admission_size)
+            }
             QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size),
         };
 
         let allowed = match operation {
             QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
-                quota_config.check_operation_allowed(current_usage, operation_size)
+                quota_config.check_operation_allowed(current_usage, admission_size)
             }
             QuotaOperation::DeleteObject => true,
         };
@@ -105,6 +110,7 @@ impl QuotaChecker {
             quota_limit: Some(quota_limit),
             operation_size,
             remaining,
+            uses_durable_reservations,
         };
 
         let duration = start_time.elapsed();
@@ -158,6 +164,26 @@ impl QuotaChecker {
             .await
     }
 
+    pub async fn set_durable_quota_config_if_incarnation(
+        &mut self,
+        bucket: &str,
+        quota: BucketQuota,
+        expected_incarnation_id: uuid::Uuid,
+        proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
+    ) -> Result<OffsetDateTime, QuotaError> {
+        let json_data = serde_json::to_vec(&quota).map_err(|e| QuotaError::InvalidConfig {
+            reason: format!("Failed to serialize quota config: {}", e),
+        })?;
+        let start_time = Instant::now();
+        let updated_at =
+            crate::bucket::metadata_sys::update_quota_if_incarnation(bucket, json_data, expected_incarnation_id, proof)
+                .await
+                .map_err(QuotaError::StorageError)?;
+
+        rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
+        Ok(updated_at)
+    }
+
     async fn set_quota_config_for_incarnation(
         &mut self,
         bucket: &str,
@@ -355,6 +381,7 @@ mod tests {
             quota_limit: None,
             operation_size: 1024,
             remaining: None,
+            uses_durable_reservations: false,
         };
 
         assert!(result.allowed);
@@ -378,4 +405,13 @@ mod tests {
         let allowed = quota.check_operation_allowed(512, 1024);
         assert!(!allowed);
     }
+
+    #[test]
+    fn legacy_quota_rejects_full_operation_while_v1_defers_net_growth() {
+        let legacy: BucketQuota = serde_json::from_str(r#"{"quota":5}"#).expect("legacy quota should parse");
+        let durable = BucketQuota::new(Some(5));
+
+        assert!(!legacy.check_operation_allowed(4, 2));
+        assert!(durable.uses_durable_reservations());
+    }
 }

crates/ecstore/src/bucket/quota/mod.rs

@@ -13,40 +13,100 @@
 // limitations under the License.
 
 pub mod checker;
+pub(crate) mod reservation;
 
 use crate::error::Result;
 use rustfs_config::{
     QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE,
     QUOTA_NOT_FOUND_ERROR_CODE,
 };
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
 use thiserror::Error;
 use time::OffsetDateTime;
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
 pub enum QuotaType {
-    /// Hard quota: reject immediately when exceeded
+    /// Hard quota accounting.
     #[default]
     #[serde(alias = "HARD", alias = "hard")]
     Hard,
 }
 
+pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
+
 /// Bucket quota configuration. quota_type defaults to Hard when omitted.
-#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)]
+#[derive(Debug, Default, Clone, PartialEq)]
 pub struct BucketQuota {
-    #[serde(default)]
     pub quota: Option<u64>,
     /// Defaults to Hard when missing.
-    #[serde(default)]
     pub quota_type: QuotaType,
+    /// Optional durable reservation protocol. The wire format gives older
+    /// nodes a zero hard quota so a mixed-version fleet fails closed.
+    pub reservation_protocol: Option<u32>,
     /// Timestamp when this quota configuration was set (for audit purposes)
-    #[serde(default, with = "time::serde::rfc3339::option")]
     pub created_at: Option<OffsetDateTime>,
     /// Accept updated_at for compatibility; not used.
-    #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
     pub updated_at: Option<OffsetDateTime>,
 }
 
+#[derive(Deserialize, Serialize)]
+struct BucketQuotaWire {
+    #[serde(default)]
+    quota: Option<u64>,
+    #[serde(default)]
+    quota_type: QuotaType,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    reservation_protocol: Option<u32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    reservation_quota: Option<u64>,
+    #[serde(default, with = "time::serde::rfc3339::option")]
+    created_at: Option<OffsetDateTime>,
+    #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
+    updated_at: Option<OffsetDateTime>,
+}
+
+impl Serialize for BucketQuota {
+    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let durable = self.uses_durable_reservations();
+        BucketQuotaWire {
+            quota: if durable { Some(0) } else { self.quota },
+            quota_type: self.quota_type.clone(),
+            reservation_protocol: self.reservation_protocol,
+            reservation_quota: if durable { self.quota } else { None },
+            created_at: self.created_at,
+            updated_at: self.updated_at,
+        }
+        .serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for BucketQuota {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let wire = BucketQuotaWire::deserialize(deserializer)?;
+        let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) {
+            Some(
+                wire.reservation_quota
+                    .ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?,
+            )
+        } else {
+            wire.quota
+        };
+        Ok(Self {
+            quota,
+            quota_type: wire.quota_type,
+            reservation_protocol: wire.reservation_protocol,
+            created_at: wire.created_at,
+            updated_at: wire.updated_at,
+        })
+    }
+}
+
 impl BucketQuota {
     /// Serialize to JSON bytes. Same format as parse_all_configs.
     pub fn marshal_msg(&self) -> Result<Vec<u8>> {
@@ -63,6 +123,7 @@ impl BucketQuota {
         Self {
             quota,
             quota_type: QuotaType::Hard,
+            reservation_protocol: quota.map(|_| QUOTA_RESERVATION_PROTOCOL_V1),
             created_at: Some(now),
             updated_at: None,
         }
@@ -72,7 +133,19 @@ impl BucketQuota {
         self.quota
     }
 
+    pub fn uses_durable_reservations(&self) -> bool {
+        self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1)
+    }
+
+    pub fn has_unsupported_reservation_protocol(&self) -> bool {
+        self.reservation_protocol
+            .is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1)
+    }
+
     pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool {
+        if operation_size == 0 {
+            return true;
+        }
         if let Some(quota_limit) = self.quota {
             current_usage.saturating_add(operation_size) <= quota_limit
         } else {
@@ -94,6 +167,7 @@ pub struct QuotaCheckResult {
     pub quota_limit: Option<u64>,
     pub operation_size: u64,
     pub remaining: Option<u64>,
+    pub uses_durable_reservations: bool,
 }
 
 #[derive(Debug)]
@@ -210,7 +284,59 @@ mod tests {
         let buf = q.marshal_msg().expect("marshal");
         let restored = BucketQuota::unmarshal(&buf).expect("unmarshal");
         assert_eq!(q.quota, restored.quota);
-        assert_eq!(q.quota_type, restored.quota_type);
+        assert_eq!(restored.quota_type, QuotaType::Hard);
+        assert_eq!(restored.reservation_protocol, Some(QUOTA_RESERVATION_PROTOCOL_V1));
+    }
+
+    #[test]
+    fn clearing_quota_keeps_the_legacy_compatible_type() {
+        let quota = BucketQuota::new(None);
+
+        assert_eq!(quota.quota_type, QuotaType::Hard);
+        assert_eq!(quota.reservation_protocol, None);
+        assert!(!quota.uses_durable_reservations());
+    }
+
+    #[test]
+    fn durable_quota_makes_legacy_nodes_fail_closed() {
+        let json = serde_json::to_vec(&BucketQuota::new(Some(2048))).expect("durable quota should serialize");
+        let quota: BucketQuota = serde_json::from_slice(&json).expect("current quota version should parse");
+        assert!(quota.uses_durable_reservations());
+        assert_eq!(quota.quota, Some(2048));
+
+        #[derive(Deserialize)]
+        enum LegacyQuotaType {
+            Hard,
+        }
+        #[derive(Deserialize)]
+        struct LegacyBucketQuota {
+            #[allow(dead_code)]
+            quota: Option<u64>,
+            #[allow(dead_code)]
+            quota_type: LegacyQuotaType,
+        }
+        let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
+            .expect("legacy readers should ignore the reservation protocol field");
+        assert_eq!(legacy.quota, Some(0));
+        assert!(matches!(legacy.quota_type, LegacyQuotaType::Hard));
+    }
+
+    #[test]
+    fn unknown_reservation_protocol_does_not_activate_v1() {
+        let quota: BucketQuota =
+            serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#)
+                .expect("future protocol should remain parseable");
+
+        assert!(!quota.uses_durable_reservations());
+        assert!(quota.has_unsupported_reservation_protocol());
+    }
+
+    #[test]
+    fn reservation_protocol_v1_requires_reservation_quota() {
+        let err = serde_json::from_str::<BucketQuota>(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":1}"#)
+            .expect_err("v1 without its authoritative quota must fail closed");
+
+        assert!(err.to_string().contains("reservation_quota is required"));
     }
 
     /// unmarshal accepts format without quota_type