fix(quota): reject oversized multipart completion (#5958)

* fix(quota): reject oversized multipart completion

* fix(arch): route quota test through app facade

crates/e2e_test/src/quota_test.rs

@@ -252,6 +252,7 @@ impl QuotaTestEnv {
 #[cfg(test)]
 mod integration_tests {
     use super::*;
+    use aws_sdk_s3::error::ProvideErrorMetadata;
 
     #[tokio::test]
     #[serial]
@@ -963,9 +964,27 @@ mod integration_tests {
             .send()
             .await;
 
-        assert!(complete_result.is_err());
+        let complete_error = complete_result.expect_err("multipart completion above quota must be rejected");
+        assert_eq!(complete_error.as_service_error().and_then(|error| error.code()), Some("InvalidRequest"));
         assert!(!env.object_exists("over_quota.txt").await?);
 
+        let staged_parts = env
+            .client
+            .list_parts()
+            .bucket(&env.bucket_name)
+            .key("over_quota.txt")
+            .upload_id(upload_id2)
+            .send()
+            .await?;
+        assert_eq!(staged_parts.parts().len(), 2, "quota rejection must preserve the multipart upload");
+        env.client
+            .abort_multipart_upload()
+            .bucket(&env.bucket_name)
+            .key("over_quota.txt")
+            .upload_id(upload_id2)
+            .send()
+            .await?;
+
         env.cleanup_bucket().await?;
 
         Ok(())

crates/ecstore/src/api/mod.rs

@@ -308,6 +308,8 @@ pub mod config {
 }
 
 pub mod data_usage {
+    #[cfg(feature = "test-util")]
+    pub use crate::data_usage::seed_bucket_usage_memory_for_test;
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
         init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
@@ -409,10 +411,10 @@ pub mod object {
     pub use crate::object_api::{
         BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
         GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
-        ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader,
-        ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len,
-        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
-        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
+        RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer,
+        get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
+        register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
     };
     pub use crate::store::{
         PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,

crates/ecstore/src/data_usage/mod.rs

@@ -1638,7 +1638,7 @@ fn preserve_unknown_dirty_usage(
     Some(preserved)
 }
 
-#[cfg(test)]
+#[cfg(any(test, feature = "test-util"))]
 async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: BucketUsageInfo, refresh_started_at: SystemTime) {
     let mut cache = memory_cache().write().await;
     if let Some(existing) = cache.get(bucket)
@@ -1650,6 +1650,19 @@ async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: Buc
     cache.insert(bucket.to_string(), cached_bucket_usage_from_backend(usage, refresh_started_at, true));
 }
 
+#[cfg(feature = "test-util")]
+pub async fn seed_bucket_usage_memory_for_test(bucket: &str, size: u64) {
+    replace_bucket_usage_memory_from_authoritative(
+        bucket,
+        BucketUsageInfo {
+            size,
+            ..Default::default()
+        },
+        SystemTime::now(),
+    )
+    .await;
+}
+
 /// Fast in-memory update for immediate quota and admin usage consistency.
 pub async fn record_bucket_object_write_memory(bucket: &str, previous_current_size: Option<u64>, new_size: u64) {
     record_bucket_object_write_memory_inner(bucket, previous_current_size, new_size, false).await;

crates/ecstore/src/error/mod.rs

@@ -204,6 +204,8 @@ pub enum StorageError {
         required: usize,
         achieved: usize,
     },
+    #[error("Bucket quota exceeded. Current usage: {current} bytes, limit: {limit} bytes")]
+    QuotaExceeded { current: u64, limit: u64 },
 
     // ── Generic ──────────────────────────────────────────────────────
     #[error("Unexpected error")]
@@ -540,6 +542,10 @@ impl Clone for StorageError {
                 required: *required,
                 achieved: *achieved,
             },
+            StorageError::QuotaExceeded { current, limit } => StorageError::QuotaExceeded {
+                current: *current,
+                limit: *limit,
+            },
         }
     }
 }
@@ -627,6 +633,7 @@ impl StorageError {
             StorageError::NotModified => StorageErrorCode::NotModified,
             StorageError::InvalidPartNumber(_) => StorageErrorCode::InvalidPartNumber,
             StorageError::NamespaceLockQuorumUnavailable { .. } => StorageErrorCode::NamespaceLockQuorumUnavailable,
+            StorageError::QuotaExceeded { .. } => StorageErrorCode::QuotaExceeded,
         }
     }
 
@@ -752,6 +759,10 @@ impl StorageError {
                 required: Default::default(),
                 achieved: Default::default(),
             }),
+            StorageErrorCode::QuotaExceeded => Some(StorageError::QuotaExceeded {
+                current: Default::default(),
+                limit: Default::default(),
+            }),
         }
     }
 }
@@ -1301,6 +1312,7 @@ mod tests {
             .to_u32(),
             0x42
         );
+        assert_eq!(StorageError::QuotaExceeded { current: 1, limit: 2 }.to_u32(), 0x53);
     }
 
     #[test]
@@ -1319,6 +1331,10 @@ mod tests {
             StorageError::from_u32(0x42),
             Some(StorageError::NamespaceLockQuorumUnavailable { .. })
         ));
+        assert!(matches!(
+            StorageError::from_u32(0x53),
+            Some(StorageError::QuotaExceeded { current: 0, limit: 0 })
+        ));
 
         // Test invalid code returns None
         assert!(StorageError::from_u32(0xFF).is_none());
@@ -1549,6 +1565,7 @@ mod tests {
             StorageError::DecommissionAlreadyRunning,
             StorageError::RebalanceAlreadyRunning,
             StorageError::OperationCanceled,
+            StorageError::QuotaExceeded { current: 1, limit: 2 },
         ];
 
         for original_error in test_errors {

crates/ecstore/src/object_api/types.rs

@@ -211,6 +211,26 @@ impl ObjectLockConfigSnapshot {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct QuotaAdmission {
+    current_usage: u64,
+    quota_limit: u64,
+}
+
+impl QuotaAdmission {
+    pub(crate) fn current_usage(self) -> u64 {
+        self.current_usage
+    }
+
+    pub(crate) fn quota_limit(self) -> u64 {
+        self.quota_limit
+    }
+
+    pub(crate) fn remaining(self) -> u64 {
+        self.quota_limit - self.current_usage
+    }
+}
+
 #[derive(Debug, Default, Clone)]
 pub struct ObjectOptions {
     // Use the maximum parity (N/2), used when saving server configuration files
@@ -275,12 +295,22 @@ pub struct ObjectOptions {
     pub want_checksum: Option<Checksum>,
     pub skip_verify_bitrot: bool,
     pub capacity_scope_token: Option<Uuid>,
+    /// Server-derived bucket-quota snapshot for commit-boundary admission.
+    pub quota_admission: Option<QuotaAdmission>,
     /// Storage-owned journal writer used by the atomic delete path. This is
     /// populated only by the `ECStore` wrapper that holds the namespace locks.
     pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
 }
 
 impl ObjectOptions {
+    pub fn set_quota_admission(&mut self, current_usage: u64, quota_limit: u64) -> bool {
+        self.quota_admission = (current_usage <= quota_limit).then_some(QuotaAdmission {
+            current_usage,
+            quota_limit,
+        });
+        self.quota_admission.is_some()
+    }
+
     pub(crate) fn overwrites_existing_version(&self) -> bool {
         self.version_id.is_some() || !self.versioned || self.version_suspended
     }

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -1860,7 +1860,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             }
 
             object_size += ext_part.size;
-            object_actual_size += ext_part.actual_size;
+            if opts.quota_admission.is_some() && ext_part.actual_size < 0 {
+                return Err(Error::PartMissingOrCorrupt);
+            }
+            object_actual_size = object_actual_size
+                .checked_add(ext_part.actual_size)
+                .ok_or(Error::PartMissingOrCorrupt)?;
 
             fi.parts.push(completed_multipart_object_part(p.part_num, ext_part));
         }
@@ -1889,6 +1894,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             }
         }
 
+        if let Some(admission) = opts.quota_admission {
+            let quota_operation_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
+            if quota_operation_size > admission.remaining() {
+                return Err(Error::QuotaExceeded {
+                    current: admission.current_usage(),
+                    limit: admission.quota_limit(),
+                });
+            }
+        }
         if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) {
             if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) {
                 fi.checksum = Some(Bytes::from(rc_crc_bytes));
@@ -2551,29 +2565,157 @@ mod tests {
             .new_multipart_upload(bucket, object, create_opts)
             .await
             .expect("multipart upload should be created");
+        let part = put_test_part(set_disks, bucket, object, &upload.upload_id, 1, content, content.len() as i64).await;
+        (upload.upload_id, vec![part])
+    }
+
+    async fn put_test_part(
+        set_disks: &Arc<SetDisks>,
+        bucket: &str,
+        object: &str,
+        upload_id: &str,
+        part_number: usize,
+        content: &[u8],
+        actual_size: i64,
+    ) -> CompletePart {
         let mut reader = PutObjReader::new(
-            HashReader::from_stream(
-                Cursor::new(content.to_vec()),
-                content.len() as i64,
-                content.len() as i64,
-                None,
-                None,
-                false,
-            )
-            .expect("hash reader should be constructed"),
+            HashReader::from_stream(Cursor::new(content.to_vec()), content.len() as i64, actual_size, None, None, false)
+                .expect("hash reader should be constructed"),
         );
         let part = set_disks
-            .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
+            .put_object_part(bucket, object, upload_id, part_number, &mut reader, &ObjectOptions::default())
             .await
             .expect("uploading the part should succeed");
-        (
-            upload.upload_id,
-            vec![CompletePart {
-                part_num: part.part_num,
-                etag: part.etag,
-                ..Default::default()
-            }],
-        )
+        CompletePart {
+            part_num: part.part_num,
+            etag: part.etag,
+            ..Default::default()
+        }
+    }
+
+    #[tokio::test]
+    async fn complete_multipart_quota_rejection_preserves_destination_and_upload() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-quota-admission-bucket";
+        let object = "object";
+        make_bucket_on_all(&disk_stores, bucket).await;
+
+        let existing_payload = b"existing object";
+        let mut existing_reader = PutObjReader::from_vec(existing_payload.to_vec());
+        let existing = set_disks
+            .put_object(bucket, object, &mut existing_reader, &ObjectOptions::default())
+            .await
+            .expect("existing object should be stored");
+
+        let payload = vec![0x51; 4096];
+        let (upload_id, parts) =
+            stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
+        let mut denied_opts = ObjectOptions::default();
+        assert!(denied_opts.set_quota_admission(100, 4195));
+
+        let err = set_disks
+            .clone()
+            .complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &denied_opts)
+            .await
+            .expect_err("completion larger than the remaining quota must be rejected");
+        assert!(matches!(
+            err,
+            StorageError::QuotaExceeded {
+                current: 100,
+                limit: 4195
+            }
+        ));
+
+        let current = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("quota rejection must preserve the existing destination");
+        assert_eq!(current.etag, existing.etag);
+        assert!(
+            set_disks
+                .check_upload_id_exists(bucket, object, &upload_id, false)
+                .await
+                .is_ok(),
+            "quota rejection must leave the multipart upload retryable"
+        );
+
+        let mut allowed_opts = ObjectOptions::default();
+        assert!(allowed_opts.set_quota_admission(100, 4196));
+        let completed = set_disks
+            .clone()
+            .complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts)
+            .await
+            .expect("completion at the exact remaining-quota boundary should succeed");
+        assert_eq!(completed.get_actual_size().expect("completed logical size should resolve"), 4096);
+    }
+
+    #[tokio::test]
+    async fn complete_multipart_quota_uses_compressed_logical_size() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-compressed-quota-bucket";
+        let object = "object";
+        make_bucket_on_all(&disk_stores, bucket).await;
+
+        let mut create_opts = ObjectOptions::default();
+        insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
+        let upload = set_disks
+            .new_multipart_upload(bucket, object, &create_opts)
+            .await
+            .expect("multipart upload should be created");
+        let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x52; 128], 8192).await;
+        let mut complete_opts = ObjectOptions::default();
+        assert!(complete_opts.set_quota_admission(0, 4096));
+
+        let err = set_disks
+            .clone()
+            .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
+            .await
+            .expect_err("logical size above the remaining quota must be rejected");
+        assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4096 }));
+        assert!(
+            set_disks
+                .check_upload_id_exists(bucket, object, &upload.upload_id, false)
+                .await
+                .is_ok(),
+            "quota rejection must leave compressed parts retryable"
+        );
+    }
+
+    #[tokio::test]
+    async fn complete_multipart_quota_rejects_invalid_logical_sizes() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-invalid-logical-size-bucket";
+        make_bucket_on_all(&disk_stores, bucket).await;
+
+        let mut create_opts = ObjectOptions::default();
+        insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
+        let mut complete_opts = ObjectOptions::default();
+        assert!(complete_opts.set_quota_admission(0, u64::MAX));
+
+        let negative_upload = set_disks
+            .new_multipart_upload(bucket, "negative", &create_opts)
+            .await
+            .expect("negative-size upload should be created");
+        let negative_part = put_test_part(&set_disks, bucket, "negative", &negative_upload.upload_id, 1, &[0x53], -1).await;
+        let negative_err = set_disks
+            .clone()
+            .complete_multipart_upload(bucket, "negative", &negative_upload.upload_id, vec![negative_part], &complete_opts)
+            .await
+            .expect_err("negative logical size must fail closed");
+        assert!(matches!(negative_err, StorageError::PartMissingOrCorrupt));
+
+        let overflow_upload = set_disks
+            .new_multipart_upload(bucket, "overflow", &create_opts)
+            .await
+            .expect("overflow upload should be created");
+        let first = put_test_part(&set_disks, bucket, "overflow", &overflow_upload.upload_id, 1, &[0x54], i64::MAX).await;
+        let second = put_test_part(&set_disks, bucket, "overflow", &overflow_upload.upload_id, 2, &[0x55], 1).await;
+        let overflow_err = set_disks
+            .clone()
+            .complete_multipart_upload(bucket, "overflow", &overflow_upload.upload_id, vec![first, second], &complete_opts)
+            .await
+            .expect_err("overflowing logical size must fail closed");
+        assert!(matches!(overflow_err, StorageError::PartMissingOrCorrupt));
     }
 
     async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {

crates/storage-api/src/error.rs

@@ -103,6 +103,7 @@ pub enum StorageErrorCode {
     SourceStalled,
     Timeout,
     InvalidPath,
+    QuotaExceeded,
 }
 
 impl StorageErrorCode {
@@ -188,6 +189,7 @@ impl StorageErrorCode {
             Self::SourceStalled => 0x50,
             Self::Timeout => 0x51,
             Self::InvalidPath => 0x52,
+            Self::QuotaExceeded => 0x53,
         }
     }
 
@@ -273,6 +275,7 @@ impl StorageErrorCode {
             0x50 => Some(Self::SourceStalled),
             0x51 => Some(Self::Timeout),
             0x52 => Some(Self::InvalidPath),
+            0x53 => Some(Self::QuotaExceeded),
             _ => None,
         }
     }
@@ -347,6 +350,7 @@ mod tests {
         (StorageErrorCode::RebalanceAlreadyRunning, 0x40),
         (StorageErrorCode::OperationCanceled, 0x41),
         (StorageErrorCode::NamespaceLockQuorumUnavailable, 0x42),
+        (StorageErrorCode::QuotaExceeded, 0x53),
     ];
 
     const DISK_PRESERVATION_ERROR_CODES: &[(StorageErrorCode, u32)] = &[