fix(quota): enforce durable hard quota reservations (#6058)

* fix(quota): enforce durable hard quota reservations

* fix(quota): close reservation bypasses

* fix(quota): isolate tests and box object futures

* fix(quota): close legacy and deferred settlement bypasses

* fix(app): keep object futures off caller stacks

* fix(metrics): preserve object operation labels

* fix(logging): retain GET trace guard contract
This commit is contained in:
cxymds
2026-08-14 14:26:00 +08:00
committed by GitHub
parent 307f50ee1b
commit d60a77b750
35 changed files with 4183 additions and 425 deletions
@@ -590,7 +590,7 @@ impl MetadataQuorumAccumulator {
})
}
pub(in crate::set_disk) fn default_write_quorum(&self) -> usize {
pub(crate) fn default_write_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
return self.total_disks;
}
@@ -3016,7 +3016,7 @@ impl SetDisks {
self.set_drive_count - self.default_parity_count
}
pub(in crate::set_disk) fn default_write_quorum(&self) -> usize {
pub(crate) fn default_write_quorum(&self) -> usize {
let mut data_count = self.set_drive_count - self.default_parity_count;
if data_count == self.default_parity_count {
data_count += 1
@@ -3025,6 +3025,89 @@ impl SetDisks {
data_count
}
pub(in crate::set_disk) async fn prepare_quota_mutation_fences(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
write_quorum: usize,
) -> crate::error::Result<(Vec<Option<DiskStore>>, Vec<Option<SnapshotLeaseToken>>)> {
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
let results = join_all(disks.iter().map(|disk| {
let disk = disk.clone();
let fence_path = fence_path.clone();
async move {
let disk = disk?;
match disk.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path).await {
Ok(token) => Some((disk, token)),
Err(_) => None,
}
}
}))
.await;
if results.iter().flatten().count() < write_quorum {
for (disk, token) in results.iter().flatten() {
let _ = disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, *token).await;
}
return Err(StorageError::ErasureWriteQuorum);
}
let mut fenced_disks = Vec::with_capacity(results.len());
let mut tokens = Vec::with_capacity(results.len());
for result in results {
match result {
Some((disk, token)) => {
fenced_disks.push(Some(disk));
tokens.push(Some(token));
}
None => {
fenced_disks.push(None);
tokens.push(None);
}
}
}
Ok((fenced_disks, tokens))
}
pub(in crate::set_disk) async fn release_quota_mutation_fences(
disks: &[Option<DiskStore>],
tokens: &[Option<SnapshotLeaseToken>],
bucket: &str,
object: &str,
write_quorum: usize,
) -> crate::error::Result<()> {
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| {
let disk = disk.as_ref()?.clone();
let token = (*token)?;
let fence_path = fence_path.clone();
Some(async move { disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, token).await })
}))
.await;
if results.iter().filter(|result| result.is_ok()).count() < write_quorum {
return Err(StorageError::ErasureWriteQuorum);
}
Ok(())
}
pub(in crate::set_disk) async fn abort_quota_reservation_after_fence(
reservation: crate::bucket::quota::reservation::QuotaReservation,
disks: &[Option<DiskStore>],
tokens: &[Option<SnapshotLeaseToken>],
bucket: &str,
object: &str,
write_quorum: usize,
fenced: bool,
) {
let safe_to_abort = !fenced
|| Self::release_quota_mutation_fences(disks, tokens, bucket, object, write_quorum)
.await
.is_ok();
if safe_to_abort {
reservation.abort().await;
} else {
reservation.defer_after_fence();
}
}
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
#[allow(clippy::type_complexity)]
pub(in crate::set_disk) async fn rename_data(
+2 -2
View File
@@ -719,8 +719,8 @@ pub(crate) use core::io_primitives::disk_call_counters;
mod ctx;
mod metadata;
mod ops;
#[cfg(test)]
pub(crate) use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(any(test, feature = "test-util"))]
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(feature = "test-util")]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
pub(crate) use ops::object::body_cache_plaintext_len;
+353 -61
View File
@@ -27,11 +27,12 @@ use super::object::{
object_transaction_fencing_requested, old_data_cleanup_receipt_path, read_object_transaction_epoch_fence,
verify_object_transaction_epoch_fence,
};
use crate::bucket::quota::reservation;
use crate::crash_inject::{self, CrashPoint};
use crate::multipart_listing::paginate_multipart_listing;
use futures::{StreamExt, stream};
use std::future::Future;
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tokio::task::JoinSet;
@@ -61,20 +62,21 @@ impl StaleMultipartCleanupGuard {
}
}
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum MultipartCommitPause {
pub enum MultipartCommitPause {
PutPartBeforeLockAcquire,
PutPartBeforeLockLost,
PutPartAfterRename,
BeforeLockLost,
BeforeQuotaRename,
BeforeTransactionEpochVerify,
BeforeObjectPublication,
AfterObjectPublication,
AfterRename,
}
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
struct MultipartCommitBarrierState {
bucket: String,
object: String,
@@ -85,27 +87,22 @@ struct MultipartCommitBarrierState {
release: tokio::sync::Semaphore,
}
#[cfg(test)]
pub(crate) struct MultipartCommitBarrier {
#[cfg(any(test, feature = "test-util"))]
pub struct MultipartCommitBarrier {
state: Arc<MultipartCommitBarrierState>,
}
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<MultipartCommitBarrierState>>>> =
std::sync::OnceLock::new();
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
impl MultipartCommitBarrier {
pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
pub fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
Self::install_for_arrivals(bucket, object, pause, 1)
}
pub(crate) fn install_for_arrivals(
bucket: &str,
object: &str,
pause: MultipartCommitPause,
expected_arrivals: usize,
) -> Self {
pub fn install_for_arrivals(bucket: &str, object: &str, pause: MultipartCommitPause, expected_arrivals: usize) -> Self {
assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival");
let state = Arc::new(MultipartCommitBarrierState {
bucket: bucket.to_string(),
@@ -126,7 +123,7 @@ impl MultipartCommitBarrier {
Self { state }
}
pub(crate) async fn wait_until_paused(&self) {
pub async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), async {
loop {
let arrived = self.state.arrived.notified();
@@ -140,12 +137,12 @@ impl MultipartCommitBarrier {
.expect("multipart completion should reach the deterministic commit barrier");
}
pub(crate) fn release(&self) {
pub fn release(&self) {
self.state.release.add_permits(self.state.expected_arrivals);
}
}
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
impl Drop for MultipartCommitBarrier {
fn drop(&mut self) {
self.release();
@@ -159,7 +156,7 @@ impl Drop for MultipartCommitBarrier {
}
}
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
let barrier = {
let mut slot = MULTIPART_COMMIT_BARRIER
@@ -1892,6 +1889,24 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
fi.parts = Vec::with_capacity(uploaded_parts.len());
let quota_context = reservation::begin(
&self.ctx,
bucket,
object,
opts.quota_admission,
opts.data_movement,
self.pool_index,
self.set_index,
)
.await?;
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
let preserve_replication_ciphertext = opts.replication_request
&& contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
if quota_context.is_enforced() && preserve_replication_ciphertext {
return Err(Error::PartMissingOrCorrupt);
}
let transformed_object = fi.is_compressed() || should_persist_encryption_original_size(&fi.metadata);
let mut object_size: usize = 0;
let mut object_actual_size: i64 = 0;
@@ -2018,15 +2033,23 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
checksum_combined.extend_from_slice(cs.raw.as_slice());
}
object_size += ext_part.size;
if opts.quota_admission.is_some() && ext_part.actual_size < 0 {
object_size = object_size.checked_add(ext_part.size).ok_or(Error::PartMissingOrCorrupt)?;
if ext_part.actual_size < 0 && (!opts.replication_request || quota_context.is_enforced()) {
return Err(Error::PartMissingOrCorrupt);
}
let normalized_actual_size = if ext_part.actual_size >= 0 && !transformed_object {
ext_part
.actual_size
.max(i64::try_from(ext_part.size).map_err(|_| Error::PartMissingOrCorrupt)?)
} else {
ext_part.actual_size
};
object_actual_size = object_actual_size
.checked_add(ext_part.actual_size)
.checked_add(normalized_actual_size)
.ok_or(Error::PartMissingOrCorrupt)?;
fi.parts.push(completed_multipart_object_part(p.part_num, ext_part));
let mut completed_part = completed_multipart_object_part(p.part_num, ext_part);
completed_part.actual_size = normalized_actual_size;
fi.parts.push(completed_part);
}
if let Some(wtcs) = opts.want_checksum.as_ref() {
@@ -2053,15 +2076,34 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
}
}
if let Some(admission) = opts.quota_admission {
let quota_operation_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
if quota_operation_size > admission.remaining() {
return Err(Error::QuotaExceeded {
current: admission.current_usage(),
limit: admission.quota_limit(),
});
}
}
let declared_replication_actual_size = opts
.replication_request
.then(|| get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP))
.flatten();
let replication_actual_size = if opts.replication_request && quota_context.is_enforced() {
let observed_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
let declared_cap = declared_replication_actual_size
.as_deref()
.map(|value| value.parse::<u64>().map_err(|_| Error::PartMissingOrCorrupt))
.transpose()?
.unwrap_or(0);
let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(&fi.metadata)
.map_err(Error::other)?
.map(u64::try_from)
.transpose()
.map_err(|_| Error::PartMissingOrCorrupt)?
.unwrap_or(0);
Some(observed_size.max(declared_cap).max(declared_encryption_size))
} else {
None
};
let quota_new_size = match replication_actual_size {
Some(size) => size.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?),
None if quota_context.is_enforced() => u64::try_from(object_actual_size)
.map_err(|_| Error::PartMissingOrCorrupt)?
.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?),
None => 0,
};
if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) {
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) {
fi.checksum = Some(Bytes::from(rc_crc_bytes));
@@ -2134,7 +2176,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
}
} else if opts.replication_request {
if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) {
if let Some(actual_size) = replication_actual_size {
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
if persist_encryption_original_size {
fi.metadata
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
}
} else if let Some(actual_size) = declared_replication_actual_size {
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone());
if persist_encryption_original_size {
fi.metadata
@@ -2324,8 +2372,41 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
} else {
None
};
let quota_old_size = if quota_context.is_enforced() {
if opts.data_movement {
quota_new_size
} else {
reservation::replaced_logical_size(&self, bucket, object, opts).await?
}
} else {
0
};
let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?;
let (commit_disks, quota_fence_tokens) = if quota_mutation_fence {
match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await {
Ok((disks, tokens)) => {
for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) {
if let Some(token) = token {
insert_str(
&mut metadata.metadata,
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
token.as_uuid().to_string(),
);
}
}
(disks, tokens)
}
Err(err) => {
quota_reservation.abort().await;
return Err(err);
}
}
} else {
(shuffle_disks.clone(), vec![None; shuffle_disks.len()])
};
let transaction_epoch =
transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas));
transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas));
let commit_set = self.clone();
let commit_bucket = bucket.to_owned();
@@ -2333,41 +2414,121 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let commit_upload_id = upload_id.to_owned();
let commit_upload_id_path = upload_id_path.clone();
let commit_version_suspended = opts.version_suspended;
let commit_versioned = opts.versioned;
let commit_version_id = opts.version_id.clone();
let commit_namespace_lock_fence = opts.namespace_lock_fence.clone();
let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
let commit_is_versioned = opts.versioned || opts.version_suspended;
let commit_capacity_scope_token = opts.capacity_scope_token;
let commit_object_lock_guard = object_lock_guard.take();
let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some();
let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some() || quota_mutation_fence;
let commit = async move {
let _object_lock_guard = commit_object_lock_guard;
let _upload_guard = upload_guard;
let mut quota_reservation = quota_reservation;
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
// Crash-consistency injection: hard power loss after the upload is fully
// staged and locked but before the authoritative rename_data commit. No
// disk has moved the staged data, so a crash here must leave any prior
// committed version byte-for-byte intact (rustfs/backlog#864) and the
// upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`.
if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) {
return Err(StorageError::Unexpected);
let pre_rename_result: Result<()> = async {
// Crash-consistency injection: hard power loss after the upload is fully
// staged and locked but before the authoritative rename_data commit. No
// disk has moved the staged data, so a crash here must leave any prior
// committed version byte-for-byte intact (rustfs/backlog#864) and the
// upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`.
if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) {
return Err(StorageError::Unexpected);
}
quota_reservation.mark_commit_started().await?;
#[cfg(any(test, feature = "test-util"))]
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeQuotaRename).await;
if quota_reservation.is_lock_lost()
|| !quota_reservation.capability_proof_matches()
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| commit_namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
|| commit_bucket_lifecycle_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
bucket: commit_bucket.clone(),
object: commit_object.clone(),
required: 1,
achieved: 0,
});
}
let restore_opts = ObjectOptions {
version_id: commit_version_id.clone(),
versioned: commit_versioned,
version_suspended: commit_version_suspended,
no_lock: true,
..Default::default()
};
commit_set
.require_current_restore_operation_id(
&commit_bucket,
&commit_object,
&restore_opts,
expected_restore_operation_id,
"complete_multipart_upload_quota_reservation",
)
.await?;
if let Some(proof) = transaction_fencing_proof.as_ref()
&& !object_transaction_fencing_fleet_proof_matches(proof)
{
return Err(Error::other(
"object transaction fencing fleet capability changed during complete_multipart_upload",
));
}
if let Some(expected) = transaction_epoch_fence {
#[cfg(test)]
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify)
.await;
verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?;
}
if quota_reservation.is_lock_lost()
|| !quota_reservation.capability_proof_matches()
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| commit_namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
|| commit_bucket_lifecycle_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
bucket: commit_bucket.clone(),
object: commit_object.clone(),
required: 1,
achieved: 0,
});
}
Ok(())
}
.await;
if let Err(err) = pre_rename_result {
SetDisks::abort_quota_reservation_after_fence(
quota_reservation,
&commit_disks,
&quota_fence_tokens,
&commit_bucket,
&commit_object,
write_quorum,
quota_mutation_fence,
)
.await;
return Err(err);
}
// The trailing `_` drops the rename_data old-size backfill
// (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit
// `get_object_info` lookup, so the backfill has no consumer here yet.
if let Some(proof) = transaction_fencing_proof.as_ref()
&& !object_transaction_fencing_fleet_proof_matches(proof)
{
return Err(Error::other(
"object transaction fencing fleet capability changed during complete_multipart_upload",
));
}
if let Some(expected) = transaction_epoch_fence {
#[cfg(test)]
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify).await;
verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?;
}
let (online_disks, convergence, op_old_dir, cleanup_disks, _) = SetDisks::rename_data(
&shuffle_disks,
let rename_result = SetDisks::rename_data(
&commit_disks,
RUSTFS_META_MULTIPART_BUCKET,
&commit_upload_id_path,
&parts_metadatas,
@@ -2375,7 +2536,24 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
&commit_object,
write_quorum,
)
.await?;
.await;
if quota_mutation_fence {
let _ = SetDisks::release_quota_mutation_fences(
&commit_disks,
&quota_fence_tokens,
&commit_bucket,
&commit_object,
write_quorum,
)
.await;
}
if rename_result.is_ok() {
quota_reservation.commit().await;
}
let (online_disks, convergence, op_old_dir, cleanup_disks, _) = match rename_result {
Ok(result) => result,
Err(err) => return Err(err.into()),
};
// Detach admission before any post-commit await: client cancellation
// must not couple durable convergence repair to cleanup work.
@@ -3162,7 +3340,7 @@ mod tests {
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
let mut denied_opts = ObjectOptions::default();
assert!(denied_opts.set_quota_admission(100, 4195));
assert!(denied_opts.set_quota_admission(100, 4180));
let err = set_disks
.clone()
@@ -3173,7 +3351,7 @@ mod tests {
err,
StorageError::QuotaExceeded {
current: 100,
limit: 4195
limit: 4180
}
));
@@ -3191,7 +3369,7 @@ mod tests {
);
let mut allowed_opts = ObjectOptions::default();
assert!(allowed_opts.set_quota_admission(100, 4196));
assert!(allowed_opts.set_quota_admission(100, 4181));
let completed = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts)
@@ -3232,6 +3410,120 @@ mod tests {
);
}
#[tokio::test]
async fn replication_quota_uses_server_observed_part_size_as_lower_bound() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-replication-quota-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let mut create_opts = ObjectOptions::default();
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
let upload = set_disks
.new_multipart_upload(bucket, object, &create_opts)
.await
.expect("replication multipart upload should be created");
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x72; 4096], 1).await;
let mut complete_opts = ObjectOptions {
replication_request: true,
..Default::default()
};
assert!(complete_opts.set_quota_admission(0, 4095));
let err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part.clone()], &complete_opts)
.await
.expect_err("a forged tiny replication logical size must not reduce quota admission");
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
assert!(
set_disks
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
.await
.is_ok(),
"quota rejection must leave replicated multipart parts retryable"
);
assert!(complete_opts.set_quota_admission(0, 4096));
let completed = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
.await
.expect("the physical safety boundary should admit the transformed replica");
assert_eq!(completed.get_actual_size().expect("replica logical size should parse"), 1);
}
#[tokio::test]
async fn direct_multipart_quota_uses_server_observed_part_size_as_lower_bound() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-direct-quota-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("multipart upload should be created");
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x73; 4096], 1).await;
let mut complete_opts = ObjectOptions::default();
assert!(complete_opts.set_quota_admission(0, 4095));
let err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
.await
.expect_err("a forged tiny direct logical size must not reduce quota admission");
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
assert!(
set_disks
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
.await
.is_ok(),
"quota rejection must leave direct multipart parts retryable"
);
}
#[tokio::test]
async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-compressed-ciphertext-quota-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let mut create_opts = ObjectOptions::default();
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
insert_str(
&mut create_opts.user_defined,
rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT,
"true".to_string(),
);
let upload = set_disks
.new_multipart_upload(bucket, object, &create_opts)
.await
.expect("ciphertext multipart upload should be created");
let payload = vec![0x74; 4096];
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &payload, 1).await;
let mut complete_opts = ObjectOptions {
replication_request: true,
..Default::default()
};
assert!(complete_opts.set_quota_admission(0, u64::MAX));
let err = set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
.await
.expect_err("ciphertext replication has no server-observed logical quota size");
assert!(matches!(err, StorageError::PartMissingOrCorrupt));
assert!(
set_disks
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
.await
.is_ok(),
"rejection must leave ciphertext multipart parts retryable"
);
}
#[tokio::test]
async fn complete_multipart_quota_rejects_invalid_logical_sizes() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+416 -10
View File
@@ -40,6 +40,7 @@ use crate::bucket::lifecycle::{
save_transition_transaction_record,
},
};
use crate::bucket::quota::reservation;
use crate::bucket::replication::{
DeleteReplicationConfigSnapshot, VersionPurgeStatusType, replication_state_to_filemeta, version_purge_status_to_filemeta,
};
@@ -57,9 +58,40 @@ use http::HeaderValue;
use rustfs_utils::path::decode_dir_object;
use std::future::Future;
use std::sync::OnceLock;
use tokio_util::sync::CancellationToken;
const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json";
struct PutObjectCommitCancellation {
token: CancellationToken,
armed: bool,
}
impl PutObjectCommitCancellation {
fn new() -> Self {
Self {
token: CancellationToken::new(),
armed: true,
}
}
fn child_token(&self) -> CancellationToken {
self.token.clone()
}
fn disarm(&mut self) {
self.armed = false;
}
}
impl Drop for PutObjectCommitCancellation {
fn drop(&mut self) {
if self.armed {
self.token.cancel();
}
}
}
#[inline]
fn duration_millis_f64(duration: std::time::Duration) -> f64 {
duration.as_secs_f64() * 1000.0
@@ -1341,6 +1373,26 @@ impl SetDisks {
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
self.put_object_with_old_current_size_boxed(bucket, object, data, opts).await
}
fn put_object_with_old_current_size_boxed<'a>(
&'a self,
bucket: &'a str,
object: &'a str,
data: &'a mut PutObjReader,
opts: &'a ObjectOptions,
) -> impl Future<Output = Result<(ObjectInfo, Option<OldCurrentSize>)>> + Send + 'a {
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts))
}
async fn put_object_with_old_current_size_inner(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
crate::hp_guard!("SetDisks::put_object");
let storage_class_config = self.storage_class_config_snapshot();
@@ -1929,8 +1981,125 @@ impl SetDisks {
} else {
None
};
let quota_context = reservation::begin(
&self.ctx,
bucket,
object,
opts.quota_admission,
opts.data_movement,
self.pool_index,
self.set_index,
)
.await?;
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
let mut replication_quota_size = None;
if opts.replication_request {
if quota_context.is_enforced() && opts.preserve_ciphertext {
return Err(Error::PartMissingOrCorrupt);
}
if quota_context.is_enforced() {
let persisted_metadata = &parts_metadatas[response_metadata_slot].metadata;
let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
let physical_size = u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?;
let transformed = contains_key_str(persisted_metadata, SUFFIX_COMPRESSION)
|| should_persist_encryption_original_size(persisted_metadata);
let declared_size = get_str(persisted_metadata, SUFFIX_ACTUAL_SIZE)
.map(|value| value.parse::<u64>().map_err(|_| Error::PartMissingOrCorrupt))
.transpose()?
.unwrap_or(0);
let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(persisted_metadata)
.map_err(Error::other)?
.map(u64::try_from)
.transpose()
.map_err(|_| Error::PartMissingOrCorrupt)?
.unwrap_or(0);
let logical_size = observed_size.max(declared_size).max(declared_encryption_size);
let persisted_size = if transformed {
logical_size
} else {
logical_size.max(physical_size)
};
replication_quota_size = Some(logical_size.max(physical_size));
actual_size = i64::try_from(persisted_size).map_err(|_| Error::PartMissingOrCorrupt)?;
for metadata in &mut parts_metadatas {
insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, persisted_size.to_string());
if should_persist_encryption_original_size(&metadata.metadata) {
metadata
.metadata
.insert("x-rustfs-encryption-original-size".to_string(), persisted_size.to_string());
}
if let Some(part) = metadata.parts.first_mut() {
part.actual_size = actual_size;
}
}
}
} else if actual_size >= 0 {
let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
let persisted_metadata = &parts_metadatas[response_metadata_slot].metadata;
let transformed = contains_key_str(persisted_metadata, SUFFIX_COMPRESSION)
|| should_persist_encryption_original_size(persisted_metadata);
let server_observed_size = if transformed {
observed_size
} else {
observed_size.max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?)
};
actual_size = i64::try_from(server_observed_size).map_err(|_| Error::PartMissingOrCorrupt)?;
for metadata in &mut parts_metadatas {
insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, server_observed_size.to_string());
if should_persist_encryption_original_size(&metadata.metadata) {
metadata
.metadata
.insert("x-rustfs-encryption-original-size".to_string(), server_observed_size.to_string());
}
if let Some(part) = metadata.parts.first_mut() {
part.actual_size = actual_size;
}
}
}
let (quota_old_size, quota_new_size) = if quota_context.is_enforced() {
let new_size = match replication_quota_size {
Some(size) => size,
None => u64::try_from(actual_size)
.map_err(|_| Error::PartMissingOrCorrupt)?
.max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?),
};
let old_size = if opts.data_movement {
new_size
} else {
reservation::replaced_logical_size(self, bucket, object, opts).await?
};
(old_size, new_size)
} else {
(0, 0)
};
let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?;
let (commit_disks, quota_fence_tokens) = if quota_mutation_fence {
match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await {
Ok((disks, tokens)) => {
for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) {
if let Some(token) = token {
insert_str(
&mut metadata.metadata,
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
token.as_uuid().to_string(),
);
}
}
(disks, tokens)
}
Err(err) => {
quota_reservation.abort().await;
return Err(err);
}
}
} else {
(shuffle_disks.clone(), vec![None; shuffle_disks.len()])
};
let transaction_epoch =
transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas));
transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas));
let commit_set = self.clone();
let commit_bucket = bucket.to_owned();
@@ -1938,18 +2107,65 @@ impl SetDisks {
let commit_tmp_dir = tmp_dir.clone();
let commit_object_lock_guard = object_lock_guard.take();
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
let detach_commit_owner = commit_object_lock_guard.is_some() || commit_bucket_lifecycle_guard.is_some();
let detach_commit_owner =
commit_object_lock_guard.is_some() || commit_bucket_lifecycle_guard.is_some() || quota_mutation_fence;
let commit_write_path_label = write_path.metric_label();
let commit_is_versioned = opts.versioned || opts.version_suspended;
let commit_versioned = opts.versioned;
let commit_version_suspended = opts.version_suspended;
let commit_version_id = opts.version_id.clone();
let commit_namespace_lock_fence = opts.namespace_lock_fence.clone();
let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
let commit_capacity_scope_token = opts.capacity_scope_token;
let commit_replication_state = replication_state_to_filemeta(&opts.put_replication_state());
tmp_cleanup_owned = true;
let commit = async move {
let commit = move |cancellation: Option<CancellationToken>| async move {
let _object_lock_guard = commit_object_lock_guard;
let _bucket_lifecycle_guard = commit_bucket_lifecycle_guard;
let mut quota_reservation = quota_reservation;
let rename_stage_start = Instant::now();
let pre_rename_result: Result<()> = async {
let pre_rename = async {
#[cfg(any(test, feature = "test-util"))]
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterQuotaReservation).await;
quota_reservation.mark_commit_started().await?;
#[cfg(any(test, feature = "test-util"))]
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::BeforeQuotaRename).await;
if quota_reservation.is_lock_lost()
|| !quota_reservation.capability_proof_matches()
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| commit_namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
|| commit_bucket_lifecycle_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
bucket: commit_bucket.clone(),
object: commit_object.clone(),
required: 1,
achieved: 0,
});
}
let restore_opts = ObjectOptions {
version_id: commit_version_id.clone(),
versioned: commit_versioned,
version_suspended: commit_version_suspended,
no_lock: true,
..Default::default()
};
commit_set
.require_current_restore_operation_id(
&commit_bucket,
&commit_object,
&restore_opts,
expected_restore_operation_id,
"put_object_quota_reservation",
)
.await?;
if let Some(proof) = transaction_fencing_proof.as_ref()
&& !object_transaction_fencing_fleet_proof_matches(proof)
{
@@ -1965,10 +2181,47 @@ impl SetDisks {
.await;
verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?;
}
if quota_reservation.is_lock_lost()
|| !quota_reservation.capability_proof_matches()
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| commit_namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
|| commit_bucket_lifecycle_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
bucket: commit_bucket.clone(),
object: commit_object.clone(),
required: 1,
achieved: 0,
});
}
Ok(())
}
.await;
};
let pre_rename_result = if let Some(cancellation) = cancellation {
tokio::select! {
biased;
_ = cancellation.cancelled() => Err(StorageError::OperationCanceled),
result = pre_rename => result,
}
} else {
pre_rename.await
};
if let Err(err) = pre_rename_result {
SetDisks::abort_quota_reservation_after_fence(
quota_reservation,
&commit_disks,
&quota_fence_tokens,
&commit_bucket,
&commit_object,
write_quorum,
quota_mutation_fence,
)
.await;
if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await {
warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data");
} else if issue3031_diag_enabled() {
@@ -1982,8 +2235,9 @@ impl SetDisks {
}
return Err(err);
}
let rename_result = SetDisks::rename_data(
&shuffle_disks,
&commit_disks,
RUSTFS_META_TMP_BUCKET,
commit_tmp_dir.as_str(),
&parts_metadatas,
@@ -1992,6 +2246,19 @@ impl SetDisks {
write_quorum,
)
.await;
if quota_mutation_fence {
let _ = SetDisks::release_quota_mutation_fences(
&commit_disks,
&quota_fence_tokens,
&commit_bucket,
&commit_object,
write_quorum,
)
.await;
}
if rename_result.is_ok() {
quota_reservation.commit().await;
}
let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = match rename_result {
Ok(commit) => commit,
Err(err) => {
@@ -2202,11 +2469,15 @@ impl SetDisks {
};
if detach_commit_owner {
tokio::spawn(commit)
let mut cancellation = PutObjectCommitCancellation::new();
let child_token = cancellation.child_token();
let result = tokio::spawn(async move { Box::pin(commit(Some(child_token))).await })
.await
.map_err(|err| Error::other(format!("put_object commit task failed: {err}")))?
.map_err(|err| Error::other(format!("put_object commit task failed: {err}")))?;
cancellation.disarm();
result
} else {
commit.await
Box::pin(commit(None)).await
}
}
.await;
@@ -3225,6 +3496,8 @@ fn transaction_fencing_gate_requested_for(requested: bool, fleet_confirmed: bool
pub enum PutObjectCommitPause {
BeforeNamespace,
AfterNamespace,
AfterQuotaReservation,
BeforeQuotaRename,
BeforeMetadata,
BeforeTransactionEpochVerify,
}
@@ -6295,6 +6568,139 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
}
}
#[cfg(test)]
mod replication_quota_safety_tests {
use super::hermetic_set_disks_support::hermetic_set_disks;
use super::*;
use std::io::Cursor;
#[tokio::test]
async fn quota_put_future_keeps_commit_state_off_the_caller_stack() {
let (_temp_dirs, _disks, set_disks) = hermetic_set_disks(4).await;
let mut reader = PutObjReader::from_vec(Vec::new());
let opts = ObjectOptions::default();
let future = set_disks.put_object_with_old_current_size("bucket", "object", &mut reader, &opts);
let future_size = std::mem::size_of_val(&future);
assert!(
future_size <= 1024,
"put_object_with_old_current_size future must stay stack-bounded, got {future_size} bytes"
);
}
#[tokio::test]
async fn replication_put_quota_uses_physical_bytes_as_a_safety_floor() {
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
let bucket = "replication-put-quota-safety";
for disk in &disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut user_defined = HashMap::new();
insert_str(
&mut user_defined,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string());
let payload = vec![0x61; 4096];
let mut denied_opts = ObjectOptions {
replication_request: true,
user_defined: user_defined.clone(),
..Default::default()
};
assert!(denied_opts.set_quota_admission(0, 4095));
let mut denied_reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false)
.expect("construct forged replication reader"),
);
let err = set_disks
.put_object(bucket, "object", &mut denied_reader, &denied_opts)
.await
.expect_err("server-observed bytes must prevent a tiny replication quota claim");
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
let mut allowed_opts = ObjectOptions {
replication_request: true,
user_defined,
..Default::default()
};
assert!(allowed_opts.set_quota_admission(0, 4096));
let mut allowed_reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false)
.expect("construct exact-boundary replication reader"),
);
let stored = set_disks
.put_object(bucket, "object", &mut allowed_reader, &allowed_opts)
.await
.expect("server-observed exact quota boundary should succeed");
assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
}
#[tokio::test]
async fn direct_put_cannot_persist_a_tiny_logical_size() {
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
let bucket = "direct-put-quota-safety";
for disk in &disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let payload = vec![0x62; 4096];
let mut denied_opts = ObjectOptions::default();
assert!(denied_opts.set_quota_admission(0, 4095));
let mut denied_reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false)
.expect("construct forged direct reader"),
);
let err = set_disks
.put_object(bucket, "object", &mut denied_reader, &denied_opts)
.await
.expect_err("server-observed bytes must prevent a tiny direct quota claim");
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
let mut allowed_opts = ObjectOptions::default();
assert!(allowed_opts.set_quota_admission(0, 4096));
let mut allowed_reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false)
.expect("construct exact-boundary direct reader"),
);
let stored = set_disks
.put_object(bucket, "object", &mut allowed_reader, &allowed_opts)
.await
.expect("server-observed exact quota boundary should succeed");
assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 4096);
}
#[tokio::test]
async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() {
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
let bucket = "ciphertext-replication-quota-safety";
for disk in &disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut user_defined = HashMap::new();
user_defined.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string());
let mut opts = ObjectOptions {
replication_request: true,
preserve_ciphertext: true,
user_defined,
..Default::default()
};
assert!(opts.set_quota_admission(0, u64::MAX));
let payload = vec![0x63; 4096];
let mut reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(payload), 4096, 4096, None, None, false)
.expect("construct ciphertext replication reader"),
);
let err = set_disks
.put_object(bucket, "object", &mut reader, &opts)
.await
.expect_err("ciphertext replication without a server-observed logical size must fail closed");
assert!(matches!(err, StorageError::PartMissingOrCorrupt));
}
}
#[cfg(test)]
mod inline_put_commit_path_tests {
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;