mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 21:26:28 +00:00
fix(quota): enforce durable hard quota reservations (#6058)
* fix(quota): enforce durable hard quota reservations * fix(quota): close reservation bypasses * fix(quota): isolate tests and box object futures * fix(quota): close legacy and deferred settlement bypasses * fix(app): keep object futures off caller stacks * fix(metrics): preserve object operation labels * fix(logging): retain GET trace guard contract
This commit is contained in:
@@ -590,7 +590,7 @@ impl MetadataQuorumAccumulator {
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn default_write_quorum(&self) -> usize {
|
||||
pub(crate) fn default_write_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
return self.total_disks;
|
||||
}
|
||||
@@ -3016,7 +3016,7 @@ impl SetDisks {
|
||||
self.set_drive_count - self.default_parity_count
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn default_write_quorum(&self) -> usize {
|
||||
pub(crate) fn default_write_quorum(&self) -> usize {
|
||||
let mut data_count = self.set_drive_count - self.default_parity_count;
|
||||
if data_count == self.default_parity_count {
|
||||
data_count += 1
|
||||
@@ -3025,6 +3025,89 @@ impl SetDisks {
|
||||
data_count
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn prepare_quota_mutation_fences(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
write_quorum: usize,
|
||||
) -> crate::error::Result<(Vec<Option<DiskStore>>, Vec<Option<SnapshotLeaseToken>>)> {
|
||||
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
|
||||
let results = join_all(disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let fence_path = fence_path.clone();
|
||||
async move {
|
||||
let disk = disk?;
|
||||
match disk.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path).await {
|
||||
Ok(token) => Some((disk, token)),
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
if results.iter().flatten().count() < write_quorum {
|
||||
for (disk, token) in results.iter().flatten() {
|
||||
let _ = disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, *token).await;
|
||||
}
|
||||
return Err(StorageError::ErasureWriteQuorum);
|
||||
}
|
||||
let mut fenced_disks = Vec::with_capacity(results.len());
|
||||
let mut tokens = Vec::with_capacity(results.len());
|
||||
for result in results {
|
||||
match result {
|
||||
Some((disk, token)) => {
|
||||
fenced_disks.push(Some(disk));
|
||||
tokens.push(Some(token));
|
||||
}
|
||||
None => {
|
||||
fenced_disks.push(None);
|
||||
tokens.push(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((fenced_disks, tokens))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn release_quota_mutation_fences(
|
||||
disks: &[Option<DiskStore>],
|
||||
tokens: &[Option<SnapshotLeaseToken>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
write_quorum: usize,
|
||||
) -> crate::error::Result<()> {
|
||||
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
|
||||
let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| {
|
||||
let disk = disk.as_ref()?.clone();
|
||||
let token = (*token)?;
|
||||
let fence_path = fence_path.clone();
|
||||
Some(async move { disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, token).await })
|
||||
}))
|
||||
.await;
|
||||
if results.iter().filter(|result| result.is_ok()).count() < write_quorum {
|
||||
return Err(StorageError::ErasureWriteQuorum);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn abort_quota_reservation_after_fence(
|
||||
reservation: crate::bucket::quota::reservation::QuotaReservation,
|
||||
disks: &[Option<DiskStore>],
|
||||
tokens: &[Option<SnapshotLeaseToken>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
write_quorum: usize,
|
||||
fenced: bool,
|
||||
) {
|
||||
let safe_to_abort = !fenced
|
||||
|| Self::release_quota_mutation_fences(disks, tokens, bucket, object, write_quorum)
|
||||
.await
|
||||
.is_ok();
|
||||
if safe_to_abort {
|
||||
reservation.abort().await;
|
||||
} else {
|
||||
reservation.defer_after_fence();
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub(in crate::set_disk) async fn rename_data(
|
||||
|
||||
@@ -719,8 +719,8 @@ pub(crate) use core::io_primitives::disk_call_counters;
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
|
||||
@@ -27,11 +27,12 @@ use super::object::{
|
||||
object_transaction_fencing_requested, old_data_cleanup_receipt_path, read_object_transaction_epoch_fence,
|
||||
verify_object_transaction_epoch_fence,
|
||||
};
|
||||
use crate::bucket::quota::reservation;
|
||||
use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::multipart_listing::paginate_multipart_listing;
|
||||
use futures::{StreamExt, stream};
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinSet;
|
||||
@@ -61,20 +62,21 @@ impl StaleMultipartCleanupGuard {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum MultipartCommitPause {
|
||||
pub enum MultipartCommitPause {
|
||||
PutPartBeforeLockAcquire,
|
||||
PutPartBeforeLockLost,
|
||||
PutPartAfterRename,
|
||||
BeforeLockLost,
|
||||
BeforeQuotaRename,
|
||||
BeforeTransactionEpochVerify,
|
||||
BeforeObjectPublication,
|
||||
AfterObjectPublication,
|
||||
AfterRename,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
struct MultipartCommitBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
@@ -85,27 +87,22 @@ struct MultipartCommitBarrierState {
|
||||
release: tokio::sync::Semaphore,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct MultipartCommitBarrier {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub struct MultipartCommitBarrier {
|
||||
state: Arc<MultipartCommitBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<MultipartCommitBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl MultipartCommitBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
|
||||
pub fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
|
||||
Self::install_for_arrivals(bucket, object, pause, 1)
|
||||
}
|
||||
|
||||
pub(crate) fn install_for_arrivals(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
pause: MultipartCommitPause,
|
||||
expected_arrivals: usize,
|
||||
) -> Self {
|
||||
pub fn install_for_arrivals(bucket: &str, object: &str, pause: MultipartCommitPause, expected_arrivals: usize) -> Self {
|
||||
assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival");
|
||||
let state = Arc::new(MultipartCommitBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
@@ -126,7 +123,7 @@ impl MultipartCommitBarrier {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
pub async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let arrived = self.state.arrived.notified();
|
||||
@@ -140,12 +137,12 @@ impl MultipartCommitBarrier {
|
||||
.expect("multipart completion should reach the deterministic commit barrier");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
pub fn release(&self) {
|
||||
self.state.release.add_permits(self.state.expected_arrivals);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl Drop for MultipartCommitBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
@@ -159,7 +156,7 @@ impl Drop for MultipartCommitBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
|
||||
let barrier = {
|
||||
let mut slot = MULTIPART_COMMIT_BARRIER
|
||||
@@ -1892,6 +1889,24 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
fi.parts = Vec::with_capacity(uploaded_parts.len());
|
||||
|
||||
let quota_context = reservation::begin(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts.quota_admission,
|
||||
opts.data_movement,
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
)
|
||||
.await?;
|
||||
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
|
||||
let preserve_replication_ciphertext = opts.replication_request
|
||||
&& contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
|
||||
if quota_context.is_enforced() && preserve_replication_ciphertext {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
let transformed_object = fi.is_compressed() || should_persist_encryption_original_size(&fi.metadata);
|
||||
|
||||
let mut object_size: usize = 0;
|
||||
let mut object_actual_size: i64 = 0;
|
||||
|
||||
@@ -2018,15 +2033,23 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
checksum_combined.extend_from_slice(cs.raw.as_slice());
|
||||
}
|
||||
|
||||
object_size += ext_part.size;
|
||||
if opts.quota_admission.is_some() && ext_part.actual_size < 0 {
|
||||
object_size = object_size.checked_add(ext_part.size).ok_or(Error::PartMissingOrCorrupt)?;
|
||||
if ext_part.actual_size < 0 && (!opts.replication_request || quota_context.is_enforced()) {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
let normalized_actual_size = if ext_part.actual_size >= 0 && !transformed_object {
|
||||
ext_part
|
||||
.actual_size
|
||||
.max(i64::try_from(ext_part.size).map_err(|_| Error::PartMissingOrCorrupt)?)
|
||||
} else {
|
||||
ext_part.actual_size
|
||||
};
|
||||
object_actual_size = object_actual_size
|
||||
.checked_add(ext_part.actual_size)
|
||||
.checked_add(normalized_actual_size)
|
||||
.ok_or(Error::PartMissingOrCorrupt)?;
|
||||
|
||||
fi.parts.push(completed_multipart_object_part(p.part_num, ext_part));
|
||||
let mut completed_part = completed_multipart_object_part(p.part_num, ext_part);
|
||||
completed_part.actual_size = normalized_actual_size;
|
||||
fi.parts.push(completed_part);
|
||||
}
|
||||
|
||||
if let Some(wtcs) = opts.want_checksum.as_ref() {
|
||||
@@ -2053,15 +2076,34 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(admission) = opts.quota_admission {
|
||||
let quota_operation_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
if quota_operation_size > admission.remaining() {
|
||||
return Err(Error::QuotaExceeded {
|
||||
current: admission.current_usage(),
|
||||
limit: admission.quota_limit(),
|
||||
});
|
||||
}
|
||||
}
|
||||
let declared_replication_actual_size = opts
|
||||
.replication_request
|
||||
.then(|| get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP))
|
||||
.flatten();
|
||||
let replication_actual_size = if opts.replication_request && quota_context.is_enforced() {
|
||||
let observed_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
let declared_cap = declared_replication_actual_size
|
||||
.as_deref()
|
||||
.map(|value| value.parse::<u64>().map_err(|_| Error::PartMissingOrCorrupt))
|
||||
.transpose()?
|
||||
.unwrap_or(0);
|
||||
let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(&fi.metadata)
|
||||
.map_err(Error::other)?
|
||||
.map(u64::try_from)
|
||||
.transpose()
|
||||
.map_err(|_| Error::PartMissingOrCorrupt)?
|
||||
.unwrap_or(0);
|
||||
Some(observed_size.max(declared_cap).max(declared_encryption_size))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let quota_new_size = match replication_actual_size {
|
||||
Some(size) => size.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?),
|
||||
None if quota_context.is_enforced() => u64::try_from(object_actual_size)
|
||||
.map_err(|_| Error::PartMissingOrCorrupt)?
|
||||
.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?),
|
||||
None => 0,
|
||||
};
|
||||
if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) {
|
||||
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) {
|
||||
fi.checksum = Some(Bytes::from(rc_crc_bytes));
|
||||
@@ -2134,7 +2176,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
|
||||
}
|
||||
} else if opts.replication_request {
|
||||
if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) {
|
||||
if let Some(actual_size) = replication_actual_size {
|
||||
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
|
||||
if persist_encryption_original_size {
|
||||
fi.metadata
|
||||
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
|
||||
}
|
||||
} else if let Some(actual_size) = declared_replication_actual_size {
|
||||
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone());
|
||||
if persist_encryption_original_size {
|
||||
fi.metadata
|
||||
@@ -2324,8 +2372,41 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let quota_old_size = if quota_context.is_enforced() {
|
||||
if opts.data_movement {
|
||||
quota_new_size
|
||||
} else {
|
||||
reservation::replaced_logical_size(&self, bucket, object, opts).await?
|
||||
}
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?;
|
||||
let (commit_disks, quota_fence_tokens) = if quota_mutation_fence {
|
||||
match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await {
|
||||
Ok((disks, tokens)) => {
|
||||
for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) {
|
||||
if let Some(token) = token {
|
||||
insert_str(
|
||||
&mut metadata.metadata,
|
||||
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||
token.as_uuid().to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
(disks, tokens)
|
||||
}
|
||||
Err(err) => {
|
||||
quota_reservation.abort().await;
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(shuffle_disks.clone(), vec![None; shuffle_disks.len()])
|
||||
};
|
||||
let transaction_epoch =
|
||||
transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas));
|
||||
transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas));
|
||||
|
||||
let commit_set = self.clone();
|
||||
let commit_bucket = bucket.to_owned();
|
||||
@@ -2333,41 +2414,121 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let commit_upload_id = upload_id.to_owned();
|
||||
let commit_upload_id_path = upload_id_path.clone();
|
||||
let commit_version_suspended = opts.version_suspended;
|
||||
let commit_versioned = opts.versioned;
|
||||
let commit_version_id = opts.version_id.clone();
|
||||
let commit_namespace_lock_fence = opts.namespace_lock_fence.clone();
|
||||
let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
|
||||
let commit_is_versioned = opts.versioned || opts.version_suspended;
|
||||
let commit_capacity_scope_token = opts.capacity_scope_token;
|
||||
let commit_object_lock_guard = object_lock_guard.take();
|
||||
let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some();
|
||||
let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some() || quota_mutation_fence;
|
||||
let commit = async move {
|
||||
let _object_lock_guard = commit_object_lock_guard;
|
||||
let _upload_guard = upload_guard;
|
||||
let mut quota_reservation = quota_reservation;
|
||||
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
// Crash-consistency injection: hard power loss after the upload is fully
|
||||
// staged and locked but before the authoritative rename_data commit. No
|
||||
// disk has moved the staged data, so a crash here must leave any prior
|
||||
// committed version byte-for-byte intact (rustfs/backlog#864) and the
|
||||
// upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`.
|
||||
if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) {
|
||||
return Err(StorageError::Unexpected);
|
||||
let pre_rename_result: Result<()> = async {
|
||||
// Crash-consistency injection: hard power loss after the upload is fully
|
||||
// staged and locked but before the authoritative rename_data commit. No
|
||||
// disk has moved the staged data, so a crash here must leave any prior
|
||||
// committed version byte-for-byte intact (rustfs/backlog#864) and the
|
||||
// upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`.
|
||||
if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) {
|
||||
return Err(StorageError::Unexpected);
|
||||
}
|
||||
quota_reservation.mark_commit_started().await?;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeQuotaRename).await;
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| commit_namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| commit_bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
bucket: commit_bucket.clone(),
|
||||
object: commit_object.clone(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
let restore_opts = ObjectOptions {
|
||||
version_id: commit_version_id.clone(),
|
||||
versioned: commit_versioned,
|
||||
version_suspended: commit_version_suspended,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
commit_set
|
||||
.require_current_restore_operation_id(
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
&restore_opts,
|
||||
expected_restore_operation_id,
|
||||
"complete_multipart_upload_quota_reservation",
|
||||
)
|
||||
.await?;
|
||||
if let Some(proof) = transaction_fencing_proof.as_ref()
|
||||
&& !object_transaction_fencing_fleet_proof_matches(proof)
|
||||
{
|
||||
return Err(Error::other(
|
||||
"object transaction fencing fleet capability changed during complete_multipart_upload",
|
||||
));
|
||||
}
|
||||
if let Some(expected) = transaction_epoch_fence {
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify)
|
||||
.await;
|
||||
verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?;
|
||||
}
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| commit_namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| commit_bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
bucket: commit_bucket.clone(),
|
||||
object: commit_object.clone(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
if let Err(err) = pre_rename_result {
|
||||
SetDisks::abort_quota_reservation_after_fence(
|
||||
quota_reservation,
|
||||
&commit_disks,
|
||||
"a_fence_tokens,
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
write_quorum,
|
||||
quota_mutation_fence,
|
||||
)
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// The trailing `_` drops the rename_data old-size backfill
|
||||
// (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit
|
||||
// `get_object_info` lookup, so the backfill has no consumer here yet.
|
||||
if let Some(proof) = transaction_fencing_proof.as_ref()
|
||||
&& !object_transaction_fencing_fleet_proof_matches(proof)
|
||||
{
|
||||
return Err(Error::other(
|
||||
"object transaction fencing fleet capability changed during complete_multipart_upload",
|
||||
));
|
||||
}
|
||||
if let Some(expected) = transaction_epoch_fence {
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify).await;
|
||||
verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?;
|
||||
}
|
||||
let (online_disks, convergence, op_old_dir, cleanup_disks, _) = SetDisks::rename_data(
|
||||
&shuffle_disks,
|
||||
let rename_result = SetDisks::rename_data(
|
||||
&commit_disks,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&commit_upload_id_path,
|
||||
&parts_metadatas,
|
||||
@@ -2375,7 +2536,24 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
&commit_object,
|
||||
write_quorum,
|
||||
)
|
||||
.await?;
|
||||
.await;
|
||||
if quota_mutation_fence {
|
||||
let _ = SetDisks::release_quota_mutation_fences(
|
||||
&commit_disks,
|
||||
"a_fence_tokens,
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
write_quorum,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
if rename_result.is_ok() {
|
||||
quota_reservation.commit().await;
|
||||
}
|
||||
let (online_disks, convergence, op_old_dir, cleanup_disks, _) = match rename_result {
|
||||
Ok(result) => result,
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
|
||||
// Detach admission before any post-commit await: client cancellation
|
||||
// must not couple durable convergence repair to cleanup work.
|
||||
@@ -3162,7 +3340,7 @@ mod tests {
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
|
||||
let mut denied_opts = ObjectOptions::default();
|
||||
assert!(denied_opts.set_quota_admission(100, 4195));
|
||||
assert!(denied_opts.set_quota_admission(100, 4180));
|
||||
|
||||
let err = set_disks
|
||||
.clone()
|
||||
@@ -3173,7 +3351,7 @@ mod tests {
|
||||
err,
|
||||
StorageError::QuotaExceeded {
|
||||
current: 100,
|
||||
limit: 4195
|
||||
limit: 4180
|
||||
}
|
||||
));
|
||||
|
||||
@@ -3191,7 +3369,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut allowed_opts = ObjectOptions::default();
|
||||
assert!(allowed_opts.set_quota_admission(100, 4196));
|
||||
assert!(allowed_opts.set_quota_admission(100, 4181));
|
||||
let completed = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts)
|
||||
@@ -3232,6 +3410,120 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replication_quota_uses_server_observed_part_size_as_lower_bound() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-replication-quota-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let mut create_opts = ObjectOptions::default();
|
||||
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &create_opts)
|
||||
.await
|
||||
.expect("replication multipart upload should be created");
|
||||
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x72; 4096], 1).await;
|
||||
let mut complete_opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(complete_opts.set_quota_admission(0, 4095));
|
||||
|
||||
let err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part.clone()], &complete_opts)
|
||||
.await
|
||||
.expect_err("a forged tiny replication logical size must not reduce quota admission");
|
||||
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
|
||||
assert!(
|
||||
set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
|
||||
.await
|
||||
.is_ok(),
|
||||
"quota rejection must leave replicated multipart parts retryable"
|
||||
);
|
||||
|
||||
assert!(complete_opts.set_quota_admission(0, 4096));
|
||||
let completed = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
|
||||
.await
|
||||
.expect("the physical safety boundary should admit the transformed replica");
|
||||
assert_eq!(completed.get_actual_size().expect("replica logical size should parse"), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_multipart_quota_uses_server_observed_part_size_as_lower_bound() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-direct-quota-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x73; 4096], 1).await;
|
||||
let mut complete_opts = ObjectOptions::default();
|
||||
assert!(complete_opts.set_quota_admission(0, 4095));
|
||||
|
||||
let err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
|
||||
.await
|
||||
.expect_err("a forged tiny direct logical size must not reduce quota admission");
|
||||
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
|
||||
assert!(
|
||||
set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
|
||||
.await
|
||||
.is_ok(),
|
||||
"quota rejection must leave direct multipart parts retryable"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-compressed-ciphertext-quota-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let mut create_opts = ObjectOptions::default();
|
||||
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
|
||||
insert_str(
|
||||
&mut create_opts.user_defined,
|
||||
rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT,
|
||||
"true".to_string(),
|
||||
);
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &create_opts)
|
||||
.await
|
||||
.expect("ciphertext multipart upload should be created");
|
||||
let payload = vec![0x74; 4096];
|
||||
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &payload, 1).await;
|
||||
let mut complete_opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(complete_opts.set_quota_admission(0, u64::MAX));
|
||||
|
||||
let err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
|
||||
.await
|
||||
.expect_err("ciphertext replication has no server-observed logical quota size");
|
||||
assert!(matches!(err, StorageError::PartMissingOrCorrupt));
|
||||
assert!(
|
||||
set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
|
||||
.await
|
||||
.is_ok(),
|
||||
"rejection must leave ciphertext multipart parts retryable"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_quota_rejects_invalid_logical_sizes() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
|
||||
@@ -40,6 +40,7 @@ use crate::bucket::lifecycle::{
|
||||
save_transition_transaction_record,
|
||||
},
|
||||
};
|
||||
use crate::bucket::quota::reservation;
|
||||
use crate::bucket::replication::{
|
||||
DeleteReplicationConfigSnapshot, VersionPurgeStatusType, replication_state_to_filemeta, version_purge_status_to_filemeta,
|
||||
};
|
||||
@@ -57,9 +58,40 @@ use http::HeaderValue;
|
||||
use rustfs_utils::path::decode_dir_object;
|
||||
use std::future::Future;
|
||||
use std::sync::OnceLock;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json";
|
||||
|
||||
struct PutObjectCommitCancellation {
|
||||
token: CancellationToken,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl PutObjectCommitCancellation {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
token: CancellationToken::new(),
|
||||
armed: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn child_token(&self) -> CancellationToken {
|
||||
self.token.clone()
|
||||
}
|
||||
|
||||
fn disarm(&mut self) {
|
||||
self.armed = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PutObjectCommitCancellation {
|
||||
fn drop(&mut self) {
|
||||
if self.armed {
|
||||
self.token.cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn duration_millis_f64(duration: std::time::Duration) -> f64 {
|
||||
duration.as_secs_f64() * 1000.0
|
||||
@@ -1341,6 +1373,26 @@ impl SetDisks {
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts).await
|
||||
}
|
||||
|
||||
fn put_object_with_old_current_size_boxed<'a>(
|
||||
&'a self,
|
||||
bucket: &'a str,
|
||||
object: &'a str,
|
||||
data: &'a mut PutObjReader,
|
||||
opts: &'a ObjectOptions,
|
||||
) -> impl Future<Output = Result<(ObjectInfo, Option<OldCurrentSize>)>> + Send + 'a {
|
||||
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts))
|
||||
}
|
||||
|
||||
async fn put_object_with_old_current_size_inner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
crate::hp_guard!("SetDisks::put_object");
|
||||
let storage_class_config = self.storage_class_config_snapshot();
|
||||
@@ -1929,8 +1981,125 @@ impl SetDisks {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let quota_context = reservation::begin(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts.quota_admission,
|
||||
opts.data_movement,
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
)
|
||||
.await?;
|
||||
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
|
||||
let mut replication_quota_size = None;
|
||||
|
||||
if opts.replication_request {
|
||||
if quota_context.is_enforced() && opts.preserve_ciphertext {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
if quota_context.is_enforced() {
|
||||
let persisted_metadata = &parts_metadatas[response_metadata_slot].metadata;
|
||||
let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
let physical_size = u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
let transformed = contains_key_str(persisted_metadata, SUFFIX_COMPRESSION)
|
||||
|| should_persist_encryption_original_size(persisted_metadata);
|
||||
let declared_size = get_str(persisted_metadata, SUFFIX_ACTUAL_SIZE)
|
||||
.map(|value| value.parse::<u64>().map_err(|_| Error::PartMissingOrCorrupt))
|
||||
.transpose()?
|
||||
.unwrap_or(0);
|
||||
let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(persisted_metadata)
|
||||
.map_err(Error::other)?
|
||||
.map(u64::try_from)
|
||||
.transpose()
|
||||
.map_err(|_| Error::PartMissingOrCorrupt)?
|
||||
.unwrap_or(0);
|
||||
let logical_size = observed_size.max(declared_size).max(declared_encryption_size);
|
||||
let persisted_size = if transformed {
|
||||
logical_size
|
||||
} else {
|
||||
logical_size.max(physical_size)
|
||||
};
|
||||
replication_quota_size = Some(logical_size.max(physical_size));
|
||||
actual_size = i64::try_from(persisted_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
for metadata in &mut parts_metadatas {
|
||||
insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, persisted_size.to_string());
|
||||
if should_persist_encryption_original_size(&metadata.metadata) {
|
||||
metadata
|
||||
.metadata
|
||||
.insert("x-rustfs-encryption-original-size".to_string(), persisted_size.to_string());
|
||||
}
|
||||
if let Some(part) = metadata.parts.first_mut() {
|
||||
part.actual_size = actual_size;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if actual_size >= 0 {
|
||||
let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
let persisted_metadata = &parts_metadatas[response_metadata_slot].metadata;
|
||||
let transformed = contains_key_str(persisted_metadata, SUFFIX_COMPRESSION)
|
||||
|| should_persist_encryption_original_size(persisted_metadata);
|
||||
let server_observed_size = if transformed {
|
||||
observed_size
|
||||
} else {
|
||||
observed_size.max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?)
|
||||
};
|
||||
actual_size = i64::try_from(server_observed_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
for metadata in &mut parts_metadatas {
|
||||
insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, server_observed_size.to_string());
|
||||
if should_persist_encryption_original_size(&metadata.metadata) {
|
||||
metadata
|
||||
.metadata
|
||||
.insert("x-rustfs-encryption-original-size".to_string(), server_observed_size.to_string());
|
||||
}
|
||||
if let Some(part) = metadata.parts.first_mut() {
|
||||
part.actual_size = actual_size;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (quota_old_size, quota_new_size) = if quota_context.is_enforced() {
|
||||
let new_size = match replication_quota_size {
|
||||
Some(size) => size,
|
||||
None => u64::try_from(actual_size)
|
||||
.map_err(|_| Error::PartMissingOrCorrupt)?
|
||||
.max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?),
|
||||
};
|
||||
let old_size = if opts.data_movement {
|
||||
new_size
|
||||
} else {
|
||||
reservation::replaced_logical_size(self, bucket, object, opts).await?
|
||||
};
|
||||
(old_size, new_size)
|
||||
} else {
|
||||
(0, 0)
|
||||
};
|
||||
let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?;
|
||||
let (commit_disks, quota_fence_tokens) = if quota_mutation_fence {
|
||||
match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await {
|
||||
Ok((disks, tokens)) => {
|
||||
for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) {
|
||||
if let Some(token) = token {
|
||||
insert_str(
|
||||
&mut metadata.metadata,
|
||||
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||
token.as_uuid().to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
(disks, tokens)
|
||||
}
|
||||
Err(err) => {
|
||||
quota_reservation.abort().await;
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(shuffle_disks.clone(), vec![None; shuffle_disks.len()])
|
||||
};
|
||||
let transaction_epoch =
|
||||
transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas));
|
||||
transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas));
|
||||
|
||||
let commit_set = self.clone();
|
||||
let commit_bucket = bucket.to_owned();
|
||||
@@ -1938,18 +2107,65 @@ impl SetDisks {
|
||||
let commit_tmp_dir = tmp_dir.clone();
|
||||
let commit_object_lock_guard = object_lock_guard.take();
|
||||
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
|
||||
let detach_commit_owner = commit_object_lock_guard.is_some() || commit_bucket_lifecycle_guard.is_some();
|
||||
let detach_commit_owner =
|
||||
commit_object_lock_guard.is_some() || commit_bucket_lifecycle_guard.is_some() || quota_mutation_fence;
|
||||
let commit_write_path_label = write_path.metric_label();
|
||||
let commit_is_versioned = opts.versioned || opts.version_suspended;
|
||||
let commit_versioned = opts.versioned;
|
||||
let commit_version_suspended = opts.version_suspended;
|
||||
let commit_version_id = opts.version_id.clone();
|
||||
let commit_namespace_lock_fence = opts.namespace_lock_fence.clone();
|
||||
let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
|
||||
let commit_capacity_scope_token = opts.capacity_scope_token;
|
||||
let commit_replication_state = replication_state_to_filemeta(&opts.put_replication_state());
|
||||
tmp_cleanup_owned = true;
|
||||
|
||||
let commit = async move {
|
||||
let commit = move |cancellation: Option<CancellationToken>| async move {
|
||||
let _object_lock_guard = commit_object_lock_guard;
|
||||
let _bucket_lifecycle_guard = commit_bucket_lifecycle_guard;
|
||||
let mut quota_reservation = quota_reservation;
|
||||
let rename_stage_start = Instant::now();
|
||||
let pre_rename_result: Result<()> = async {
|
||||
let pre_rename = async {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterQuotaReservation).await;
|
||||
quota_reservation.mark_commit_started().await?;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::BeforeQuotaRename).await;
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| commit_namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| commit_bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
bucket: commit_bucket.clone(),
|
||||
object: commit_object.clone(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
let restore_opts = ObjectOptions {
|
||||
version_id: commit_version_id.clone(),
|
||||
versioned: commit_versioned,
|
||||
version_suspended: commit_version_suspended,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
commit_set
|
||||
.require_current_restore_operation_id(
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
&restore_opts,
|
||||
expected_restore_operation_id,
|
||||
"put_object_quota_reservation",
|
||||
)
|
||||
.await?;
|
||||
if let Some(proof) = transaction_fencing_proof.as_ref()
|
||||
&& !object_transaction_fencing_fleet_proof_matches(proof)
|
||||
{
|
||||
@@ -1965,10 +2181,47 @@ impl SetDisks {
|
||||
.await;
|
||||
verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?;
|
||||
}
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| commit_namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| commit_bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
bucket: commit_bucket.clone(),
|
||||
object: commit_object.clone(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
};
|
||||
let pre_rename_result = if let Some(cancellation) = cancellation {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => Err(StorageError::OperationCanceled),
|
||||
result = pre_rename => result,
|
||||
}
|
||||
} else {
|
||||
pre_rename.await
|
||||
};
|
||||
if let Err(err) = pre_rename_result {
|
||||
SetDisks::abort_quota_reservation_after_fence(
|
||||
quota_reservation,
|
||||
&commit_disks,
|
||||
"a_fence_tokens,
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
write_quorum,
|
||||
quota_mutation_fence,
|
||||
)
|
||||
.await;
|
||||
if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await {
|
||||
warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data");
|
||||
} else if issue3031_diag_enabled() {
|
||||
@@ -1982,8 +2235,9 @@ impl SetDisks {
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let rename_result = SetDisks::rename_data(
|
||||
&shuffle_disks,
|
||||
&commit_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
commit_tmp_dir.as_str(),
|
||||
&parts_metadatas,
|
||||
@@ -1992,6 +2246,19 @@ impl SetDisks {
|
||||
write_quorum,
|
||||
)
|
||||
.await;
|
||||
if quota_mutation_fence {
|
||||
let _ = SetDisks::release_quota_mutation_fences(
|
||||
&commit_disks,
|
||||
"a_fence_tokens,
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
write_quorum,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
if rename_result.is_ok() {
|
||||
quota_reservation.commit().await;
|
||||
}
|
||||
let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = match rename_result {
|
||||
Ok(commit) => commit,
|
||||
Err(err) => {
|
||||
@@ -2202,11 +2469,15 @@ impl SetDisks {
|
||||
};
|
||||
|
||||
if detach_commit_owner {
|
||||
tokio::spawn(commit)
|
||||
let mut cancellation = PutObjectCommitCancellation::new();
|
||||
let child_token = cancellation.child_token();
|
||||
let result = tokio::spawn(async move { Box::pin(commit(Some(child_token))).await })
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("put_object commit task failed: {err}")))?
|
||||
.map_err(|err| Error::other(format!("put_object commit task failed: {err}")))?;
|
||||
cancellation.disarm();
|
||||
result
|
||||
} else {
|
||||
commit.await
|
||||
Box::pin(commit(None)).await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
@@ -3225,6 +3496,8 @@ fn transaction_fencing_gate_requested_for(requested: bool, fleet_confirmed: bool
|
||||
pub enum PutObjectCommitPause {
|
||||
BeforeNamespace,
|
||||
AfterNamespace,
|
||||
AfterQuotaReservation,
|
||||
BeforeQuotaRename,
|
||||
BeforeMetadata,
|
||||
BeforeTransactionEpochVerify,
|
||||
}
|
||||
@@ -6295,6 +6568,139 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod replication_quota_safety_tests {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks;
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_put_future_keeps_commit_state_off_the_caller_stack() {
|
||||
let (_temp_dirs, _disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let mut reader = PutObjReader::from_vec(Vec::new());
|
||||
let opts = ObjectOptions::default();
|
||||
|
||||
let future = set_disks.put_object_with_old_current_size("bucket", "object", &mut reader, &opts);
|
||||
let future_size = std::mem::size_of_val(&future);
|
||||
|
||||
assert!(
|
||||
future_size <= 1024,
|
||||
"put_object_with_old_current_size future must stay stack-bounded, got {future_size} bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replication_put_quota_uses_physical_bytes_as_a_safety_floor() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "replication-put-quota-safety";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string());
|
||||
let payload = vec![0x61; 4096];
|
||||
|
||||
let mut denied_opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
user_defined: user_defined.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(denied_opts.set_quota_admission(0, 4095));
|
||||
let mut denied_reader = PutObjReader::new(
|
||||
HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false)
|
||||
.expect("construct forged replication reader"),
|
||||
);
|
||||
let err = set_disks
|
||||
.put_object(bucket, "object", &mut denied_reader, &denied_opts)
|
||||
.await
|
||||
.expect_err("server-observed bytes must prevent a tiny replication quota claim");
|
||||
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
|
||||
|
||||
let mut allowed_opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
user_defined,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(allowed_opts.set_quota_admission(0, 4096));
|
||||
let mut allowed_reader = PutObjReader::new(
|
||||
HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false)
|
||||
.expect("construct exact-boundary replication reader"),
|
||||
);
|
||||
let stored = set_disks
|
||||
.put_object(bucket, "object", &mut allowed_reader, &allowed_opts)
|
||||
.await
|
||||
.expect("server-observed exact quota boundary should succeed");
|
||||
assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_put_cannot_persist_a_tiny_logical_size() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "direct-put-quota-safety";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let payload = vec![0x62; 4096];
|
||||
let mut denied_opts = ObjectOptions::default();
|
||||
assert!(denied_opts.set_quota_admission(0, 4095));
|
||||
let mut denied_reader = PutObjReader::new(
|
||||
HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false)
|
||||
.expect("construct forged direct reader"),
|
||||
);
|
||||
let err = set_disks
|
||||
.put_object(bucket, "object", &mut denied_reader, &denied_opts)
|
||||
.await
|
||||
.expect_err("server-observed bytes must prevent a tiny direct quota claim");
|
||||
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
|
||||
|
||||
let mut allowed_opts = ObjectOptions::default();
|
||||
assert!(allowed_opts.set_quota_admission(0, 4096));
|
||||
let mut allowed_reader = PutObjReader::new(
|
||||
HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false)
|
||||
.expect("construct exact-boundary direct reader"),
|
||||
);
|
||||
let stored = set_disks
|
||||
.put_object(bucket, "object", &mut allowed_reader, &allowed_opts)
|
||||
.await
|
||||
.expect("server-observed exact quota boundary should succeed");
|
||||
assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 4096);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "ciphertext-replication-quota-safety";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
|
||||
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string());
|
||||
let mut opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
preserve_ciphertext: true,
|
||||
user_defined,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(opts.set_quota_admission(0, u64::MAX));
|
||||
let payload = vec![0x63; 4096];
|
||||
let mut reader = PutObjReader::new(
|
||||
HashReader::from_stream(Cursor::new(payload), 4096, 4096, None, None, false)
|
||||
.expect("construct ciphertext replication reader"),
|
||||
);
|
||||
let err = set_disks
|
||||
.put_object(bucket, "object", &mut reader, &opts)
|
||||
.await
|
||||
.expect_err("ciphertext replication without a server-observed logical size must fail closed");
|
||||
assert!(matches!(err, StorageError::PartMissingOrCorrupt));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod inline_put_commit_path_tests {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||
|
||||
Reference in New Issue
Block a user