From bd5685f8a2c502015e4b353b38dadc980d008e13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E7=99=BB=E5=B1=B1?= Date: Wed, 12 Aug 2026 16:49:45 +0800 Subject: [PATCH] fix(quota): enforce durable hard quota reservations --- crates/ecstore/src/api/mod.rs | 10 +- crates/ecstore/src/bucket/metadata_sys.rs | 143 ++- crates/ecstore/src/bucket/quota/checker.rs | 20 + crates/ecstore/src/bucket/quota/mod.rs | 130 +- .../ecstore/src/bucket/quota/reservation.rs | 1080 +++++++++++++++++ .../src/cluster/rpc/peer_rest_client.rs | 38 + crates/ecstore/src/data_usage/mod.rs | 90 +- crates/ecstore/src/disk/local.rs | 186 ++- crates/ecstore/src/disk/mod.rs | 36 + crates/ecstore/src/disk/os.rs | 9 + .../ecstore/src/services/notification_sys.rs | 195 ++- .../src/set_disk/core/io_primitives.rs | 87 +- crates/ecstore/src/set_disk/ops/multipart.rs | 347 +++++- crates/ecstore/src/set_disk/ops/object.rs | 344 +++++- crates/ecstore/src/store/init.rs | 26 + crates/ecstore/src/store/list_objects.rs | 2 +- crates/protocols/src/swift/object.rs | 81 +- docs/architecture/compat-cleanup-register.md | 1 - rustfs/src/admin/handlers/quota.rs | 29 +- rustfs/src/admin/handlers/site_replication.rs | 33 +- rustfs/src/admin/storage_api.rs | 17 +- rustfs/src/app/gating_test_env.rs | 32 + rustfs/src/app/multipart_usecase.rs | 202 +-- rustfs/src/app/object_usecase.rs | 546 ++++++++- rustfs/src/app/storage_api.rs | 11 + rustfs/src/error.rs | 15 + rustfs/src/storage/rpc/node_service.rs | 142 ++- rustfs/src/storage/rpc/node_service/disk.rs | 61 +- rustfs/src/storage/storage_api.rs | 20 +- scripts/check_s3s_footprint.sh | 2 +- 30 files changed, 3602 insertions(+), 333 deletions(-) create mode 100644 crates/ecstore/src/bucket/quota/reservation.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 4b3162313..d8b294dae 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -131,6 +131,8 @@ pub mod bucket { } pub mod metadata_sys { + #[cfg(feature = "test-util")] + pub use crate::bucket::metadata_sys::ConfigWriteLockProbe; pub use crate::bucket::metadata_sys::{ BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy, @@ -140,7 +142,7 @@ pub mod bucket { get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, - update_under_transaction_lock, + update_quota_if_incarnation, update_under_transaction_lock, }; } @@ -316,7 +318,7 @@ pub mod data_usage { DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage, init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached, - load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, + load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory, refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer, @@ -404,7 +406,8 @@ pub mod metrics { pub mod notification { pub use crate::services::notification_sys::{ - NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys, + CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof, + cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe, }; } @@ -464,6 +467,7 @@ pub mod set_disk { #[cfg(feature = "test-util")] pub mod test_util { + pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test; pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; } } diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 59f5aae37..40aa0bd77 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -50,6 +50,72 @@ use uuid::Uuid; const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60); +#[cfg(any(test, feature = "test-util"))] +struct ConfigWriteLockProbeState { + bucket: String, + arrived: tokio::sync::Notify, +} + +#[cfg(any(test, feature = "test-util"))] +static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock>>> = std::sync::OnceLock::new(); + +#[cfg(any(test, feature = "test-util"))] +pub struct ConfigWriteLockProbe { + state: Arc, +} + +#[cfg(any(test, feature = "test-util"))] +impl ConfigWriteLockProbe { + pub fn install(bucket: &str) -> Self { + let state = Arc::new(ConfigWriteLockProbeState { + bucket: bucket.to_string(), + arrived: tokio::sync::Notify::new(), + }); + let mut probes = CONFIG_WRITE_LOCK_PROBES + .get_or_init(|| StdMutex::new(Vec::new())) + .lock() + .expect("config write lock probe mutex should not poison"); + assert!( + !probes.iter().any(|current| current.bucket == state.bucket), + "config write lock probe must be unique for a bucket" + ); + probes.push(Arc::clone(&state)); + drop(probes); + Self { state } + } + + pub async fn wait_until_attempted(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("bucket config update should attempt the transaction lock"); + } +} + +#[cfg(any(test, feature = "test-util"))] +impl Drop for ConfigWriteLockProbe { + fn drop(&mut self) { + let mut probes = CONFIG_WRITE_LOCK_PROBES + .get_or_init(|| StdMutex::new(Vec::new())) + .lock() + .expect("config write lock probe mutex should not poison"); + probes.retain(|state| !Arc::ptr_eq(state, &self.state)); + } +} + +#[cfg(any(test, feature = "test-util"))] +fn notify_config_write_lock_attempt(bucket: &str) { + let probe = CONFIG_WRITE_LOCK_PROBES + .get_or_init(|| StdMutex::new(Vec::new())) + .lock() + .expect("config write lock probe mutex should not poison") + .iter() + .find(|probe| probe.bucket == bucket) + .cloned(); + if let Some(probe) = probe { + probe.arrived.notify_one(); + } +} + #[derive(Clone, Copy)] enum MetadataLoadMode { Initial, @@ -590,6 +656,31 @@ pub async fn update_under_transaction_lock( update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await } +pub async fn update_quota_if_incarnation( + bucket: &str, + data: Vec, + expected_incarnation_id: Uuid, + proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken, +) -> Result { + let sys = get_bucket_metadata_sys()?; + let guard = Box::pin(acquire_config_write_guard_for_incarnation( + sys.clone(), + bucket, + Some(expected_incarnation_id), + )) + .await?; + if !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof) { + return Err(Error::NamespaceLockQuorumUnavailable { + mode: "quota_capability", + bucket: bucket.to_string(), + object: rustfs_config::QUOTA_CONFIG_FILE.to_string(), + required: 1, + achieved: 0, + }); + } + update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await +} + pub async fn update_bucket_targets_under_transaction_lock( guard: &BucketMetadataMutationGuard, bucket: &str, @@ -734,7 +825,26 @@ async fn acquire_transaction_lock_with_sys( let lock = api .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket)) .await?; - Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?) + let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()); + #[cfg(any(test, feature = "test-util"))] + { + tokio::pin!(acquire); + let mut notified = false; + let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) { + std::task::Poll::Pending => { + if !notified { + notify_config_write_lock_attempt(bucket); + notified = true; + } + std::task::Poll::Pending + } + std::task::Poll::Ready(result) => std::task::Poll::Ready(result), + }) + .await?; + Ok(guard) + } + #[cfg(not(any(test, feature = "test-util")))] + Ok(acquire.await?) } /// The lock resource name is deliberately still the `bucket-targets` one it @@ -889,6 +999,37 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in( } } +/// Re-read the quota configuration and bucket incarnation from the same +/// authoritative metadata blob while the caller holds the bucket metadata +/// transaction read lock. +pub(crate) async fn get_quota_config_and_incarnation_from_disk_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result<(Option, Uuid, OffsetDateTime)> { + let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); + + match bucket_meta_sys + .read_authoritative_metadata_from_disk_under_transaction_lock(bucket) + .await? + { + BucketMetadataAuthority::Authoritative(metadata) + if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() => + { + Ok(( + metadata.quota_config.clone(), + metadata.bucket_incarnation_id, + metadata.quota_config_updated_at, + )) + } + BucketMetadataAuthority::Authoritative(_) => { + Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}"))) + } + BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())), + BucketMetadataAuthority::Fabricated => Err(Error::other(format!("bucket quota metadata is not authoritative: {bucket}"))), + } +} + pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; diff --git a/crates/ecstore/src/bucket/quota/checker.rs b/crates/ecstore/src/bucket/quota/checker.rs index 38b4e1423..d30e91935 100644 --- a/crates/ecstore/src/bucket/quota/checker.rs +++ b/crates/ecstore/src/bucket/quota/checker.rs @@ -158,6 +158,26 @@ impl QuotaChecker { .await } + pub async fn set_durable_quota_config_if_incarnation( + &mut self, + bucket: &str, + quota: BucketQuota, + expected_incarnation_id: uuid::Uuid, + proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken, + ) -> Result { + let json_data = serde_json::to_vec("a).map_err(|e| QuotaError::InvalidConfig { + reason: format!("Failed to serialize quota config: {}", e), + })?; + let start_time = Instant::now(); + let updated_at = + crate::bucket::metadata_sys::update_quota_if_incarnation(bucket, json_data, expected_incarnation_id, proof) + .await + .map_err(QuotaError::StorageError)?; + + rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed()); + Ok(updated_at) + } + async fn set_quota_config_for_incarnation( &mut self, bucket: &str, diff --git a/crates/ecstore/src/bucket/quota/mod.rs b/crates/ecstore/src/bucket/quota/mod.rs index 9172fd652..a4c40faf5 100644 --- a/crates/ecstore/src/bucket/quota/mod.rs +++ b/crates/ecstore/src/bucket/quota/mod.rs @@ -13,40 +13,97 @@ // limitations under the License. pub mod checker; +pub(crate) mod reservation; use crate::error::Result; use rustfs_config::{ QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE, QUOTA_NOT_FOUND_ERROR_CODE, }; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; use time::OffsetDateTime; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] pub enum QuotaType { - /// Hard quota: reject immediately when exceeded + /// Hard quota accounting. #[default] #[serde(alias = "HARD", alias = "hard")] Hard, } +pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1; + /// Bucket quota configuration. quota_type defaults to Hard when omitted. -#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)] +#[derive(Debug, Default, Clone, PartialEq)] pub struct BucketQuota { - #[serde(default)] pub quota: Option, /// Defaults to Hard when missing. - #[serde(default)] pub quota_type: QuotaType, + /// Optional durable reservation protocol. The wire format gives older + /// nodes a zero hard quota so a mixed-version fleet fails closed. + pub reservation_protocol: Option, /// Timestamp when this quota configuration was set (for audit purposes) - #[serde(default, with = "time::serde::rfc3339::option")] pub created_at: Option, /// Accept updated_at for compatibility; not used. - #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] pub updated_at: Option, } +#[derive(Deserialize, Serialize)] +struct BucketQuotaWire { + #[serde(default)] + quota: Option, + #[serde(default)] + quota_type: QuotaType, + #[serde(default, skip_serializing_if = "Option::is_none")] + reservation_protocol: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + reservation_quota: Option, + #[serde(default, with = "time::serde::rfc3339::option")] + created_at: Option, + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + updated_at: Option, +} + +impl Serialize for BucketQuota { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + let durable = self.uses_durable_reservations(); + BucketQuotaWire { + quota: if durable { Some(0) } else { self.quota }, + quota_type: self.quota_type.clone(), + reservation_protocol: self.reservation_protocol, + reservation_quota: if durable { self.quota } else { None }, + created_at: self.created_at, + updated_at: self.updated_at, + } + .serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for BucketQuota { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let wire = BucketQuotaWire::deserialize(deserializer)?; + let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) { + wire.reservation_quota + } else { + wire.quota + }; + Ok(Self { + quota, + quota_type: wire.quota_type, + reservation_protocol: wire.reservation_protocol, + created_at: wire.created_at, + updated_at: wire.updated_at, + }) + } +} + impl BucketQuota { /// Serialize to JSON bytes. Same format as parse_all_configs. pub fn marshal_msg(&self) -> Result> { @@ -63,6 +120,7 @@ impl BucketQuota { Self { quota, quota_type: QuotaType::Hard, + reservation_protocol: quota.map(|_| QUOTA_RESERVATION_PROTOCOL_V1), created_at: Some(now), updated_at: None, } @@ -72,7 +130,19 @@ impl BucketQuota { self.quota } + pub fn uses_durable_reservations(&self) -> bool { + self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) + } + + pub fn has_unsupported_reservation_protocol(&self) -> bool { + self.reservation_protocol + .is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1) + } + pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool { + if operation_size == 0 { + return true; + } if let Some(quota_limit) = self.quota { current_usage.saturating_add(operation_size) <= quota_limit } else { @@ -210,7 +280,51 @@ mod tests { let buf = q.marshal_msg().expect("marshal"); let restored = BucketQuota::unmarshal(&buf).expect("unmarshal"); assert_eq!(q.quota, restored.quota); - assert_eq!(q.quota_type, restored.quota_type); + assert_eq!(restored.quota_type, QuotaType::Hard); + assert_eq!(restored.reservation_protocol, Some(QUOTA_RESERVATION_PROTOCOL_V1)); + } + + #[test] + fn clearing_quota_keeps_the_legacy_compatible_type() { + let quota = BucketQuota::new(None); + + assert_eq!(quota.quota_type, QuotaType::Hard); + assert_eq!(quota.reservation_protocol, None); + assert!(!quota.uses_durable_reservations()); + } + + #[test] + fn durable_quota_makes_legacy_nodes_fail_closed() { + let json = serde_json::to_vec(&BucketQuota::new(Some(2048))).expect("durable quota should serialize"); + let quota: BucketQuota = serde_json::from_slice(&json).expect("current quota version should parse"); + assert!(quota.uses_durable_reservations()); + assert_eq!(quota.quota, Some(2048)); + + #[derive(Deserialize)] + enum LegacyQuotaType { + Hard, + } + #[derive(Deserialize)] + struct LegacyBucketQuota { + #[allow(dead_code)] + quota: Option, + #[allow(dead_code)] + quota_type: LegacyQuotaType, + } + let legacy = serde_json::from_slice::(&json) + .expect("legacy readers should ignore the reservation protocol field"); + assert_eq!(legacy.quota, Some(0)); + assert!(matches!(legacy.quota_type, LegacyQuotaType::Hard)); + } + + #[test] + fn unknown_reservation_protocol_does_not_activate_v1() { + let quota: BucketQuota = + serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#) + .expect("future protocol should remain parseable"); + + assert!(!quota.uses_durable_reservations()); + assert!(quota.has_unsupported_reservation_protocol()); } /// unmarshal accepts format without quota_type diff --git a/crates/ecstore/src/bucket/quota/reservation.rs b/crates/ecstore/src/bucket/quota/reservation.rs new file mode 100644 index 000000000..e1119d12c --- /dev/null +++ b/crates/ecstore/src/bucket/quota/reservation.rs @@ -0,0 +1,1080 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::bucket::metadata_sys; +use crate::config::com::{CONFIG_PREFIX, read_config_no_lock, save_config_with_opts}; +use crate::data_usage::compute_bucket_usage; +use crate::disk::RUSTFS_META_BUCKET; +use crate::disk::{DiskAPI, error::DiskError}; +use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found}; +use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission}; +use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; +use crate::storage_api_contracts::namespace::NamespaceLocking; +use crate::storage_api_contracts::object::ObjectOperations; +use crate::store::ECStore; +use futures::{StreamExt, stream}; +use rustfs_lock::NamespaceLockGuard; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; +use time::OffsetDateTime; +use tracing::warn; +use uuid::Uuid; + +const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1; +const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64; +const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128; +const ORPHAN_PROBE_CONCURRENCY: usize = 32; +const EVENT_QUOTA_LEDGER_SETTLEMENT: &str = "quota_ledger_settlement"; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_QUOTA: &str = "quota"; + +#[cfg(any(test, feature = "test-util"))] +static FAIL_NEXT_LEDGER_SAVE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +// Lock order: caller-held destination object/upload, bucket metadata +// transaction (read), operation reservation, then quota ledger. + +#[cfg(not(any(test, feature = "test-util")))] +const ORPHAN_MIN_AGE_SECONDS: i64 = 30; +#[cfg(any(test, feature = "test-util"))] +const ORPHAN_MIN_AGE_SECONDS: i64 = 0; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct PersistedReservation { + object: String, + old_size: u64, + new_size: u64, + created_at: i64, + #[serde(default)] + pool_index: Option, + #[serde(default)] + set_index: Option, + #[serde(default)] + commit_started: bool, +} + +impl PersistedReservation { + fn growth(&self) -> u64 { + self.new_size.saturating_sub(self.old_size) + } + + fn target(&self) -> Option<(usize, usize)> { + self.pool_index.zip(self.set_index) + } + + fn matches_expected(&self, expected: &Self) -> bool { + self.object == expected.object && self.old_size == expected.old_size && self.new_size == expected.new_size + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct QuotaLedger { + version: u8, + bucket_incarnation: Uuid, + quota_revision_unix_nanos: i128, + accounted_usage: u64, + reservations: BTreeMap, + #[serde(default)] + reconcile_required: bool, + #[serde(default)] + reap_cursor: Option, +} + +impl QuotaLedger { + fn new(bucket_incarnation: Uuid, quota_revision: OffsetDateTime, accounted_usage: u64) -> Self { + Self { + version: QUOTA_LEDGER_FORMAT_VERSION, + bucket_incarnation, + quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(), + accounted_usage, + reservations: BTreeMap::new(), + reconcile_required: false, + reap_cursor: None, + } + } + + fn matches(&self, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> bool { + self.bucket_incarnation == bucket_incarnation && self.quota_revision_unix_nanos == quota_revision.unix_timestamp_nanos() + } + + fn admitted_usage(&self) -> Result { + let reserved_growth = self.reservations.values().try_fold(0_u64, |total, reservation| { + total + .checked_add(reservation.growth()) + .ok_or(StorageError::PartMissingOrCorrupt) + })?; + if reserved_growth > self.accounted_usage { + return Err(StorageError::PartMissingOrCorrupt); + } + Ok(self.accounted_usage) + } + + fn reserve(&mut self, operation_id: Uuid, reservation: PersistedReservation) -> Result<()> { + self.accounted_usage = self + .accounted_usage + .checked_add(reservation.growth()) + .ok_or(StorageError::PartMissingOrCorrupt)?; + self.reservations.insert(operation_id, reservation); + Ok(()) + } + + fn commit(&mut self, operation_id: Uuid, expected: &PersistedReservation) -> Result<()> { + let Some(reservation) = self.reservations.remove(&operation_id) else { + return Err(StorageError::PartMissingOrCorrupt); + }; + if !reservation.matches_expected(expected) { + return Err(StorageError::PartMissingOrCorrupt); + } + if reservation.new_size < reservation.old_size { + self.reconcile_required = true; + } + Ok(()) + } + + fn abort(&mut self, operation_id: Uuid, expected: &PersistedReservation) -> Result<()> { + let Some(reservation) = self.reservations.remove(&operation_id) else { + return Ok(()); + }; + if !reservation.matches_expected(expected) { + return Err(StorageError::PartMissingOrCorrupt); + } + self.accounted_usage = self + .accounted_usage + .checked_sub(reservation.growth()) + .ok_or(StorageError::PartMissingOrCorrupt)?; + Ok(()) + } + + fn mark_commit_started(&mut self, operation_id: Uuid, expected: &PersistedReservation) -> Result<()> { + let reservation = self + .reservations + .get_mut(&operation_id) + .ok_or(StorageError::PartMissingOrCorrupt)?; + if !reservation.matches_expected(expected) { + return Err(StorageError::PartMissingOrCorrupt); + } + reservation.commit_started = true; + Ok(()) + } + + fn should_reconcile_after_denial(&self) -> bool { + self.reservations.is_empty() + } + + fn reap_candidates(&self, now: i64) -> (Vec, Option) { + let aged = self + .reservations + .iter() + .filter(|(_, reservation)| { + reservation.created_at > now || now.saturating_sub(reservation.created_at) >= ORPHAN_MIN_AGE_SECONDS + }) + .map(|(operation_id, _)| *operation_id) + .collect::>(); + let start = self + .reap_cursor + .and_then(|cursor| aged.iter().position(|operation_id| *operation_id > cursor)) + .unwrap_or(0); + let candidates = aged + .iter() + .cycle() + .skip(start) + .take(aged.len().min(MAX_ORPHAN_PROBES_PER_WRITE)) + .copied() + .collect::>(); + let next_cursor = candidates.last().copied(); + (candidates, next_cursor) + } +} + +pub(crate) struct QuotaContext { + store: Option>, + bucket: String, + object: String, + ledger_object: String, + bucket_incarnation: Option, + quota_revision: Option, + quota_limit: Option, + capability_proof: Option, + snapshot_admission: Option, + metadata_guard: Option, + pool_index: Option, + set_index: Option, +} + +impl QuotaContext { + pub(crate) fn is_enforced(&self) -> bool { + self.quota_limit.is_some() + } + + pub(crate) async fn reserve(self, old_size: u64, new_size: u64) -> Result { + let Some(quota_limit) = self.quota_limit else { + return Ok(QuotaReservation::unlimited(self.metadata_guard)); + }; + if let Some(admission) = self.snapshot_admission { + let growth = new_size.saturating_sub(old_size); + if growth > admission.remaining() { + return Err(StorageError::QuotaExceeded { + current: admission.current_usage(), + limit: admission.quota_limit(), + }); + } + return Ok(QuotaReservation::unlimited(self.metadata_guard)); + } + let store = self.store.ok_or(StorageError::PartMissingOrCorrupt)?; + let bucket_incarnation = self.bucket_incarnation.ok_or(StorageError::PartMissingOrCorrupt)?; + let quota_revision = self.quota_revision.ok_or(StorageError::PartMissingOrCorrupt)?; + let operation_id = Uuid::new_v4(); + let operation_lock_object = operation_lock_object(&self.ledger_object, operation_id); + let operation_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &operation_lock_object).await?; + let operation_guard = operation_lock.get_write_lock(get_lock_acquire_timeout()).await?; + let reservation = PersistedReservation { + object: self.object, + old_size, + new_size, + created_at: OffsetDateTime::now_utc().unix_timestamp(), + pool_index: self.pool_index, + set_index: self.set_index, + commit_started: false, + }; + let ledger_data = LedgerReservationData { + store: Arc::clone(&store), + bucket: self.bucket, + ledger_object: self.ledger_object, + operation_id, + reservation: reservation.clone(), + }; + let metadata_guard = self.metadata_guard; + let capability_proof = self.capability_proof; + + tokio::spawn(async move { + reap_stale_reservations(Arc::clone(&store), &ledger_data.bucket, &ledger_data.ledger_object).await?; + + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?; + let mut ledger = load_current_ledger_locked( + Arc::clone(&store), + &ledger_data.bucket, + &ledger_data.ledger_object, + bucket_incarnation, + quota_revision, + ) + .await?; + + let growth = reservation.growth(); + let mut current_usage = ledger.admitted_usage()?; + let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?; + if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() { + reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?; + current_usage = ledger.admitted_usage()?; + expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?; + } + if growth > 0 && expected_usage > quota_limit { + return Err(StorageError::QuotaExceeded { + current: current_usage, + limit: quota_limit, + }); + } + if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: ledger_data.bucket.clone(), + object: ledger_data.ledger_object.clone(), + required: 1, + achieved: 0, + }); + } + ledger.reserve(operation_id, reservation)?; + save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?; + + Ok(QuotaReservation { + ledger: Some(ledger_data), + operation_guard: Some(operation_guard), + metadata_guard, + capability_proof, + state: ReservationState::Pending, + }) + }) + .await + .map_err(|err| StorageError::other(format!("quota ledger reservation task failed: {err}")))? + } +} + +#[derive(Clone)] +struct LedgerReservationData { + store: Arc, + bucket: String, + ledger_object: String, + operation_id: Uuid, + reservation: PersistedReservation, +} + +pub(crate) struct QuotaReservation { + ledger: Option, + operation_guard: Option, + metadata_guard: Option, + capability_proof: Option, + state: ReservationState, +} + +#[derive(Clone, Copy)] +enum ReservationState { + Pending, + CommitStarted, + Committed, + FenceReleaseUncertain, +} + +impl QuotaReservation { + fn unlimited(metadata_guard: Option) -> Self { + Self { + ledger: None, + operation_guard: None, + metadata_guard, + capability_proof: None, + state: ReservationState::Pending, + } + } + + pub(crate) fn is_lock_lost(&self) -> bool { + self.operation_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) + || self.metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) + } + + pub(crate) fn capability_proof_matches(&self) -> bool { + self.capability_proof + .as_ref() + .is_none_or(crate::services::notification_sys::cross_pool_fence_fleet_proof_matches) + } + + pub(crate) async fn mark_commit_started(&mut self) -> Result<()> { + if !self.capability_proof_matches() { + let ledger = self.ledger.as_ref().ok_or(StorageError::PartMissingOrCorrupt)?; + return Err(quota_capability_error(&ledger.bucket, &ledger.ledger_object)); + } + if let Some(ledger) = self.ledger.as_ref() { + mark_commit_started(ledger).await?; + } + self.state = ReservationState::CommitStarted; + Ok(()) + } + + pub(crate) async fn commit(mut self) { + self.state = ReservationState::Committed; + let Some(ledger) = self.ledger.as_ref() else { + return; + }; + crate::store::list_objects::observe_list_objects_mutation(&ledger.store, &ledger.bucket).await; + match settle(ledger, true).await { + Ok(()) => self.ledger = None, + Err(err) => log_deferred_settlement(ledger, "commit_deferred", &err), + } + } + + pub(crate) async fn abort(mut self) { + let Some(ledger) = self.ledger.as_ref() else { + return; + }; + match settle(ledger, false).await { + Ok(()) => self.ledger = None, + Err(err) => log_deferred_settlement(ledger, "abort_deferred", &err), + } + } + + pub(crate) fn defer_after_fence(mut self) { + self.state = ReservationState::FenceReleaseUncertain; + } +} + +fn should_settle_on_drop(state: ReservationState) -> bool { + !matches!(state, ReservationState::CommitStarted | ReservationState::FenceReleaseUncertain) +} + +impl Drop for QuotaReservation { + fn drop(&mut self) { + let Some(ledger) = self.ledger.take() else { + return; + }; + if !should_settle_on_drop(self.state) { + return; + } + let committed = matches!(self.state, ReservationState::Committed); + let operation_guard = self.operation_guard.take(); + let metadata_guard = self.metadata_guard.take(); + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + runtime.spawn(async move { + let _operation_guard = operation_guard; + let _metadata_guard = metadata_guard; + if let Err(err) = settle(&ledger, committed).await { + log_deferred_settlement(&ledger, "background_retry_failed", &err); + } + }); + } +} + +pub(crate) async fn begin( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, + object: &str, + _snapshot_admission: Option, + pool_index: usize, + set_index: usize, +) -> Result { + if crate::bucket::utils::is_meta_bucketname(bucket) { + return Ok(QuotaContext { + store: None, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: None, + quota_revision: None, + quota_limit: None, + capability_proof: None, + snapshot_admission: None, + metadata_guard: None, + pool_index: None, + set_index: None, + }); + } + #[cfg(test)] + if let Some(snapshot_admission) = _snapshot_admission { + return Ok(QuotaContext { + store: None, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: None, + quota_revision: None, + quota_limit: Some(snapshot_admission.quota_limit()), + capability_proof: None, + snapshot_admission: Some(snapshot_admission), + metadata_guard: None, + pool_index: Some(pool_index), + set_index: Some(set_index), + }); + } + #[cfg(any(test, feature = "test-util"))] + if ctx.bucket_metadata_sys().is_none() { + return Ok(QuotaContext { + store: None, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: None, + quota_revision: None, + quota_limit: None, + capability_proof: None, + snapshot_admission: None, + metadata_guard: None, + pool_index: None, + set_index: None, + }); + } + + let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?; + let (quota, bucket_incarnation, quota_revision) = + metadata_sys::get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await?; + if metadata_guard.is_lock_lost() { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_config", + bucket: bucket.to_string(), + object: ledger_object(bucket), + required: 1, + achieved: 0, + }); + } + if quota + .as_ref() + .is_some_and(|quota| quota.has_unsupported_reservation_protocol()) + { + return Err(StorageError::PartMissingOrCorrupt); + } + let capability_proof = if quota.as_ref().is_some_and(|quota| quota.uses_durable_reservations()) { + Some( + crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof() + .ok_or_else(|| quota_capability_error(bucket, &ledger_object(bucket)))?, + ) + } else { + None + }; + let quota_limit = quota + .filter(crate::bucket::quota::BucketQuota::uses_durable_reservations) + .and_then(|quota| quota.quota); + let store = if quota_limit.is_some() { + Some(metadata_sys::object_store_in(ctx).await?) + } else { + None + }; + Ok(QuotaContext { + store, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: Some(bucket_incarnation), + quota_revision: Some(quota_revision), + quota_limit, + capability_proof, + snapshot_admission: None, + metadata_guard: Some(metadata_guard), + pool_index: Some(pool_index), + set_index: Some(set_index), + }) +} + +fn quota_capability_error(bucket: &str, object: &str) -> StorageError { + StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_capability", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + } +} + +pub(crate) async fn replaced_logical_size(set_disks: &SetDisks, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { + if opts.versioned && !opts.version_suspended && opts.version_id.is_none() { + return Ok(0); + } + let version_id = opts + .version_id + .clone() + .or_else(|| opts.version_suspended.then(|| Uuid::nil().to_string())); + let lookup_opts = ObjectOptions { + version_id, + no_lock: true, + metadata_cache_safe: false, + versioned: opts.versioned, + version_suspended: opts.version_suspended, + ..Default::default() + }; + match set_disks.get_object_info(bucket, object, &lookup_opts).await { + Ok(info) if info.delete_marker => Ok(0), + Ok(info) => logical_object_size(&info), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(0), + Err(err) => Err(err), + } +} + +fn logical_object_size(info: &ObjectInfo) -> Result { + crate::data_usage::quota_object_size(info) +} + +async fn mark_commit_started(data: &LedgerReservationData) -> Result<()> { + let data = data.clone(); + tokio::spawn(async move { + let ledger_lock = data.store.new_ns_lock(RUSTFS_META_BUCKET, &data.ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?; + let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?; + ledger.mark_commit_started(data.operation_id, &data.reservation)?; + save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await + }) + .await + .map_err(|err| StorageError::other(format!("quota commit marker task failed: {err}")))? +} + +async fn settle(data: &LedgerReservationData, committed: bool) -> Result<()> { + let store = Arc::clone(&data.store); + let ledger_object = data.ledger_object.clone(); + let operation_id = data.operation_id; + let reservation = data.reservation.clone(); + let bucket = data.bucket.clone(); + tokio::spawn(async move { + fence_namespace_mutations(&store, &bucket, &reservation.object, reservation.target()).await?; + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_object, None).await?; + let mut ledger = load_ledger_locked(Arc::clone(&store), &ledger_object).await?; + if committed { + ledger.commit(operation_id, &reservation)?; + } else { + ledger.abort(operation_id, &reservation)?; + } + save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await + }) + .await + .map_err(|err| StorageError::other(format!("quota ledger settlement task failed: {err}")))? +} + +async fn load_current_ledger_locked( + store: Arc, + bucket: &str, + ledger_object: &str, + bucket_incarnation: Uuid, + quota_revision: OffsetDateTime, +) -> Result { + match load_ledger_locked(Arc::clone(&store), ledger_object).await { + Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger), + Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => { + let usage = exact_bucket_usage(&store, bucket).await?; + Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage)) + } + Ok(_) => Err(StorageError::PartMissingOrCorrupt), + Err(StorageError::ConfigNotFound) => { + let usage = exact_bucket_usage(&store, bucket).await?; + Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage)) + } + Err(err) => Err(err), + } +} + +async fn reap_stale_reservations(store: Arc, bucket: &str, ledger_object: &str) -> Result<()> { + let now = now_unix(); + let (candidates, reconcile_required, next_cursor) = { + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, ledger_object).await?; + let _ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + match load_ledger_locked(Arc::clone(&store), ledger_object).await { + Ok(ledger) => { + let (candidates, next_cursor) = ledger.reap_candidates(now); + (candidates, ledger.reconcile_required, next_cursor) + } + Err(StorageError::ConfigNotFound) => (Vec::new(), false, None), + Err(err) => return Err(err), + } + }; + if candidates.is_empty() && !reconcile_required { + return Ok(()); + } + + let probe_results = stream::iter(candidates) + .map(|operation_id| { + let store = Arc::clone(&store); + async move { + let lock_object = operation_lock_object(ledger_object, operation_id); + let operation_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &lock_object).await?; + Ok::<_, StorageError>( + operation_lock + .get_write_lock_quiet(Duration::from_millis(50)) + .await + .ok() + .map(|guard| (operation_id, guard)), + ) + } + }) + .buffer_unordered(ORPHAN_PROBE_CONCURRENCY) + .collect::>() + .await; + let mut orphan_guards = Vec::new(); + for result in probe_results { + if let Some(guard) = result? { + orphan_guards.push(guard); + if orphan_guards.len() == MAX_ORPHANS_REAPED_PER_WRITE { + break; + } + } + } + + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&store, RUSTFS_META_BUCKET, ledger_object, None).await?; + let mut ledger = load_ledger_locked(Arc::clone(&store), ledger_object).await?; + let cursor_changed = next_cursor.is_some() && ledger.reap_cursor != next_cursor; + if next_cursor.is_some() { + ledger.reap_cursor = next_cursor; + } + let orphan_ids = orphan_guards + .iter() + .map(|(operation_id, _)| *operation_id) + .collect::>(); + let orphan_commit_targets = orphan_ids + .iter() + .filter_map(|operation_id| ledger.reservations.get(operation_id)) + .filter(|reservation| reservation.commit_started) + .map(|reservation| (reservation.object.clone(), reservation.target())) + .collect::>(); + for (object, target) in orphan_commit_targets { + fence_namespace_mutations(&store, bucket, &object, target).await?; + } + let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?; + if ledger.reservations.is_empty() && ledger.reconcile_required { + reconcile_exact(&store, bucket, &mut ledger).await?; + removed = true; + } + if !removed && !cursor_changed { + return Ok(()); + } + save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await +} + +fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result { + let mut removed = false; + for operation_id in operation_ids { + let Some(reservation) = ledger.reservations.get(operation_id).cloned() else { + continue; + }; + if reservation.commit_started { + ledger.reservations.remove(operation_id); + ledger.reconcile_required = true; + } else { + ledger.abort(*operation_id, &reservation)?; + } + removed = true; + } + Ok(removed) +} + +async fn reconcile_exact(store: &Arc, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> { + if !ledger.reservations.is_empty() { + return Err(StorageError::PartMissingOrCorrupt); + } + ledger.accounted_usage = exact_bucket_usage(store, bucket).await?; + ledger.reconcile_required = false; + Ok(()) +} + +async fn exact_bucket_usage(store: &Arc, bucket: &str) -> Result { + crate::store::list_objects::observe_list_objects_mutation(store, bucket).await; + Ok(compute_bucket_usage(Arc::clone(store), bucket).await?.size) +} + +fn ledger_object(bucket: &str) -> String { + format!("{CONFIG_PREFIX}/quota-ledger/{bucket}.json") +} + +fn operation_lock_object(ledger_object: &str, operation_id: Uuid) -> String { + format!("{ledger_object}.operations/{operation_id}") +} + +fn now_unix() -> i64 { + OffsetDateTime::now_utc().unix_timestamp() +} + +async fn fence_namespace_mutations( + store: &Arc, + bucket: &str, + object: &str, + target: Option<(usize, usize)>, +) -> Result<()> { + crate::bucket::utils::check_object_args(bucket, object)?; + let sets = match target { + Some((pool_index, set_index)) => { + let set = store + .pools + .get(pool_index) + .and_then(|pool| pool.disk_set.get(set_index)) + .cloned() + .ok_or(StorageError::PartMissingOrCorrupt)?; + vec![set] + } + None => store.pools.iter().map(|pool| pool.get_disks_by_key(object)).collect(), + }; + for set in sets { + let write_quorum = set.default_write_quorum(); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + let revoke_results = stream::iter(disks) + .map(|disk| { + let fence_path = fence_path.clone(); + async move { + let result = disk + .release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, crate::disk::SnapshotLeaseToken::revoke_all()) + .await; + (disk, result) + } + }) + .buffer_unordered(ORPHAN_PROBE_CONCURRENCY) + .collect::>() + .await; + let revoked_disks = revoke_results + .into_iter() + .filter_map(|(disk, result)| result.is_ok().then_some(disk)) + .collect::>(); + if revoked_disks.len() < write_quorum { + return Err(StorageError::ErasureWriteQuorum); + } + + let drain_results = stream::iter(revoked_disks) + .map(|disk| async move { + match disk.acquire_snapshot_lease(bucket, object).await { + Ok(token) => disk.release_snapshot_lease(bucket, object, token).await, + Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => Ok(()), + Err(err) => Err(err), + } + }) + .buffer_unordered(ORPHAN_PROBE_CONCURRENCY) + .collect::>() + .await; + if drain_results.iter().filter(|result| result.is_ok()).count() < write_quorum { + return Err(StorageError::ErasureWriteQuorum); + } + } + Ok(()) +} + +#[cfg(test)] +pub(crate) async fn fence_namespace_mutations_for_test( + store: &Arc, + bucket: &str, + object: &str, + target: Option<(usize, usize)>, +) -> Result<()> { + fence_namespace_mutations(store, bucket, object, target).await +} + +async fn load_ledger_locked(store: Arc, ledger_object: &str) -> Result { + let data = read_config_no_lock(store, ledger_object).await?; + let ledger: QuotaLedger = serde_json::from_slice(&data)?; + if ledger.version != QUOTA_LEDGER_FORMAT_VERSION { + return Err(StorageError::CorruptedFormat); + } + ledger.admitted_usage()?; + Ok(ledger) +} + +async fn save_ledger_locked( + store: Arc, + ledger_object: &str, + ledger: &QuotaLedger, + ledger_guard: &NamespaceLockGuard, +) -> Result<()> { + if ledger_guard.is_lock_lost() { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_ledger", + bucket: RUSTFS_META_BUCKET.to_string(), + object: ledger_object.to_string(), + required: 1, + achieved: 0, + }); + } + #[cfg(any(test, feature = "test-util"))] + if FAIL_NEXT_LEDGER_SAVE.swap(false, std::sync::atomic::Ordering::SeqCst) { + return Err(StorageError::Unexpected); + } + let mut opts = ObjectOptions { + max_parity: true, + no_lock: true, + ..Default::default() + }; + let _ = opts.set_quota_admission(0, u64::MAX); + opts.add_namespace_lock_guard(ledger_guard); + save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await +} + +#[cfg(any(test, feature = "test-util"))] +pub fn fail_next_quota_ledger_save_for_test() { + FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst); +} + +fn log_deferred_settlement(data: &LedgerReservationData, state: &'static str, err: &StorageError) { + warn!( + event = EVENT_QUOTA_LEDGER_SETTLEMENT, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_QUOTA, + state, + bucket = %data.bucket, + operation_id = %data.operation_id, + error = %err, + "quota ledger settlement deferred" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ledger(accounted_usage: u64) -> QuotaLedger { + QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage) + } + + #[test] + fn ledger_rejects_reserved_growth_overflow() { + let mut ledger = ledger(u64::MAX); + let result = ledger.reserve( + Uuid::new_v4(), + PersistedReservation { + object: "object".to_string(), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ); + + assert!(matches!(result, Err(StorageError::PartMissingOrCorrupt))); + } + + #[test] + fn legacy_reservation_without_topology_uses_conservative_fallback() { + let reservation: PersistedReservation = + serde_json::from_str(r#"{"object":"object","old_size":0,"new_size":1,"created_at":0,"commit_started":true}"#) + .expect("legacy reservation should deserialize"); + + assert_eq!(reservation.target(), None); + } + + #[test] + fn ledger_rejects_persisted_reservations_larger_than_accounted_usage() { + let mut ledger = ledger(0); + ledger.reservations.insert( + Uuid::new_v4(), + PersistedReservation { + object: "object".to_string(), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ); + + assert!(matches!(ledger.admitted_usage(), Err(StorageError::PartMissingOrCorrupt))); + } + + #[test] + fn ledger_accounts_overwrite_delta_and_reserved_growth() { + let mut ledger = ledger(10); + let operation_id = Uuid::new_v4(); + let overwrite = PersistedReservation { + object: "object".to_string(), + old_size: 8, + new_size: 5, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }; + ledger + .reserve(operation_id, overwrite.clone()) + .expect("shrinking overwrite should reserve"); + ledger + .reserve( + Uuid::new_v4(), + PersistedReservation { + object: "new-object".to_string(), + old_size: 0, + new_size: 7, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ) + .expect("new object should reserve positive growth"); + + assert_eq!( + ledger + .admitted_usage() + .expect("ledger usage should count positive growth only"), + 17 + ); + ledger + .commit(operation_id, &overwrite) + .expect("overwrite should settle exactly"); + assert_eq!(ledger.accounted_usage, 17); + assert!(ledger.reconcile_required); + assert_eq!(ledger.admitted_usage().expect("remaining reservation should stay counted"), 17); + } + + #[test] + fn commit_started_reservation_stays_precharged_until_reconciled() { + let mut ledger = ledger(10); + let operation_id = Uuid::new_v4(); + ledger + .reserve( + operation_id, + PersistedReservation { + object: "object".to_string(), + old_size: 0, + new_size: 7, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ) + .expect("new object should reserve positive growth"); + assert_eq!(ledger.accounted_usage, 17); + + let expected = ledger + .reservations + .get(&operation_id) + .expect("reservation should exist") + .clone(); + ledger + .mark_commit_started(operation_id, &expected) + .expect("commit marker should persist"); + + assert_eq!(ledger.accounted_usage, 17); + assert!(!ledger.should_reconcile_after_denial()); + } + + #[test] + fn commit_started_orphans_make_progress_across_bounded_batches() { + let mut ledger = ledger(65); + let operation_ids = (0..65).map(|_| Uuid::new_v4()).collect::>(); + for operation_id in &operation_ids { + ledger.reservations.insert( + *operation_id, + PersistedReservation { + object: format!("object-{operation_id}"), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: true, + }, + ); + } + + assert!(remove_orphan_reservations(&mut ledger, &operation_ids[..64]).expect("first orphan batch should apply")); + assert_eq!(ledger.reservations.len(), 1); + assert!(ledger.reconcile_required); + assert!(remove_orphan_reservations(&mut ledger, &operation_ids[64..]).expect("final orphan batch should apply")); + assert!(ledger.reservations.is_empty()); + } + + #[test] + fn orphan_probe_cursor_rotates_across_the_bounded_window() { + let mut ledger = ledger(129); + let operation_ids = (1..=129).map(Uuid::from_u128).collect::>(); + for operation_id in &operation_ids { + ledger.reservations.insert( + *operation_id, + PersistedReservation { + object: format!("object-{operation_id}"), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ); + } + + let (first, cursor) = ledger.reap_candidates(1); + assert_eq!(first.len(), MAX_ORPHAN_PROBES_PER_WRITE); + assert_eq!(first.first(), operation_ids.first()); + ledger.reap_cursor = cursor; + + let (second, _) = ledger.reap_candidates(1); + assert_eq!(second.first(), operation_ids.last()); + } + + #[test] + fn uncertain_fence_release_does_not_schedule_abort_on_drop() { + assert!(!should_settle_on_drop(ReservationState::FenceReleaseUncertain)); + assert!(!should_settle_on_drop(ReservationState::CommitStarted)); + assert!(should_settle_on_drop(ReservationState::Pending)); + assert!(should_settle_on_drop(ReservationState::Committed)); + } +} diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 46f246d6b..1426fc91f 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -248,6 +248,16 @@ fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) Ok(server_epoch) } +fn decode_cross_pool_fence_capability(expected_member: &str, result: &[u8]) -> Result<(u32, Uuid)> { + let version = result + .get(..4) + .and_then(|value| value.try_into().ok()) + .map(u32::from_be_bytes) + .ok_or_else(|| Error::other("peer returned an invalid cross-pool fence capability version"))?; + let epoch = decode_remote_version_state_capability(expected_member, &result[4..])?; + Ok((version, epoch)) +} + #[derive(Clone, Debug)] pub struct PeerLiveEventsBatch { pub events: Vec, @@ -1288,6 +1298,16 @@ impl PeerRestClient { Ok((self.topology_member.clone(), epoch)) } + pub async fn probe_cross_pool_fence(&self, topology_fingerprint: String) -> Result<(String, u32, Uuid)> { + let mut probe = rustfs_protos::CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX.to_vec(); + probe.extend_from_slice(Uuid::new_v4().as_bytes()); + let result = self + .heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe) + .await?; + let (supported_version, epoch) = decode_cross_pool_fence_capability(&self.topology_member, &result)?; + Ok((self.topology_member.clone(), supported_version, epoch)) + } + pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> { self.finalize_result( async { @@ -2738,6 +2758,24 @@ mod tests { assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err()); } + #[test] + fn cross_pool_fence_capability_decoder_fails_closed() { + let epoch = Uuid::new_v4(); + let result = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", epoch.as_bytes()) + .expect("small capability response should encode"); + assert_eq!( + decode_cross_pool_fence_capability("node-a:9000", &result).expect("valid capability should decode"), + (1, epoch) + ); + for malformed in [&[][..], &[0, 0, 0][..], &result[..result.len() - 1]] { + assert!(decode_cross_pool_fence_capability("node-a:9000", malformed).is_err()); + } + assert!(decode_cross_pool_fence_capability("node-b:9000", &result).is_err()); + let nil = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", Uuid::nil().as_bytes()) + .expect("small capability response should encode"); + assert!(decode_cross_pool_fence_capability("node-a:9000", &nil).is_err()); + } + struct TierMutationResponseFixture<'a> { version: u32, phase: TierMutationRpcPhase, diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 41495f068..ecef9c403 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -1355,7 +1355,7 @@ impl BucketUsageAccumulator { return Ok(()); } - let object_size = object.size.max(0) as u64; + let object_size = quota_object_size(object)?; self.current_live_versions = self.current_live_versions.saturating_add(1); self.size_histogram.add(object_size); self.total_size = self.total_size.saturating_add(object_size); @@ -1385,6 +1385,20 @@ impl BucketUsageAccumulator { } } +pub fn quota_object_size(object: &ObjectInfo) -> Result { + let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?; + let persisted_part_size = if object.parts.is_empty() { + u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)? + } else { + object.parts.iter().try_fold(0_u64, |total, part| { + let actual_size = u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let part_size = actual_size.max(u64::try_from(part.size).map_err(|_| Error::PartMissingOrCorrupt)?); + total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt) + })? + }; + Ok(logical_size.max(persisted_part_size)) +} + type UsageVersionPage = StorageListObjectVersionsInfo; pub async fn compute_bucket_usage(store: Arc, bucket_name: &str) -> Result { @@ -3124,6 +3138,80 @@ mod tests { assert_eq!(usage.object_versions_histogram.get("BETWEEN_1000_AND_10000"), Some(&1)); } + #[test] + fn bucket_usage_uses_the_larger_of_logical_and_physical_size() { + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "4096".to_string()); + let object = ObjectInfo { + name: "compressed".to_string(), + size: 128, + user_defined: Arc::new(metadata), + ..Default::default() + }; + let mut usage = BucketUsageAccumulator::default(); + usage + .record("bucket", &object) + .expect("valid compressed metadata should be counted"); + assert_eq!(usage.finish().size, 4096); + + let mut framed_metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut framed_metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + rustfs_utils::http::insert_str(&mut framed_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let framed = ObjectInfo { + name: "framed".to_string(), + size: 17, + user_defined: Arc::new(framed_metadata), + ..Default::default() + }; + assert_eq!(quota_object_size(&framed).expect("physical framing must remain quota-accounted"), 17); + + let mut corrupt_metadata = (*object.user_defined).clone(); + rustfs_utils::http::insert_str(&mut corrupt_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "-1".to_string()); + let corrupt = ObjectInfo { + user_defined: Arc::new(corrupt_metadata), + ..object + }; + assert!( + matches!( + BucketUsageAccumulator::default().record("bucket", &corrupt), + Err(Error::PartMissingOrCorrupt) + ), + "negative logical metadata must not become a smaller quota baseline" + ); + + let mut poisoned_metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut poisoned_metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + rustfs_utils::http::insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let poisoned = ObjectInfo { + name: "legacy-swift-metadata".to_string(), + size: 4096, + user_defined: Arc::new(poisoned_metadata), + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 4096, + actual_size: 4096, + ..Default::default() + }]), + ..Default::default() + }; + assert_eq!( + quota_object_size(&poisoned).expect("persisted part accounting must bound legacy user metadata"), + 4096 + ); + } + #[tokio::test] #[serial] async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() { diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 5a42245d8..8cdd45b89 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -22,17 +22,18 @@ use crate::disk::{ BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, - PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, - RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, - conv_part_err_to_int, + PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, + ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, + SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int, endpoint::Endpoint, error::{DiskError, Error, FileAccessDeniedWithContext, Result}, error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}, format::FormatV3, fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename}, - os, + is_quota_mutation_fence_path, os, os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source}, + quota_mutation_fence_path, }; use crate::erasure::coding::{self, bitrot_verify}; use crate::runtime::sources as runtime_sources; @@ -55,9 +56,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Debug; use std::io::{Error as IoError, SeekFrom}; -#[cfg(target_os = "linux")] -use std::sync::atomic::AtomicBool; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; use std::{ @@ -4751,6 +4750,25 @@ struct SnapshotLeaseEntry { tokens: HashSet, pending_delete: Option, deleting: bool, + mutation_fence: Option>, +} + +#[derive(Default)] +struct QuotaMutationFenceState { + revoked: AtomicBool, + running: AtomicUsize, + notify: Notify, +} + +struct QuotaMutationFenceClaim { + state: Arc, +} + +impl Drop for QuotaMutationFenceClaim { + fn drop(&mut self) { + self.state.running.fetch_sub(1, Ordering::AcqRel); + self.state.notify.notify_waiters(); + } } #[derive(Default)] @@ -7315,6 +7333,34 @@ fn normalize_path_components(path: impl AsRef) -> PathBuf { } impl LocalDisk { + async fn claim_quota_mutation_fence( + &self, + volume: &str, + path: &str, + token: SnapshotLeaseToken, + ) -> Result> { + let key = SnapshotLeaseKey { + volume: RUSTFS_META_BUCKET.to_string(), + path: quota_mutation_fence_path(volume, path), + }; + let state = { + let registry = self.snapshot_leases.lock().await; + let entry = registry.entries.get(&key).ok_or(DiskError::FileNotFound)?; + let state = entry.mutation_fence.as_ref().ok_or(DiskError::FileNotFound)?; + if !entry.tokens.contains(&token) || state.revoked.load(Ordering::Acquire) { + return Err(DiskError::FileNotFound); + } + state.running.fetch_add(1, Ordering::AcqRel); + Arc::clone(state) + }; + if state.revoked.load(Ordering::Acquire) { + state.running.fetch_sub(1, Ordering::AcqRel); + state.notify.notify_waiters(); + return Err(DiskError::FileNotFound); + } + Ok(Arc::new(QuotaMutationFenceClaim { state })) + } + async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result { let path = format!("{object}/{data_dir}"); let data_path = self.io_get_object_path(volume, &path)?; @@ -8648,7 +8694,30 @@ impl DiskAPI for LocalDisk { // optimistic; this lease establishes the local commit/delete order and // remains owned by any blocking syscall that outlives async cancellation. let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; + let quota_fence_token = + match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { + Some(value) => { + let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; + Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) + } + None if rustfs_utils::http::metadata_compat::contains_key_str( + &fi.metadata, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + ) => + { + return Err(DiskError::FileCorrupt); + } + None => None, + }; + rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); + let quota_fence_claim = match quota_fence_token { + Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), + None => None, + }; let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; + if let Some(claim) = quota_fence_claim { + mutation_lease.attach_external_guard(claim); + } if fi.is_legacy_indexed_delete_marker() { fi.erasure.index = 0; } @@ -9638,11 +9707,26 @@ impl DiskAPI for LocalDisk { } async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result { - let file_path = self.io_get_object_path(volume, path)?; let key = SnapshotLeaseKey { volume: volume.to_string(), path: path.to_string(), }; + if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) { + let mut registry = self.snapshot_leases.lock().await; + let entry = registry.entries.entry(key).or_default(); + let state = entry + .mutation_fence + .get_or_insert_with(|| Arc::new(QuotaMutationFenceState::default())); + if state.revoked.load(Ordering::Acquire) { + return Err(DiskError::FileNotFound); + } + let token = SnapshotLeaseToken::new(); + entry.tokens.insert(token); + return Ok(token); + } + + let file_path = self.io_get_object_path(volume, path)?; + let _mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, volume, &file_path).await; let token = { let mut registry = self.snapshot_leases.lock().await; if registry.entries.get(&key).is_some_and(|entry| entry.deleting) { @@ -9670,6 +9754,48 @@ impl DiskAPI for LocalDisk { volume: volume.to_string(), path: path.to_string(), }; + if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) { + if !token.is_revoke_all() { + let mut registry = self.snapshot_leases.lock().await; + let Some(entry) = registry.entries.get_mut(&key) else { + return Ok(()); + }; + entry.tokens.remove(&token); + let removable = entry.tokens.is_empty() + && entry + .mutation_fence + .as_ref() + .is_none_or(|state| state.running.load(Ordering::Acquire) == 0); + if removable { + registry.entries.remove(&key); + } + return Ok(()); + } + let state = { + let mut registry = self.snapshot_leases.lock().await; + let Some(entry) = registry.entries.get_mut(&key) else { + return Ok(()); + }; + let Some(state) = entry.mutation_fence.as_ref().cloned() else { + registry.entries.remove(&key); + return Ok(()); + }; + state.revoked.store(true, Ordering::Release); + entry.tokens.clear(); + state + }; + loop { + let notified = state.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if state.running.load(Ordering::Acquire) == 0 { + break; + } + notified.await; + } + self.snapshot_leases.lock().await.entries.remove(&key); + return Ok(()); + } let opts = { let mut registry = self.snapshot_leases.lock().await; let Some(entry) = registry.entries.get_mut(&key) else { @@ -18723,6 +18849,48 @@ mod test { assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound))); } + #[tokio::test] + async fn quota_mutation_fence_revoke_waits_for_active_claim_and_rejects_late_claims() { + use tempfile::tempdir; + + let root_dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let bucket = "quota-fence-volume"; + let object = "object"; + let fence_path = quota_mutation_fence_path(bucket, object); + let token = disk + .acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path) + .await + .expect("quota mutation token should be prepared"); + let claim = disk + .claim_quota_mutation_fence(bucket, object, token) + .await + .expect("prepared token should be claimable"); + + let release_disk = Arc::clone(&disk); + let mut release = tokio::spawn(async move { + release_disk + .release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, SnapshotLeaseToken::revoke_all()) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut release).await.is_err(), + "revoke must wait until an already claimed mutation has finished" + ); + + drop(claim); + tokio::time::timeout(Duration::from_secs(1), release) + .await + .expect("revoke should wake after the final claim drops") + .expect("revoke task should not panic") + .expect("revoke should succeed"); + assert!(matches!( + disk.claim_quota_mutation_fence(bucket, object, token).await, + Err(DiskError::FileNotFound) + )); + } + #[tokio::test] async fn delete_version_keeps_later_part_until_snapshot_release() { use tempfile::tempdir; diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 0f9c28bee..c701052cb 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -71,6 +71,28 @@ use time::OffsetDateTime; use tokio::io::{AsyncRead, AsyncWrite}; use uuid::Uuid; +const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/"; +pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token"; + +pub(crate) fn quota_mutation_fence_path(bucket: &str, object: &str) -> String { + use sha2::{Digest, Sha256}; + + let mut input = Vec::with_capacity(bucket.len() + object.len() + 1); + input.extend_from_slice(bucket.as_bytes()); + input.push(0); + input.extend_from_slice(object.as_bytes()); + let digest = Sha256::digest(input); + format!( + "{QUOTA_MUTATION_FENCE_PREFIX}{}", + hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower) + ) +} + +pub(crate) fn is_quota_mutation_fence_path(path: &str) -> bool { + path.strip_prefix(QUOTA_MUTATION_FENCE_PREFIX) + .is_some_and(|digest| digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit())) +} + pub type DiskStore = Arc; pub type FileReader = Box; @@ -95,6 +117,20 @@ impl SnapshotLeaseToken { pub fn as_bytes(&self) -> &[u8; 16] { self.0.as_bytes() } + + pub(crate) fn as_uuid(self) -> Uuid { + self.0 + } + + #[doc(hidden)] + pub fn revoke_all() -> Self { + Self(Uuid::nil()) + } + + #[doc(hidden)] + pub fn is_revoke_all(self) -> bool { + self.0.is_nil() + } } impl Default for SnapshotLeaseToken { diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index bd3ec77a5..fc9c8027c 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -306,12 +306,20 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc { pub(crate) struct NamespaceMutationLease { _namespace_guard: OwnedMutexGuard<()>, _volume_guard: Option>, + external_guard: Mutex>>, +} + +impl NamespaceMutationLease { + pub(crate) fn attach_external_guard(&self, guard: Arc) { + *self.external_guard.lock() = Some(guard); + } } async fn acquire_namespace_mutation_lease(path: &Path) -> Arc { Arc::new(NamespaceMutationLease { _namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await, _volume_guard: None, + external_guard: Mutex::new(None), }) } @@ -327,6 +335,7 @@ pub(crate) async fn acquire_rename_data_mutation_lease( Arc::new(NamespaceMutationLease { _namespace_guard: namespace_guard, _volume_guard: Some(volume_guard), + external_guard: Mutex::new(None), }) } diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index b08211853..272acc88f 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -44,12 +44,14 @@ const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification"; const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation"; +const EVENT_NOTIFICATION_CAPABILITY_PROBE: &str = "notification_capability_probe"; const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100); const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10); const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30); +const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1; /// Cached result from the last successful admin call to a peer. struct PeerAdminCache { @@ -95,15 +97,15 @@ lazy_static! { } #[derive(Clone)] -struct RemoteVersionStateFleetProof { +struct FleetCapabilityProof { topology_fingerprint: String, peer_epochs: Arc>, expires_at: Instant, } -impl RemoteVersionStateFleetProof { - fn token(&self) -> RemoteVersionStateFleetProofToken { - RemoteVersionStateFleetProofToken { +impl FleetCapabilityProof { + fn token(&self) -> FleetCapabilityProofToken { + FleetCapabilityProofToken { topology_fingerprint: self.topology_fingerprint.clone(), peer_epochs: self.peer_epochs.clone(), } @@ -111,37 +113,41 @@ impl RemoteVersionStateFleetProof { } #[derive(Clone, PartialEq, Eq)] -pub(crate) struct RemoteVersionStateFleetProofToken { +struct FleetCapabilityProofToken { topology_fingerprint: String, peer_epochs: Arc>, } #[derive(Default)] -struct RemoteVersionStateFleetProofState { - proof: Option, +struct FleetCapabilityProofState { + proof: Option, topology_conflict: bool, } -static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock> = OnceLock::new(); +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken); + +#[derive(Clone, PartialEq, Eq)] +pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken); + +static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock> = OnceLock::new(); +static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock> = OnceLock::new(); static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock = OnceLock::new(); -fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock { - REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default())) +fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock { + CROSS_POOL_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) } -fn replace_remote_version_state_fleet_proof(proof: Option) { - replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof); +fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock { + REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) } -fn replace_remote_version_state_fleet_proof_in( - slot: &std::sync::RwLock, - proof: Option, -) { +fn replace_fleet_capability_proof(slot: &std::sync::RwLock, proof: Option) { slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof; } -fn publish_remote_version_state_probe_result( - slot: &std::sync::RwLock, +fn publish_fleet_capability_probe_result( + slot: &std::sync::RwLock, topology_fingerprint: &str, result: Result>, observed_at: Instant, @@ -155,7 +161,7 @@ fn publish_remote_version_state_probe_result( .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs) .map(|proof| Arc::clone(&proof.peer_epochs)) .unwrap_or_else(|| Arc::new(peer_epochs)); - state.proof = Some(RemoteVersionStateFleetProof { + state.proof = Some(FleetCapabilityProof { topology_fingerprint: topology_fingerprint.to_string(), peer_epochs, expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL, @@ -163,7 +169,7 @@ fn publish_remote_version_state_probe_result( None } Err(err) => { - replace_remote_version_state_fleet_proof_in(slot, None); + replace_fleet_capability_proof(slot, None); Some(err) } } @@ -174,27 +180,44 @@ pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option Option { - if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) { +) -> Option { + if state.topology_conflict || !fleet_capability_proof_valid_at(state.proof.as_ref(), expected_topology, now) { return None; } - state.proof.as_ref().map(RemoteVersionStateFleetProof::token) + state.proof.as_ref().map(FleetCapabilityProof::token) } pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool { + fleet_capability_proof_matches(remote_version_state_fleet_proof_slot(), &proof.0) +} + +pub fn acquire_cross_pool_fence_fleet_proof() -> Option { + let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?; + let state = cross_pool_fence_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(CrossPoolFenceFleetProofToken) +} + +pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToken) -> bool { + fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0) +} + +fn fleet_capability_proof_matches( + slot: &std::sync::RwLock, + proof: &FleetCapabilityProofToken, +) -> bool { let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else { return false; }; - let state = remote_version_state_fleet_proof_slot() - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner); + let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner); if state.topology_conflict { return false; } @@ -206,11 +229,7 @@ pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStat }) } -fn remote_version_state_fleet_proof_valid_at( - proof: Option<&RemoteVersionStateFleetProof>, - expected_topology: &str, - now: Instant, -) -> bool { +fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool { proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at) } @@ -224,11 +243,11 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap, pe pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) { - let mut state = remote_version_state_fleet_proof_slot() - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner); - state.topology_conflict = true; - state.proof = None; + for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] { + let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner); + state.topology_conflict = true; + state.proof = None; + } } return; } @@ -249,13 +268,23 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { } None => Err(Error::other("remote version state fleet capability notification system is unavailable")), }; + let fence_result = match get_global_notification_sys() { + Some(notification_sys) => timeout( + REMOTE_VERSION_STATE_PROBE_TIMEOUT, + notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint), + ) + .await + .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))), + None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")), + }; let topology_conflict = remote_version_state_fleet_proof_slot() .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .topology_conflict; if topology_conflict { - replace_remote_version_state_fleet_proof(None); - } else if let Some(err) = publish_remote_version_state_probe_result( + replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None); + replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None); + } else if let Some(err) = publish_fleet_capability_probe_result( remote_version_state_fleet_proof_slot(), &topology_fingerprint, result, @@ -263,6 +292,24 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { ) { debug!(error = %err, "remote version state fleet capability probe failed closed"); } + if !topology_conflict + && let Some(err) = publish_fleet_capability_probe_result( + cross_pool_fence_fleet_proof_slot(), + &topology_fingerprint, + fence_result, + Instant::now(), + ) + { + debug!( + event = EVENT_NOTIFICATION_CAPABILITY_PROBE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + capability = "cross_pool_fence_v1", + state = "failed_closed", + error = %err, + "notification capability probe" + ); + } sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await; } }); @@ -330,6 +377,27 @@ impl NotificationSys { } Ok(peer_epochs) } + + async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result> { + if self.peer_clients.len() != self.peer_topology_hosts.len() { + return Err(Error::other("cross-pool fence capability fleet membership is incomplete")); + } + let probes = self.peer_clients.iter().map(|client| async { + let client = client + .as_ref() + .ok_or_else(|| Error::other("cross-pool fence capability peer is unreachable"))?; + client.probe_cross_pool_fence(topology_fingerprint.to_string()).await + }); + let mut peer_epochs = BTreeMap::new(); + for result in join_all(probes).await { + let (peer, version, epoch) = result?; + if version < CROSS_POOL_FENCE_SUPPORTED_VERSION { + return Err(Error::other("cross-pool fence capability version is unsupported")); + } + insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?; + } + Ok(peer_epochs) + } } pub struct NotificationPeerErr { @@ -2145,16 +2213,16 @@ mod tests { let now = Instant::now(); let mut peer_epochs = BTreeMap::new(); peer_epochs.insert("peer-a".to_string(), Uuid::new_v4()); - let proof = RemoteVersionStateFleetProof { + let proof = FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(peer_epochs), expires_at: now + Duration::from_secs(1), }; - assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now)); - assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now)); - assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at)); - assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now)); + assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now)); + assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now)); + assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-a", proof.expires_at)); + assert!(!fleet_capability_proof_valid_at(None, "topology-a", now)); } #[test] @@ -2168,25 +2236,25 @@ mod tests { #[test] fn remote_version_state_fleet_proof_accepts_single_node_membership() { let now = Instant::now(); - let proof = RemoteVersionStateFleetProof { + let proof = FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(BTreeMap::new()), expires_at: now + Duration::from_secs(1), }; - assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now)); + assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now)); } #[test] fn remote_version_state_fleet_proof_token_changes_with_process_epoch() { let now = Instant::now(); - let proof = RemoteVersionStateFleetProof { + let proof = FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), expires_at: now + Duration::from_secs(1), }; let captured = proof.token(); - let restarted = RemoteVersionStateFleetProof { + let restarted = FleetCapabilityProof { topology_fingerprint: proof.topology_fingerprint.clone(), peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), expires_at: proof.expires_at, @@ -2197,11 +2265,11 @@ mod tests { #[test] fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() { - let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()); + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); let now = Instant::now(); let epoch = Uuid::new_v4(); let peers = BTreeMap::from([("peer-a".to_string(), epoch)]); - assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none()); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none()); let original = slot .read() .expect("proof slot should not poison") @@ -2210,9 +2278,7 @@ mod tests { .expect("successful probe should publish proof") .token(); - assert!( - publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none() - ); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()); let renewed = slot .read() .expect("proof slot should not poison") @@ -2224,8 +2290,7 @@ mod tests { let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]); assert!( - publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2)) - .is_none() + publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2)).is_none() ); let replaced = slot .read() @@ -2240,18 +2305,18 @@ mod tests { #[test] fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() { let now = Instant::now(); - let mut state = RemoteVersionStateFleetProofState { - proof: Some(RemoteVersionStateFleetProof { + let mut state = FleetCapabilityProofState { + proof: Some(FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(BTreeMap::new()), expires_at: now + Duration::from_secs(1), }), topology_conflict: false, }; - assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some()); + assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some()); state.topology_conflict = true; - assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none()); + assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_none()); } #[test] @@ -2267,19 +2332,19 @@ mod tests { #[test] fn remote_version_state_fleet_probe_failure_revokes_previous_proof() { - let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()); + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); let now = Instant::now(); let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]); - assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); assert!(slot.read().expect("proof slot should not poison").proof.is_some()); assert!( - publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some() + publish_fleet_capability_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some() ); assert!(slot.read().expect("proof slot should not poison").proof.is_none()); let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]); - assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); assert!(slot.read().expect("proof slot should not poison").proof.is_some()); } diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index a6c3095e0..816e25d22 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -538,7 +538,7 @@ impl MetadataQuorumAccumulator { }) } - pub(in crate::set_disk) fn default_write_quorum(&self) -> usize { + pub(crate) fn default_write_quorum(&self) -> usize { if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { return self.total_disks; } @@ -2816,7 +2816,7 @@ impl SetDisks { self.set_drive_count - self.default_parity_count } - pub(in crate::set_disk) fn default_write_quorum(&self) -> usize { + pub(crate) fn default_write_quorum(&self) -> usize { let mut data_count = self.set_drive_count - self.default_parity_count; if data_count == self.default_parity_count { data_count += 1 @@ -2825,6 +2825,89 @@ impl SetDisks { data_count } + pub(in crate::set_disk) async fn prepare_quota_mutation_fences( + disks: &[Option], + bucket: &str, + object: &str, + write_quorum: usize, + ) -> crate::error::Result<(Vec>, Vec>)> { + let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + let results = join_all(disks.iter().map(|disk| { + let disk = disk.clone(); + let fence_path = fence_path.clone(); + async move { + let disk = disk?; + match disk.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path).await { + Ok(token) => Some((disk, token)), + Err(_) => None, + } + } + })) + .await; + if results.iter().flatten().count() < write_quorum { + for (disk, token) in results.iter().flatten() { + let _ = disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, *token).await; + } + return Err(StorageError::ErasureWriteQuorum); + } + let mut fenced_disks = Vec::with_capacity(results.len()); + let mut tokens = Vec::with_capacity(results.len()); + for result in results { + match result { + Some((disk, token)) => { + fenced_disks.push(Some(disk)); + tokens.push(Some(token)); + } + None => { + fenced_disks.push(None); + tokens.push(None); + } + } + } + Ok((fenced_disks, tokens)) + } + + pub(in crate::set_disk) async fn release_quota_mutation_fences( + disks: &[Option], + tokens: &[Option], + bucket: &str, + object: &str, + write_quorum: usize, + ) -> crate::error::Result<()> { + let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| { + let disk = disk.as_ref()?.clone(); + let token = (*token)?; + let fence_path = fence_path.clone(); + Some(async move { disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, token).await }) + })) + .await; + if results.iter().filter(|result| result.is_ok()).count() < write_quorum { + return Err(StorageError::ErasureWriteQuorum); + } + Ok(()) + } + + pub(in crate::set_disk) async fn abort_quota_reservation_after_fence( + reservation: crate::bucket::quota::reservation::QuotaReservation, + disks: &[Option], + tokens: &[Option], + bucket: &str, + object: &str, + write_quorum: usize, + fenced: bool, + ) { + let safe_to_abort = !fenced + || Self::release_quota_mutation_fences(disks, tokens, bucket, object, write_quorum) + .await + .is_ok(); + if safe_to_abort { + reservation.abort().await; + } else { + reservation.defer_after_fence(); + } + } + #[tracing::instrument(level = "debug", skip(disks, file_infos))] #[allow(clippy::type_complexity)] pub(in crate::set_disk) async fn rename_data( diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 2d7a5ff27..4f9e255ba 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -22,6 +22,7 @@ use super::super::*; use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards}; +use crate::bucket::quota::reservation; use crate::crash_inject::{self, CrashPoint}; use crate::multipart_listing::paginate_multipart_listing; use futures::{StreamExt, stream}; @@ -1760,6 +1761,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { fi.parts = Vec::with_capacity(uploaded_parts.len()); + let quota_context = + reservation::begin(&self.ctx, bucket, object, opts.quota_admission, self.pool_index, self.set_index).await?; + let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some(); + let preserve_replication_ciphertext = opts.replication_request + && contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT); + if quota_context.is_enforced() && preserve_replication_ciphertext { + return Err(Error::PartMissingOrCorrupt); + } + let transformed_object = fi.is_compressed() || should_persist_encryption_original_size(&fi.metadata); + let mut object_size: usize = 0; let mut object_actual_size: i64 = 0; @@ -1883,15 +1894,23 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { checksum_combined.extend_from_slice(cs.raw.as_slice()); } - object_size += ext_part.size; - if opts.quota_admission.is_some() && ext_part.actual_size < 0 { + object_size = object_size.checked_add(ext_part.size).ok_or(Error::PartMissingOrCorrupt)?; + if ext_part.actual_size < 0 && (!opts.replication_request || quota_context.is_enforced()) { return Err(Error::PartMissingOrCorrupt); } + let normalized_actual_size = if ext_part.actual_size >= 0 && !transformed_object { + ext_part + .actual_size + .max(i64::try_from(ext_part.size).map_err(|_| Error::PartMissingOrCorrupt)?) + } else { + ext_part.actual_size + }; object_actual_size = object_actual_size - .checked_add(ext_part.actual_size) + .checked_add(normalized_actual_size) .ok_or(Error::PartMissingOrCorrupt)?; - - fi.parts.push(completed_multipart_object_part(p.part_num, ext_part)); + let mut completed_part = completed_multipart_object_part(p.part_num, ext_part); + completed_part.actual_size = normalized_actual_size; + fi.parts.push(completed_part); } if let Some(wtcs) = opts.want_checksum.as_ref() { @@ -1918,15 +1937,34 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } } - if let Some(admission) = opts.quota_admission { - let quota_operation_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; - if quota_operation_size > admission.remaining() { - return Err(Error::QuotaExceeded { - current: admission.current_usage(), - limit: admission.quota_limit(), - }); - } - } + let declared_replication_actual_size = opts + .replication_request + .then(|| get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP)) + .flatten(); + let replication_actual_size = if opts.replication_request && quota_context.is_enforced() { + let observed_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let declared_cap = declared_replication_actual_size + .as_deref() + .map(|value| value.parse::().map_err(|_| Error::PartMissingOrCorrupt)) + .transpose()? + .unwrap_or(0); + let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(&fi.metadata) + .map_err(Error::other)? + .map(u64::try_from) + .transpose() + .map_err(|_| Error::PartMissingOrCorrupt)? + .unwrap_or(0); + Some(observed_size.max(declared_cap).max(declared_encryption_size)) + } else { + None + }; + let quota_new_size = match replication_actual_size { + Some(size) => size.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?), + None if quota_context.is_enforced() => u64::try_from(object_actual_size) + .map_err(|_| Error::PartMissingOrCorrupt)? + .max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?), + None => 0, + }; if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) { if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) { fi.checksum = Some(Bytes::from(rc_crc_bytes)); @@ -1979,7 +2017,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } if opts.replication_request { - if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) { + if let Some(actual_size) = replication_actual_size { + insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); + if persist_encryption_original_size { + fi.metadata + .insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string()); + } + } else if let Some(actual_size) = declared_replication_actual_size { insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone()); if persist_encryption_original_size { fi.metadata @@ -2080,11 +2124,142 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { return Err(StorageError::Unexpected); } - // The trailing `_` drops the rename_data old-size backfill - // (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit - // `get_object_info` lookup, so the backfill has no consumer here yet. - let (online_disks, convergence, op_old_dir, cleanup_disks, _) = Self::rename_data( - &shuffle_disks, + let quota_old_size = if quota_context.is_enforced() { + reservation::replaced_logical_size(&self, bucket, object, opts).await? + } else { + 0 + }; + let mut quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?; + let (commit_disks, quota_fence_tokens) = if quota_mutation_fence { + match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await { + Ok((disks, tokens)) => { + for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) { + if let Some(token) = token { + insert_str( + &mut metadata.metadata, + crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + token.as_uuid().to_string(), + ); + } + } + (disks, tokens) + } + Err(err) => { + quota_reservation.abort().await; + return Err(err); + } + } + } else { + (shuffle_disks.clone(), vec![None; shuffle_disks.len()]) + }; + if quota_reservation.is_lock_lost() + || !quota_reservation.capability_proof_matches() + || object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + Self::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + bucket, + object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + if let Err(err) = ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts) { + Self::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + bucket, + object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(err); + } + if let Err(err) = self + .require_current_restore_operation_id( + bucket, + object, + opts, + expected_restore_operation_id, + "complete_multipart_upload_quota_reservation", + ) + .await + { + Self::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + bucket, + object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(err); + } + + if let Err(err) = quota_reservation.mark_commit_started().await { + Self::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + bucket, + object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(err); + } + if quota_reservation.is_lock_lost() + || object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || opts + .bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + Self::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + bucket, + object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + let rename_result = Self::rename_data( + &commit_disks, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, &parts_metadatas, @@ -2092,7 +2267,17 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { object, write_quorum, ) - .await?; + .await; + if quota_mutation_fence { + let _ = Self::release_quota_mutation_fences(&commit_disks, "a_fence_tokens, bucket, object, write_quorum).await; + } + if rename_result.is_ok() { + quota_reservation.commit().await; + } + let (online_disks, convergence, op_old_dir, cleanup_disks, _) = match rename_result { + Ok(result) => result, + Err(err) => return Err(err.into()), + }; // Detach admission before any post-commit await: client cancellation // must not couple durable convergence repair to cleanup work. @@ -2635,7 +2820,7 @@ mod tests { let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await; let mut denied_opts = ObjectOptions::default(); - assert!(denied_opts.set_quota_admission(100, 4195)); + assert!(denied_opts.set_quota_admission(100, 4180)); let err = set_disks .clone() @@ -2646,7 +2831,7 @@ mod tests { err, StorageError::QuotaExceeded { current: 100, - limit: 4195 + limit: 4180 } )); @@ -2664,7 +2849,7 @@ mod tests { ); let mut allowed_opts = ObjectOptions::default(); - assert!(allowed_opts.set_quota_admission(100, 4196)); + assert!(allowed_opts.set_quota_admission(100, 4181)); let completed = set_disks .clone() .complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts) @@ -2705,6 +2890,120 @@ mod tests { ); } + #[tokio::test] + async fn replication_quota_uses_server_observed_part_size_as_lower_bound() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-replication-quota-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let mut create_opts = ObjectOptions::default(); + insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string()); + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("replication multipart upload should be created"); + let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x72; 4096], 1).await; + let mut complete_opts = ObjectOptions { + replication_request: true, + ..Default::default() + }; + assert!(complete_opts.set_quota_admission(0, 4095)); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part.clone()], &complete_opts) + .await + .expect_err("a forged tiny replication logical size must not reduce quota admission"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + assert!( + set_disks + .check_upload_id_exists(bucket, object, &upload.upload_id, false) + .await + .is_ok(), + "quota rejection must leave replicated multipart parts retryable" + ); + + assert!(complete_opts.set_quota_admission(0, 4096)); + let completed = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect("the physical safety boundary should admit the transformed replica"); + assert_eq!(completed.get_actual_size().expect("replica logical size should parse"), 1); + } + + #[tokio::test] + async fn direct_multipart_quota_uses_server_observed_part_size_as_lower_bound() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-direct-quota-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let upload = set_disks + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("multipart upload should be created"); + let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x73; 4096], 1).await; + let mut complete_opts = ObjectOptions::default(); + assert!(complete_opts.set_quota_admission(0, 4095)); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect_err("a forged tiny direct logical size must not reduce quota admission"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + assert!( + set_disks + .check_upload_id_exists(bucket, object, &upload.upload_id, false) + .await + .is_ok(), + "quota rejection must leave direct multipart parts retryable" + ); + } + + #[tokio::test] + async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-compressed-ciphertext-quota-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let mut create_opts = ObjectOptions::default(); + insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string()); + insert_str( + &mut create_opts.user_defined, + rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, + "true".to_string(), + ); + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("ciphertext multipart upload should be created"); + let payload = vec![0x74; 4096]; + let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &payload, 1).await; + let mut complete_opts = ObjectOptions { + replication_request: true, + ..Default::default() + }; + assert!(complete_opts.set_quota_admission(0, u64::MAX)); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect_err("ciphertext replication has no server-observed logical quota size"); + assert!(matches!(err, StorageError::PartMissingOrCorrupt)); + assert!( + set_disks + .check_upload_id_exists(bucket, object, &upload.upload_id, false) + .await + .is_ok(), + "rejection must leave ciphertext multipart parts retryable" + ); + } + #[tokio::test] async fn complete_multipart_quota_rejects_invalid_logical_sizes() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 20d271cd9..8095f8cdb 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -40,6 +40,7 @@ use crate::bucket::lifecycle::{ save_transition_transaction_record, }, }; +use crate::bucket::quota::reservation; use crate::bucket::replication::{ DeleteReplicationConfigSnapshot, VersionPurgeStatusType, replication_state_to_filemeta, version_purge_status_to_filemeta, }; @@ -1495,9 +1496,215 @@ impl SetDisks { }); } + let quota_context = + reservation::begin(&self.ctx, bucket, object, opts.quota_admission, self.pool_index, self.set_index).await?; + let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some(); + let mut replication_quota_size = None; + + if opts.replication_request { + if quota_context.is_enforced() && opts.preserve_ciphertext { + return Err(Error::PartMissingOrCorrupt); + } + if quota_context.is_enforced() { + let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let physical_size = u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let transformed = contains_key_str(&user_defined, SUFFIX_COMPRESSION) + || parts_metadatas + .first() + .is_some_and(|metadata| should_persist_encryption_original_size(&metadata.metadata)); + let declared_size = get_str(&user_defined, SUFFIX_ACTUAL_SIZE) + .map(|value| value.parse::().map_err(|_| Error::PartMissingOrCorrupt)) + .transpose()? + .unwrap_or(0); + let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(&user_defined) + .map_err(Error::other)? + .map(u64::try_from) + .transpose() + .map_err(|_| Error::PartMissingOrCorrupt)? + .unwrap_or(0); + let logical_size = observed_size.max(declared_size).max(declared_encryption_size); + let persisted_size = if transformed { + logical_size + } else { + logical_size.max(physical_size) + }; + replication_quota_size = Some(logical_size.max(physical_size)); + actual_size = i64::try_from(persisted_size).map_err(|_| Error::PartMissingOrCorrupt)?; + for metadata in &mut parts_metadatas { + insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, persisted_size.to_string()); + if should_persist_encryption_original_size(&metadata.metadata) { + metadata + .metadata + .insert("x-rustfs-encryption-original-size".to_string(), persisted_size.to_string()); + } + if let Some(part) = metadata.parts.first_mut() { + part.actual_size = actual_size; + } + } + } + } else if actual_size >= 0 { + let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let transformed = contains_key_str(&user_defined, SUFFIX_COMPRESSION) + || parts_metadatas + .first() + .is_some_and(|metadata| should_persist_encryption_original_size(&metadata.metadata)); + let server_observed_size = if transformed { + observed_size + } else { + observed_size.max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?) + }; + actual_size = i64::try_from(server_observed_size).map_err(|_| Error::PartMissingOrCorrupt)?; + for metadata in &mut parts_metadatas { + insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, server_observed_size.to_string()); + if should_persist_encryption_original_size(&metadata.metadata) { + metadata + .metadata + .insert("x-rustfs-encryption-original-size".to_string(), server_observed_size.to_string()); + } + if let Some(part) = metadata.parts.first_mut() { + part.actual_size = actual_size; + } + } + } + + let (quota_old_size, quota_new_size) = if quota_context.is_enforced() { + let new_size = match replication_quota_size { + Some(size) => size, + None => u64::try_from(actual_size) + .map_err(|_| Error::PartMissingOrCorrupt)? + .max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?), + }; + (reservation::replaced_logical_size(self, bucket, object, opts).await?, new_size) + } else { + (0, 0) + }; + let mut quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?; + let (commit_disks, quota_fence_tokens) = if quota_mutation_fence { + match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await { + Ok((disks, tokens)) => { + for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) { + if let Some(token) = token { + insert_str( + &mut metadata.metadata, + crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + token.as_uuid().to_string(), + ); + } + } + (disks, tokens) + } + Err(err) => { + quota_reservation.abort().await; + return Err(err); + } + } + } else { + (shuffle_disks.clone(), vec![None; shuffle_disks.len()]) + }; + if quota_reservation.is_lock_lost() + || !quota_reservation.capability_proof_matches() + || object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || opts + .bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { + Self::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + bucket, + object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + if let Err(err) = self + .require_current_restore_operation_id( + bucket, + object, + opts, + expected_restore_operation_id, + "put_object_quota_reservation", + ) + .await + { + Self::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + bucket, + object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(err); + } + let rename_stage_start = Instant::now(); - let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = Self::rename_data( - &shuffle_disks, + #[cfg(any(test, feature = "test-util"))] + pause_put_object_commit(bucket, object, PutObjectCommitPause::AfterQuotaReservation).await; + if let Err(err) = quota_reservation.mark_commit_started().await { + Self::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + bucket, + object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(err); + } + #[cfg(any(test, feature = "test-util"))] + pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeQuotaRename).await; + if quota_reservation.is_lock_lost() + || object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || opts + .bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { + Self::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + bucket, + object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + let rename_result = Self::rename_data( + &commit_disks, RUSTFS_META_TMP_BUCKET, tmp_dir.as_str(), &parts_metadatas, @@ -1505,7 +1712,18 @@ impl SetDisks { object, write_quorum, ) - .await?; + .await; + if quota_mutation_fence { + let _ = + Self::release_quota_mutation_fences(&commit_disks, "a_fence_tokens, bucket, object, write_quorum).await; + } + if rename_result.is_ok() { + quota_reservation.commit().await; + } + let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = match rename_result { + Ok(result) => result, + Err(err) => return Err(err.into()), + }; // Do this before any post-commit await so request cancellation cannot // bypass best-effort admission. A process crash before admission // remains subject to the existing scanner reconciliation path. @@ -2636,6 +2854,8 @@ fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: boo pub enum PutObjectCommitPause { BeforeNamespace, AfterNamespace, + AfterQuotaReservation, + BeforeQuotaRename, BeforeMetadata, } @@ -5568,6 +5788,124 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support { } } +#[cfg(test)] +mod replication_quota_safety_tests { + use super::hermetic_set_disks_support::hermetic_set_disks; + use super::*; + use std::io::Cursor; + + #[tokio::test] + async fn replication_put_quota_uses_physical_bytes_as_a_safety_floor() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "replication-put-quota-safety"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut user_defined = HashMap::new(); + insert_str( + &mut user_defined, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string()); + let payload = vec![0x61; 4096]; + + let mut denied_opts = ObjectOptions { + replication_request: true, + user_defined: user_defined.clone(), + ..Default::default() + }; + assert!(denied_opts.set_quota_admission(0, 4095)); + let mut denied_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false) + .expect("construct forged replication reader"), + ); + let err = set_disks + .put_object(bucket, "object", &mut denied_reader, &denied_opts) + .await + .expect_err("server-observed bytes must prevent a tiny replication quota claim"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + + let mut allowed_opts = ObjectOptions { + replication_request: true, + user_defined, + ..Default::default() + }; + assert!(allowed_opts.set_quota_admission(0, 4096)); + let mut allowed_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false) + .expect("construct exact-boundary replication reader"), + ); + let stored = set_disks + .put_object(bucket, "object", &mut allowed_reader, &allowed_opts) + .await + .expect("server-observed exact quota boundary should succeed"); + assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1); + } + + #[tokio::test] + async fn direct_put_cannot_persist_a_tiny_logical_size() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "direct-put-quota-safety"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let payload = vec![0x62; 4096]; + let mut denied_opts = ObjectOptions::default(); + assert!(denied_opts.set_quota_admission(0, 4095)); + let mut denied_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false) + .expect("construct forged direct reader"), + ); + let err = set_disks + .put_object(bucket, "object", &mut denied_reader, &denied_opts) + .await + .expect_err("server-observed bytes must prevent a tiny direct quota claim"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + + let mut allowed_opts = ObjectOptions::default(); + assert!(allowed_opts.set_quota_admission(0, 4096)); + let mut allowed_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false) + .expect("construct exact-boundary direct reader"), + ); + let stored = set_disks + .put_object(bucket, "object", &mut allowed_reader, &allowed_opts) + .await + .expect("server-observed exact quota boundary should succeed"); + assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 4096); + } + + #[tokio::test] + async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "ciphertext-replication-quota-safety"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut user_defined = HashMap::new(); + user_defined.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()); + insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string()); + let mut opts = ObjectOptions { + replication_request: true, + preserve_ciphertext: true, + user_defined, + ..Default::default() + }; + assert!(opts.set_quota_admission(0, u64::MAX)); + let payload = vec![0x63; 4096]; + let mut reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload), 4096, 4096, None, None, false) + .expect("construct ciphertext replication reader"), + ); + let err = set_disks + .put_object(bucket, "object", &mut reader, &opts) + .await + .expect_err("ciphertext replication without a server-observed logical size must fail closed"); + assert!(matches!(err, StorageError::PartMissingOrCorrupt)); + } +} + #[cfg(test)] mod get_object_downstream_close_accounting_tests { use super::hermetic_set_disks_support::hermetic_set_disks; diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index a979328e7..a33aeb8e7 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -1175,6 +1175,32 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn quota_object_fence_ignores_an_unrelated_offline_pool() { + let temp_dir = tempfile::tempdir().expect("create quota fence store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "quota-object-fence", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = format!("quota-object-fence-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create quota fence bucket"); + store.pools[1].disk_set[0].disks.write().await.fill(None); + + crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, Some((0, 0))) + .await + .expect("the selected pool fence should ignore an unrelated offline pool"); + let err = crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, None) + .await + .expect_err("legacy reservations must conservatively fence every pool"); + assert!(matches!(err, StorageError::ErasureWriteQuorum)); + + shutdown.cancel(); + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn tag_updates_skip_active_rebalance_source_pool() { diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 2e8ee9968..10fc766a2 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -641,7 +641,7 @@ pub(crate) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) { .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta))); } -pub(super) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 { +pub(crate) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 { observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default() } diff --git a/crates/protocols/src/swift/object.rs b/crates/protocols/src/swift/object.rs index 85581a323..e871cfba2 100644 --- a/crates/protocols/src/swift/object.rs +++ b/crates/protocols/src/swift/object.rs @@ -67,10 +67,39 @@ const LOG_COMPONENT_PROTOCOLS: &str = "protocols"; const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object"; const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state"; const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at"; +const USER_METADATA_PREFIX: &str = "x-amz-meta-"; /// Maximum object size in bytes (5GB - Swift default) const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024; +fn stored_swift_user_metadata_key(key: &str) -> String { + if rustfs_utils::http::is_internal_key(key) + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-amz-") + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-") + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-") + { + format!("{USER_METADATA_PREFIX}{key}") + } else { + key.to_string() + } +} + +fn swift_user_metadata(headers: &HeaderMap) -> Option> { + let mut metadata = HashMap::new(); + let mut present = false; + for (header_name, header_value) in headers.iter() { + let header_name = header_name.as_str().to_lowercase(); + let Some(key) = header_name.strip_prefix("x-object-meta-") else { + continue; + }; + present = true; + if let Ok(value) = header_value.to_str() { + metadata.insert(stored_swift_user_metadata_key(key), value.to_string()); + } + } + present.then_some(metadata) +} + /// Object key translator for Swift object names /// /// Handles URL encoding/decoding and path normalization for Swift object keys. @@ -303,15 +332,7 @@ where let bucket = mapper.swift_to_s3_bucket(container, &project_id); // 5. Extract Swift metadata from X-Object-Meta-* headers - let mut user_metadata = HashMap::new(); - for (header_name, header_value) in headers.iter() { - let header_str = header_name.as_str().to_lowercase(); - if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") - && let Ok(value_str) = header_value.to_str() - { - user_metadata.insert(meta_key.to_string(), value_str.to_string()); - } - } + let mut user_metadata = swift_user_metadata(headers).unwrap_or_default(); // 6. Extract Content-Type if provided if let Some(content_type) = headers.get("content-type") @@ -739,15 +760,7 @@ pub async fn update_object_metadata( } // 8. Extract new metadata from X-Object-Meta-* headers - let mut new_metadata = HashMap::new(); - for (header_name, header_value) in headers.iter() { - let header_str = header_name.as_str().to_lowercase(); - if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") - && let Ok(value_str) = header_value.to_str() - { - new_metadata.insert(meta_key.to_string(), value_str.to_string()); - } - } + let mut new_metadata = swift_user_metadata(headers).unwrap_or_default(); // 9. Also update Content-Type if provided if let Some(content_type) = headers.get("content-type") @@ -889,19 +902,8 @@ pub async fn copy_object( let mut new_metadata = (*src_info.user_defined).clone(); // 11. If custom metadata headers provided, use those instead (Swift behavior) - let mut has_custom_meta = false; - for (header_name, header_value) in headers.iter() { - let header_str = header_name.as_str().to_lowercase(); - if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") { - if !has_custom_meta { - // First custom meta header - clear source metadata - new_metadata.clear(); - has_custom_meta = true; - } - if let Ok(value_str) = header_value.to_str() { - new_metadata.insert(meta_key.to_string(), value_str.to_string()); - } - } + if let Some(custom_metadata) = swift_user_metadata(headers) { + new_metadata = custom_metadata; } // 12. Also check for Content-Type override @@ -1123,6 +1125,23 @@ mod tests { assert!(ObjectKeyMapper::validate_object_name("unicode-文件.txt").is_ok()); } + #[test] + fn swift_user_metadata_cannot_materialize_internal_storage_keys() { + let mut headers = HeaderMap::new(); + headers.insert("x-object-meta-x-rustfs-internal-actual-size", "1".parse().expect("valid metadata value")); + headers.insert("x-object-meta-description", "safe".parse().expect("valid metadata value")); + let metadata = swift_user_metadata(&headers).expect("custom metadata should be detected"); + + assert_eq!(metadata.get("x-amz-meta-x-rustfs-internal-actual-size").map(String::as_str), Some("1")); + assert_eq!(metadata.get("description").map(String::as_str), Some("safe")); + assert!(!metadata.contains_key("x-rustfs-internal-actual-size")); + assert_eq!( + stored_swift_user_metadata_key("x-minio-encryption-original-size"), + "x-amz-meta-x-minio-encryption-original-size" + ); + assert_eq!(stored_swift_user_metadata_key("description"), "description"); + } + #[test] fn test_validate_object_name_empty() { let result = ObjectKeyMapper::validate_object_name(""); diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index e1a0b8040..3e24b7db9 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -16,7 +16,6 @@ for later deletion. - `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: mixed-version deployments continue writing version 1 snapshots until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2. - `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration. - `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract. -- `cross-pool-fence-v1` authenticated unsupported advertisement: predeployment servers recognize the versioned cross-pool fence capability probe but report support version 0, allowing a later all-peer probe to distinguish predeployment nodes without activating a second lock domain. Replace the unsupported advertisement only when composite lock acquisition, a cluster-wide activation fence, complete fleet proof, commit-time proof revalidation, and fail-closed revocation ship together. - `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration. - `rustfs-5509` FileInfo positional MessagePack decoding: beta.11 serialized 28 fields, while beta.12 inserted transition-version fields in the middle and serialized an incompatible 30-field array. New releases write named maps and retain readers for both shipped array layouts so direct and rolling upgrades can read either release. Remove the positional-array readers after every supported direct-upgrade release writes named maps and no retained RPC payload can contain a pre-map FileInfo array. - `rustfs-5416` Helm distributed startup wait setting: charts that predate explicit local endpoint identity expose startupWaitTimeoutSeconds for their peer DNS/TCP init gate. The new chart keeps the value accepted but ignores it after moving startup convergence into RustFS. Remove the value and its documentation after the minimum supported direct-upgrade chart includes localEndpointHost.autoInject and no longer renders the peer gate. diff --git a/rustfs/src/admin/handlers/quota.rs b/rustfs/src/admin/handlers/quota.rs index 35124b038..7dc2621be 100644 --- a/rustfs/src/admin/handlers/quota.rs +++ b/rustfs/src/admin/handlers/quota.rs @@ -22,6 +22,7 @@ use crate::admin::storage_api::bucket::metadata_sys::{self, BucketMetadataSys}; use crate::admin::storage_api::bucket::quota::checker::QuotaChecker; use crate::admin::storage_api::bucket::quota::{BucketQuota, QuotaError, QuotaOperation}; use crate::auth::{check_key_valid, get_session_token}; +use crate::error::ApiError; use crate::server::ADMIN_PREFIX; use hyper::{Method, StatusCode}; use matchit::Params; @@ -293,16 +294,36 @@ impl Operation for SetBucketQuotaHandler { return Err(s3_error!(InvalidArgument, "{}", rustfs_config::QUOTA_INVALID_TYPE_ERROR_MSG)); } + let fleet_proof = if request.quota.is_some() { + Some(crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| { + S3Error::with_message( + s3s::S3ErrorCode::ServiceUnavailable, + "durable quota capability is not confirmed across the cluster".to_string(), + ) + })?) + } else { + None + }; + let quota = BucketQuota::new(request.quota); let metadata_sys_lock = bucket_metadata_from_context() .ok_or_else(|| s3_error!(InternalError, "{}", rustfs_config::QUOTA_METADATA_SYSTEM_ERROR_MSG))?; let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); - let updated_at = quota_checker - .set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id) - .await - .map_err(|e| s3_error!(InternalError, "failed to set quota: {}", e))?; + let updated_at = match fleet_proof.as_ref() { + Some(fleet_proof) => { + quota_checker + .set_durable_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id, fleet_proof) + .await + } + None => { + quota_checker + .set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id) + .await + } + } + .map_err(ApiError::from)?; if let Err(err) = site_replication_bucket_meta_hook(SRBucketMeta { bucket: bucket.clone(), diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 85d529836..bf946c0c4 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -29,6 +29,7 @@ use crate::admin::storage_api::bucket::metadata::{ BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG, }; use crate::admin::storage_api::bucket::metadata_sys; +use crate::admin::storage_api::bucket::quota::BucketQuota; use crate::admin::storage_api::bucket::replication; use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; use crate::admin::storage_api::bucket::target_sys::BucketTargetSys; @@ -7895,9 +7896,35 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { if !skip_config_write { if let Some(data) = data { - metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) - .await - .map_err(ApiError::from)?; + if item.r#type == "quota-config" { + let quota: BucketQuota = serde_json::from_slice(&data) + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid bucket quota: {e}")))?; + if quota.has_unsupported_reservation_protocol() { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "unsupported bucket quota reservation protocol".to_string(), + )); + } + if quota.uses_durable_reservations() { + let proof = crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| { + S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "durable quota capability is not confirmed across the cluster".to_string(), + ) + })?; + metadata_sys::update_quota_if_incarnation(&item.bucket, data, expected_incarnation_id, &proof) + .await + .map_err(ApiError::from)?; + } else { + metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + } + } else { + metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + } } else { metadata_sys::delete_if_incarnation(&item.bucket, config_file, expected_incarnation_id) .await diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index fd01de999..17841d1de 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -64,7 +64,9 @@ mod ecstore_metrics { } mod ecstore_notification { - pub(crate) use crate::storage::storage_api::ecstore_notification::NotificationSys; + pub(crate) use crate::storage::storage_api::ecstore_notification::{ + CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof, + }; } #[allow(unused_imports)] @@ -112,6 +114,10 @@ pub(crate) type TierCreds = ecstore_tier::tier_admin::TierCreds; pub(crate) type TierType = ecstore_tier::tier_config::TierType; pub(crate) type TierConfigUpdateError = crate::storage::storage_api::TierConfigUpdateError; +pub(crate) fn acquire_cross_pool_fence_fleet_proof() -> Option { + ecstore_notification::acquire_cross_pool_fence_fleet_proof() +} + pub(crate) mod runtime_sources { pub(crate) type DailyAllTierStats = super::DailyAllTierStats; pub(crate) type ECStore = super::ECStore; @@ -296,6 +302,15 @@ pub(crate) mod metadata_sys { super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await } + pub(crate) async fn update_quota_if_incarnation( + bucket: &str, + data: Vec, + expected_incarnation_id: uuid::Uuid, + proof: &super::ecstore_notification::CrossPoolFenceFleetProofToken, + ) -> Result { + super::ecstore_bucket::metadata_sys::update_quota_if_incarnation(bucket, data, expected_incarnation_id, proof).await + } + pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result { super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await } diff --git a/rustfs/src/app/gating_test_env.rs b/rustfs/src/app/gating_test_env.rs index e25a4471e..f8d5bfd9b 100644 --- a/rustfs/src/app/gating_test_env.rs +++ b/rustfs/src/app/gating_test_env.rs @@ -23,6 +23,9 @@ //! `ECStore` and one metadata-sys initialization exist per test binary. use super::storage_api::test::bucket::metadata_sys; +use super::storage_api::test::bucket::quota::BucketQuota; +use super::storage_api::test::bucket::quota::checker::QuotaChecker; +use super::storage_api::test::contract::bucket::MakeBucketOptions; use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions}; use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints}; use super::{context::AppContext, object_traffic_health::ObjectTrafficHealth}; @@ -87,6 +90,17 @@ pub(crate) async fn shared_gating_ecstore() -> Arc { crate::storage::storage_api::new_global_notification_sys(endpoint_pools.clone()) .await .expect("initialize notification system for gating test env"); + let topology_fingerprint = + crate::storage::storage_api::heal_control_startup_consumer::heal_topology_fingerprint(&endpoint_pools) + .expect("single-node gating topology should hash"); + crate::storage::storage_api::start_remote_version_state_fleet_probe(topology_fingerprint); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while crate::storage::storage_api::ecstore_notification::acquire_cross_pool_fence_fleet_proof().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .expect("single-node cross-pool fence capability proof should publish"); let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) @@ -107,6 +121,24 @@ pub(crate) async fn shared_gating_ecstore() -> Arc { ecstore } +pub(crate) async fn durable_quota_test_bucket(prefix: &str, limit: u64) -> (Arc, String) { + let store = shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let bucket = format!("{prefix:.30}-{}", uuid::Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create durable quota test bucket"); + super::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 0).await; + let metadata_sys = + crate::app::storage_api::test::get_global_bucket_metadata_sys().expect("test app context should expose bucket metadata"); + QuotaChecker::new(metadata_sys) + .set_quota_config(&bucket, BucketQuota::new(Some(limit))) + .await + .expect("configure durable quota test bucket"); + (store, bucket) +} + pub(crate) async fn shared_gating_ambient() -> Arc { let store = shared_gating_ecstore().await; if let Some(ambient) = crate::runtime_sources::current_app_context() { diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index cc4d0b798..c97590c4f 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -33,7 +33,7 @@ use super::storage_api::multipart_usecase::contract::multipart::{CompletePart, M use super::storage_api::multipart_usecase::contract::object::{ObjectIO as _, ObjectOperations as _}; use super::storage_api::multipart_usecase::contract::range::HTTPRangeSpec; use super::storage_api::multipart_usecase::data_usage::{ - record_bucket_object_version_write_memory, record_bucket_object_write_memory, + quota_object_size, record_bucket_object_version_write_memory, record_bucket_object_write_memory, }; use super::storage_api::multipart_usecase::error::{StorageError, is_err_object_not_found, is_err_version_not_found}; use super::storage_api::multipart_usecase::helper::OperationHelper; @@ -63,10 +63,10 @@ use super::storage_api::multipart_usecase::{ }; use crate::app::object_data_cache::{ ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success, - invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation, + invalidate_object_data_cache_before_mutation, }; use crate::app::object_usecase::{ - acquire_copy_bucket_lifecycle_locks, build_put_like_object_lock_metadata, map_quota_check_outcome, + acquire_copy_bucket_lifecycle_locks, apply_quota_admission, build_put_like_object_lock_metadata, map_quota_check_outcome, validate_existing_object_lock_for_write, }; use crate::app::runtime_sources::{ @@ -82,6 +82,8 @@ use rustfs_io_metrics::record_s3_op; use rustfs_s3_ops::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; +#[cfg(test)] +use rustfs_utils::http::insert_header; use rustfs_utils::http::{ SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_header, get_source_scheme, @@ -225,12 +227,8 @@ fn internal_object_info_lookup_opts(mut opts: ObjectOptions) -> ObjectOptions { opts } -fn logical_object_size(info: &ObjectInfo) -> Result { - u64::try_from(info.get_actual_size()?).map_err(|_| StorageError::PartMissingOrCorrupt) -} - fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result { - match logical_object_size(info) { + match quota_object_size(info) { Ok(size) => Ok(size), Err(err) if fail_closed => Err(ApiError::from(err).into()), Err(_) => Ok(info.size.max(0) as u64), @@ -506,11 +504,7 @@ impl DefaultMultipartUsecase { Ok(existing_obj_info) => { validate_existing_object_lock_for_write(&existing_obj_info, ¤t_opts)?; let physical_size = existing_obj_info.size.max(0) as u64; - let logical_size = if opts.replication_request { - Ok(physical_size) - } else { - logical_object_size(&existing_obj_info) - }; + let logical_size = quota_object_size(&existing_obj_info); Some((physical_size, logical_size)) } Err(err) => { @@ -559,32 +553,19 @@ impl DefaultMultipartUsecase { }; let quota_metadata_sys = self.bucket_metadata_sys(); + let mut quota_enabled = false; if let Some(metadata_sys) = quota_metadata_sys.as_ref() { let quota_checker = QuotaChecker::new(metadata_sys.clone()); let check_result = map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?; - // Ciphertext-passthrough replication parts use a different size basis and retain - // the existing post-commit accounting path until they carry a trusted logical-size proof. - if !opts.replication_request - && let Some(quota_limit) = check_result.quota_limit - { - let installed = check_result - .current_usage - .is_some_and(|current_usage| opts.set_quota_admission(current_usage, quota_limit)); - if !installed { - return Err(S3Error::with_message( - S3ErrorCode::ServiceUnavailable, - "Bucket quota check temporarily unavailable, please retry".to_string(), - )); - } - } + quota_enabled = check_result.quota_limit.is_some(); + apply_quota_admission(&mut opts, &check_result)?; } let previous_current_size = match previous_current_sizes { - Some((physical_size, _)) if opts.replication_request => Some(physical_size), - Some((_, Ok(logical_size))) => Some(logical_size), - Some((_, Err(err))) if opts.quota_admission.is_some() => return Err(ApiError::from(err).into()), - Some((physical_size, Err(_))) => Some(physical_size), + Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size), + Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err).into()), + Some((physical_size, _)) => Some(physical_size), None => None, }; @@ -596,37 +577,8 @@ impl DefaultMultipartUsecase { let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await; record_capacity_write(Some(capacity_scope_token)).await; - if let Some(metadata_sys) = quota_metadata_sys.as_ref() { - if opts.replication_request { - let quota_checker = QuotaChecker::new(metadata_sys.clone()); - match quota_checker - .check_quota(&bucket, QuotaOperation::PutObject, obj_info.size.max(0) as u64) - .await - { - Ok(check_result) if !check_result.allowed => { - let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await; - let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await; - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!( - "Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes", - check_result.current_usage.unwrap_or(0), - check_result.quota_limit.unwrap_or(0) - ), - )); - } - Err(err) => { - warn!("Quota check failed for bucket {} after multipart completion: {}", bucket, err); - } - Ok(_) => {} - } - } - - let committed_size = if opts.replication_request { - obj_info.size.max(0) as u64 - } else { - quota_accounting_object_size(&obj_info, opts.quota_admission.is_some())? - }; + if quota_metadata_sys.is_some() { + let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?; if versioned { record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { @@ -817,6 +769,20 @@ impl DefaultMultipartUsecase { let ciphertext_passthrough = replication_authorized && get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") && rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some(); + if ciphertext_passthrough && let Some(metadata_sys) = self.bucket_metadata_sys() { + let check_result = map_quota_check_outcome( + &bucket, + QuotaChecker::new(metadata_sys) + .check_quota(&bucket, QuotaOperation::PutObject, 0) + .await, + )?; + if check_result.quota_limit.is_some() { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(), + )); + } + } if ciphertext_passthrough { insert_str(&mut metadata, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, "true".to_string()); } @@ -1650,6 +1616,24 @@ mod tests { assert_eq!(quota_accounting_object_size(&info, true).expect("logical size should resolve"), 8192); assert_eq!(quota_accounting_object_size(&info, false).expect("logical size should resolve"), 8192); + + let mut poisoned_metadata = HashMap::new(); + insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string()); + insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let poisoned = ObjectInfo { + size: 17, + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 4096, + actual_size: 4096, + ..Default::default() + }]), + user_defined: Arc::new(poisoned_metadata), + ..Default::default() + }; + assert_eq!( + quota_accounting_object_size(&poisoned, true).expect("persisted part size must be charged"), + 4096 + ); } #[test] @@ -2058,30 +2042,14 @@ mod tests { #[tokio::test] #[serial_test::serial] async fn compressed_complete_records_logical_quota_usage_and_overwrite_delta() { - use crate::app::storage_api::multipart_usecase::bucket::quota::BucketQuota; - use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; - use crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test; - - let store = crate::app::gating_test_env::shared_gating_ecstore().await; - crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; - - let bucket = format!("compressed-complete-quota-{}", Uuid::new_v4()); + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("compressed-complete-quota", 16_384).await; let object = "object"; - store - .make_bucket(&bucket, &MakeBucketOptions::default()) - .await - .expect("create compressed quota bucket"); - seed_bucket_usage_memory_for_test(&bucket, 0).await; let usecase = DefaultMultipartUsecase::from_global(); let metadata_sys = usecase .bucket_metadata_sys() .expect("test app context should expose bucket metadata"); - let mut quota_checker = QuotaChecker::new(metadata_sys); - quota_checker - .set_quota_config(&bucket, BucketQuota::new(Some(16_384))) - .await - .expect("configure bucket quota"); + let quota_checker = QuotaChecker::new(metadata_sys); for (actual_size, payload_byte) in [(8192_i64, 0x61), (4096_i64, 0x62)] { let mut create_opts = ObjectOptions::default(); @@ -2126,6 +2094,78 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial] + async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() { + let (_store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await; + let usecase = DefaultMultipartUsecase::from_global(); + let input = CreateMultipartUploadInput::builder() + .bucket(bucket) + .key("object".to_string()) + .build() + .expect("create multipart request should build"); + let mut request = build_request(input, Method::POST); + insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + request + .headers + .insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256")); + request.extensions.insert(crate::storage::access::ReqInfo { + replication_request_authorized: true, + ..Default::default() + }); + + let err = usecase + .execute_create_multipart_upload(request) + .await + .expect_err("quota-enabled ciphertext multipart replication should fail before upload creation"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[tokio::test] + #[serial_test::serial] + async fn concurrent_completions_share_durable_bucket_quota_reservations() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-complete-quota", 6000).await; + + let usecase = DefaultMultipartUsecase::from_global(); + + let mut inputs = Vec::new(); + for object in ["first", "second"] { + let upload = store + .new_multipart_upload(&bucket, object, &ObjectOptions::default()) + .await + .expect("create concurrent multipart upload"); + let mut reader = PutObjReader::from_vec(vec![0x71; 4096]); + let part = store + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .expect("stage concurrent multipart part"); + inputs.push( + CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .upload_id(upload.upload_id) + .multipart_upload(Some(CompletedMultipartUpload { + parts: Some(vec![CompletedPart { + part_number: Some(1), + e_tag: part.etag.map(|etag| to_s3s_etag(&etag)), + ..Default::default() + }]), + })) + .build() + .expect("build concurrent completion input"), + ); + } + + let first_usecase = usecase.clone(); + let first = first_usecase.execute_complete_multipart_upload(build_request(inputs.remove(0), Method::POST)); + let second = usecase.execute_complete_multipart_upload(build_request(inputs.remove(0), Method::POST)); + let (first, second) = tokio::join!(first, second); + + assert_eq!(usize::from(first.is_ok()) + usize::from(second.is_ok()), 1); + let denied = first.err().or_else(|| second.err()).expect("one completion must be denied"); + assert_eq!(denied.code(), &S3ErrorCode::InvalidRequest); + } + #[tokio::test] #[serial_test::serial] async fn rejected_empty_parts_preserve_existing_object_and_staging() { diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 8e420e97e..4f665b303 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -24,6 +24,8 @@ use super::storage_api::object_usecase::access::{ has_bypass_governance_header, load_bucket_generation_from_store, recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref, }; +#[cfg(test)] +use super::storage_api::object_usecase::bucket::quota::BucketQuota; use super::storage_api::object_usecase::bucket::quota::checker::QuotaChecker; #[cfg(test)] use super::storage_api::object_usecase::bucket::replication::{ReplicationState, replication_statuses_map}; @@ -63,8 +65,9 @@ use super::storage_api::object_usecase::contract::namespace::NamespaceLocking; use super::storage_api::object_usecase::contract::object::{ObjectIO as _, ObjectOperations as _}; use super::storage_api::object_usecase::contract::range::HTTPRangeSpec; use super::storage_api::object_usecase::data_usage::{ - record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, - record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, + quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, + record_bucket_object_version_write_memory, record_bucket_object_write_memory, + record_bucket_object_write_unknown_previous_memory, }; use super::storage_api::object_usecase::deadlock_detector; use super::storage_api::object_usecase::ecfs::FS; @@ -134,6 +137,8 @@ use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_fo use rustfs_s3select_api::object_store::bytes_stream; use rustfs_targets::{EventName, get_request_host, get_request_port, get_request_user_agent}; use rustfs_utils::CompressionAlgorithm; +#[cfg(test)] +use rustfs_utils::http::insert_header; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, @@ -537,6 +542,43 @@ pub(super) fn map_quota_check_outcome(bucket: &str, outcome: Result S3Result<()> { + let Some(quota_limit) = result.quota_limit else { + return Ok(()); + }; + let Some(current_usage) = result.current_usage else { + return Err(S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "Bucket quota check temporarily unavailable, please retry".to_string(), + )); + }; + if current_usage <= quota_limit { + let _ = opts.set_quota_admission(current_usage, quota_limit); + } + Ok(()) +} + +fn ensure_object_size_within_quota(result: &QuotaCheckResult, new_size: u64) -> S3Result<()> { + let (Some(current_usage), Some(quota_limit)) = (result.current_usage, result.quota_limit) else { + return Ok(()); + }; + if new_size > quota_limit { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"), + )); + } + Ok(()) +} + +fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result { + match quota_object_size(info) { + Ok(size) => Ok(size), + Err(err) if fail_closed => Err(ApiError::from(err).into()), + Err(_) => Ok(info.size.max(0) as u64), + } +} + fn request_uses_aws_chunked(headers: &HeaderMap) -> bool { let has_aws_chunked = |header_name: &str| { headers @@ -3854,13 +3896,6 @@ fn put_object_extract_limits() -> ArchiveLimits { ArchiveLimits::default() } -fn put_object_extract_quota_exceeded(current_usage: u64, quota_limit: u64) -> S3Error { - S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"), - ) -} - fn validate_put_object_extract_entry_count(count: usize, limits: ArchiveLimits) -> S3Result<()> { if count > limits.max_entries { return Err(s3_error!( @@ -4083,12 +4118,12 @@ impl DefaultObjectUsecase { .unwrap_or_else(|| RustFSBufferConfig::default().base_config.default_unknown) } - async fn check_bucket_quota(&self, bucket: &str, op: QuotaOperation, size: u64) -> S3Result<()> { + async fn check_bucket_quota(&self, bucket: &str, op: QuotaOperation, size: u64) -> S3Result> { let Some(metadata_sys) = self.bucket_metadata_sys() else { - return Ok(()); + return Ok(None); }; let quota_checker = QuotaChecker::new(metadata_sys); - map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(|_| ()) + map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(Some) } fn build_memory_bytes_blob( @@ -5593,8 +5628,16 @@ impl DefaultObjectUsecase { // Resolve the authoritative decoded/plain object length (rejecting negative/unknown) before anything else consumes it. let mut size = resolve_put_object_authoritative_size(&req.headers, content_length)?; - // Bucket-quota admission runs exactly once, and only now that the authoritative object length is known. `size` is the same basis the settle phase records via ObjectInfo.size (actual, pre-compression/pre-encryption logical size), NOT the aws-chunked wire Content-Length. When no quota is configured this stays a zero-extra-I/O fast path; once a hard quota is set, checker/config/usage faults fail closed with a retryable error. - self.check_bucket_quota(&bucket, quota_operation, size as u64).await?; + // The app check preserves the existing S3 error contract; the storage + // commit path reserves the exact net logical growth under its locks. + let quota_check = self.check_bucket_quota(&bucket, quota_operation, 0).await?; + let quota_enabled = quota_check.as_ref().is_some_and(|result| result.quota_limit.is_some()); + if quota_enabled && ciphertext_passthrough { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(), + )); + } let ingress_stage_start = std::time::Instant::now(); let should_compress = @@ -5760,6 +5803,9 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; + if let Some(quota_check) = quota_check.as_ref() { + apply_quota_admission(&mut opts, quota_check)?; + } apply_bucket_generation_guard(&req, &bucket, &mut opts)?; apply_put_request_object_lock_opts( &bucket, @@ -5778,7 +5824,7 @@ impl DefaultObjectUsecase { // replication), the lookup is skipped and accounting is backfilled from // the dst xl.meta that rename_data already reads, saving a full-disk // metadata fanout per PUT. - let prelookup_required = version_id.is_some() || object_lock_checks_required(&bucket).await; + let prelookup_required = quota_enabled || version_id.is_some() || object_lock_checks_required(&bucket).await; // Outer None = prelookup skipped (accounting comes from the commit // backfill); Some(inner) = the previous current size as observed by the // lookup, with the pre-#1009 semantics kept bit-for-bit. @@ -5795,7 +5841,11 @@ impl DefaultObjectUsecase { Some(match previous_current_info { Ok(existing_obj_info) => { validate_existing_object_lock_for_write(&existing_obj_info, &opts)?; - Some(existing_obj_info.size.max(0) as u64) + Some(if quota_enabled { + quota_object_size(&existing_obj_info).map_err(ApiError::from)? + } else { + existing_obj_info.size.max(0) as u64 + }) } Err(err) => { if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { @@ -5809,6 +5859,12 @@ impl DefaultObjectUsecase { }; let actual_size = size; + if !ciphertext_passthrough && let Some(quota_check) = quota_check.as_ref() { + ensure_object_size_within_quota( + quota_check, + u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + )?; + } let mut md5hex = if let Some(base64_md5) = content_md5 { let md5 = base64_simd::STANDARD @@ -6077,12 +6133,13 @@ impl DefaultObjectUsecase { // backfill reproduces the lookup's observation bit for bit (latest // version's ObjectInfo.size — 0 for a delete-marker latest — or // not-found → None). + let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?; match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size)) { Some(previous_current_size) => { if put_versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; + record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { - record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; + record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; } } None => { @@ -6098,7 +6155,7 @@ impl DefaultObjectUsecase { put_versioned, "put_object old-size backfill unknown; recording degraded usage delta" ); - record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, put_versioned).await; + record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, put_versioned).await; } } @@ -7172,7 +7229,7 @@ impl DefaultObjectUsecase { if _self_copy_lock_guard.is_some() { current_opts.no_lock = true; } - let previous_current_size = match store.get_object_info(&bucket, &key, ¤t_opts).await { + let previous_current_sizes = match store.get_object_info(&bucket, &key, ¤t_opts).await { Ok(existing_obj_info) => { validate_existing_object_lock_for_write(&existing_obj_info, &dst_opts)?; if let Some(expected) = expected_current_version_id.as_deref() @@ -7180,7 +7237,7 @@ impl DefaultObjectUsecase { { return Err(s3_error!(PreconditionFailed)); } - Some(existing_obj_info.size.max(0) as u64) + Some((existing_obj_info.size.max(0) as u64, quota_object_size(&existing_obj_info))) } Err(err) => { if expected_current_version_id.is_some() { @@ -7501,8 +7558,23 @@ impl DefaultObjectUsecase { src_info.user_defined = Arc::new(user_defined); - self.check_bucket_quota(&bucket, QuotaOperation::CopyObject, src_info.size as u64) - .await?; + let quota_check = self.check_bucket_quota(&bucket, QuotaOperation::CopyObject, 0).await?; + let quota_enabled = quota_check.as_ref().is_some_and(|result| result.quota_limit.is_some()); + if let Some(quota_check) = quota_check.as_ref() { + apply_quota_admission(&mut dst_opts, quota_check)?; + } + let previous_current_size = match previous_current_sizes { + Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size), + Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err).into()), + Some((physical_size, _)) => Some(physical_size), + None => None, + }; + if let Some(quota_check) = quota_check.as_ref() { + ensure_object_size_within_quota( + quota_check, + u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + )?; + } let has_bucket_metadata = self.bucket_metadata_sys().is_some(); let cache_adapter = self.object_data_cache(); let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; @@ -7526,10 +7598,11 @@ impl DefaultObjectUsecase { let dest_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; // Update quota tracking after successful copy if has_bucket_metadata { + let committed_size = quota_accounting_object_size(&oi, quota_enabled)?; if dest_versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; + record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { - record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; + record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; } } @@ -9117,8 +9190,7 @@ impl DefaultObjectUsecase { } validate_object_key(&key, "PUT")?; validate_table_catalog_object_mutation(&bucket, &key).await?; - self.check_bucket_quota(&bucket, QuotaOperation::PutObject, size as u64) - .await?; + let _ = self.check_bucket_quota(&bucket, QuotaOperation::PutObject, 0).await?; // Apply adaptive buffer sizing based on file size for optimal streaming performance. // Uses workload profile configuration (enabled by default) to select appropriate buffer size. @@ -9174,14 +9246,17 @@ impl DefaultObjectUsecase { let extract_options = resolve_put_object_extract_options(&req.headers)?; let extract_limits = put_object_extract_limits(); - let extract_quota_snapshot = if let Some(metadata_sys) = self.bucket_metadata_sys() { + let extract_quota_check = if let Some(metadata_sys) = self.bucket_metadata_sys() { let quota_checker = QuotaChecker::new(metadata_sys); let check_result = map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?; - check_result.current_usage.zip(check_result.quota_limit) + Some(check_result) } else { None }; + let extract_quota_enabled = extract_quota_check + .as_ref() + .is_some_and(|result| result.quota_limit.is_some()); let version_id = match event_version_id { Some(v) => v.to_string(), None => String::new(), @@ -9261,11 +9336,6 @@ impl DefaultObjectUsecase { .checked_add(entry_size) .ok_or_else(|| s3_error!(InvalidArgument, "Archive total unpacked size overflowed while processing entries"))?; validate_put_object_extract_total_size(total_unpacked_size, extract_limits)?; - if let Some((current_usage, quota_limit)) = extract_quota_snapshot - && current_usage.saturating_add(total_unpacked_size) > quota_limit - { - return Err(put_object_extract_quota_exceeded(current_usage, quota_limit)); - } let mut size = i64::try_from(entry_size).map_err(|_| s3_error!(InvalidArgument, "Archive entry size does not fit into i64"))?; // mtime 0 means "unset" in tar headers, and xl.meta cannot represent an @@ -9309,6 +9379,9 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; + if let Some(quota_check) = extract_quota_check.as_ref() { + apply_quota_admission(&mut opts, quota_check)?; + } opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id; opts.object_lock_config_snapshot = Some(Arc::clone(&object_lock_config_snapshot)); let pax_authorization = @@ -9452,19 +9525,18 @@ impl DefaultObjectUsecase { return Err(ApiError::from(e).into()); } }; + let committed_size = quota_accounting_object_size(&obj_info, extract_quota_enabled)?; let extract_versioned = BucketVersioningSys::prefix_enabled(&bucket, &fpath).await; match previous_current_size_from_backfill(backfilled_old_current_size) { Some(previous_current_size) => { if extract_versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64) - .await; + record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { - record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; + record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; } } None => { - record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, extract_versioned) - .await; + record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, extract_versioned).await; } } let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await; @@ -15611,13 +15683,6 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); } - #[test] - fn put_object_extract_quota_exceeded_matches_existing_error_shape() { - let err = put_object_extract_quota_exceeded(10, 8); - assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); - assert_eq!(err.message(), Some("Bucket quota exceeded. Current usage: 10 bytes, limit: 8 bytes")); - } - #[tokio::test] async fn execute_put_object_rejects_post_object_sse_kms_from_input() { let input = PutObjectInput::builder() @@ -17343,6 +17408,44 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial] + async fn quota_rejects_ciphertext_replication_before_polling_the_body() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let (_store, bucket) = + crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-replication-early-reject", 4096).await; + let body_polled = Arc::new(AtomicBool::new(false)); + let body_polled_in_stream = Arc::clone(&body_polled); + let body = StreamingBlob::wrap(futures::stream::once(async move { + body_polled_in_stream.store(true, Ordering::Release); + Ok::(Bytes::from_static(b"ciphertext")) + })); + let input = PutObjectInput::builder() + .bucket(bucket) + .key("object".to_string()) + .body(Some(body)) + .content_length(Some(10)) + .build() + .expect("ciphertext replication PUT input should build"); + let mut request = build_request(input, Method::PUT); + insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + request + .headers + .insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256")); + request.extensions.insert(crate::storage::access::ReqInfo { + replication_request_authorized: true, + ..Default::default() + }); + + let err = DefaultObjectUsecase::from_global() + .execute_put_object(&FS::new(), request) + .await + .expect_err("quota-enabled ciphertext replication should fail at ingress"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(!body_polled.load(Ordering::Acquire), "rejected ciphertext body must not be consumed"); + } + #[test] fn quota_admission_allows_within_limit() { let result = map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write"); @@ -17353,6 +17456,340 @@ mod tests { assert_eq!(result.remaining, Some(512)); } + #[tokio::test] + #[serial_test::serial] + async fn concurrent_puts_share_durable_bucket_quota_reservations() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-put-quota", 6000).await; + + let first_opts = ObjectOptions::default(); + let second_opts = ObjectOptions::default(); + let first_store = Arc::clone(&store); + let first_bucket = bucket.clone(); + let first = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x73; 4096]); + first_store.put_object(&first_bucket, "first", &mut reader, &first_opts).await + }); + let second = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x74; 4096]); + store.put_object(&bucket, "second", &mut reader, &second_opts).await + }); + let (first, second) = tokio::join!(first, second); + let first = first.expect("first PUT task should not panic"); + let second = second.expect("second PUT task should not panic"); + + assert_eq!(usize::from(first.is_ok()) + usize::from(second.is_ok()), 1); + let denied = first.err().or_else(|| second.err()).expect("one PUT must be denied"); + assert!(matches!( + denied, + StorageError::QuotaExceeded { + current: 4096, + limit: 6000 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn concurrent_within_limit_puts_keep_independent_mutation_fences() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-fence-quota", 8192).await; + let first_barrier = PutObjectCommitBarrier::install(&bucket, "first", PutObjectCommitPause::BeforeQuotaRename); + let second_barrier = PutObjectCommitBarrier::install(&bucket, "second", PutObjectCommitPause::BeforeQuotaRename); + + let first_store = Arc::clone(&store); + let first_bucket = bucket.clone(); + let first = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x75; 4096]); + first_store + .put_object(&first_bucket, "first", &mut reader, &ObjectOptions::default()) + .await + }); + let second_store = Arc::clone(&store); + let second_bucket = bucket.clone(); + let second = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x76; 4096]); + second_store + .put_object(&second_bucket, "second", &mut reader, &ObjectOptions::default()) + .await + }); + + first_barrier.wait_until_paused().await; + second_barrier.wait_until_paused().await; + first_barrier.release(); + second_barrier.release(); + + first + .await + .expect("first PUT task should not panic") + .expect("first within-limit PUT should commit"); + second + .await + .expect("second PUT task should not panic") + .expect("second within-limit PUT should commit"); + } + + #[tokio::test] + #[serial_test::serial] + async fn durable_quota_reclaims_overwrites_and_deleted_bytes() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("quota-delta-reconcile", 4096).await; + + for byte in [0x41, 0x42] { + let mut reader = PutObjReader::from_vec(vec![byte; 4096]); + store + .put_object(&bucket, "object", &mut reader, &ObjectOptions::default()) + .await + .expect("same-size overwrite must consume no additional quota"); + } + + store + .delete_object(&bucket, "object", ObjectOptions::default()) + .await + .expect("delete quota-tracked object"); + let mut replacement = PutObjReader::from_vec(vec![0x43; 4096]); + store + .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) + .await + .expect("deleted bytes must be reclaimed before rejecting a replacement"); + + let mut excess = PutObjReader::from_vec(vec![0x44]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("one byte beyond the reclaimed exact quota must be denied"); + assert!(matches!( + err, + StorageError::QuotaExceeded { + current: 4096, + limit: 4096 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn cancelled_put_releases_durable_quota_reservation() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("cancelled-put-quota", 4096).await; + + let barrier = PutObjectCommitBarrier::install(&bucket, "cancelled", PutObjectCommitPause::AfterQuotaReservation); + let cancelled_store = Arc::clone(&store); + let cancelled_bucket = bucket.clone(); + let cancelled = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x51; 4096]); + cancelled_store + .put_object(&cancelled_bucket, "cancelled", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + cancelled.abort(); + let cancelled_result = cancelled.await; + assert!(cancelled_result.is_err(), "the paused request must be cancelled"); + drop(barrier); + + let mut replacement = PutObjReader::from_vec(vec![0x52; 4096]); + store + .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) + .await + .expect("cancelling before commit must release the complete reservation"); + } + + #[tokio::test] + #[serial_test::serial] + async fn cancelled_put_after_commit_marker_is_reconciled() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("cancelled-spawned-put-quota", 4096).await; + let commit_barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename); + let first_store = Arc::clone(&store); + let first_bucket = bucket.clone(); + let first = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x53; 4096]); + first_store + .put_object(&first_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + commit_barrier.wait_until_paused().await; + first.abort(); + assert!(first.await.is_err(), "the outer request task must be cancelled"); + drop(commit_barrier); + + store + .get_object_info(&bucket, "object", &ObjectOptions::default()) + .await + .expect_err("cancelling before rename must not commit the object"); + let mut replacement = PutObjReader::from_vec(vec![0x54; 4096]); + store + .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) + .await + .expect("the next admission must reap the abandoned commit marker"); + } + + #[tokio::test] + #[serial_test::serial] + async fn committed_put_survives_quota_ledger_settlement_failure() { + use crate::app::storage_api::test::set_disk::{ + PutObjectCommitBarrier, PutObjectCommitPause, fail_next_quota_ledger_save_for_test, + }; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("settlement-failure-quota", 4096).await; + let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename); + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x59; 4096]); + put_store + .put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + fail_next_quota_ledger_save_for_test(); + barrier.release(); + put.await + .expect("PUT task should not panic") + .expect("a post-commit ledger failure must not change the successful write result"); + let stored = store + .get_object_info(&bucket, "object", &ObjectOptions::default()) + .await + .expect("the committed object must remain visible"); + assert_eq!(stored.size, 4096); + } + + #[tokio::test] + #[serial_test::serial] + async fn suspended_null_version_overwrite_uses_exact_quota_delta() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("suspended-version-quota", 6200).await; + let mut versioned_reader = PutObjReader::from_vec(vec![0x61; 4096]); + store + .put_object( + &bucket, + "object", + &mut versioned_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("write UUID version"); + + for (size, byte) in [(1024, 0x62), (2048, 0x63)] { + let mut reader = PutObjReader::from_vec(vec![byte; size]); + store + .put_object( + &bucket, + "object", + &mut reader, + &ObjectOptions { + version_suspended: true, + ..Default::default() + }, + ) + .await + .expect("suspended write should replace only the exact null version"); + } + + let mut excess = PutObjReader::from_vec(vec![0x64; 57]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("UUID plus replacement null version must consume 6144 bytes"); + assert!(matches!( + err, + StorageError::QuotaExceeded { + current: 6144, + limit: 6200 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn durable_quota_reservation_observes_lowered_config_revision() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("lowered-quota-revision", 8192).await; + let mut initial = PutObjReader::from_vec(vec![0x71; 4096]); + store + .put_object(&bucket, "initial", &mut initial, &ObjectOptions::default()) + .await + .expect("write under original quota"); + + let metadata_sys = DefaultObjectUsecase::from_global() + .bucket_metadata_sys() + .expect("test app context should expose bucket metadata"); + QuotaChecker::new(metadata_sys) + .set_quota_config(&bucket, BucketQuota::new(Some(4096))) + .await + .expect("lower bucket quota"); + let mut excess = PutObjReader::from_vec(vec![0x72]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("reservation must not use the stale larger quota revision"); + assert!(matches!( + err, + StorageError::QuotaExceeded { + current: 4096, + limit: 4096 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn quota_enable_waits_for_unlimited_commit() { + use crate::app::storage_api::test::metadata_sys::ConfigWriteLockProbe; + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("quota-config-fence", 8192).await; + let metadata_sys = DefaultObjectUsecase::from_global() + .bucket_metadata_sys() + .expect("test app context should expose bucket metadata"); + QuotaChecker::new(Arc::clone(&metadata_sys)) + .set_quota_config(&bucket, BucketQuota::new(None)) + .await + .expect("clear quota before the fenced write"); + let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::AfterQuotaReservation); + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x73; 4096]); + put_store + .put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + + let update_probe = ConfigWriteLockProbe::install(&bucket); + let update_bucket = bucket.clone(); + let update = tokio::spawn(async move { + QuotaChecker::new(metadata_sys) + .set_quota_config(&update_bucket, BucketQuota::new(Some(0))) + .await + }); + update_probe.wait_until_attempted().await; + assert!( + !update.is_finished(), + "quota mutation must wait for the reservation's metadata transaction guard" + ); + + barrier.release(); + put.await + .expect("PUT task should not panic") + .expect("the write linearized before the quota update must commit"); + update + .await + .expect("quota update task should not panic") + .expect("quota update should proceed after commit"); + + let mut excess = PutObjReader::from_vec(vec![0x74]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("writes after the zero-byte quota update must be denied"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 4096, limit: 0 })); + } + #[test] fn quota_admission_rejects_over_limit() { let err = map_quota_check_outcome("bucket", Ok(quota_result(false))).expect_err("an over-limit result rejects the write"); @@ -17372,6 +17809,23 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); } + #[test] + fn early_quota_filter_rejects_only_an_individually_impossible_object() { + let stale_full_usage = QuotaCheckResult { + allowed: true, + current_usage: Some(4096), + quota_limit: Some(4096), + operation_size: 0, + remaining: Some(0), + }; + + ensure_object_size_within_quota(&stale_full_usage, 4096) + .expect("commit-time ledger must decide whether stale usage was reclaimed"); + let err = ensure_object_size_within_quota(&stale_full_usage, 4097) + .expect_err("an object larger than the whole quota can never fit"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + #[test] fn quota_admission_fails_closed_on_unknown_authoritative_usage() { let err = map_quota_check_outcome( diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index cfb5c5493..e0bef7005 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -47,6 +47,8 @@ pub(crate) mod capacity { pub(crate) mod data_usage { use std::sync::Arc; + pub(crate) use crate::storage::storage_api::ecstore_data_usage::quota_object_size; + pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) { crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await; } @@ -1212,4 +1214,13 @@ pub(crate) mod test { pub(crate) use crate::storage::storage_api::{ ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, }; + pub(crate) mod set_disk { + pub(crate) use crate::storage::storage_api::ecstore_set_disk::{ + PutObjectCommitBarrier, PutObjectCommitPause, fail_next_quota_ledger_save_for_test, + }; + } + + pub(crate) mod metadata_sys { + pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::ConfigWriteLockProbe; + } } diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 49ac36319..4eb6621c6 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -398,6 +398,7 @@ impl From for ApiError { QuotaError::ConfigNotFound { .. } => S3ErrorCode::NoSuchBucket, QuotaError::UsageUnavailable { .. } => S3ErrorCode::ServiceUnavailable, QuotaError::InvalidConfig { .. } => S3ErrorCode::InvalidArgument, + QuotaError::StorageError(StorageError::NamespaceLockQuorumUnavailable { .. }) => S3ErrorCode::ServiceUnavailable, QuotaError::StorageError(_) => S3ErrorCode::InternalError, }; @@ -551,6 +552,20 @@ mod tests { assert_eq!(api_error.message, "The service is unavailable. Please retry."); } + #[test] + fn stale_quota_capability_maps_to_retryable_error() { + let api_error = ApiError::from(QuotaError::StorageError(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_capability", + bucket: "bucket".to_string(), + object: rustfs_config::QUOTA_CONFIG_FILE.to_string(), + required: 1, + achieved: 0, + })); + + assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable); + assert_eq!(api_error.message, "The service is unavailable. Please retry."); + } + #[test] fn test_kms_cryptographic_error_is_not_retryable() { let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::cryptographic_error( diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 131f3ce1f..9043908d2 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -153,9 +153,7 @@ fn remove_heal_control_replay( static HEAL_CONTROL_REPLAY_CACHE: OnceLock>>> = OnceLock::new(); static NODE_CAPABILITY_SERVER_EPOCH: LazyLock = LazyLock::new(Uuid::new_v4); -// RUSTFS_COMPAT_TODO(cross-pool-fence-v1): advertise unsupported during predeployment. Remove after composite acquisition, -// activation fencing, fleet proof, commit-time proof revalidation, and fail-closed revocation ship together. -const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 0; +const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1; fn admit_heal_control_replay( replay_cache: &mut HashMap>, @@ -2207,7 +2205,7 @@ mod tests { use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo, HealEndpoint}; use crate::storage::storage_api::set_tonic_canonical_body_digest; use crate::storage::storage_api::{ - Endpoint, + Endpoint, RUSTFS_META_BUCKET, SnapshotLeaseToken, ecstore_layout::{EndpointServerPools, Endpoints, PoolEndpoints}, }; use bytes::Bytes; @@ -2945,14 +2943,22 @@ mod tests { } #[tokio::test] - async fn snapshot_lease_acquire_and_renew_handlers_fail_closed() { + #[serial_test::serial] + async fn snapshot_lease_handlers_forward_to_local_disk() { + let temp_dir = tempfile::tempdir().expect("snapshot lease RPC test directory"); + let env = rustfs_test_utils::TestECStoreEnv::builder() + .base_dir(temp_dir.path()) + .init_bucket_metadata(false) + .build() + .await; let service = make_server(); - let disk = "http://node-a:9000/data/rustfs0".to_string(); + let disk = env.disk_paths[0].to_string_lossy().into_owned(); + let fence_path = format!("tmp/quota-mutation-fences/{}", "0".repeat(64)); let mut acquire = Request::new(SnapshotLeaseRequest { disk: disk.clone(), - volume: "v".into(), - path: "p".into(), + volume: RUSTFS_META_BUCKET.into(), + path: fence_path.clone(), ttl_ms: 60_000, }); let acquire_body = @@ -2962,14 +2968,18 @@ mod tests { let acquire = service .acquire_snapshot_lease(acquire) .await - .expect("disabled acquire should return a protocol response") + .expect("acquire should return a protocol response") .into_inner(); + assert!(acquire.success, "local disk should acquire the mutation fence"); + assert_eq!(acquire.protocol_version, 1); + assert!(acquire.error.is_none()); + let token = SnapshotLeaseToken::from_slice(&acquire.token).expect("acquire should return a valid token"); let mut renew = Request::new(SnapshotLeaseRenewRequest { - disk, - volume: "v".into(), - path: "p".into(), - token: vec![1; 16].into(), + disk: disk.clone(), + volume: RUSTFS_META_BUCKET.into(), + path: fence_path.clone(), + token: token.as_bytes().to_vec().into(), ttl_ms: 60_000, }); let renew_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(renew.get_ref()) @@ -2979,15 +2989,103 @@ mod tests { let renew = service .renew_snapshot_lease(renew) .await - .expect("disabled renew should return a protocol response") + .expect("renew should return a protocol response") .into_inner(); + assert!(renew.success, "local disk should renew the mutation fence"); + assert_eq!(renew.protocol_version, 1); + assert!(renew.error.is_none()); + let renewed = SnapshotLeaseToken::from_slice(&renew.token).expect("renew should return a valid token"); + assert_ne!(renewed, token); - for response in [acquire, renew] { - assert!(!response.success); - assert!(response.token.is_empty()); - assert_eq!(response.protocol_version, 1); - assert_eq!(response.error, Some(DiskError::UnsupportedDisk.into())); - } + let mut release = Request::new(SnapshotLeaseReleaseRequest { + disk: disk.clone(), + volume: RUSTFS_META_BUCKET.into(), + path: fence_path.clone(), + token: renewed.as_bytes().to_vec().into(), + }); + let release_body = rustfs_protos::canonical_snapshot_lease_release_request_body(release.get_ref()) + .expect("release request body should encode"); + set_tonic_canonical_body_digest(&mut release, &release_body).expect("release digest metadata should encode"); + mark_v2_authenticated(&mut release); + let release = service + .release_snapshot_lease(release) + .await + .expect("release should return a protocol response") + .into_inner(); + assert!(release.success, "local disk should release the renewed token"); + assert!(release.error.is_none()); + + let mut acquire = Request::new(SnapshotLeaseRequest { + disk: disk.clone(), + volume: RUSTFS_META_BUCKET.into(), + path: fence_path.clone(), + ttl_ms: 60_000, + }); + let acquire_body = + rustfs_protos::canonical_snapshot_lease_request_body(acquire.get_ref()).expect("acquire request body should encode"); + set_tonic_canonical_body_digest(&mut acquire, &acquire_body).expect("acquire digest metadata should encode"); + mark_v2_authenticated(&mut acquire); + let active_token = service + .acquire_snapshot_lease(acquire) + .await + .expect("second acquire should return a protocol response") + .into_inner(); + assert!(active_token.success); + + let mut revoke = Request::new(SnapshotLeaseReleaseRequest { + disk: disk.clone(), + volume: RUSTFS_META_BUCKET.into(), + path: fence_path.clone(), + token: SnapshotLeaseToken::revoke_all().as_bytes().to_vec().into(), + }); + let revoke_body = rustfs_protos::canonical_snapshot_lease_release_request_body(revoke.get_ref()) + .expect("revoke-all request body should encode"); + set_tonic_canonical_body_digest(&mut revoke, &revoke_body).expect("revoke-all digest metadata should encode"); + mark_v2_authenticated(&mut revoke); + let revoke = service + .release_snapshot_lease(revoke) + .await + .expect("revoke-all should return a protocol response") + .into_inner(); + assert!(revoke.success, "nil revoke-all sentinel must reach the local disk"); + assert!(revoke.error.is_none()); + + let mut stale_renew = Request::new(SnapshotLeaseRenewRequest { + disk: disk.clone(), + volume: RUSTFS_META_BUCKET.into(), + path: fence_path.clone(), + token: active_token.token, + ttl_ms: 60_000, + }); + let stale_renew_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(stale_renew.get_ref()) + .expect("stale renew request body should encode"); + set_tonic_canonical_body_digest(&mut stale_renew, &stale_renew_body).expect("stale renew digest metadata should encode"); + mark_v2_authenticated(&mut stale_renew); + let stale_renew = service + .renew_snapshot_lease(stale_renew) + .await + .expect("stale renew should return a protocol response") + .into_inner(); + assert!(!stale_renew.success, "revoke-all must invalidate active tokens"); + assert!(stale_renew.token.is_empty()); + assert!(stale_renew.error.is_some()); + + let mut malformed = Request::new(SnapshotLeaseRenewRequest { + disk, + volume: RUSTFS_META_BUCKET.into(), + path: fence_path, + token: vec![1; 15].into(), + ttl_ms: 60_000, + }); + let malformed_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(malformed.get_ref()) + .expect("malformed renew request body should encode"); + set_tonic_canonical_body_digest(&mut malformed, &malformed_body).expect("malformed renew digest metadata should encode"); + mark_v2_authenticated(&mut malformed); + let malformed = service + .renew_snapshot_lease(malformed) + .await + .expect_err("a malformed non-nil token must be rejected"); + assert_eq!(malformed.code(), tonic::Code::InvalidArgument); } #[tokio::test] @@ -3342,7 +3440,7 @@ mod tests { } #[tokio::test] - async fn cross_pool_fence_probe_authenticates_unsupported_rollout_state() { + async fn cross_pool_fence_probe_authenticates_supported_v1_state() { let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string()); let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true); assert!( @@ -3407,7 +3505,7 @@ mod tests { assert!(response.success); assert_eq!(response.error_info, None); - assert_eq!(&response.result[..4], &0_u32.to_be_bytes()); + assert_eq!(&response.result[..4], &super::CROSS_POOL_FENCE_SUPPORTED_VERSION.to_be_bytes()); let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..]) .expect("capability identity should decode"); assert_eq!(topology_member, "node-a:9000"); diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index 0e0f2defe..d1dc1c8a1 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -37,19 +37,28 @@ const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512; const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024; const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1; +fn snapshot_lease_response(result: Result) -> Response { + match result { + Ok(token) => Response::new(SnapshotLeaseResponse { + success: true, + token: token.as_bytes().to_vec().into(), + protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION, + error: None, + }), + Err(err) => Response::new(SnapshotLeaseResponse { + success: false, + token: Bytes::new(), + protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION, + error: Some(err.into()), + }), + } +} + struct DecodedRpcPayload { value: T, from_msgpack: bool, } -fn snapshot_lease_disabled_response() -> SnapshotLeaseResponse { - SnapshotLeaseResponse { - success: false, - token: Bytes::new(), - protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION, - error: Some(DiskError::UnsupportedDisk.into()), - } -} fn decode_msgpack_or_json( binary: &[u8], json: &str, @@ -241,7 +250,12 @@ impl NodeService { rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref()), "acquire_snapshot_lease", )?; - Ok(Response::new(snapshot_lease_disabled_response())) + let request = request.into_inner(); + let result = match self.find_disk(&request.disk).await { + Some(disk) => disk.acquire_snapshot_lease(&request.volume, &request.path).await, + None => Err(DiskError::other("cannot find disk")), + }; + Ok(snapshot_lease_response(result)) } pub(super) async fn handle_renew_snapshot_lease( @@ -253,7 +267,14 @@ impl NodeService { rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref()), "renew_snapshot_lease", )?; - Ok(Response::new(snapshot_lease_disabled_response())) + let request = request.into_inner(); + let token = + SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?; + let result = match self.find_disk(&request.disk).await { + Some(disk) => disk.renew_snapshot_lease(&request.volume, &request.path, token).await, + None => Err(DiskError::other("cannot find disk")), + }; + Ok(snapshot_lease_response(result)) } pub(super) async fn handle_release_snapshot_lease( @@ -266,8 +287,11 @@ impl NodeService { "release_snapshot_lease", )?; let request = request.into_inner(); - let token = - SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?; + let token = if request.token.as_ref() == SnapshotLeaseToken::revoke_all().as_bytes() { + SnapshotLeaseToken::revoke_all() + } else { + SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))? + }; let Some(disk) = self.find_disk(&request.disk).await else { return Ok(Response::new(SnapshotLeaseMutationResponse { success: false, @@ -1494,7 +1518,7 @@ mod tests { use super::{ compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, encode_batch_read_version_response_payloads, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, - encode_read_multiple_response_payloads, encode_rename_data_response_payloads, snapshot_lease_disabled_response, + encode_read_multiple_response_payloads, encode_rename_data_response_payloads, }; use crate::storage::storage_api::ReadMultipleResp; use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp; @@ -1509,17 +1533,6 @@ mod tests { count: u32, } - #[test] - fn snapshot_lease_acquire_and_renew_fail_closed() { - let response = snapshot_lease_disabled_response(); - let expected_error = DiskError::UnsupportedDisk.into(); - - assert!(!response.success); - assert!(response.token.is_empty()); - assert_eq!(response.protocol_version, 1); - assert_eq!(response.error, Some(expected_error)); - } - #[test] fn decode_msgpack_or_json_prefers_binary_payload() { let payload = SamplePayload { diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index d5dfc3af5..d7453f5c5 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -428,7 +428,7 @@ pub(crate) mod ecstore_config { pub(crate) mod ecstore_data_usage { pub(crate) use rustfs_ecstore::api::data_usage::{ apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached, - load_data_usage_from_backend, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, + load_data_usage_from_backend, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend, }; @@ -486,7 +486,9 @@ pub(crate) mod ecstore_metrics { #[allow(unused_imports)] pub(crate) mod ecstore_notification { pub(crate) use rustfs_ecstore::api::notification::{ - NotificationSys, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe, + CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof, + cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys, + start_remote_version_state_fleet_probe, }; } @@ -544,6 +546,10 @@ pub(crate) mod ecstore_test_support { } pub(crate) mod ecstore_set_disk { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::set_disk::test_util::{ + PutObjectCommitBarrier, PutObjectCommitPause, fail_next_quota_ledger_save_for_test, + }; pub(crate) use rustfs_ecstore::api::set_disk::{ DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class, }; @@ -1128,7 +1134,9 @@ pub(crate) trait StorageDiskRpcExt { ) -> DiskResult<()>; async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult; async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>; + async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult; async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()>; + async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult; async fn stat_volume(&self, volume: &str) -> DiskResult; async fn list_volumes(&self) -> DiskResult>; async fn make_volume(&self, volume: &str) -> DiskResult<()>; @@ -1256,10 +1264,18 @@ where ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await } + async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult { + ecstore_disk::DiskAPI::acquire_snapshot_lease(self, volume, path).await + } + async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()> { ecstore_disk::DiskAPI::release_snapshot_lease(self, volume, path, token).await } + async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult { + ecstore_disk::DiskAPI::renew_snapshot_lease(self, volume, path, token).await + } + async fn stat_volume(&self, volume: &str) -> DiskResult { ecstore_disk::DiskAPI::stat_volume(self, volume).await } diff --git a/scripts/check_s3s_footprint.sh b/scripts/check_s3s_footprint.sh index 30cdcb3f0..56fb60d35 100755 --- a/scripts/check_s3s_footprint.sh +++ b/scripts/check_s3s_footprint.sh @@ -26,7 +26,7 @@ cd "$(dirname "$0")/.." # Excludes crates/e2e_test/ — test infrastructure legitimately uses s3s # to verify S3 behavior and does not widen the production s3s surface. S3S_IMPORT_FILES_BASELINE=213 -S3_ERROR_LINES_BASELINE=1621 +S3_ERROR_LINES_BASELINE=1620 S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::' E2E_TEST_GLOB='--glob=!crates/e2e_test/**'