diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 4b3162313..ffc175659 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -131,6 +131,8 @@ pub mod bucket { } pub mod metadata_sys { + #[cfg(feature = "test-util")] + pub use crate::bucket::metadata_sys::ConfigWriteLockProbe; pub use crate::bucket::metadata_sys::{ BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy, @@ -140,7 +142,7 @@ pub mod bucket { get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, - update_under_transaction_lock, + update_quota_if_incarnation, update_under_transaction_lock, }; } @@ -316,7 +318,7 @@ pub mod data_usage { DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage, init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached, - load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, + load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory, refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer, @@ -403,8 +405,11 @@ pub mod metrics { } pub mod notification { + #[cfg(any(test, feature = "test-util"))] + pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test; pub use crate::services::notification_sys::{ - NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys, + CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof, + cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe, }; } @@ -464,7 +469,8 @@ pub mod set_disk { #[cfg(feature = "test-util")] pub mod test_util { - pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test; + pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause}; } } diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 59f5aae37..40aa0bd77 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -50,6 +50,72 @@ use uuid::Uuid; const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60); +#[cfg(any(test, feature = "test-util"))] +struct ConfigWriteLockProbeState { + bucket: String, + arrived: tokio::sync::Notify, +} + +#[cfg(any(test, feature = "test-util"))] +static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock>>> = std::sync::OnceLock::new(); + +#[cfg(any(test, feature = "test-util"))] +pub struct ConfigWriteLockProbe { + state: Arc, +} + +#[cfg(any(test, feature = "test-util"))] +impl ConfigWriteLockProbe { + pub fn install(bucket: &str) -> Self { + let state = Arc::new(ConfigWriteLockProbeState { + bucket: bucket.to_string(), + arrived: tokio::sync::Notify::new(), + }); + let mut probes = CONFIG_WRITE_LOCK_PROBES + .get_or_init(|| StdMutex::new(Vec::new())) + .lock() + .expect("config write lock probe mutex should not poison"); + assert!( + !probes.iter().any(|current| current.bucket == state.bucket), + "config write lock probe must be unique for a bucket" + ); + probes.push(Arc::clone(&state)); + drop(probes); + Self { state } + } + + pub async fn wait_until_attempted(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("bucket config update should attempt the transaction lock"); + } +} + +#[cfg(any(test, feature = "test-util"))] +impl Drop for ConfigWriteLockProbe { + fn drop(&mut self) { + let mut probes = CONFIG_WRITE_LOCK_PROBES + .get_or_init(|| StdMutex::new(Vec::new())) + .lock() + .expect("config write lock probe mutex should not poison"); + probes.retain(|state| !Arc::ptr_eq(state, &self.state)); + } +} + +#[cfg(any(test, feature = "test-util"))] +fn notify_config_write_lock_attempt(bucket: &str) { + let probe = CONFIG_WRITE_LOCK_PROBES + .get_or_init(|| StdMutex::new(Vec::new())) + .lock() + .expect("config write lock probe mutex should not poison") + .iter() + .find(|probe| probe.bucket == bucket) + .cloned(); + if let Some(probe) = probe { + probe.arrived.notify_one(); + } +} + #[derive(Clone, Copy)] enum MetadataLoadMode { Initial, @@ -590,6 +656,31 @@ pub async fn update_under_transaction_lock( update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await } +pub async fn update_quota_if_incarnation( + bucket: &str, + data: Vec, + expected_incarnation_id: Uuid, + proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken, +) -> Result { + let sys = get_bucket_metadata_sys()?; + let guard = Box::pin(acquire_config_write_guard_for_incarnation( + sys.clone(), + bucket, + Some(expected_incarnation_id), + )) + .await?; + if !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof) { + return Err(Error::NamespaceLockQuorumUnavailable { + mode: "quota_capability", + bucket: bucket.to_string(), + object: rustfs_config::QUOTA_CONFIG_FILE.to_string(), + required: 1, + achieved: 0, + }); + } + update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await +} + pub async fn update_bucket_targets_under_transaction_lock( guard: &BucketMetadataMutationGuard, bucket: &str, @@ -734,7 +825,26 @@ async fn acquire_transaction_lock_with_sys( let lock = api .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket)) .await?; - Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?) + let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()); + #[cfg(any(test, feature = "test-util"))] + { + tokio::pin!(acquire); + let mut notified = false; + let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) { + std::task::Poll::Pending => { + if !notified { + notify_config_write_lock_attempt(bucket); + notified = true; + } + std::task::Poll::Pending + } + std::task::Poll::Ready(result) => std::task::Poll::Ready(result), + }) + .await?; + Ok(guard) + } + #[cfg(not(any(test, feature = "test-util")))] + Ok(acquire.await?) } /// The lock resource name is deliberately still the `bucket-targets` one it @@ -889,6 +999,37 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in( } } +/// Re-read the quota configuration and bucket incarnation from the same +/// authoritative metadata blob while the caller holds the bucket metadata +/// transaction read lock. +pub(crate) async fn get_quota_config_and_incarnation_from_disk_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result<(Option, Uuid, OffsetDateTime)> { + let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); + + match bucket_meta_sys + .read_authoritative_metadata_from_disk_under_transaction_lock(bucket) + .await? + { + BucketMetadataAuthority::Authoritative(metadata) + if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() => + { + Ok(( + metadata.quota_config.clone(), + metadata.bucket_incarnation_id, + metadata.quota_config_updated_at, + )) + } + BucketMetadataAuthority::Authoritative(_) => { + Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}"))) + } + BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())), + BucketMetadataAuthority::Fabricated => Err(Error::other(format!("bucket quota metadata is not authoritative: {bucket}"))), + } +} + pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; diff --git a/crates/ecstore/src/bucket/quota/checker.rs b/crates/ecstore/src/bucket/quota/checker.rs index 38b4e1423..ebb1b4dbb 100644 --- a/crates/ecstore/src/bucket/quota/checker.rs +++ b/crates/ecstore/src/bucket/quota/checker.rs @@ -52,6 +52,7 @@ impl QuotaChecker { ) -> Result { let start_time = Instant::now(); let quota_config = self.get_quota_config(bucket).await?; + let uses_durable_reservations = quota_config.uses_durable_reservations(); // If no quota limit is set, allow operation let quota_limit = match quota_config.quota { @@ -67,6 +68,7 @@ impl QuotaChecker { quota_limit: None, operation_size, remaining: None, + uses_durable_reservations, }); } Some(q) => q, @@ -74,14 +76,17 @@ impl QuotaChecker { let current_usage = self.get_real_time_usage(bucket).await?; + let admission_size = if uses_durable_reservations { 0 } else { operation_size }; let expected_usage = match operation { - QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => current_usage + operation_size, + QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => { + current_usage.saturating_add(admission_size) + } QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size), }; let allowed = match operation { QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => { - quota_config.check_operation_allowed(current_usage, operation_size) + quota_config.check_operation_allowed(current_usage, admission_size) } QuotaOperation::DeleteObject => true, }; @@ -105,6 +110,7 @@ impl QuotaChecker { quota_limit: Some(quota_limit), operation_size, remaining, + uses_durable_reservations, }; let duration = start_time.elapsed(); @@ -158,6 +164,26 @@ impl QuotaChecker { .await } + pub async fn set_durable_quota_config_if_incarnation( + &mut self, + bucket: &str, + quota: BucketQuota, + expected_incarnation_id: uuid::Uuid, + proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken, + ) -> Result { + let json_data = serde_json::to_vec("a).map_err(|e| QuotaError::InvalidConfig { + reason: format!("Failed to serialize quota config: {}", e), + })?; + let start_time = Instant::now(); + let updated_at = + crate::bucket::metadata_sys::update_quota_if_incarnation(bucket, json_data, expected_incarnation_id, proof) + .await + .map_err(QuotaError::StorageError)?; + + rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed()); + Ok(updated_at) + } + async fn set_quota_config_for_incarnation( &mut self, bucket: &str, @@ -355,6 +381,7 @@ mod tests { quota_limit: None, operation_size: 1024, remaining: None, + uses_durable_reservations: false, }; assert!(result.allowed); @@ -378,4 +405,13 @@ mod tests { let allowed = quota.check_operation_allowed(512, 1024); assert!(!allowed); } + + #[test] + fn legacy_quota_rejects_full_operation_while_v1_defers_net_growth() { + let legacy: BucketQuota = serde_json::from_str(r#"{"quota":5}"#).expect("legacy quota should parse"); + let durable = BucketQuota::new(Some(5)); + + assert!(!legacy.check_operation_allowed(4, 2)); + assert!(durable.uses_durable_reservations()); + } } diff --git a/crates/ecstore/src/bucket/quota/mod.rs b/crates/ecstore/src/bucket/quota/mod.rs index 9172fd652..157c750e4 100644 --- a/crates/ecstore/src/bucket/quota/mod.rs +++ b/crates/ecstore/src/bucket/quota/mod.rs @@ -13,40 +13,100 @@ // limitations under the License. pub mod checker; +pub(crate) mod reservation; use crate::error::Result; use rustfs_config::{ QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE, QUOTA_NOT_FOUND_ERROR_CODE, }; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; use thiserror::Error; use time::OffsetDateTime; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] pub enum QuotaType { - /// Hard quota: reject immediately when exceeded + /// Hard quota accounting. #[default] #[serde(alias = "HARD", alias = "hard")] Hard, } +pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1; + /// Bucket quota configuration. quota_type defaults to Hard when omitted. -#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)] +#[derive(Debug, Default, Clone, PartialEq)] pub struct BucketQuota { - #[serde(default)] pub quota: Option, /// Defaults to Hard when missing. - #[serde(default)] pub quota_type: QuotaType, + /// Optional durable reservation protocol. The wire format gives older + /// nodes a zero hard quota so a mixed-version fleet fails closed. + pub reservation_protocol: Option, /// Timestamp when this quota configuration was set (for audit purposes) - #[serde(default, with = "time::serde::rfc3339::option")] pub created_at: Option, /// Accept updated_at for compatibility; not used. - #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] pub updated_at: Option, } +#[derive(Deserialize, Serialize)] +struct BucketQuotaWire { + #[serde(default)] + quota: Option, + #[serde(default)] + quota_type: QuotaType, + #[serde(default, skip_serializing_if = "Option::is_none")] + reservation_protocol: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + reservation_quota: Option, + #[serde(default, with = "time::serde::rfc3339::option")] + created_at: Option, + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + updated_at: Option, +} + +impl Serialize for BucketQuota { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + let durable = self.uses_durable_reservations(); + BucketQuotaWire { + quota: if durable { Some(0) } else { self.quota }, + quota_type: self.quota_type.clone(), + reservation_protocol: self.reservation_protocol, + reservation_quota: if durable { self.quota } else { None }, + created_at: self.created_at, + updated_at: self.updated_at, + } + .serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for BucketQuota { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let wire = BucketQuotaWire::deserialize(deserializer)?; + let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) { + Some( + wire.reservation_quota + .ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?, + ) + } else { + wire.quota + }; + Ok(Self { + quota, + quota_type: wire.quota_type, + reservation_protocol: wire.reservation_protocol, + created_at: wire.created_at, + updated_at: wire.updated_at, + }) + } +} + impl BucketQuota { /// Serialize to JSON bytes. Same format as parse_all_configs. pub fn marshal_msg(&self) -> Result> { @@ -63,6 +123,7 @@ impl BucketQuota { Self { quota, quota_type: QuotaType::Hard, + reservation_protocol: quota.map(|_| QUOTA_RESERVATION_PROTOCOL_V1), created_at: Some(now), updated_at: None, } @@ -72,7 +133,19 @@ impl BucketQuota { self.quota } + pub fn uses_durable_reservations(&self) -> bool { + self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) + } + + pub fn has_unsupported_reservation_protocol(&self) -> bool { + self.reservation_protocol + .is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1) + } + pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool { + if operation_size == 0 { + return true; + } if let Some(quota_limit) = self.quota { current_usage.saturating_add(operation_size) <= quota_limit } else { @@ -94,6 +167,7 @@ pub struct QuotaCheckResult { pub quota_limit: Option, pub operation_size: u64, pub remaining: Option, + pub uses_durable_reservations: bool, } #[derive(Debug)] @@ -210,7 +284,59 @@ mod tests { let buf = q.marshal_msg().expect("marshal"); let restored = BucketQuota::unmarshal(&buf).expect("unmarshal"); assert_eq!(q.quota, restored.quota); - assert_eq!(q.quota_type, restored.quota_type); + assert_eq!(restored.quota_type, QuotaType::Hard); + assert_eq!(restored.reservation_protocol, Some(QUOTA_RESERVATION_PROTOCOL_V1)); + } + + #[test] + fn clearing_quota_keeps_the_legacy_compatible_type() { + let quota = BucketQuota::new(None); + + assert_eq!(quota.quota_type, QuotaType::Hard); + assert_eq!(quota.reservation_protocol, None); + assert!(!quota.uses_durable_reservations()); + } + + #[test] + fn durable_quota_makes_legacy_nodes_fail_closed() { + let json = serde_json::to_vec(&BucketQuota::new(Some(2048))).expect("durable quota should serialize"); + let quota: BucketQuota = serde_json::from_slice(&json).expect("current quota version should parse"); + assert!(quota.uses_durable_reservations()); + assert_eq!(quota.quota, Some(2048)); + + #[derive(Deserialize)] + enum LegacyQuotaType { + Hard, + } + #[derive(Deserialize)] + struct LegacyBucketQuota { + #[allow(dead_code)] + quota: Option, + #[allow(dead_code)] + quota_type: LegacyQuotaType, + } + let legacy = serde_json::from_slice::(&json) + .expect("legacy readers should ignore the reservation protocol field"); + assert_eq!(legacy.quota, Some(0)); + assert!(matches!(legacy.quota_type, LegacyQuotaType::Hard)); + } + + #[test] + fn unknown_reservation_protocol_does_not_activate_v1() { + let quota: BucketQuota = + serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#) + .expect("future protocol should remain parseable"); + + assert!(!quota.uses_durable_reservations()); + assert!(quota.has_unsupported_reservation_protocol()); + } + + #[test] + fn reservation_protocol_v1_requires_reservation_quota() { + let err = serde_json::from_str::(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":1}"#) + .expect_err("v1 without its authoritative quota must fail closed"); + + assert!(err.to_string().contains("reservation_quota is required")); } /// unmarshal accepts format without quota_type diff --git a/crates/ecstore/src/bucket/quota/reservation.rs b/crates/ecstore/src/bucket/quota/reservation.rs new file mode 100644 index 000000000..175785487 --- /dev/null +++ b/crates/ecstore/src/bucket/quota/reservation.rs @@ -0,0 +1,1113 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::bucket::metadata_sys; +use crate::config::com::{CONFIG_PREFIX, read_config_no_lock, save_config_with_opts}; +use crate::data_usage::compute_bucket_usage; +use crate::disk::RUSTFS_META_BUCKET; +use crate::disk::{DiskAPI, error::DiskError}; +use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found}; +use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission}; +use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; +use crate::storage_api_contracts::namespace::NamespaceLocking; +use crate::storage_api_contracts::object::ObjectOperations; +use crate::store::ECStore; +use futures::{StreamExt, stream}; +use rustfs_lock::NamespaceLockGuard; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; +use time::OffsetDateTime; +use tracing::warn; +use uuid::Uuid; + +const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1; +const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64; +const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128; +const ORPHAN_PROBE_CONCURRENCY: usize = 32; +const EVENT_QUOTA_LEDGER_SETTLEMENT: &str = "quota_ledger_settlement"; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_QUOTA: &str = "quota"; + +#[cfg(any(test, feature = "test-util"))] +static FAIL_NEXT_LEDGER_SAVE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +// Lock order: caller-held destination object/upload, bucket metadata +// transaction (read), operation reservation, then quota ledger. + +#[cfg(not(any(test, feature = "test-util")))] +const ORPHAN_MIN_AGE_SECONDS: i64 = 30; +#[cfg(any(test, feature = "test-util"))] +const ORPHAN_MIN_AGE_SECONDS: i64 = 0; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct PersistedReservation { + object: String, + old_size: u64, + new_size: u64, + created_at: i64, + #[serde(default)] + pool_index: Option, + #[serde(default)] + set_index: Option, + #[serde(default)] + commit_started: bool, +} + +impl PersistedReservation { + fn growth(&self) -> u64 { + self.new_size.saturating_sub(self.old_size) + } + + fn target(&self) -> Option<(usize, usize)> { + self.pool_index.zip(self.set_index) + } + + fn matches_expected(&self, expected: &Self) -> bool { + self.object == expected.object && self.old_size == expected.old_size && self.new_size == expected.new_size + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct QuotaLedger { + version: u8, + bucket_incarnation: Uuid, + quota_revision_unix_nanos: i128, + accounted_usage: u64, + reservations: BTreeMap, + #[serde(default)] + reconcile_required: bool, + #[serde(default)] + reap_cursor: Option, +} + +impl QuotaLedger { + fn new(bucket_incarnation: Uuid, quota_revision: OffsetDateTime, accounted_usage: u64) -> Self { + Self { + version: QUOTA_LEDGER_FORMAT_VERSION, + bucket_incarnation, + quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(), + accounted_usage, + reservations: BTreeMap::new(), + reconcile_required: false, + reap_cursor: None, + } + } + + fn matches(&self, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> bool { + self.bucket_incarnation == bucket_incarnation && self.quota_revision_unix_nanos == quota_revision.unix_timestamp_nanos() + } + + fn admitted_usage(&self) -> Result { + let reserved_growth = self.reservations.values().try_fold(0_u64, |total, reservation| { + total + .checked_add(reservation.growth()) + .ok_or(StorageError::PartMissingOrCorrupt) + })?; + if reserved_growth > self.accounted_usage { + return Err(StorageError::PartMissingOrCorrupt); + } + Ok(self.accounted_usage) + } + + fn reserve(&mut self, operation_id: Uuid, reservation: PersistedReservation) -> Result<()> { + self.accounted_usage = self + .accounted_usage + .checked_add(reservation.growth()) + .ok_or(StorageError::PartMissingOrCorrupt)?; + self.reservations.insert(operation_id, reservation); + Ok(()) + } + + fn commit(&mut self, operation_id: Uuid, expected: &PersistedReservation) -> Result<()> { + let Some(reservation) = self.reservations.remove(&operation_id) else { + return Err(StorageError::PartMissingOrCorrupt); + }; + if !reservation.matches_expected(expected) { + return Err(StorageError::PartMissingOrCorrupt); + } + if reservation.new_size < reservation.old_size { + self.reconcile_required = true; + } + Ok(()) + } + + fn abort(&mut self, operation_id: Uuid, expected: &PersistedReservation) -> Result<()> { + let Some(reservation) = self.reservations.remove(&operation_id) else { + return Ok(()); + }; + if !reservation.matches_expected(expected) { + return Err(StorageError::PartMissingOrCorrupt); + } + self.accounted_usage = self + .accounted_usage + .checked_sub(reservation.growth()) + .ok_or(StorageError::PartMissingOrCorrupt)?; + Ok(()) + } + + fn mark_commit_started(&mut self, operation_id: Uuid, expected: &PersistedReservation) -> Result<()> { + let reservation = self + .reservations + .get_mut(&operation_id) + .ok_or(StorageError::PartMissingOrCorrupt)?; + if !reservation.matches_expected(expected) { + return Err(StorageError::PartMissingOrCorrupt); + } + reservation.commit_started = true; + Ok(()) + } + + fn should_reconcile_after_denial(&self) -> bool { + self.reservations.is_empty() + } + + fn reap_candidates(&self, now: i64) -> (Vec, Option) { + let aged = self + .reservations + .iter() + .filter(|(_, reservation)| { + reservation.created_at > now || now.saturating_sub(reservation.created_at) >= ORPHAN_MIN_AGE_SECONDS + }) + .map(|(operation_id, _)| *operation_id) + .collect::>(); + let start = self + .reap_cursor + .and_then(|cursor| aged.iter().position(|operation_id| *operation_id > cursor)) + .unwrap_or(0); + let candidates = aged + .iter() + .cycle() + .skip(start) + .take(aged.len().min(MAX_ORPHAN_PROBES_PER_WRITE)) + .copied() + .collect::>(); + let next_cursor = candidates.last().copied(); + (candidates, next_cursor) + } +} + +pub(crate) struct QuotaContext { + store: Option>, + bucket: String, + object: String, + ledger_object: String, + bucket_incarnation: Option, + quota_revision: Option, + quota_limit: Option, + capability_proof: Option, + snapshot_admission: Option, + legacy_data_movement: bool, + metadata_guard: Option, + pool_index: Option, + set_index: Option, +} + +impl QuotaContext { + pub(crate) fn is_enforced(&self) -> bool { + self.quota_limit.is_some() + } + + pub(crate) async fn reserve(self, old_size: u64, new_size: u64) -> Result { + let Some(quota_limit) = self.quota_limit else { + return Ok(QuotaReservation::unlimited(self.metadata_guard)); + }; + if let Some(admission) = self.snapshot_admission { + let growth = new_size.saturating_sub(old_size); + if growth > admission.remaining() { + return Err(StorageError::QuotaExceeded { + current: admission.current_usage(), + limit: admission.quota_limit(), + }); + } + return Ok(QuotaReservation::unlimited(self.metadata_guard)); + } + if self.legacy_data_movement { + if new_size > old_size { + return Err(StorageError::PartMissingOrCorrupt); + } + return Ok(QuotaReservation::unlimited(self.metadata_guard)); + } + let store = self.store.ok_or(StorageError::PartMissingOrCorrupt)?; + let bucket_incarnation = self.bucket_incarnation.ok_or(StorageError::PartMissingOrCorrupt)?; + let quota_revision = self.quota_revision.ok_or(StorageError::PartMissingOrCorrupt)?; + let operation_id = Uuid::new_v4(); + let operation_lock_object = operation_lock_object(&self.ledger_object, operation_id); + let operation_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &operation_lock_object).await?; + let operation_guard = operation_lock.get_write_lock(get_lock_acquire_timeout()).await?; + let reservation = PersistedReservation { + object: self.object, + old_size, + new_size, + created_at: OffsetDateTime::now_utc().unix_timestamp(), + pool_index: self.pool_index, + set_index: self.set_index, + commit_started: false, + }; + let ledger_data = LedgerReservationData { + store: Arc::clone(&store), + bucket: self.bucket, + ledger_object: self.ledger_object, + operation_id, + reservation: reservation.clone(), + }; + let metadata_guard = self.metadata_guard; + let capability_proof = self.capability_proof; + + tokio::spawn(async move { + reap_stale_reservations(Arc::clone(&store), &ledger_data.bucket, &ledger_data.ledger_object).await?; + + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?; + let mut ledger = load_current_ledger_locked( + Arc::clone(&store), + &ledger_data.bucket, + &ledger_data.ledger_object, + bucket_incarnation, + quota_revision, + ) + .await?; + + let growth = reservation.growth(); + let mut current_usage = ledger.admitted_usage()?; + let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?; + if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() { + reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?; + current_usage = ledger.admitted_usage()?; + expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?; + } + if growth > 0 && expected_usage > quota_limit { + return Err(StorageError::QuotaExceeded { + current: current_usage, + limit: quota_limit, + }); + } + if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: ledger_data.bucket.clone(), + object: ledger_data.ledger_object.clone(), + required: 1, + achieved: 0, + }); + } + ledger.reserve(operation_id, reservation)?; + save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?; + + Ok(QuotaReservation { + ledger: Some(ledger_data), + operation_guard: Some(operation_guard), + metadata_guard, + capability_proof, + state: ReservationState::Pending, + }) + }) + .await + .map_err(|err| StorageError::other(format!("quota ledger reservation task failed: {err}")))? + } +} + +#[derive(Clone)] +struct LedgerReservationData { + store: Arc, + bucket: String, + ledger_object: String, + operation_id: Uuid, + reservation: PersistedReservation, +} + +pub(crate) struct QuotaReservation { + ledger: Option, + operation_guard: Option, + metadata_guard: Option, + capability_proof: Option, + state: ReservationState, +} + +#[derive(Clone, Copy)] +enum ReservationState { + Pending, + CommitStarted, + Committed, + FenceReleaseUncertain, +} + +impl QuotaReservation { + fn unlimited(metadata_guard: Option) -> Self { + Self { + ledger: None, + operation_guard: None, + metadata_guard, + capability_proof: None, + state: ReservationState::Pending, + } + } + + pub(crate) fn is_lock_lost(&self) -> bool { + self.operation_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) + || self.metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) + } + + pub(crate) fn capability_proof_matches(&self) -> bool { + self.capability_proof + .as_ref() + .is_none_or(crate::services::notification_sys::cross_pool_fence_fleet_proof_matches) + } + + pub(crate) async fn mark_commit_started(&mut self) -> Result<()> { + if !self.capability_proof_matches() { + let ledger = self.ledger.as_ref().ok_or(StorageError::PartMissingOrCorrupt)?; + return Err(quota_capability_error(&ledger.bucket, &ledger.ledger_object)); + } + if let Some(ledger) = self.ledger.as_ref() { + mark_commit_started(ledger).await?; + } + self.state = ReservationState::CommitStarted; + Ok(()) + } + + pub(crate) async fn commit(mut self) { + self.state = ReservationState::Committed; + let Some(ledger) = self.ledger.as_ref() else { + return; + }; + crate::store::list_objects::observe_list_objects_mutation(&ledger.store, &ledger.bucket).await; + match settle(ledger, true).await { + Ok(()) => self.ledger = None, + Err(err) => log_deferred_settlement(ledger, "commit_deferred", &err), + } + } + + pub(crate) async fn abort(mut self) { + let Some(ledger) = self.ledger.as_ref() else { + return; + }; + match settle(ledger, false).await { + Ok(()) => self.ledger = None, + Err(err) => log_deferred_settlement(ledger, "abort_deferred", &err), + } + } + + pub(crate) fn defer_after_fence(mut self) { + self.state = ReservationState::FenceReleaseUncertain; + } +} + +fn should_settle_on_drop(state: ReservationState) -> bool { + !matches!(state, ReservationState::CommitStarted | ReservationState::FenceReleaseUncertain) +} + +impl Drop for QuotaReservation { + fn drop(&mut self) { + let Some(ledger) = self.ledger.take() else { + return; + }; + if !should_settle_on_drop(self.state) { + return; + } + let committed = matches!(self.state, ReservationState::Committed); + let operation_guard = self.operation_guard.take(); + let metadata_guard = self.metadata_guard.take(); + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + runtime.spawn(async move { + let _operation_guard = operation_guard; + let _metadata_guard = metadata_guard; + if let Err(err) = settle(&ledger, committed).await { + log_deferred_settlement(&ledger, "background_retry_failed", &err); + } + }); + } +} + +pub(crate) async fn begin( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, + object: &str, + snapshot_admission: Option, + data_movement: bool, + pool_index: usize, + set_index: usize, +) -> Result { + if crate::bucket::utils::is_meta_bucketname(bucket) { + return Ok(QuotaContext { + store: None, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: None, + quota_revision: None, + quota_limit: None, + capability_proof: None, + snapshot_admission: None, + legacy_data_movement: false, + metadata_guard: None, + pool_index: None, + set_index: None, + }); + } + #[cfg(test)] + if let Some(snapshot_admission) = snapshot_admission { + return Ok(QuotaContext { + store: None, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: None, + quota_revision: None, + quota_limit: Some(snapshot_admission.quota_limit()), + capability_proof: None, + snapshot_admission: Some(snapshot_admission), + legacy_data_movement: false, + metadata_guard: None, + pool_index: Some(pool_index), + set_index: Some(set_index), + }); + } + #[cfg(any(test, feature = "test-util"))] + if ctx.bucket_metadata_sys().is_none() { + return Ok(QuotaContext { + store: None, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: None, + quota_revision: None, + quota_limit: None, + capability_proof: None, + snapshot_admission: None, + legacy_data_movement: false, + metadata_guard: None, + pool_index: None, + set_index: None, + }); + } + + let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?; + let (quota, bucket_incarnation, quota_revision) = + metadata_sys::get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await?; + if metadata_guard.is_lock_lost() { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_config", + bucket: bucket.to_string(), + object: ledger_object(bucket), + required: 1, + achieved: 0, + }); + } + if quota + .as_ref() + .is_some_and(|quota| quota.has_unsupported_reservation_protocol()) + { + return Err(StorageError::PartMissingOrCorrupt); + } + let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations()); + let capability_proof = if durable_quota.is_some() { + Some( + crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof() + .ok_or_else(|| quota_capability_error(bucket, &ledger_object(bucket)))?, + ) + } else { + None + }; + let durable_quota_limit = durable_quota.and_then(|quota| quota.quota); + let snapshot_admission = match quota.as_ref().filter(|quota| !quota.uses_durable_reservations()) { + Some(quota) => match (quota.quota, snapshot_admission) { + (Some(limit), Some(admission)) if admission.quota_limit() == limit => Some(admission), + (Some(_), None) if data_movement => None, + (Some(_), _) => return Err(StorageError::PartMissingOrCorrupt), + (None, _) => None, + }, + None => None, + }; + let legacy_data_movement = durable_quota_limit.is_none() + && quota.as_ref().and_then(|quota| quota.quota).is_some() + && snapshot_admission.is_none() + && data_movement; + let quota_limit = durable_quota_limit + .or_else(|| snapshot_admission.map(QuotaAdmission::quota_limit)) + .or_else(|| { + legacy_data_movement + .then(|| quota.as_ref().and_then(|quota| quota.quota)) + .flatten() + }); + let store = if durable_quota_limit.is_some() { + Some(metadata_sys::object_store_in(ctx).await?) + } else { + None + }; + Ok(QuotaContext { + store, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: Some(bucket_incarnation), + quota_revision: Some(quota_revision), + quota_limit, + capability_proof, + snapshot_admission, + legacy_data_movement, + metadata_guard: Some(metadata_guard), + pool_index: Some(pool_index), + set_index: Some(set_index), + }) +} + +fn quota_capability_error(bucket: &str, object: &str) -> StorageError { + StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_capability", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + } +} + +pub(crate) async fn replaced_logical_size(set_disks: &SetDisks, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { + if opts.versioned && !opts.version_suspended && opts.version_id.is_none() { + return Ok(0); + } + let version_id = opts + .version_id + .clone() + .or_else(|| opts.version_suspended.then(|| Uuid::nil().to_string())); + let lookup_opts = ObjectOptions { + version_id, + no_lock: true, + metadata_cache_safe: false, + versioned: opts.versioned, + version_suspended: opts.version_suspended, + ..Default::default() + }; + match set_disks.get_object_info(bucket, object, &lookup_opts).await { + Ok(info) if info.delete_marker => Ok(0), + Ok(info) => logical_object_size(&info), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(0), + Err(err) => Err(err), + } +} + +fn logical_object_size(info: &ObjectInfo) -> Result { + crate::data_usage::quota_object_size(info) +} + +async fn mark_commit_started(data: &LedgerReservationData) -> Result<()> { + let data = data.clone(); + tokio::spawn(async move { + let ledger_lock = data.store.new_ns_lock(RUSTFS_META_BUCKET, &data.ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?; + let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?; + ledger.mark_commit_started(data.operation_id, &data.reservation)?; + save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await + }) + .await + .map_err(|err| StorageError::other(format!("quota commit marker task failed: {err}")))? +} + +async fn settle(data: &LedgerReservationData, committed: bool) -> Result<()> { + let store = Arc::clone(&data.store); + let ledger_object = data.ledger_object.clone(); + let operation_id = data.operation_id; + let reservation = data.reservation.clone(); + tokio::spawn(async move { + // The commit/abort path releases its own object fence before settlement. + // Do not revoke all tokens here: a deferred retry can run after the + // object lock is released and would otherwise revoke a later write's + // newly acquired fence for the same object. + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_object, None).await?; + let mut ledger = load_ledger_locked(Arc::clone(&store), &ledger_object).await?; + if committed { + ledger.commit(operation_id, &reservation)?; + } else { + ledger.abort(operation_id, &reservation)?; + } + save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await + }) + .await + .map_err(|err| StorageError::other(format!("quota ledger settlement task failed: {err}")))? +} + +async fn load_current_ledger_locked( + store: Arc, + bucket: &str, + ledger_object: &str, + bucket_incarnation: Uuid, + quota_revision: OffsetDateTime, +) -> Result { + match load_ledger_locked(Arc::clone(&store), ledger_object).await { + Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger), + Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => { + let usage = exact_bucket_usage(&store, bucket).await?; + Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage)) + } + Ok(_) => Err(StorageError::PartMissingOrCorrupt), + Err(StorageError::ConfigNotFound) => { + let usage = exact_bucket_usage(&store, bucket).await?; + Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage)) + } + Err(err) => Err(err), + } +} + +async fn reap_stale_reservations(store: Arc, bucket: &str, ledger_object: &str) -> Result<()> { + let now = now_unix(); + let (candidates, reconcile_required, next_cursor) = { + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, ledger_object).await?; + let _ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + match load_ledger_locked(Arc::clone(&store), ledger_object).await { + Ok(ledger) => { + let (candidates, next_cursor) = ledger.reap_candidates(now); + (candidates, ledger.reconcile_required, next_cursor) + } + Err(StorageError::ConfigNotFound) => (Vec::new(), false, None), + Err(err) => return Err(err), + } + }; + if candidates.is_empty() && !reconcile_required { + return Ok(()); + } + + let probe_results = stream::iter(candidates) + .map(|operation_id| { + let store = Arc::clone(&store); + async move { + let lock_object = operation_lock_object(ledger_object, operation_id); + let operation_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &lock_object).await?; + Ok::<_, StorageError>( + operation_lock + .get_write_lock_quiet(Duration::from_millis(50)) + .await + .ok() + .map(|guard| (operation_id, guard)), + ) + } + }) + .buffer_unordered(ORPHAN_PROBE_CONCURRENCY) + .collect::>() + .await; + let mut orphan_guards = Vec::new(); + for result in probe_results { + if let Some(guard) = result? { + orphan_guards.push(guard); + if orphan_guards.len() == MAX_ORPHANS_REAPED_PER_WRITE { + break; + } + } + } + + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&store, RUSTFS_META_BUCKET, ledger_object, None).await?; + let mut ledger = load_ledger_locked(Arc::clone(&store), ledger_object).await?; + let cursor_changed = next_cursor.is_some() && ledger.reap_cursor != next_cursor; + if next_cursor.is_some() { + ledger.reap_cursor = next_cursor; + } + let orphan_ids = orphan_guards + .iter() + .map(|(operation_id, _)| *operation_id) + .collect::>(); + let orphan_commit_targets = orphan_ids + .iter() + .filter_map(|operation_id| ledger.reservations.get(operation_id)) + .filter(|reservation| reservation.commit_started) + .map(|reservation| (reservation.object.clone(), reservation.target())) + .collect::>(); + for (object, target) in orphan_commit_targets { + fence_namespace_mutations(&store, bucket, &object, target).await?; + } + let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?; + if ledger.reservations.is_empty() && ledger.reconcile_required { + reconcile_exact(&store, bucket, &mut ledger).await?; + removed = true; + } + if !removed && !cursor_changed { + return Ok(()); + } + save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await +} + +fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result { + let mut removed = false; + for operation_id in operation_ids { + let Some(reservation) = ledger.reservations.get(operation_id).cloned() else { + continue; + }; + if reservation.commit_started { + ledger.reservations.remove(operation_id); + ledger.reconcile_required = true; + } else { + ledger.abort(*operation_id, &reservation)?; + } + removed = true; + } + Ok(removed) +} + +async fn reconcile_exact(store: &Arc, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> { + if !ledger.reservations.is_empty() { + return Err(StorageError::PartMissingOrCorrupt); + } + ledger.accounted_usage = exact_bucket_usage(store, bucket).await?; + ledger.reconcile_required = false; + Ok(()) +} + +async fn exact_bucket_usage(store: &Arc, bucket: &str) -> Result { + crate::store::list_objects::observe_list_objects_mutation(store, bucket).await; + Ok(compute_bucket_usage(Arc::clone(store), bucket).await?.size) +} + +fn ledger_object(bucket: &str) -> String { + format!("{CONFIG_PREFIX}/quota-ledger/{bucket}.json") +} + +fn operation_lock_object(ledger_object: &str, operation_id: Uuid) -> String { + format!("{ledger_object}.operations/{operation_id}") +} + +fn now_unix() -> i64 { + OffsetDateTime::now_utc().unix_timestamp() +} + +async fn fence_namespace_mutations( + store: &Arc, + bucket: &str, + object: &str, + target: Option<(usize, usize)>, +) -> Result<()> { + crate::bucket::utils::check_object_args(bucket, object)?; + let sets = match target { + Some((pool_index, set_index)) => { + let set = store + .pools + .get(pool_index) + .and_then(|pool| pool.disk_set.get(set_index)) + .cloned() + .ok_or(StorageError::PartMissingOrCorrupt)?; + vec![set] + } + None => store.pools.iter().map(|pool| pool.get_disks_by_key(object)).collect(), + }; + for set in sets { + let write_quorum = set.default_write_quorum(); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + let revoke_results = stream::iter(disks) + .map(|disk| { + let fence_path = fence_path.clone(); + async move { + let result = disk + .release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, crate::disk::SnapshotLeaseToken::revoke_all()) + .await; + (disk, result) + } + }) + .buffer_unordered(ORPHAN_PROBE_CONCURRENCY) + .collect::>() + .await; + let revoked_disks = revoke_results + .into_iter() + .filter_map(|(disk, result)| result.is_ok().then_some(disk)) + .collect::>(); + if revoked_disks.len() < write_quorum { + return Err(StorageError::ErasureWriteQuorum); + } + + let drain_results = stream::iter(revoked_disks) + .map(|disk| async move { + match disk.acquire_snapshot_lease(bucket, object).await { + Ok(token) => disk.release_snapshot_lease(bucket, object, token).await, + Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => Ok(()), + Err(err) => Err(err), + } + }) + .buffer_unordered(ORPHAN_PROBE_CONCURRENCY) + .collect::>() + .await; + if drain_results.iter().filter(|result| result.is_ok()).count() < write_quorum { + return Err(StorageError::ErasureWriteQuorum); + } + } + Ok(()) +} + +#[cfg(test)] +pub(crate) async fn fence_namespace_mutations_for_test( + store: &Arc, + bucket: &str, + object: &str, + target: Option<(usize, usize)>, +) -> Result<()> { + fence_namespace_mutations(store, bucket, object, target).await +} + +async fn load_ledger_locked(store: Arc, ledger_object: &str) -> Result { + let data = read_config_no_lock(store, ledger_object).await?; + let ledger: QuotaLedger = serde_json::from_slice(&data)?; + if ledger.version != QUOTA_LEDGER_FORMAT_VERSION { + return Err(StorageError::CorruptedFormat); + } + ledger.admitted_usage()?; + Ok(ledger) +} + +async fn save_ledger_locked( + store: Arc, + ledger_object: &str, + ledger: &QuotaLedger, + ledger_guard: &NamespaceLockGuard, +) -> Result<()> { + if ledger_guard.is_lock_lost() { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_ledger", + bucket: RUSTFS_META_BUCKET.to_string(), + object: ledger_object.to_string(), + required: 1, + achieved: 0, + }); + } + #[cfg(any(test, feature = "test-util"))] + if FAIL_NEXT_LEDGER_SAVE.swap(false, std::sync::atomic::Ordering::SeqCst) { + return Err(StorageError::Unexpected); + } + let mut opts = ObjectOptions { + max_parity: true, + no_lock: true, + ..Default::default() + }; + let _ = opts.set_quota_admission(0, u64::MAX); + opts.add_namespace_lock_guard(ledger_guard); + save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await +} + +#[cfg(any(test, feature = "test-util"))] +pub fn fail_next_quota_ledger_save_for_test() { + FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst); +} + +fn log_deferred_settlement(data: &LedgerReservationData, state: &'static str, err: &StorageError) { + warn!( + event = EVENT_QUOTA_LEDGER_SETTLEMENT, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_QUOTA, + state, + bucket = %data.bucket, + operation_id = %data.operation_id, + error = %err, + "quota ledger settlement deferred" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ledger(accounted_usage: u64) -> QuotaLedger { + QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage) + } + + #[test] + fn ledger_rejects_reserved_growth_overflow() { + let mut ledger = ledger(u64::MAX); + let result = ledger.reserve( + Uuid::new_v4(), + PersistedReservation { + object: "object".to_string(), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ); + + assert!(matches!(result, Err(StorageError::PartMissingOrCorrupt))); + } + + #[test] + fn legacy_reservation_without_topology_uses_conservative_fallback() { + let reservation: PersistedReservation = + serde_json::from_str(r#"{"object":"object","old_size":0,"new_size":1,"created_at":0,"commit_started":true}"#) + .expect("legacy reservation should deserialize"); + + assert_eq!(reservation.target(), None); + } + + #[test] + fn ledger_rejects_persisted_reservations_larger_than_accounted_usage() { + let mut ledger = ledger(0); + ledger.reservations.insert( + Uuid::new_v4(), + PersistedReservation { + object: "object".to_string(), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ); + + assert!(matches!(ledger.admitted_usage(), Err(StorageError::PartMissingOrCorrupt))); + } + + #[test] + fn ledger_accounts_overwrite_delta_and_reserved_growth() { + let mut ledger = ledger(10); + let operation_id = Uuid::new_v4(); + let overwrite = PersistedReservation { + object: "object".to_string(), + old_size: 8, + new_size: 5, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }; + ledger + .reserve(operation_id, overwrite.clone()) + .expect("shrinking overwrite should reserve"); + ledger + .reserve( + Uuid::new_v4(), + PersistedReservation { + object: "new-object".to_string(), + old_size: 0, + new_size: 7, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ) + .expect("new object should reserve positive growth"); + + assert_eq!( + ledger + .admitted_usage() + .expect("ledger usage should count positive growth only"), + 17 + ); + ledger + .commit(operation_id, &overwrite) + .expect("overwrite should settle exactly"); + assert_eq!(ledger.accounted_usage, 17); + assert!(ledger.reconcile_required); + assert_eq!(ledger.admitted_usage().expect("remaining reservation should stay counted"), 17); + } + + #[test] + fn commit_started_reservation_stays_precharged_until_reconciled() { + let mut ledger = ledger(10); + let operation_id = Uuid::new_v4(); + ledger + .reserve( + operation_id, + PersistedReservation { + object: "object".to_string(), + old_size: 0, + new_size: 7, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ) + .expect("new object should reserve positive growth"); + assert_eq!(ledger.accounted_usage, 17); + + let expected = ledger + .reservations + .get(&operation_id) + .expect("reservation should exist") + .clone(); + ledger + .mark_commit_started(operation_id, &expected) + .expect("commit marker should persist"); + + assert_eq!(ledger.accounted_usage, 17); + assert!(!ledger.should_reconcile_after_denial()); + } + + #[test] + fn commit_started_orphans_make_progress_across_bounded_batches() { + let mut ledger = ledger(65); + let operation_ids = (0..65).map(|_| Uuid::new_v4()).collect::>(); + for operation_id in &operation_ids { + ledger.reservations.insert( + *operation_id, + PersistedReservation { + object: format!("object-{operation_id}"), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: true, + }, + ); + } + + assert!(remove_orphan_reservations(&mut ledger, &operation_ids[..64]).expect("first orphan batch should apply")); + assert_eq!(ledger.reservations.len(), 1); + assert!(ledger.reconcile_required); + assert!(remove_orphan_reservations(&mut ledger, &operation_ids[64..]).expect("final orphan batch should apply")); + assert!(ledger.reservations.is_empty()); + } + + #[test] + fn orphan_probe_cursor_rotates_across_the_bounded_window() { + let mut ledger = ledger(129); + let operation_ids = (1..=129).map(Uuid::from_u128).collect::>(); + for operation_id in &operation_ids { + ledger.reservations.insert( + *operation_id, + PersistedReservation { + object: format!("object-{operation_id}"), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ); + } + + let (first, cursor) = ledger.reap_candidates(1); + assert_eq!(first.len(), MAX_ORPHAN_PROBES_PER_WRITE); + assert_eq!(first.first(), operation_ids.first()); + ledger.reap_cursor = cursor; + + let (second, _) = ledger.reap_candidates(1); + assert_eq!(second.first(), operation_ids.last()); + } + + #[test] + fn uncertain_fence_release_does_not_schedule_abort_on_drop() { + assert!(!should_settle_on_drop(ReservationState::FenceReleaseUncertain)); + assert!(!should_settle_on_drop(ReservationState::CommitStarted)); + assert!(should_settle_on_drop(ReservationState::Pending)); + assert!(should_settle_on_drop(ReservationState::Committed)); + } +} diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 46f246d6b..1426fc91f 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -248,6 +248,16 @@ fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) Ok(server_epoch) } +fn decode_cross_pool_fence_capability(expected_member: &str, result: &[u8]) -> Result<(u32, Uuid)> { + let version = result + .get(..4) + .and_then(|value| value.try_into().ok()) + .map(u32::from_be_bytes) + .ok_or_else(|| Error::other("peer returned an invalid cross-pool fence capability version"))?; + let epoch = decode_remote_version_state_capability(expected_member, &result[4..])?; + Ok((version, epoch)) +} + #[derive(Clone, Debug)] pub struct PeerLiveEventsBatch { pub events: Vec, @@ -1288,6 +1298,16 @@ impl PeerRestClient { Ok((self.topology_member.clone(), epoch)) } + pub async fn probe_cross_pool_fence(&self, topology_fingerprint: String) -> Result<(String, u32, Uuid)> { + let mut probe = rustfs_protos::CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX.to_vec(); + probe.extend_from_slice(Uuid::new_v4().as_bytes()); + let result = self + .heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe) + .await?; + let (supported_version, epoch) = decode_cross_pool_fence_capability(&self.topology_member, &result)?; + Ok((self.topology_member.clone(), supported_version, epoch)) + } + pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> { self.finalize_result( async { @@ -2738,6 +2758,24 @@ mod tests { assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err()); } + #[test] + fn cross_pool_fence_capability_decoder_fails_closed() { + let epoch = Uuid::new_v4(); + let result = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", epoch.as_bytes()) + .expect("small capability response should encode"); + assert_eq!( + decode_cross_pool_fence_capability("node-a:9000", &result).expect("valid capability should decode"), + (1, epoch) + ); + for malformed in [&[][..], &[0, 0, 0][..], &result[..result.len() - 1]] { + assert!(decode_cross_pool_fence_capability("node-a:9000", malformed).is_err()); + } + assert!(decode_cross_pool_fence_capability("node-b:9000", &result).is_err()); + let nil = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", Uuid::nil().as_bytes()) + .expect("small capability response should encode"); + assert!(decode_cross_pool_fence_capability("node-a:9000", &nil).is_err()); + } + struct TierMutationResponseFixture<'a> { version: u32, phase: TierMutationRpcPhase, diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index d94aca006..cd34cd22a 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -1445,6 +1445,71 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> { file_info.validate_for_metadata_read().map_err(Into::into) } +impl RemoteDisk { + #[tracing::instrument(level = "trace", skip_all)] + pub(crate) async fn rename_data_borrowed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result { + trace!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + src_volume, + src_path, + dst_volume, + dst_path, + op = "rename_data", + state = "started", + "Remote disk RPC started" + ); + + self.execute_with_timeout_for_op( + "rename_data", + || async { + let file_info = compat_json(fi)?; + let file_info_bin = encode_file_info_msgpack(fi)?; + let mut client = self + .get_client() + .await + .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; + let mut request = Request::new(RenameDataRequest { + disk: self.endpoint.to_string(), + src_volume: src_volume.to_string(), + src_path: src_path.to_string(), + file_info, + dst_volume: dst_volume.to_string(), + dst_path: dst_path.to_string(), + file_info_bin: file_info_bin.into(), + }); + let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref()); + attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?; + + let response = client.rename_data(request).await?.into_inner(); + + if !response.success { + return Err(response.error.unwrap_or_default().into()); + } + + let rename_data_resp = decode_msgpack_or_json::( + &response.rename_data_resp_bin, + &response.rename_data_resp, + "RenameDataResp", + )?; + + Ok(rename_data_resp) + }, + get_max_timeout_duration(), + ) + .await + } +} + #[async_trait::async_trait] impl DiskAPI for RemoteDisk { #[tracing::instrument(level = "trace", skip_all)] @@ -2370,58 +2435,8 @@ impl DiskAPI for RemoteDisk { dst_volume: &str, dst_path: &str, ) -> Result { - trace!( - event = EVENT_REMOTE_DISK_RPC, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REMOTE_DISK, - endpoint = %self.endpoint, - src_volume, - src_path, - dst_volume, - dst_path, - op = "rename_data", - state = "started", - "Remote disk RPC started" - ); - - self.execute_with_timeout_for_op( - "rename_data", - || async { - let file_info = compat_json(&fi)?; - let file_info_bin = encode_file_info_msgpack(&fi)?; - let mut client = self - .get_client() - .await - .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; - let mut request = Request::new(RenameDataRequest { - disk: self.endpoint.to_string(), - src_volume: src_volume.to_string(), - src_path: src_path.to_string(), - file_info, - dst_volume: dst_volume.to_string(), - dst_path: dst_path.to_string(), - file_info_bin: file_info_bin.into(), - }); - let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref()); - attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?; - - let response = client.rename_data(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let rename_data_resp = decode_msgpack_or_json::( - &response.rename_data_resp_bin, - &response.rename_data_resp, - "RenameDataResp", - )?; - - Ok(rename_data_resp) - }, - get_max_timeout_duration(), - ) - .await + self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path) + .await } #[tracing::instrument(level = "trace", skip_all)] diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 41495f068..3d0c122de 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -1355,7 +1355,7 @@ impl BucketUsageAccumulator { return Ok(()); } - let object_size = object.size.max(0) as u64; + let object_size = quota_object_size(object)?; self.current_live_versions = self.current_live_versions.saturating_add(1); self.size_histogram.add(object_size); self.total_size = self.total_size.saturating_add(object_size); @@ -1385,6 +1385,31 @@ impl BucketUsageAccumulator { } } +pub fn quota_object_size(object: &ObjectInfo) -> Result { + let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?; + let persisted_part_size = if object.parts.is_empty() { + u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)? + } else { + object.parts.iter().try_fold(0_u64, |total, part| { + // Compressed streaming objects persist -1 when the transformed + // part size is unknown. The physical part size remains a valid + // quota floor; reject only non-negative values that overflow. + let actual_size = if part.actual_size < 0 { + if object.is_compressed() { + 0 + } else { + return Err(Error::PartMissingOrCorrupt); + } + } else { + u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)? + }; + let part_size = actual_size.max(u64::try_from(part.size).map_err(|_| Error::PartMissingOrCorrupt)?); + total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt) + })? + }; + Ok(logical_size.max(persisted_part_size)) +} + type UsageVersionPage = StorageListObjectVersionsInfo; pub async fn compute_bucket_usage(store: Arc, bucket_name: &str) -> Result { @@ -3124,6 +3149,102 @@ mod tests { assert_eq!(usage.object_versions_histogram.get("BETWEEN_1000_AND_10000"), Some(&1)); } + #[test] + fn bucket_usage_uses_the_larger_of_logical_and_physical_size() { + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "4096".to_string()); + let object = ObjectInfo { + name: "compressed".to_string(), + size: 128, + user_defined: Arc::new(metadata), + ..Default::default() + }; + let mut usage = BucketUsageAccumulator::default(); + usage + .record("bucket", &object) + .expect("valid compressed metadata should be counted"); + assert_eq!(usage.finish().size, 4096); + + let mut framed_metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut framed_metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + rustfs_utils::http::insert_str(&mut framed_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let framed = ObjectInfo { + name: "framed".to_string(), + size: 17, + user_defined: Arc::new(framed_metadata), + ..Default::default() + }; + assert_eq!(quota_object_size(&framed).expect("physical framing must remain quota-accounted"), 17); + + let legacy_compressed_part = ObjectInfo { + name: "legacy-compressed-part".to_string(), + size: 1, + user_defined: Arc::new((*framed.user_defined).clone()), + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 1, + actual_size: -1, + ..Default::default() + }]), + ..Default::default() + }; + assert_eq!( + quota_object_size(&legacy_compressed_part).expect("unknown compressed part size is a valid sentinel"), + 1 + ); + + let uncompressed_negative_part = ObjectInfo { + name: "uncompressed-negative-part".to_string(), + size: 1, + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 1, + actual_size: -1, + ..Default::default() + }]), + ..Default::default() + }; + assert!(matches!(quota_object_size(&uncompressed_negative_part), Err(Error::PartMissingOrCorrupt))); + + let mut corrupt_metadata = (*object.user_defined).clone(); + rustfs_utils::http::insert_str(&mut corrupt_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "-1".to_string()); + let corrupt = ObjectInfo { + user_defined: Arc::new(corrupt_metadata), + ..object + }; + assert!(matches!(quota_object_size(&corrupt), Err(Error::PartMissingOrCorrupt))); + + let mut poisoned_metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut poisoned_metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + rustfs_utils::http::insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let poisoned = ObjectInfo { + name: "legacy-swift-metadata".to_string(), + size: 4096, + user_defined: Arc::new(poisoned_metadata), + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 4096, + actual_size: 4096, + ..Default::default() + }]), + ..Default::default() + }; + assert_eq!( + quota_object_size(&poisoned).expect("persisted part accounting must bound legacy user metadata"), + 4096 + ); + } + #[tokio::test] #[serial] async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() { diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 0606b55e6..111b8fabc 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -241,6 +241,40 @@ pub fn get_drive_list_dir_timeout() -> Duration { ) } +pub(crate) trait DiskStoreRenameDataExt { + async fn rename_data_borrowed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result; +} + +impl DiskStoreRenameDataExt for LocalDiskWrapper { + async fn rename_data_borrowed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result { + self.track_disk_health_mutation( + "rename_data", + DiskMetricMutation::Write, + || async { + self.disk + .rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path) + .await + }, + get_max_timeout_duration(), + ) + .await + } +} + pub fn get_drive_walkdir_timeout() -> Duration { get_drive_timeout_duration( rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, @@ -2017,13 +2051,8 @@ impl DiskAPI for LocalDiskWrapper { dst_volume: &str, dst_path: &str, ) -> Result { - self.track_disk_health_mutation( - "rename_data", - DiskMetricMutation::Write, - || async { self.disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await }, - get_max_timeout_duration(), - ) - .await + self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path) + .await } async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result> { diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 64e945d1d..1dc3a79c4 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -27,17 +27,18 @@ use crate::disk::{ BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, - PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, - RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, - conv_part_err_to_int, + PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, + ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, + SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int, endpoint::Endpoint, error::{DiskError, Error, FileAccessDeniedWithContext, Result}, error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}, format::FormatV3, fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename}, - os, + is_quota_mutation_fence_path, os, os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source}, + quota_mutation_fence_path, }; use crate::erasure::coding::{self, bitrot_verify}; use crate::runtime::sources as runtime_sources; @@ -60,9 +61,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Debug; use std::io::{Error as IoError, SeekFrom}; -#[cfg(target_os = "linux")] -use std::sync::atomic::AtomicBool; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; use std::{ @@ -2029,14 +2028,17 @@ static RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT: std::sync::Mutex; #[cfg(test)] +type RenameDataPublicationHookKey = (PathBuf, String, String); +#[cfg(test)] static INLINE_PREPARATION_BEFORE_BACKUP: std::sync::LazyLock>> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); #[cfg(test)] static INLINE_BEFORE_FILE_SYNC_ADMISSION: std::sync::LazyLock>> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); #[cfg(test)] -static RENAME_DATA_AFTER_FIRST_PUBLICATION: std::sync::LazyLock>> = - std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); +static RENAME_DATA_AFTER_FIRST_PUBLICATION: std::sync::LazyLock< + std::sync::Mutex>, +> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); #[cfg(test)] static OWNED_FILE_WRITE_BEFORE_OPEN: std::sync::LazyLock>> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); @@ -2108,11 +2110,11 @@ fn set_inline_before_file_sync_admission(dst_path: &str, hook: impl FnOnce() + S } #[cfg(test)] -fn set_rename_data_after_first_publication(dst_path: &str, hook: impl FnOnce() + Send + 'static) { +fn set_rename_data_after_first_publication(root: &Path, dst_volume: &str, dst_path: &str, hook: impl FnOnce() + Send + 'static) { RENAME_DATA_AFTER_FIRST_PUBLICATION .lock() .expect("test publication hook lock should not be poisoned") - .insert(dst_path.to_string(), Box::new(hook)); + .insert((root.to_path_buf(), dst_volume.to_string(), dst_path.to_string()), Box::new(hook)); } #[cfg(test)] @@ -2264,11 +2266,11 @@ fn run_inline_before_file_sync_admission(dst_path: &str) { } #[cfg(test)] -fn run_rename_data_after_first_publication(dst_path: &str) { +fn run_rename_data_after_first_publication(root: &Path, dst_volume: &str, dst_path: &str) { let hook = RENAME_DATA_AFTER_FIRST_PUBLICATION .lock() .expect("test publication hook lock should not be poisoned") - .remove(dst_path); + .remove(&(root.to_path_buf(), dst_volume.to_string(), dst_path.to_string())); if let Some(hook) = hook { hook(); } @@ -2366,9 +2368,6 @@ async fn remove_dst_base_before_commit( #[cfg(not(test))] fn run_inline_preparation_before_backup(_dst_path: &str) {} -#[cfg(not(test))] -fn run_rename_data_after_first_publication(_dst_path: &str) {} - #[cfg(not(test))] fn should_fail_after_delete_data_staged(_path: &str) -> bool { false @@ -4756,6 +4755,25 @@ struct SnapshotLeaseEntry { tokens: HashSet, pending_delete: Option, deleting: bool, + mutation_fence: Option>, +} + +#[derive(Default)] +struct QuotaMutationFenceState { + revoked: AtomicBool, + running: AtomicUsize, + notify: Notify, +} + +struct QuotaMutationFenceClaim { + state: Arc, +} + +impl Drop for QuotaMutationFenceClaim { + fn drop(&mut self) { + self.state.running.fetch_sub(1, Ordering::AcqRel); + self.state.notify.notify_waiters(); + } } #[derive(Default)] @@ -7320,6 +7338,34 @@ fn normalize_path_components(path: impl AsRef) -> PathBuf { } impl LocalDisk { + async fn claim_quota_mutation_fence( + &self, + volume: &str, + path: &str, + token: SnapshotLeaseToken, + ) -> Result> { + let key = SnapshotLeaseKey { + volume: RUSTFS_META_BUCKET.to_string(), + path: quota_mutation_fence_path(volume, path), + }; + let state = { + let registry = self.snapshot_leases.lock().await; + let entry = registry.entries.get(&key).ok_or(DiskError::FileNotFound)?; + let state = entry.mutation_fence.as_ref().ok_or(DiskError::FileNotFound)?; + if !entry.tokens.contains(&token) || state.revoked.load(Ordering::Acquire) { + return Err(DiskError::FileNotFound); + } + state.running.fetch_add(1, Ordering::AcqRel); + Arc::clone(state) + }; + if state.revoked.load(Ordering::Acquire) { + state.running.fetch_sub(1, Ordering::AcqRel); + state.notify.notify_waiters(); + return Err(DiskError::FileNotFound); + } + Ok(Arc::new(QuotaMutationFenceClaim { state })) + } + async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result { let path = format!("{object}/{data_dir}"); let data_path = self.io_get_object_path(volume, &path)?; @@ -8643,17 +8689,41 @@ impl DiskAPI for LocalDisk { &self, src_volume: &str, src_path: &str, - mut fi: FileInfo, + fi: FileInfo, dst_volume: &str, dst_path: &str, ) -> Result { crate::hp_guard!("LocalDisk::rename_data"); + let mut fi = fi; // A non-force DeleteBucket must not remove a directory while a local // object commit is publishing into it. The peer's empty scan remains // optimistic; this lease establishes the local commit/delete order and // remains owned by any blocking syscall that outlives async cancellation. let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; + let quota_fence_token = + match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { + Some(value) => { + let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; + Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) + } + None if rustfs_utils::http::metadata_compat::contains_key_str( + &fi.metadata, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + ) => + { + return Err(DiskError::FileCorrupt); + } + None => None, + }; + rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); + let quota_fence_claim = match quota_fence_token { + Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), + None => None, + }; let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; + if let Some(claim) = quota_fence_claim { + mutation_lease.attach_external_guard(claim); + } if fi.is_legacy_indexed_delete_marker() { fi.erasure.index = 0; } @@ -8946,8 +9016,9 @@ impl DiskAPI for LocalDisk { .await?; return Err(err); } + #[cfg(test)] if has_data_dir_path.is_some() { - run_rename_data_after_first_publication(dst_path); + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); } // Crash-consistency injection: hard power loss after the data dir @@ -9380,7 +9451,8 @@ impl DiskAPI for LocalDisk { let _ = remove_file_if_exists(staged_backup); return Err(err); } - run_rename_data_after_first_publication(dst_path); + #[cfg(test)] + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); if sync { file_sync_admission = Some( os::acquire_file_sync_admission(self.file_sync_permits.clone()) @@ -9643,11 +9715,26 @@ impl DiskAPI for LocalDisk { } async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result { - let file_path = self.io_get_object_path(volume, path)?; let key = SnapshotLeaseKey { volume: volume.to_string(), path: path.to_string(), }; + if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) { + let mut registry = self.snapshot_leases.lock().await; + let entry = registry.entries.entry(key).or_default(); + let state = entry + .mutation_fence + .get_or_insert_with(|| Arc::new(QuotaMutationFenceState::default())); + if state.revoked.load(Ordering::Acquire) { + return Err(DiskError::FileNotFound); + } + let token = SnapshotLeaseToken::new(); + entry.tokens.insert(token); + return Ok(token); + } + + let file_path = self.io_get_object_path(volume, path)?; + let _mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, volume, &file_path).await; let token = { let mut registry = self.snapshot_leases.lock().await; if registry.entries.get(&key).is_some_and(|entry| entry.deleting) { @@ -9675,6 +9762,48 @@ impl DiskAPI for LocalDisk { volume: volume.to_string(), path: path.to_string(), }; + if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) { + if !token.is_revoke_all() { + let mut registry = self.snapshot_leases.lock().await; + let Some(entry) = registry.entries.get_mut(&key) else { + return Ok(()); + }; + entry.tokens.remove(&token); + let removable = entry.tokens.is_empty() + && entry + .mutation_fence + .as_ref() + .is_none_or(|state| state.running.load(Ordering::Acquire) == 0); + if removable { + registry.entries.remove(&key); + } + return Ok(()); + } + let state = { + let mut registry = self.snapshot_leases.lock().await; + let Some(entry) = registry.entries.get_mut(&key) else { + return Ok(()); + }; + let Some(state) = entry.mutation_fence.as_ref().cloned() else { + registry.entries.remove(&key); + return Ok(()); + }; + state.revoked.store(true, Ordering::Release); + entry.tokens.clear(); + state + }; + loop { + let notified = state.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if state.running.load(Ordering::Acquire) == 0 { + break; + } + notified.await; + } + self.snapshot_leases.lock().await.entries.remove(&key); + return Ok(()); + } let opts = { let mut registry = self.snapshot_leases.lock().await; let Some(entry) = registry.entries.get_mut(&key) else { @@ -10453,6 +10582,19 @@ impl DiskAPI for LocalDisk { } } +impl LocalDisk { + pub(crate) async fn rename_data_borrowed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result { + ::rename_data(self, src_volume, src_path, fi.clone(), dst_volume, dst_path).await + } +} + async fn wait_for_startup_cleanup_signal( startup_cleanup_ready: &AtomicU32, startup_cleanup_notify: &Notify, @@ -13007,7 +13149,7 @@ mod test { let replacement_staging_parent_for_hook = replacement_staging_parent.clone(); let staged_metadata_for_hook = staged_metadata.clone(); let replacement_staged_metadata_for_hook = replacement_staged_metadata.clone(); - set_rename_data_after_first_publication(object, move || { + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { std::fs::rename(&object_dir_for_hook, &replacement_dir_for_hook) .expect_err("the destination object identity must remain pinned until xl.meta commits"); std::fs::rename(&staging_parent_for_hook, &replacement_staging_parent_for_hook) @@ -13274,7 +13416,7 @@ mod test { let replacement_dir_for_hook = replacement_dir.clone(); let staged_metadata_for_hook = staged_metadata.clone(); let replacement_staged_metadata_for_hook = replacement_staged_metadata.clone(); - set_rename_data_after_first_publication(object, move || { + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { std::fs::rename(&object_dir_for_hook, &replacement_dir_for_hook) .expect_err("the destination object identity must remain pinned after publishing its rollback backup"); std::fs::rename(&staged_metadata_for_hook, &replacement_staged_metadata_for_hook) @@ -13640,7 +13782,7 @@ mod test { let (entered_tx, entered_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel(); - set_rename_data_after_first_publication(object, move || { + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { entered_tx.send(()).expect("signal first publication"); release_rx.recv().expect("wait while delete_volume is blocked"); }); @@ -14234,7 +14376,7 @@ mod test { let (published_tx, published_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel(); - set_rename_data_after_first_publication(object, move || { + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { published_tx.send(()).expect("signal backup publication"); release_rx.recv().expect("wait for lock-order assertion"); }); @@ -18889,6 +19031,48 @@ mod test { assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound))); } + #[tokio::test] + async fn quota_mutation_fence_revoke_waits_for_active_claim_and_rejects_late_claims() { + use tempfile::tempdir; + + let root_dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let bucket = "quota-fence-volume"; + let object = "object"; + let fence_path = quota_mutation_fence_path(bucket, object); + let token = disk + .acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path) + .await + .expect("quota mutation token should be prepared"); + let claim = disk + .claim_quota_mutation_fence(bucket, object, token) + .await + .expect("prepared token should be claimable"); + + let release_disk = Arc::clone(&disk); + let mut release = tokio::spawn(async move { + release_disk + .release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, SnapshotLeaseToken::revoke_all()) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut release).await.is_err(), + "revoke must wait until an already claimed mutation has finished" + ); + + drop(claim); + tokio::time::timeout(Duration::from_secs(1), release) + .await + .expect("revoke should wake after the final claim drops") + .expect("revoke task should not panic") + .expect("revoke should succeed"); + assert!(matches!( + disk.claim_quota_mutation_fence(bucket, object, token).await, + Err(DiskError::FileNotFound) + )); + } + #[tokio::test] async fn delete_version_keeps_later_part_until_snapshot_release() { use tempfile::tempdir; diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 26047fdfc..428bd61a9 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -55,6 +55,7 @@ pub fn part_transaction_path(part_path: &str) -> String { use crate::cluster::rpc::RemoteDisk; use crate::cluster::rpc::build_internode_data_transport_from_env; +use crate::disk::disk_store::DiskStoreRenameDataExt; use crate::disk::disk_store::LocalDiskWrapper; use crate::disk::health_state::RuntimeDriveHealthState; use crate::disk::local::ScanGuard; @@ -72,6 +73,28 @@ use time::OffsetDateTime; use tokio::io::{AsyncRead, AsyncWrite}; use uuid::Uuid; +const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/"; +pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token"; + +pub(crate) fn quota_mutation_fence_path(bucket: &str, object: &str) -> String { + use sha2::{Digest, Sha256}; + + let mut input = Vec::with_capacity(bucket.len() + object.len() + 1); + input.extend_from_slice(bucket.as_bytes()); + input.push(0); + input.extend_from_slice(object.as_bytes()); + let digest = Sha256::digest(input); + format!( + "{QUOTA_MUTATION_FENCE_PREFIX}{}", + hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower) + ) +} + +pub(crate) fn is_quota_mutation_fence_path(path: &str) -> bool { + path.strip_prefix(QUOTA_MUTATION_FENCE_PREFIX) + .is_some_and(|digest| digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit())) +} + pub type DiskStore = Arc; pub type FileReader = Box; @@ -96,6 +119,20 @@ impl SnapshotLeaseToken { pub fn as_bytes(&self) -> &[u8; 16] { self.0.as_bytes() } + + pub(crate) fn as_uuid(self) -> Uuid { + self.0 + } + + #[doc(hidden)] + pub fn revoke_all() -> Self { + Self(Uuid::nil()) + } + + #[doc(hidden)] + pub fn is_revoke_all(self) -> bool { + self.0.is_nil() + } } impl Default for SnapshotLeaseToken { @@ -398,10 +435,8 @@ impl DiskAPI for Disk { dst_volume: &str, dst_path: &str, ) -> Result { - match self { - Disk::Local(local_disk) => local_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await, - Disk::Remote(remote_disk) => remote_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await, - } + self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path) + .await } #[tracing::instrument(level = "trace", skip_all)] @@ -631,6 +666,30 @@ impl DiskAPI for Disk { } } +impl Disk { + pub(crate) async fn rename_data_borrowed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result { + match self { + Disk::Local(local_disk) => { + local_disk + .rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path) + .await + } + Disk::Remote(remote_disk) => { + remote_disk + .rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path) + .await + } + } + } +} + impl Disk { pub async fn ns_scanner_server_epoch(&self) -> Result> { match self { diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index bd3ec77a5..fc9c8027c 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -306,12 +306,20 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc { pub(crate) struct NamespaceMutationLease { _namespace_guard: OwnedMutexGuard<()>, _volume_guard: Option>, + external_guard: Mutex>>, +} + +impl NamespaceMutationLease { + pub(crate) fn attach_external_guard(&self, guard: Arc) { + *self.external_guard.lock() = Some(guard); + } } async fn acquire_namespace_mutation_lease(path: &Path) -> Arc { Arc::new(NamespaceMutationLease { _namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await, _volume_guard: None, + external_guard: Mutex::new(None), }) } @@ -327,6 +335,7 @@ pub(crate) async fn acquire_rename_data_mutation_lease( Arc::new(NamespaceMutationLease { _namespace_guard: namespace_guard, _volume_guard: Some(volume_guard), + external_guard: Mutex::new(None), }) } diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index c63dbc401..41fc15970 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -44,12 +44,14 @@ const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification"; const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation"; +const EVENT_NOTIFICATION_CAPABILITY_PROBE: &str = "notification_capability_probe"; const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100); const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10); const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30); +const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1; /// Cached result from the last successful admin call to a peer. struct PeerAdminCache { @@ -95,15 +97,15 @@ lazy_static! { } #[derive(Clone)] -struct RemoteVersionStateFleetProof { +struct FleetCapabilityProof { topology_fingerprint: String, peer_epochs: Arc>, expires_at: Instant, } -impl RemoteVersionStateFleetProof { - fn token(&self) -> RemoteVersionStateFleetProofToken { - RemoteVersionStateFleetProofToken { +impl FleetCapabilityProof { + fn token(&self) -> FleetCapabilityProofToken { + FleetCapabilityProofToken { topology_fingerprint: self.topology_fingerprint.clone(), peer_epochs: self.peer_epochs.clone(), } @@ -111,37 +113,41 @@ impl RemoteVersionStateFleetProof { } #[derive(Clone, PartialEq, Eq)] -pub(crate) struct RemoteVersionStateFleetProofToken { +struct FleetCapabilityProofToken { topology_fingerprint: String, peer_epochs: Arc>, } #[derive(Default)] -struct RemoteVersionStateFleetProofState { - proof: Option, +struct FleetCapabilityProofState { + proof: Option, topology_conflict: bool, } -static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock> = OnceLock::new(); +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken); + +#[derive(Clone, PartialEq, Eq)] +pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken); + +static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock> = OnceLock::new(); +static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock> = OnceLock::new(); static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock = OnceLock::new(); -fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock { - REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default())) +fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock { + CROSS_POOL_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) } -fn replace_remote_version_state_fleet_proof(proof: Option) { - replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof); +fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock { + REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) } -fn replace_remote_version_state_fleet_proof_in( - slot: &std::sync::RwLock, - proof: Option, -) { +fn replace_fleet_capability_proof(slot: &std::sync::RwLock, proof: Option) { slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof; } -fn publish_remote_version_state_probe_result( - slot: &std::sync::RwLock, +fn publish_fleet_capability_probe_result( + slot: &std::sync::RwLock, topology_fingerprint: &str, result: Result>, observed_at: Instant, @@ -155,7 +161,7 @@ fn publish_remote_version_state_probe_result( .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs) .map(|proof| Arc::clone(&proof.peer_epochs)) .unwrap_or_else(|| Arc::new(peer_epochs)); - state.proof = Some(RemoteVersionStateFleetProof { + state.proof = Some(FleetCapabilityProof { topology_fingerprint: topology_fingerprint.to_string(), peer_epochs, expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL, @@ -163,7 +169,7 @@ fn publish_remote_version_state_probe_result( None } Err(err) => { - replace_remote_version_state_fleet_proof_in(slot, None); + replace_fleet_capability_proof(slot, None); Some(err) } } @@ -174,27 +180,60 @@ pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option Option { - if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) { +) -> Option { + if state.topology_conflict || !fleet_capability_proof_valid_at(state.proof.as_ref(), expected_topology, now) { return None; } - state.proof.as_ref().map(RemoteVersionStateFleetProof::token) + state.proof.as_ref().map(FleetCapabilityProof::token) } pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool { + fleet_capability_proof_matches(remote_version_state_fleet_proof_slot(), &proof.0) +} + +pub fn acquire_cross_pool_fence_fleet_proof() -> Option { + let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?; + let state = cross_pool_fence_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(CrossPoolFenceFleetProofToken) +} + +pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToken) -> bool { + fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0) +} + +#[cfg(any(test, feature = "test-util"))] +pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool { + let mut state = cross_pool_fence_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(current) = state.proof.as_ref() else { + return false; + }; + state.proof = Some(FleetCapabilityProof { + topology_fingerprint: current.topology_fingerprint.clone(), + peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()), + expires_at: current.expires_at, + }); + true +} + +fn fleet_capability_proof_matches( + slot: &std::sync::RwLock, + proof: &FleetCapabilityProofToken, +) -> bool { let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else { return false; }; - let state = remote_version_state_fleet_proof_slot() - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner); + let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner); if state.topology_conflict { return false; } @@ -206,13 +245,17 @@ pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStat }) } +fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool { + proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at) +} + #[cfg(test)] pub(crate) struct RemoteVersionStateFleetProofGuard; #[cfg(test)] impl Drop for RemoteVersionStateFleetProofGuard { fn drop(&mut self) { - replace_remote_version_state_fleet_proof(None); + replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None); } } @@ -227,7 +270,7 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp Err(_) => panic!("remote version state test topology is already bound to another fingerprint"), } let peer_epochs = BTreeMap::new(); - if let Some(err) = publish_remote_version_state_probe_result( + if let Some(err) = publish_fleet_capability_probe_result( remote_version_state_fleet_proof_slot(), topology_fingerprint, Ok(peer_epochs), @@ -238,14 +281,6 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp RemoteVersionStateFleetProofGuard } -fn remote_version_state_fleet_proof_valid_at( - proof: Option<&RemoteVersionStateFleetProof>, - expected_topology: &str, - now: Instant, -) -> bool { - proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at) -} - fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap, peer: String, epoch: Uuid) -> Result<()> { if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() { return Err(Error::other("remote version state capability peer identity is invalid")); @@ -256,11 +291,11 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap, pe pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) { - let mut state = remote_version_state_fleet_proof_slot() - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner); - state.topology_conflict = true; - state.proof = None; + for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] { + let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner); + state.topology_conflict = true; + state.proof = None; + } } return; } @@ -281,13 +316,23 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { } None => Err(Error::other("remote version state fleet capability notification system is unavailable")), }; + let fence_result = match get_global_notification_sys() { + Some(notification_sys) => timeout( + REMOTE_VERSION_STATE_PROBE_TIMEOUT, + notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint), + ) + .await + .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))), + None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")), + }; let topology_conflict = remote_version_state_fleet_proof_slot() .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .topology_conflict; if topology_conflict { - replace_remote_version_state_fleet_proof(None); - } else if let Some(err) = publish_remote_version_state_probe_result( + replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None); + replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None); + } else if let Some(err) = publish_fleet_capability_probe_result( remote_version_state_fleet_proof_slot(), &topology_fingerprint, result, @@ -295,6 +340,24 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { ) { debug!(error = %err, "remote version state fleet capability probe failed closed"); } + if !topology_conflict + && let Some(err) = publish_fleet_capability_probe_result( + cross_pool_fence_fleet_proof_slot(), + &topology_fingerprint, + fence_result, + Instant::now(), + ) + { + debug!( + event = EVENT_NOTIFICATION_CAPABILITY_PROBE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + capability = "cross_pool_fence_v1", + state = "failed_closed", + error = %err, + "notification capability probe" + ); + } sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await; } }); @@ -362,6 +425,27 @@ impl NotificationSys { } Ok(peer_epochs) } + + async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result> { + if self.peer_clients.len() != self.peer_topology_hosts.len() { + return Err(Error::other("cross-pool fence capability fleet membership is incomplete")); + } + let probes = self.peer_clients.iter().map(|client| async { + let client = client + .as_ref() + .ok_or_else(|| Error::other("cross-pool fence capability peer is unreachable"))?; + client.probe_cross_pool_fence(topology_fingerprint.to_string()).await + }); + let mut peer_epochs = BTreeMap::new(); + for result in join_all(probes).await { + let (peer, version, epoch) = result?; + if version < CROSS_POOL_FENCE_SUPPORTED_VERSION { + return Err(Error::other("cross-pool fence capability version is unsupported")); + } + insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?; + } + Ok(peer_epochs) + } } pub struct NotificationPeerErr { @@ -2177,16 +2261,16 @@ mod tests { let now = Instant::now(); let mut peer_epochs = BTreeMap::new(); peer_epochs.insert("peer-a".to_string(), Uuid::new_v4()); - let proof = RemoteVersionStateFleetProof { + let proof = FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(peer_epochs), expires_at: now + Duration::from_secs(1), }; - assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now)); - assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now)); - assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at)); - assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now)); + assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now)); + assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now)); + assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-a", proof.expires_at)); + assert!(!fleet_capability_proof_valid_at(None, "topology-a", now)); } #[test] @@ -2200,25 +2284,25 @@ mod tests { #[test] fn remote_version_state_fleet_proof_accepts_single_node_membership() { let now = Instant::now(); - let proof = RemoteVersionStateFleetProof { + let proof = FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(BTreeMap::new()), expires_at: now + Duration::from_secs(1), }; - assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now)); + assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now)); } #[test] fn remote_version_state_fleet_proof_token_changes_with_process_epoch() { let now = Instant::now(); - let proof = RemoteVersionStateFleetProof { + let proof = FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), expires_at: now + Duration::from_secs(1), }; let captured = proof.token(); - let restarted = RemoteVersionStateFleetProof { + let restarted = FleetCapabilityProof { topology_fingerprint: proof.topology_fingerprint.clone(), peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), expires_at: proof.expires_at, @@ -2229,11 +2313,11 @@ mod tests { #[test] fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() { - let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()); + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); let now = Instant::now(); let epoch = Uuid::new_v4(); let peers = BTreeMap::from([("peer-a".to_string(), epoch)]); - assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none()); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none()); let original = slot .read() .expect("proof slot should not poison") @@ -2242,9 +2326,7 @@ mod tests { .expect("successful probe should publish proof") .token(); - assert!( - publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none() - ); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()); let renewed = slot .read() .expect("proof slot should not poison") @@ -2256,8 +2338,7 @@ mod tests { let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]); assert!( - publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2)) - .is_none() + publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2)).is_none() ); let replaced = slot .read() @@ -2272,18 +2353,18 @@ mod tests { #[test] fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() { let now = Instant::now(); - let mut state = RemoteVersionStateFleetProofState { - proof: Some(RemoteVersionStateFleetProof { + let mut state = FleetCapabilityProofState { + proof: Some(FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(BTreeMap::new()), expires_at: now + Duration::from_secs(1), }), topology_conflict: false, }; - assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some()); + assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some()); state.topology_conflict = true; - assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none()); + assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_none()); } #[test] @@ -2299,19 +2380,19 @@ mod tests { #[test] fn remote_version_state_fleet_probe_failure_revokes_previous_proof() { - let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()); + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); let now = Instant::now(); let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]); - assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); assert!(slot.read().expect("proof slot should not poison").proof.is_some()); assert!( - publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some() + publish_fleet_capability_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some() ); assert!(slot.read().expect("proof slot should not poison").proof.is_none()); let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]); - assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); assert!(slot.read().expect("proof slot should not poison").proof.is_some()); } diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 1fd1b02d5..521cc1a33 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -46,6 +46,7 @@ use crate::diagnostics::get::{ GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, }; +use crate::disk::disk_store::DiskStoreRenameDataExt; use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX; use crate::disk::{ DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, @@ -590,7 +591,7 @@ impl MetadataQuorumAccumulator { }) } - pub(in crate::set_disk) fn default_write_quorum(&self) -> usize { + pub(crate) fn default_write_quorum(&self) -> usize { if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { return self.total_disks; } @@ -3011,12 +3012,41 @@ impl RenameConvergence { } } +pub(in crate::set_disk) struct RenameDataCommit { + pub(in crate::set_disk) online_disks: Vec>, + pub(in crate::set_disk) convergence: RenameConvergence, + pub(in crate::set_disk) data_dir: Option, + pub(in crate::set_disk) cleanup_disks: Vec>, + pub(in crate::set_disk) old_current_size: Option, + pub(in crate::set_disk) committed_file_info: FileInfo, +} + +type RenameDataLegacyTuple = ( + Vec>, + RenameConvergence, + Option, + Vec>, + Option, +); + +impl RenameDataCommit { + fn into_legacy_tuple(self) -> RenameDataLegacyTuple { + ( + self.online_disks, + self.convergence, + self.data_dir, + self.cleanup_disks, + self.old_current_size, + ) + } +} + impl SetDisks { pub(in crate::set_disk) fn default_read_quorum(&self) -> usize { self.set_drive_count - self.default_parity_count } - pub(in crate::set_disk) fn default_write_quorum(&self) -> usize { + pub(crate) fn default_write_quorum(&self) -> usize { let mut data_count = self.set_drive_count - self.default_parity_count; if data_count == self.default_parity_count { data_count += 1 @@ -3025,6 +3055,97 @@ impl SetDisks { data_count } + pub(in crate::set_disk) async fn prepare_quota_mutation_fences( + disks: &[Option], + bucket: &str, + object: &str, + write_quorum: usize, + ) -> crate::error::Result<(Vec>, Vec>)> { + let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + let results = join_all(disks.iter().map(|disk| { + let disk = disk.clone(); + let fence_path = fence_path.clone(); + async move { + let disk = disk?; + match disk.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path).await { + Ok(token) => Some((disk, token)), + Err(_) => None, + } + } + })) + .await; + if results.iter().flatten().count() < write_quorum { + for (disk, token) in results.iter().flatten() { + let _ = disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, *token).await; + } + return Err(StorageError::ErasureWriteQuorum); + } + let mut fenced_disks = Vec::with_capacity(results.len()); + let mut tokens = Vec::with_capacity(results.len()); + for result in results { + match result { + Some((disk, token)) => { + fenced_disks.push(Some(disk)); + tokens.push(Some(token)); + } + None => { + fenced_disks.push(None); + tokens.push(None); + } + } + } + Ok((fenced_disks, tokens)) + } + + pub(in crate::set_disk) async fn release_quota_mutation_fences( + disks: &[Option], + tokens: &[Option], + bucket: &str, + object: &str, + write_quorum: usize, + ) -> crate::error::Result<()> { + let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| { + let disk = disk.as_ref()?.clone(); + let token = (*token)?; + let fence_path = fence_path.clone(); + Some(async move { disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, token).await }) + })) + .await; + if results.iter().filter(|result| result.is_ok()).count() < write_quorum { + return Err(StorageError::ErasureWriteQuorum); + } + Ok(()) + } + + pub(in crate::set_disk) fn assign_rename_data_indexes(file_infos: &mut [FileInfo]) { + for (index, file_info) in file_infos.iter_mut().enumerate() { + if file_info.erasure.index == 0 { + file_info.erasure.index = index + 1; + } + } + } + + pub(in crate::set_disk) async fn abort_quota_reservation_after_fence( + reservation: crate::bucket::quota::reservation::QuotaReservation, + disks: &[Option], + tokens: &[Option], + bucket: &str, + object: &str, + write_quorum: usize, + fenced: bool, + ) { + let safe_to_abort = !fenced + || Self::release_quota_mutation_fences(disks, tokens, bucket, object, write_quorum) + .await + .is_ok(); + if safe_to_abort { + reservation.abort().await; + } else { + reservation.defer_after_fence(); + } + } + #[tracing::instrument(level = "debug", skip(disks, file_infos))] #[allow(clippy::type_complexity)] pub(in crate::set_disk) async fn rename_data( @@ -3035,13 +3156,22 @@ impl SetDisks { dst_bucket: &str, dst_object: &str, write_quorum: usize, - ) -> disk::error::Result<( - Vec>, - RenameConvergence, - Option, - Vec>, - Option, - )> { + ) -> disk::error::Result { + Self::rename_data_owned(disks, src_bucket, src_object, file_infos.to_vec(), dst_bucket, dst_object, write_quorum) + .await + .map(RenameDataCommit::into_legacy_tuple) + } + + #[tracing::instrument(level = "debug", skip(disks, file_infos))] + pub(in crate::set_disk) async fn rename_data_owned( + disks: &[Option], + src_bucket: &str, + src_object: &str, + file_infos: Vec, + dst_bucket: &str, + dst_object: &str, + write_quorum: usize, + ) -> disk::error::Result { if let Some(file_info) = disks .iter() .zip(file_infos.iter()) @@ -3066,7 +3196,7 @@ impl SetDisks { let disk_count = disks.len(); let fanout_disks = disks.to_vec(); - let fanout_file_infos = file_infos.to_vec(); + let fanout_file_infos = file_infos; let fanout_src_bucket = src_bucket.clone(); let fanout_src_object = src_object.clone(); let fanout_dst_bucket = dst_bucket.clone(); @@ -3078,9 +3208,9 @@ impl SetDisks { let fanout = tokio::spawn(async move { let futures = fanout_disks .into_iter() - .zip(fanout_file_infos) + .zip(fanout_file_infos.iter()) .enumerate() - .map(|(i, (disk, mut file_info))| { + .map(|(i, (disk, file_info))| { let src_bucket = fanout_src_bucket.clone(); let src_object = fanout_src_object.clone(); let dst_object = fanout_dst_object.clone(); @@ -3097,11 +3227,15 @@ impl SetDisks { }; let is_delete_marker = file_info.is_canonical_delete_marker(); - if file_info.erasure.index == 0 { - file_info.erasure.index = i + 1; - } - - if !is_delete_marker && !file_info.has_valid_erasure_geometry() { + let mut local_file_info; + let file_info = if file_info.erasure.index == 0 { + local_file_info = file_info.clone(); + local_file_info.erasure.index = i + 1; + &local_file_info + } else { + file_info + }; + if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) { return Err(DiskError::FileCorrupt); } @@ -3109,12 +3243,13 @@ impl SetDisks { // A no-op immediately-ready future in production. Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await; - disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) + disk.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) .await }) .catch_unwind() }); - join_all(futures).await + let results = join_all(futures).await; + (results, fanout_file_infos) }); let mut disk_versions = vec![None; disk_count]; @@ -3122,7 +3257,7 @@ impl SetDisks { let mut cleanup_data_dirs = vec![None; disk_count]; let mut old_current_sizes = vec![None; disk_count]; - let results = fanout.await.map_err(|_| DiskError::Unexpected)?; + let (results, mut file_infos) = fanout.await.map_err(|_| DiskError::Unexpected)?; for (idx, result) in results.iter().enumerate() { match result { @@ -3178,7 +3313,7 @@ impl SetDisks { } if let Some(disk) = disks[i].as_ref() { - let fi = file_infos[i].clone(); + let fi = std::mem::take(&mut file_infos[i]); let old_data_dir = data_dirs[i]; let disk = disk.clone(); let dst_bucket = dst_bucket.clone(); @@ -3301,6 +3436,8 @@ impl SetDisks { let convergence = Self::classify_rename_convergence(&disk_versions, &errs); let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum); let online_disks = Self::eval_disks(disks, &errs); + let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?; + let committed_file_info = std::mem::take(&mut file_infos[committed_slot]); let cleanup_disks = if let Some(data_dir) = data_dir { disks .iter() @@ -3318,7 +3455,14 @@ impl SetDisks { vec![None; disks.len()] }; - Ok((online_disks, convergence, data_dir, cleanup_disks, old_current_size)) + Ok(RenameDataCommit { + online_disks, + convergence, + data_dir, + cleanup_disks, + old_current_size, + committed_file_info, + }) } /// rustfs/backlog#1009: reduce the per-disk observations of the diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 27e382a3e..87f4624ef 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -719,8 +719,8 @@ pub(crate) use core::io_primitives::disk_call_counters; mod ctx; mod metadata; mod ops; -#[cfg(test)] -pub(crate) use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; +#[cfg(any(test, feature = "test-util"))] +pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; #[cfg(feature = "test-util")] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; pub(crate) use ops::object::body_cache_plaintext_len; diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 089515ec6..20e1f0fd6 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::super::*; +use crate::disk::disk_store::DiskStoreRenameDataExt; use crate::io_support::bitrot::object_mmap_read_enabled; use crate::storage_api_contracts::namespace::NamespaceLocking as _; use tracing::trace; @@ -1164,10 +1165,10 @@ impl SetDisks { let rename_result = if should_fail_heal_rename(bucket, object, index) { Err(DiskError::Unexpected) } else { - disk.rename_data( + disk.rename_data_borrowed( RUSTFS_META_TMP_BUCKET, &tmp_id, - parts_metadata[index].clone(), + &parts_metadata[index], bucket, object, ) diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 236c8af3b..9684cc1f6 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -27,11 +27,12 @@ use super::object::{ object_transaction_fencing_requested, old_data_cleanup_receipt_path, read_object_transaction_epoch_fence, verify_object_transaction_epoch_fence, }; +use crate::bucket::quota::reservation; use crate::crash_inject::{self, CrashPoint}; use crate::multipart_listing::paginate_multipart_listing; use futures::{StreamExt, stream}; use std::future::Future; -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use tokio::task::JoinSet; @@ -61,20 +62,21 @@ impl StaleMultipartCleanupGuard { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] #[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum MultipartCommitPause { +pub enum MultipartCommitPause { PutPartBeforeLockAcquire, PutPartBeforeLockLost, PutPartAfterRename, BeforeLockLost, + BeforeQuotaRename, BeforeTransactionEpochVerify, BeforeObjectPublication, AfterObjectPublication, AfterRename, } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] struct MultipartCommitBarrierState { bucket: String, object: String, @@ -85,27 +87,22 @@ struct MultipartCommitBarrierState { release: tokio::sync::Semaphore, } -#[cfg(test)] -pub(crate) struct MultipartCommitBarrier { +#[cfg(any(test, feature = "test-util"))] +pub struct MultipartCommitBarrier { state: Arc, } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] impl MultipartCommitBarrier { - pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self { + pub fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self { Self::install_for_arrivals(bucket, object, pause, 1) } - pub(crate) fn install_for_arrivals( - bucket: &str, - object: &str, - pause: MultipartCommitPause, - expected_arrivals: usize, - ) -> Self { + pub fn install_for_arrivals(bucket: &str, object: &str, pause: MultipartCommitPause, expected_arrivals: usize) -> Self { assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival"); let state = Arc::new(MultipartCommitBarrierState { bucket: bucket.to_string(), @@ -126,7 +123,7 @@ impl MultipartCommitBarrier { Self { state } } - pub(crate) async fn wait_until_paused(&self) { + pub async fn wait_until_paused(&self) { tokio::time::timeout(Duration::from_secs(30), async { loop { let arrived = self.state.arrived.notified(); @@ -140,12 +137,12 @@ impl MultipartCommitBarrier { .expect("multipart completion should reach the deterministic commit barrier"); } - pub(crate) fn release(&self) { + pub fn release(&self) { self.state.release.add_permits(self.state.expected_arrivals); } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] impl Drop for MultipartCommitBarrier { fn drop(&mut self) { self.release(); @@ -159,7 +156,7 @@ impl Drop for MultipartCommitBarrier { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) { let barrier = { let mut slot = MULTIPART_COMMIT_BARRIER @@ -1892,6 +1889,41 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { fi.parts = Vec::with_capacity(uploaded_parts.len()); + let quota_context = reservation::begin( + &self.ctx, + bucket, + object, + opts.quota_admission, + opts.data_movement, + self.pool_index, + self.set_index, + ) + .await?; + let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some(); + let preserve_replication_ciphertext = opts.replication_request + && contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT); + if quota_context.is_enforced() && preserve_replication_ciphertext { + return Err(Error::PartMissingOrCorrupt); + } + let transformed_object = fi.is_compressed() || should_persist_encryption_original_size(&fi.metadata); + let data_movement_actual_size = if opts.data_movement { + rustfs_utils::http::get_consistent_str(&opts.user_defined, SUFFIX_ACTUAL_SIZE) + .map(|value| { + value + .parse::() + .ok() + .filter(|value| *value >= 0) + .ok_or(Error::PartMissingOrCorrupt) + }) + .transpose()? + } else { + None + }; + let data_movement_actual_size_u64 = data_movement_actual_size + .map(u64::try_from) + .transpose() + .map_err(|_| Error::PartMissingOrCorrupt)?; + let mut object_size: usize = 0; let mut object_actual_size: i64 = 0; @@ -2018,17 +2050,29 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { checksum_combined.extend_from_slice(cs.raw.as_slice()); } - object_size += ext_part.size; - if opts.quota_admission.is_some() && ext_part.actual_size < 0 { + object_size = object_size.checked_add(ext_part.size).ok_or(Error::PartMissingOrCorrupt)?; + let unknown_actual_size_allowed = opts.data_movement && transformed_object && data_movement_actual_size.is_some() + || opts.replication_request && !quota_context.is_enforced(); + if ext_part.actual_size < 0 && !unknown_actual_size_allowed { return Err(Error::PartMissingOrCorrupt); } + let normalized_actual_size = if ext_part.actual_size >= 0 && !transformed_object { + ext_part + .actual_size + .max(i64::try_from(ext_part.size).map_err(|_| Error::PartMissingOrCorrupt)?) + } else { + ext_part.actual_size + }; object_actual_size = object_actual_size - .checked_add(ext_part.actual_size) + .checked_add(normalized_actual_size) .ok_or(Error::PartMissingOrCorrupt)?; - - fi.parts.push(completed_multipart_object_part(p.part_num, ext_part)); + let mut completed_part = completed_multipart_object_part(p.part_num, ext_part); + completed_part.actual_size = normalized_actual_size; + fi.parts.push(completed_part); + } + if !transformed_object && data_movement_actual_size.is_some_and(|actual_size| actual_size < object_actual_size) { + return Err(Error::PartMissingOrCorrupt); } - if let Some(wtcs) = opts.want_checksum.as_ref() { if checksum_type.full_object_requested() { if wtcs.encoded != checksum.encoded { @@ -2053,15 +2097,35 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } } - if let Some(admission) = opts.quota_admission { - let quota_operation_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; - if quota_operation_size > admission.remaining() { - return Err(Error::QuotaExceeded { - current: admission.current_usage(), - limit: admission.quota_limit(), - }); - } - } + let declared_replication_actual_size = opts + .replication_request + .then(|| get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP)) + .flatten(); + let replication_actual_size = if opts.replication_request && quota_context.is_enforced() { + let observed_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let declared_cap = declared_replication_actual_size + .as_deref() + .map(|value| value.parse::().map_err(|_| Error::PartMissingOrCorrupt)) + .transpose()? + .unwrap_or(0); + let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(&fi.metadata) + .map_err(Error::other)? + .map(u64::try_from) + .transpose() + .map_err(|_| Error::PartMissingOrCorrupt)? + .unwrap_or(0); + Some(observed_size.max(declared_cap).max(declared_encryption_size)) + } else { + None + }; + let data_movement_quota_size = data_movement_actual_size_u64.filter(|_| quota_context.is_enforced()); + let quota_new_size = match data_movement_quota_size.or(replication_actual_size) { + Some(size) => size.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?), + None if quota_context.is_enforced() => u64::try_from(object_actual_size) + .map_err(|_| Error::PartMissingOrCorrupt)? + .max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?), + None => 0, + }; if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) { if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) { fi.checksum = Some(Bytes::from(rc_crc_bytes)); @@ -2113,20 +2177,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { ); } - let data_movement_actual_size = if opts.data_movement { - rustfs_utils::http::get_consistent_str(&opts.user_defined, SUFFIX_ACTUAL_SIZE) - .map(|value| { - value - .parse::() - .ok() - .filter(|value| *value >= 0) - .ok_or_else(|| Error::other("data movement actual size metadata is invalid")) - }) - .transpose()? - } else { - None - }; - if let Some(actual_size) = data_movement_actual_size { insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); if persist_encryption_original_size { @@ -2134,7 +2184,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string()); } } else if opts.replication_request { - if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) { + if let Some(actual_size) = replication_actual_size { + insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); + if persist_encryption_original_size { + fi.metadata + .insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string()); + } + } else if let Some(actual_size) = declared_replication_actual_size { insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone()); if persist_encryption_original_size { fi.metadata @@ -2324,8 +2380,41 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } else { None }; + + let quota_old_size = if quota_context.is_enforced() { + if opts.data_movement { + quota_new_size + } else { + reservation::replaced_logical_size(&self, bucket, object, opts).await? + } + } else { + 0 + }; + let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?; + let (commit_disks, quota_fence_tokens) = if quota_mutation_fence { + match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await { + Ok((disks, tokens)) => { + for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) { + if let Some(token) = token { + insert_str( + &mut metadata.metadata, + crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + token.as_uuid().to_string(), + ); + } + } + (disks, tokens) + } + Err(err) => { + quota_reservation.abort().await; + return Err(err); + } + } + } else { + (shuffle_disks.clone(), vec![None; shuffle_disks.len()]) + }; let transaction_epoch = - transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas)); + transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas)); let commit_set = self.clone(); let commit_bucket = bucket.to_owned(); @@ -2333,49 +2422,152 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let commit_upload_id = upload_id.to_owned(); let commit_upload_id_path = upload_id_path.clone(); let commit_version_suspended = opts.version_suspended; + let commit_versioned = opts.versioned; + let commit_version_id = opts.version_id.clone(); + let commit_namespace_lock_fence = opts.namespace_lock_fence.clone(); + let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone(); let commit_is_versioned = opts.versioned || opts.version_suspended; let commit_capacity_scope_token = opts.capacity_scope_token; let commit_object_lock_guard = object_lock_guard.take(); - let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some(); + let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some() || quota_mutation_fence; let commit = async move { let _object_lock_guard = commit_object_lock_guard; let _upload_guard = upload_guard; + let mut quota_reservation = quota_reservation; let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); - // Crash-consistency injection: hard power loss after the upload is fully - // staged and locked but before the authoritative rename_data commit. No - // disk has moved the staged data, so a crash here must leave any prior - // committed version byte-for-byte intact (rustfs/backlog#864) and the - // upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`. - if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) { - return Err(StorageError::Unexpected); + let pre_rename_result: Result<()> = async { + // Crash-consistency injection: hard power loss after the upload is fully + // staged and locked but before the authoritative rename_data commit. No + // disk has moved the staged data, so a crash here must leave any prior + // committed version byte-for-byte intact (rustfs/backlog#864) and the + // upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`. + if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) { + return Err(StorageError::Unexpected); + } + quota_reservation.mark_commit_started().await?; + #[cfg(any(test, feature = "test-util"))] + pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeQuotaRename).await; + if quota_reservation.is_lock_lost() + || !quota_reservation.capability_proof_matches() + || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || commit_namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || commit_bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: commit_bucket.clone(), + object: commit_object.clone(), + required: 1, + achieved: 0, + }); + } + let restore_opts = ObjectOptions { + version_id: commit_version_id.clone(), + versioned: commit_versioned, + version_suspended: commit_version_suspended, + no_lock: true, + ..Default::default() + }; + commit_set + .require_current_restore_operation_id( + &commit_bucket, + &commit_object, + &restore_opts, + expected_restore_operation_id, + "complete_multipart_upload_quota_reservation", + ) + .await?; + if let Some(proof) = transaction_fencing_proof.as_ref() + && !object_transaction_fencing_fleet_proof_matches(proof) + { + return Err(Error::other( + "object transaction fencing fleet capability changed during complete_multipart_upload", + )); + } + if let Some(expected) = transaction_epoch_fence { + #[cfg(test)] + pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify) + .await; + verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?; + } + if quota_reservation.is_lock_lost() + || !quota_reservation.capability_proof_matches() + || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || commit_namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || commit_bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: commit_bucket.clone(), + object: commit_object.clone(), + required: 1, + achieved: 0, + }); + } + Ok(()) + } + .await; + if let Err(err) = pre_rename_result { + SetDisks::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + &commit_bucket, + &commit_object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(err); } // The trailing `_` drops the rename_data old-size backfill // (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit // `get_object_info` lookup, so the backfill has no consumer here yet. - if let Some(proof) = transaction_fencing_proof.as_ref() - && !object_transaction_fencing_fleet_proof_matches(proof) - { - return Err(Error::other( - "object transaction fencing fleet capability changed during complete_multipart_upload", - )); - } - if let Some(expected) = transaction_epoch_fence { - #[cfg(test)] - pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify).await; - verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?; - } - let (online_disks, convergence, op_old_dir, cleanup_disks, _) = SetDisks::rename_data( - &shuffle_disks, + Self::assign_rename_data_indexes(&mut parts_metadatas); + let rename_result = SetDisks::rename_data_owned( + &commit_disks, RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, - &parts_metadatas, + parts_metadatas, &commit_bucket, &commit_object, write_quorum, ) - .await?; + .await; + if quota_mutation_fence { + let _ = SetDisks::release_quota_mutation_fences( + &commit_disks, + "a_fence_tokens, + &commit_bucket, + &commit_object, + write_quorum, + ) + .await; + } + if rename_result.is_ok() { + quota_reservation.commit().await; + } + let rename_commit = match rename_result { + Ok(result) => result, + Err(err) => return Err(err.into()), + }; + let online_disks = rename_commit.online_disks; + let convergence = rename_commit.convergence; + let op_old_dir = rename_commit.data_dir; + let cleanup_disks = rename_commit.cleanup_disks; + let committed_file_info = rename_commit.committed_file_info; // Detach admission before any post-commit await: client cancellation // must not couple durable convergence repair to cleanup work. @@ -2420,9 +2612,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { return Err(StorageError::Unexpected); } - if let Some(committed_slot) = online_disks.iter().position(Option::is_some) { - fi = parts_metadatas[committed_slot].clone(); - } + fi = committed_file_info; let committed_dir = fi.data_dir.unwrap_or_default().to_string(); commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &online_disks); @@ -2927,12 +3117,34 @@ mod tests { content: &[u8], actual_size: i64, ) -> CompletePart { + put_test_part_with_opts( + set_disks, + bucket, + object, + upload_id, + part_number, + (content, actual_size), + &ObjectOptions::default(), + ) + .await + } + + async fn put_test_part_with_opts( + set_disks: &Arc, + bucket: &str, + object: &str, + upload_id: &str, + part_number: usize, + part: (&[u8], i64), + opts: &ObjectOptions, + ) -> CompletePart { + let (content, actual_size) = part; let mut reader = PutObjReader::new( HashReader::from_stream(Cursor::new(content.to_vec()), content.len() as i64, actual_size, None, None, false) .expect("hash reader should be constructed"), ); let part = set_disks - .put_object_part(bucket, object, upload_id, part_number, &mut reader, &ObjectOptions::default()) + .put_object_part(bucket, object, upload_id, part_number, &mut reader, opts) .await .expect("uploading the part should succeed"); CompletePart { @@ -3162,7 +3374,7 @@ mod tests { let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await; let mut denied_opts = ObjectOptions::default(); - assert!(denied_opts.set_quota_admission(100, 4195)); + assert!(denied_opts.set_quota_admission(100, 4180)); let err = set_disks .clone() @@ -3173,7 +3385,7 @@ mod tests { err, StorageError::QuotaExceeded { current: 100, - limit: 4195 + limit: 4180 } )); @@ -3191,7 +3403,7 @@ mod tests { ); let mut allowed_opts = ObjectOptions::default(); - assert!(allowed_opts.set_quota_admission(100, 4196)); + assert!(allowed_opts.set_quota_admission(100, 4181)); let completed = set_disks .clone() .complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts) @@ -3232,6 +3444,120 @@ mod tests { ); } + #[tokio::test] + async fn replication_quota_uses_server_observed_part_size_as_lower_bound() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-replication-quota-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let mut create_opts = ObjectOptions::default(); + insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string()); + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("replication multipart upload should be created"); + let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x72; 4096], 1).await; + let mut complete_opts = ObjectOptions { + replication_request: true, + ..Default::default() + }; + assert!(complete_opts.set_quota_admission(0, 4095)); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part.clone()], &complete_opts) + .await + .expect_err("a forged tiny replication logical size must not reduce quota admission"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + assert!( + set_disks + .check_upload_id_exists(bucket, object, &upload.upload_id, false) + .await + .is_ok(), + "quota rejection must leave replicated multipart parts retryable" + ); + + assert!(complete_opts.set_quota_admission(0, 4096)); + let completed = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect("the physical safety boundary should admit the transformed replica"); + assert_eq!(completed.get_actual_size().expect("replica logical size should parse"), 1); + } + + #[tokio::test] + async fn direct_multipart_quota_uses_server_observed_part_size_as_lower_bound() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-direct-quota-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let upload = set_disks + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("multipart upload should be created"); + let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x73; 4096], 1).await; + let mut complete_opts = ObjectOptions::default(); + assert!(complete_opts.set_quota_admission(0, 4095)); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect_err("a forged tiny direct logical size must not reduce quota admission"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + assert!( + set_disks + .check_upload_id_exists(bucket, object, &upload.upload_id, false) + .await + .is_ok(), + "quota rejection must leave direct multipart parts retryable" + ); + } + + #[tokio::test] + async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-compressed-ciphertext-quota-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let mut create_opts = ObjectOptions::default(); + insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string()); + insert_str( + &mut create_opts.user_defined, + rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, + "true".to_string(), + ); + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("ciphertext multipart upload should be created"); + let payload = vec![0x74; 4096]; + let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &payload, 1).await; + let mut complete_opts = ObjectOptions { + replication_request: true, + ..Default::default() + }; + assert!(complete_opts.set_quota_admission(0, u64::MAX)); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect_err("ciphertext replication has no server-observed logical quota size"); + assert!(matches!(err, StorageError::PartMissingOrCorrupt)); + assert!( + set_disks + .check_upload_id_exists(bucket, object, &upload.upload_id, false) + .await + .is_ok(), + "rejection must leave ciphertext multipart parts retryable" + ); + } + #[tokio::test] async fn complete_multipart_quota_rejects_invalid_logical_sizes() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; @@ -3554,7 +3880,6 @@ mod tests { async fn data_movement_complete_accepts_unknown_compressed_part_actual_size() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; let bucket = "data-movement-unknown-actual-size-bucket"; - let object = "object"; make_bucket_on_all(&disk_stores, bucket).await; let mut metadata = HashMap::new(); @@ -3568,6 +3893,63 @@ mod tests { rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, "source-generation".to_string(), ); + + for (object, quota_limit) in [("without-quota", None), ("with-quota", Some(u64::MAX))] { + let create_opts = ObjectOptions { + data_movement: true, + user_defined: metadata.clone(), + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("data movement upload should be created"); + + let mut completed_parts = Vec::new(); + for (number, actual_size) in [(1, -1), (2, 1)] { + let mut reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(vec![number as u8]), 1, actual_size, None, None, false) + .expect("part reader should be constructed"), + ); + let part = set_disks + .put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &create_opts) + .await + .expect("data movement part should be written"); + completed_parts.push(CompletePart { + part_num: number, + etag: part.etag, + ..Default::default() + }); + } + + let missing_size_err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts.clone(), &create_opts) + .await + .expect_err("data movement completion must require an authoritative total size"); + assert!(matches!(missing_size_err, StorageError::PartMissingOrCorrupt)); + + let mut complete_opts = create_opts.clone(); + rustfs_utils::http::insert_str( + &mut complete_opts.user_defined, + rustfs_utils::http::SUFFIX_ACTUAL_SIZE, + "2".to_string(), + ); + if let Some(quota_limit) = quota_limit { + assert!(complete_opts.set_quota_admission(0, quota_limit)); + } + let completion = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &complete_opts) + .await; + + let completed = completion.expect("data movement completion should accept the persisted unknown-size sentinel"); + + assert_eq!(completed.parts[0].actual_size, -1); + assert_eq!(completed.get_actual_size().expect("completed object actual size"), 2); + } + + let object = "all-unknown-with-quota"; let create_opts = ObjectOptions { data_movement: true, user_defined: metadata.clone(), @@ -3577,43 +3959,93 @@ mod tests { .new_multipart_upload(bucket, object, &create_opts) .await .expect("data movement upload should be created"); - - let mut completed_parts = Vec::new(); - for (number, actual_size) in [(1, -1), (2, 1)] { - let mut reader = PutObjReader::new( - HashReader::from_stream(Cursor::new(vec![number as u8]), 1, actual_size, None, None, false) - .expect("part reader should be constructed"), - ); - let part = set_disks - .put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &create_opts) - .await - .expect("data movement part should be written"); - completed_parts.push(CompletePart { - part_num: number, - etag: part.etag, - ..Default::default() - }); - } - - rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "2".to_string()); + let part = put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x40], -1), &create_opts).await; + let mut complete_opts = create_opts; + rustfs_utils::http::insert_str(&mut complete_opts.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "2".to_string()); + assert!(complete_opts.set_quota_admission(0, u64::MAX)); let completed = set_disks .clone() - .complete_multipart_upload( - bucket, - object, - &upload.upload_id, - completed_parts, - &ObjectOptions { - data_movement: true, - user_defined: metadata, - ..Default::default() - }, - ) + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) .await - .expect("data movement completion should accept the persisted unknown-size sentinel"); - + .expect("quota must accept an all-unknown data movement upload with an authoritative total"); assert_eq!(completed.parts[0].actual_size, -1); assert_eq!(completed.get_actual_size().expect("completed object actual size"), 2); + + let object = "legacy-zero-fallback"; + let create_opts = ObjectOptions { + data_movement: true, + user_defined: metadata.clone(), + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("data movement upload should be created"); + let part = + put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x41; 128], 128), &create_opts).await; + let mut complete_opts = create_opts; + rustfs_utils::http::insert_str( + &mut complete_opts.user_defined, + rustfs_utils::http::SUFFIX_ACTUAL_SIZE, + "100".to_string(), + ); + assert!(complete_opts.set_quota_admission(0, u64::MAX)); + let completed = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect("legacy zero-to-physical part fallback must remain migratable"); + assert_eq!(completed.parts[0].actual_size, 128); + assert_eq!(completed.get_actual_size().expect("completed object actual size"), 100); + + let object = "inconsistent-untransformed-total"; + let mut untransformed_metadata = metadata.clone(); + rustfs_utils::http::remove_str(&mut untransformed_metadata, rustfs_utils::http::SUFFIX_COMPRESSION); + let create_opts = ObjectOptions { + data_movement: true, + user_defined: untransformed_metadata, + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("data movement upload should be created"); + let part = put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x42; 2], 2), &create_opts).await; + let mut complete_opts = create_opts; + rustfs_utils::http::insert_str(&mut complete_opts.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect_err("authoritative total below known logical part sizes must fail closed"); + assert!(matches!(err, StorageError::PartMissingOrCorrupt)); + + for (object, declared_size) in [("invalid-total", "invalid"), ("negative-total", "-1")] { + let create_opts = ObjectOptions { + data_movement: true, + user_defined: metadata.clone(), + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("data movement upload should be created"); + let part = + put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x41], 1), &create_opts).await; + let mut complete_opts = create_opts.clone(); + rustfs_utils::http::insert_str( + &mut complete_opts.user_defined, + rustfs_utils::http::SUFFIX_ACTUAL_SIZE, + declared_size.to_string(), + ); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect_err("invalid authoritative total size must fail closed"); + assert!(matches!(err, StorageError::PartMissingOrCorrupt)); + } } async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index cd450dac3..95fe3438a 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -40,6 +40,7 @@ use crate::bucket::lifecycle::{ save_transition_transaction_record, }, }; +use crate::bucket::quota::reservation; use crate::bucket::replication::{ DeleteReplicationConfigSnapshot, VersionPurgeStatusType, replication_state_to_filemeta, version_purge_status_to_filemeta, }; @@ -57,18 +58,45 @@ use http::HeaderValue; use rustfs_utils::path::decode_dir_object; use std::future::Future; use std::sync::OnceLock; +use tokio_util::sync::CancellationToken; const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json"; +struct PutObjectCommitCancellation { + token: CancellationToken, + armed: bool, +} + +impl PutObjectCommitCancellation { + fn new() -> Self { + Self { + token: CancellationToken::new(), + armed: true, + } + } + + fn child_token(&self) -> CancellationToken { + self.token.clone() + } + + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for PutObjectCommitCancellation { + fn drop(&mut self) { + if self.armed { + self.token.cancel(); + } + } +} + #[inline] fn duration_millis_f64(duration: std::time::Duration) -> f64 { duration.as_secs_f64() * 1000.0 } -fn committed_response_metadata_slot(committed_disks: &[Option], fallback_slot: usize) -> usize { - committed_disks.iter().position(Option::is_some).unwrap_or(fallback_slot) -} - pub(in crate::set_disk::ops) fn assign_object_transaction_epoch( shuffle_disks: &[Option], parts_metadatas: &mut [FileInfo], @@ -207,38 +235,6 @@ mod duration_metrics_tests { } } -#[cfg(test)] -mod put_metadata_tests { - use super::*; - - #[test] - fn committed_file_info_follows_exact_quorum_success_slot() { - let mut first_success = FileInfo::new("bucket/object", 2, 2); - first_success.name = "first-success".to_string(); - let mut second_success = first_success.clone(); - second_success.name = "second-success".to_string(); - let mut parts_metadata = [FileInfo::default(), first_success, second_success, FileInfo::default()]; - let committed_disks = [None, Some(()), Some(()), None]; - - assert_eq!( - committed_disks.iter().filter(|disk| disk.is_some()).count(), - 2, - "fixture must meet exact quorum" - ); - let selected_slot = committed_response_metadata_slot(&committed_disks, 3); - let selected = std::mem::take(&mut parts_metadata[selected_slot]); - - assert_eq!(selected.name, "first-success"); - assert_eq!(parts_metadata[1], FileInfo::default(), "selected metadata should move without cloning"); - assert_eq!(parts_metadata[2].name, "second-success", "other committed metadata must remain available"); - assert_eq!( - committed_response_metadata_slot::<()>(&[None, None, None, None], 3), - 3, - "a violated post-commit success-mask invariant must not turn a durable PUT into an error" - ); - } -} - fn is_restore_control_metadata(key: &str) -> bool { key.eq_ignore_ascii_case(X_AMZ_RESTORE.as_str()) || key.eq_ignore_ascii_case(rustfs_utils::http::headers::AMZ_RESTORE_EXPIRY_DAYS) @@ -1341,6 +1337,26 @@ impl SetDisks { object: &str, data: &mut PutObjReader, opts: &ObjectOptions, + ) -> Result<(ObjectInfo, Option)> { + self.put_object_with_old_current_size_boxed(bucket, object, data, opts).await + } + + fn put_object_with_old_current_size_boxed<'a>( + &'a self, + bucket: &'a str, + object: &'a str, + data: &'a mut PutObjReader, + opts: &'a ObjectOptions, + ) -> impl Future)>> + Send + 'a { + Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts)) + } + + async fn put_object_with_old_current_size_inner( + &self, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, ) -> Result<(ObjectInfo, Option)> { crate::hp_guard!("SetDisks::put_object"); let storage_class_config = self.storage_class_config_snapshot(); @@ -1929,8 +1945,125 @@ impl SetDisks { } else { None }; + + let quota_context = reservation::begin( + &self.ctx, + bucket, + object, + opts.quota_admission, + opts.data_movement, + self.pool_index, + self.set_index, + ) + .await?; + let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some(); + let mut replication_quota_size = None; + + if opts.replication_request { + if quota_context.is_enforced() && opts.preserve_ciphertext { + return Err(Error::PartMissingOrCorrupt); + } + if quota_context.is_enforced() { + let persisted_metadata = &parts_metadatas[response_metadata_slot].metadata; + let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let physical_size = u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let transformed = contains_key_str(persisted_metadata, SUFFIX_COMPRESSION) + || should_persist_encryption_original_size(persisted_metadata); + let declared_size = get_str(persisted_metadata, SUFFIX_ACTUAL_SIZE) + .map(|value| value.parse::().map_err(|_| Error::PartMissingOrCorrupt)) + .transpose()? + .unwrap_or(0); + let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(persisted_metadata) + .map_err(Error::other)? + .map(u64::try_from) + .transpose() + .map_err(|_| Error::PartMissingOrCorrupt)? + .unwrap_or(0); + let logical_size = observed_size.max(declared_size).max(declared_encryption_size); + let persisted_size = if transformed { + logical_size + } else { + logical_size.max(physical_size) + }; + replication_quota_size = Some(logical_size.max(physical_size)); + actual_size = i64::try_from(persisted_size).map_err(|_| Error::PartMissingOrCorrupt)?; + for metadata in &mut parts_metadatas { + insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, persisted_size.to_string()); + if should_persist_encryption_original_size(&metadata.metadata) { + metadata + .metadata + .insert("x-rustfs-encryption-original-size".to_string(), persisted_size.to_string()); + } + if let Some(part) = metadata.parts.first_mut() { + part.actual_size = actual_size; + } + } + } + } else if actual_size >= 0 { + let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let persisted_metadata = &parts_metadatas[response_metadata_slot].metadata; + let transformed = contains_key_str(persisted_metadata, SUFFIX_COMPRESSION) + || should_persist_encryption_original_size(persisted_metadata); + let server_observed_size = if transformed { + observed_size + } else { + observed_size.max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?) + }; + actual_size = i64::try_from(server_observed_size).map_err(|_| Error::PartMissingOrCorrupt)?; + for metadata in &mut parts_metadatas { + insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, server_observed_size.to_string()); + if should_persist_encryption_original_size(&metadata.metadata) { + metadata + .metadata + .insert("x-rustfs-encryption-original-size".to_string(), server_observed_size.to_string()); + } + if let Some(part) = metadata.parts.first_mut() { + part.actual_size = actual_size; + } + } + } + + let (quota_old_size, quota_new_size) = if quota_context.is_enforced() { + let new_size = match replication_quota_size { + Some(size) => size, + None => u64::try_from(actual_size) + .map_err(|_| Error::PartMissingOrCorrupt)? + .max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?), + }; + let old_size = if opts.data_movement { + new_size + } else { + reservation::replaced_logical_size(self, bucket, object, opts).await? + }; + (old_size, new_size) + } else { + (0, 0) + }; + let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?; + let (commit_disks, quota_fence_tokens) = if quota_mutation_fence { + match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await { + Ok((disks, tokens)) => { + for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) { + if let Some(token) = token { + insert_str( + &mut metadata.metadata, + crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + token.as_uuid().to_string(), + ); + } + } + (disks, tokens) + } + Err(err) => { + quota_reservation.abort().await; + return Err(err); + } + } + } else { + (shuffle_disks.clone(), vec![None; shuffle_disks.len()]) + }; let transaction_epoch = - transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas)); + transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas)); let commit_set = self.clone(); let commit_bucket = bucket.to_owned(); @@ -1938,18 +2071,65 @@ impl SetDisks { let commit_tmp_dir = tmp_dir.clone(); let commit_object_lock_guard = object_lock_guard.take(); let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take(); - let detach_commit_owner = commit_object_lock_guard.is_some() || commit_bucket_lifecycle_guard.is_some(); + let detach_commit_owner = + commit_object_lock_guard.is_some() || commit_bucket_lifecycle_guard.is_some() || quota_mutation_fence; let commit_write_path_label = write_path.metric_label(); let commit_is_versioned = opts.versioned || opts.version_suspended; + let commit_versioned = opts.versioned; + let commit_version_suspended = opts.version_suspended; + let commit_version_id = opts.version_id.clone(); + let commit_namespace_lock_fence = opts.namespace_lock_fence.clone(); + let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone(); let commit_capacity_scope_token = opts.capacity_scope_token; let commit_replication_state = replication_state_to_filemeta(&opts.put_replication_state()); tmp_cleanup_owned = true; - let commit = async move { + let commit = move |cancellation: Option| async move { let _object_lock_guard = commit_object_lock_guard; let _bucket_lifecycle_guard = commit_bucket_lifecycle_guard; + let mut quota_reservation = quota_reservation; let rename_stage_start = Instant::now(); - let pre_rename_result: Result<()> = async { + let pre_rename = async { + #[cfg(any(test, feature = "test-util"))] + pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterQuotaReservation).await; + quota_reservation.mark_commit_started().await?; + #[cfg(any(test, feature = "test-util"))] + pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::BeforeQuotaRename).await; + if quota_reservation.is_lock_lost() + || !quota_reservation.capability_proof_matches() + || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || commit_namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || commit_bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: commit_bucket.clone(), + object: commit_object.clone(), + required: 1, + achieved: 0, + }); + } + let restore_opts = ObjectOptions { + version_id: commit_version_id.clone(), + versioned: commit_versioned, + version_suspended: commit_version_suspended, + no_lock: true, + ..Default::default() + }; + commit_set + .require_current_restore_operation_id( + &commit_bucket, + &commit_object, + &restore_opts, + expected_restore_operation_id, + "put_object_quota_reservation", + ) + .await?; if let Some(proof) = transaction_fencing_proof.as_ref() && !object_transaction_fencing_fleet_proof_matches(proof) { @@ -1965,10 +2145,47 @@ impl SetDisks { .await; verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?; } + if quota_reservation.is_lock_lost() + || !quota_reservation.capability_proof_matches() + || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || commit_namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || commit_bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: commit_bucket.clone(), + object: commit_object.clone(), + required: 1, + achieved: 0, + }); + } Ok(()) - } - .await; + }; + let pre_rename_result = if let Some(cancellation) = cancellation { + tokio::select! { + biased; + _ = cancellation.cancelled() => Err(StorageError::OperationCanceled), + result = pre_rename => result, + } + } else { + pre_rename.await + }; if let Err(err) = pre_rename_result { + SetDisks::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + &commit_bucket, + &commit_object, + write_quorum, + quota_mutation_fence, + ) + .await; if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { @@ -1982,17 +2199,32 @@ impl SetDisks { } return Err(err); } - let rename_result = SetDisks::rename_data( - &shuffle_disks, + + Self::assign_rename_data_indexes(&mut parts_metadatas); + let rename_result = SetDisks::rename_data_owned( + &commit_disks, RUSTFS_META_TMP_BUCKET, commit_tmp_dir.as_str(), - &parts_metadatas, + parts_metadatas, &commit_bucket, &commit_object, write_quorum, ) .await; - let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = match rename_result { + if quota_mutation_fence { + let _ = SetDisks::release_quota_mutation_fences( + &commit_disks, + "a_fence_tokens, + &commit_bucket, + &commit_object, + write_quorum, + ) + .await; + } + if rename_result.is_ok() { + quota_reservation.commit().await; + } + let rename_commit = match rename_result { Ok(commit) => commit, Err(err) => { if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { @@ -2009,6 +2241,12 @@ impl SetDisks { return Err(err.into()); } }; + let online_disks = rename_commit.online_disks; + let convergence = rename_commit.convergence; + let op_old_dir = rename_commit.data_dir; + let cleanup_disks = rename_commit.cleanup_disks; + let old_current_size = rename_commit.old_current_size; + let mut fi = rename_commit.committed_file_info; // Do this before any post-commit await so request cancellation cannot // bypass best-effort admission. A process crash before admission // remains subject to the existing scanner reconciliation path. @@ -2117,9 +2355,6 @@ impl SetDisks { } } - let committed_metadata_slot = committed_response_metadata_slot(&online_disks, response_metadata_slot); - let mut fi = std::mem::take(&mut parts_metadatas[committed_metadata_slot]); - if is_compressed { record_compression_total_memory(actual_size as u64, w_size as u64).await; } @@ -2202,11 +2437,15 @@ impl SetDisks { }; if detach_commit_owner { - tokio::spawn(commit) + let mut cancellation = PutObjectCommitCancellation::new(); + let child_token = cancellation.child_token(); + let result = tokio::spawn(async move { Box::pin(commit(Some(child_token))).await }) .await - .map_err(|err| Error::other(format!("put_object commit task failed: {err}")))? + .map_err(|err| Error::other(format!("put_object commit task failed: {err}")))?; + cancellation.disarm(); + result } else { - commit.await + Box::pin(commit(None)).await } } .await; @@ -3225,6 +3464,8 @@ fn transaction_fencing_gate_requested_for(requested: bool, fleet_confirmed: bool pub enum PutObjectCommitPause { BeforeNamespace, AfterNamespace, + AfterQuotaReservation, + BeforeQuotaRename, BeforeMetadata, BeforeTransactionEpochVerify, } @@ -6295,6 +6536,139 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support { } } +#[cfg(test)] +mod replication_quota_safety_tests { + use super::hermetic_set_disks_support::hermetic_set_disks; + use super::*; + use std::io::Cursor; + + #[tokio::test] + async fn quota_put_future_keeps_commit_state_off_the_caller_stack() { + let (_temp_dirs, _disks, set_disks) = hermetic_set_disks(4).await; + let mut reader = PutObjReader::from_vec(Vec::new()); + let opts = ObjectOptions::default(); + + let future = set_disks.put_object_with_old_current_size("bucket", "object", &mut reader, &opts); + let future_size = std::mem::size_of_val(&future); + + assert!( + future_size <= 1024, + "put_object_with_old_current_size future must stay stack-bounded, got {future_size} bytes" + ); + } + + #[tokio::test] + async fn replication_put_quota_uses_physical_bytes_as_a_safety_floor() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "replication-put-quota-safety"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut user_defined = HashMap::new(); + insert_str( + &mut user_defined, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string()); + let payload = vec![0x61; 4096]; + + let mut denied_opts = ObjectOptions { + replication_request: true, + user_defined: user_defined.clone(), + ..Default::default() + }; + assert!(denied_opts.set_quota_admission(0, 4095)); + let mut denied_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false) + .expect("construct forged replication reader"), + ); + let err = set_disks + .put_object(bucket, "object", &mut denied_reader, &denied_opts) + .await + .expect_err("server-observed bytes must prevent a tiny replication quota claim"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + + let mut allowed_opts = ObjectOptions { + replication_request: true, + user_defined, + ..Default::default() + }; + assert!(allowed_opts.set_quota_admission(0, 4096)); + let mut allowed_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false) + .expect("construct exact-boundary replication reader"), + ); + let stored = set_disks + .put_object(bucket, "object", &mut allowed_reader, &allowed_opts) + .await + .expect("server-observed exact quota boundary should succeed"); + assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1); + } + + #[tokio::test] + async fn direct_put_cannot_persist_a_tiny_logical_size() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "direct-put-quota-safety"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let payload = vec![0x62; 4096]; + let mut denied_opts = ObjectOptions::default(); + assert!(denied_opts.set_quota_admission(0, 4095)); + let mut denied_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false) + .expect("construct forged direct reader"), + ); + let err = set_disks + .put_object(bucket, "object", &mut denied_reader, &denied_opts) + .await + .expect_err("server-observed bytes must prevent a tiny direct quota claim"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + + let mut allowed_opts = ObjectOptions::default(); + assert!(allowed_opts.set_quota_admission(0, 4096)); + let mut allowed_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false) + .expect("construct exact-boundary direct reader"), + ); + let stored = set_disks + .put_object(bucket, "object", &mut allowed_reader, &allowed_opts) + .await + .expect("server-observed exact quota boundary should succeed"); + assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 4096); + } + + #[tokio::test] + async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "ciphertext-replication-quota-safety"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut user_defined = HashMap::new(); + user_defined.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()); + insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string()); + let mut opts = ObjectOptions { + replication_request: true, + preserve_ciphertext: true, + user_defined, + ..Default::default() + }; + assert!(opts.set_quota_admission(0, u64::MAX)); + let payload = vec![0x63; 4096]; + let mut reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload), 4096, 4096, None, None, false) + .expect("construct ciphertext replication reader"), + ); + let err = set_disks + .put_object(bucket, "object", &mut reader, &opts) + .await + .expect_err("ciphertext replication without a server-observed logical size must fail closed"); + assert!(matches!(err, StorageError::PartMissingOrCorrupt)); + } +} + #[cfg(test)] mod inline_put_commit_path_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 077caf714..1e4b706d2 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -1174,6 +1174,32 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn quota_object_fence_ignores_an_unrelated_offline_pool() { + let temp_dir = tempfile::tempdir().expect("create quota fence store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "quota-object-fence", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = format!("quota-object-fence-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create quota fence bucket"); + store.pools[1].disk_set[0].disks.write().await.fill(None); + + crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, Some((0, 0))) + .await + .expect("the selected pool fence should ignore an unrelated offline pool"); + let err = crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, None) + .await + .expect_err("legacy reservations must conservatively fence every pool"); + assert!(matches!(err, StorageError::ErasureWriteQuorum)); + + shutdown.cancel(); + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn tag_updates_skip_active_rebalance_source_pool() { diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 2e8ee9968..10fc766a2 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -641,7 +641,7 @@ pub(crate) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) { .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta))); } -pub(super) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 { +pub(crate) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 { observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default() } diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index ac6b520f0..0ba89a7ec 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -4741,9 +4741,10 @@ mod tests { #[tokio::test] #[serial_test::serial(body_cache_hook)] async fn select_snapshot_rejects_latest_versioned_delete_marker_during_prepare() { - let (_first_dirs, first_set) = make_local_set_disks(4, 2).await; - let (_second_dirs, second_set) = make_local_set_disks(4, 2).await; - let store = new_prepared_reader_test_store(&[Arc::clone(&first_set), Arc::clone(&second_set)]).await; + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await; let bucket = "select-snapshot-latest-delete-marker"; let object = "versioned-object.bin"; let versioned_opts = ObjectOptions { diff --git a/crates/protocols/src/swift/handler.rs b/crates/protocols/src/swift/handler.rs index 0d9cdc455..b8018ab5a 100644 --- a/crates/protocols/src/swift/handler.rs +++ b/crates/protocols/src/swift/handler.rs @@ -748,9 +748,10 @@ async fn handle_authenticated_request( } for (key, value) in info.user_defined.iter() { - if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } @@ -817,9 +818,10 @@ async fn handle_authenticated_request( // Add custom metadata headers (X-Object-Meta-*) for (key, value) in info.user_defined.iter() { - if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } @@ -856,9 +858,10 @@ async fn handle_authenticated_request( // Add custom metadata headers (X-Object-Meta-*) for (key, value) in info.user_defined.iter() { - if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } @@ -1168,9 +1171,10 @@ async fn handle_object_get( } for (key, value) in info.user_defined.iter() { - if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } @@ -1237,9 +1241,10 @@ async fn handle_object_get( if key == "x-delete-at" { // Add X-Delete-At header directly (not as X-Object-Meta-*) response = response.header("x-delete-at", value.as_str()); - } else if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + } else if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } @@ -1293,9 +1298,10 @@ async fn handle_object_head( if key == "x-delete-at" { // Add X-Delete-At header directly (not as X-Object-Meta-*) response = response.header("x-delete-at", value.as_str()); - } else if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + } else if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } diff --git a/crates/protocols/src/swift/object.rs b/crates/protocols/src/swift/object.rs index 85581a323..7169b152a 100644 --- a/crates/protocols/src/swift/object.rs +++ b/crates/protocols/src/swift/object.rs @@ -67,10 +67,57 @@ const LOG_COMPONENT_PROTOCOLS: &str = "protocols"; const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object"; const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state"; const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at"; +const USER_METADATA_PREFIX: &str = "x-amz-meta-"; /// Maximum object size in bytes (5GB - Swift default) const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024; +fn stored_swift_user_metadata_key(key: &str) -> String { + if rustfs_utils::http::is_internal_key(key) + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-amz-") + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-") + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-") + { + format!("{USER_METADATA_PREFIX}{key}") + } else { + key.to_string() + } +} + +pub(super) fn swift_response_user_metadata_key(key: &str) -> Option<&str> { + if rustfs_utils::http::is_internal_key(key) + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-") + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-") + { + return None; + } + if let Some(unescaped) = key.strip_prefix(USER_METADATA_PREFIX) + && (rustfs_utils::http::is_internal_key(unescaped) + || rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-amz-") + || rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-rustfs-encryption-") + || rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-minio-encryption-")) + { + return Some(unescaped); + } + Some(key) +} + +fn swift_user_metadata(headers: &HeaderMap) -> Option> { + let mut metadata = HashMap::new(); + let mut present = false; + for (header_name, header_value) in headers.iter() { + let header_name = header_name.as_str().to_lowercase(); + let Some(key) = header_name.strip_prefix("x-object-meta-") else { + continue; + }; + present = true; + if let Ok(value) = header_value.to_str() { + metadata.insert(stored_swift_user_metadata_key(key), value.to_string()); + } + } + present.then_some(metadata) +} + /// Object key translator for Swift object names /// /// Handles URL encoding/decoding and path normalization for Swift object keys. @@ -303,15 +350,7 @@ where let bucket = mapper.swift_to_s3_bucket(container, &project_id); // 5. Extract Swift metadata from X-Object-Meta-* headers - let mut user_metadata = HashMap::new(); - for (header_name, header_value) in headers.iter() { - let header_str = header_name.as_str().to_lowercase(); - if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") - && let Ok(value_str) = header_value.to_str() - { - user_metadata.insert(meta_key.to_string(), value_str.to_string()); - } - } + let mut user_metadata = swift_user_metadata(headers).unwrap_or_default(); // 6. Extract Content-Type if provided if let Some(content_type) = headers.get("content-type") @@ -739,15 +778,7 @@ pub async fn update_object_metadata( } // 8. Extract new metadata from X-Object-Meta-* headers - let mut new_metadata = HashMap::new(); - for (header_name, header_value) in headers.iter() { - let header_str = header_name.as_str().to_lowercase(); - if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") - && let Ok(value_str) = header_value.to_str() - { - new_metadata.insert(meta_key.to_string(), value_str.to_string()); - } - } + let mut new_metadata = swift_user_metadata(headers).unwrap_or_default(); // 9. Also update Content-Type if provided if let Some(content_type) = headers.get("content-type") @@ -889,19 +920,8 @@ pub async fn copy_object( let mut new_metadata = (*src_info.user_defined).clone(); // 11. If custom metadata headers provided, use those instead (Swift behavior) - let mut has_custom_meta = false; - for (header_name, header_value) in headers.iter() { - let header_str = header_name.as_str().to_lowercase(); - if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") { - if !has_custom_meta { - // First custom meta header - clear source metadata - new_metadata.clear(); - has_custom_meta = true; - } - if let Ok(value_str) = header_value.to_str() { - new_metadata.insert(meta_key.to_string(), value_str.to_string()); - } - } + if let Some(custom_metadata) = swift_user_metadata(headers) { + new_metadata = custom_metadata; } // 12. Also check for Content-Type override @@ -1123,6 +1143,36 @@ mod tests { assert!(ObjectKeyMapper::validate_object_name("unicode-文件.txt").is_ok()); } + #[test] + fn swift_user_metadata_cannot_materialize_internal_storage_keys() { + let mut headers = HeaderMap::new(); + headers.insert("x-object-meta-x-rustfs-internal-actual-size", "1".parse().expect("valid metadata value")); + headers.insert("x-object-meta-description", "safe".parse().expect("valid metadata value")); + let metadata = swift_user_metadata(&headers).expect("custom metadata should be detected"); + + assert_eq!(metadata.get("x-amz-meta-x-rustfs-internal-actual-size").map(String::as_str), Some("1")); + assert_eq!(metadata.get("description").map(String::as_str), Some("safe")); + assert!(!metadata.contains_key("x-rustfs-internal-actual-size")); + assert_eq!( + stored_swift_user_metadata_key("x-minio-encryption-original-size"), + "x-amz-meta-x-minio-encryption-original-size" + ); + assert_eq!(stored_swift_user_metadata_key("description"), "description"); + } + + #[test] + fn swift_user_metadata_response_mapping_is_reversible_and_filters_internal_keys() { + assert_eq!( + swift_response_user_metadata_key("x-amz-meta-x-rustfs-internal-actual-size"), + Some("x-rustfs-internal-actual-size") + ); + assert_eq!(swift_response_user_metadata_key("x-amz-meta-x-amz-checksum"), Some("x-amz-checksum")); + assert_eq!(swift_response_user_metadata_key("x-amz-meta-description"), Some("x-amz-meta-description")); + assert_eq!(swift_response_user_metadata_key("description"), Some("description")); + assert_eq!(swift_response_user_metadata_key("x-rustfs-internal-actual-size"), None); + assert_eq!(swift_response_user_metadata_key("x-minio-internal-actual-size"), None); + } + #[test] fn test_validate_object_name_empty() { let result = ObjectKeyMapper::validate_object_name(""); diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 019a1fe64..6b0586ecf 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -4023,7 +4023,7 @@ mod tests { use crate::scanner_budget::ScannerCycleBudgetConfig; use crate::scanner_folder::ScannerItem; use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats}; - use crate::storage_api::scan::{BucketOperations as _, MakeBucketOptions, ObjectIO as _}; + use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _}; use crate::{ DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions, ScannerPutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, @@ -4248,16 +4248,20 @@ mod tests { async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() { let (_temp_dir, store) = setup_two_pool_scanner_store().await; let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple()); - store.pools[1].disk_set[0] + store .make_bucket(&bucket, &MakeBucketOptions::default()) .await - .expect("bucket should be created only in the second pool"); + .expect("bucket and its authoritative metadata should be created"); let body = b"second-only"; let mut reader = ScannerPutObjReader::from_vec(body.to_vec()); - store.pools[1].disk_set[0] + store.pools[1] .put_object(&bucket, "pool-b", &mut reader, &ScannerObjectOptions::default()) .await .expect("object should be written only to the second pool"); + store.pools[0] + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("bucket should be removed from the first pool only"); let ctx = CancellationToken::new(); let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()); diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index 634f20e45..e77033aa2 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -278,7 +278,7 @@ pub(crate) mod scan { SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, }; #[cfg(test)] - pub(crate) use super::storage_contracts::{MakeBucketOptions, ObjectIO}; + pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO}; } pub(crate) mod scanner_io { diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 93f958ef2..9f721f17b 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -16,7 +16,6 @@ for later deletion. - `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2. - `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration. - `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract. -- `cross-pool-fence-v1` authenticated unsupported advertisement: predeployment servers recognize the versioned cross-pool fence capability probe but report support version 0, allowing a later all-peer probe to distinguish predeployment nodes without activating a second lock domain. Replace the unsupported advertisement only when composite lock acquisition, a cluster-wide activation fence, complete fleet proof, commit-time proof revalidation, and fail-closed revocation ship together. - `replacement-recovery-status-v1` replacement recovery status capability: new peers treat an unimplemented ReplacementRecoveryStatus RPC as an explicitly non-definitive rolling-upgrade response, so Admin v4 cannot claim distributed replacement completion from old peers. Remove the fallback after the minimum supported RustFS peer version implements ReplacementRecoveryStatus. - `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration. - `rustfs-5509` FileInfo positional MessagePack decoding: beta.11 serialized 28 fields, while beta.12 inserted transition-version fields in the middle and serialized an incompatible 30-field array. New releases write named maps and retain readers for both shipped array layouts so direct and rolling upgrades can read either release. Remove the positional-array readers after every supported direct-upgrade release writes named maps and no retained RPC payload can contain a pre-map FileInfo array. diff --git a/docs/operations/object-io-tuning-ab-matrix.md b/docs/operations/object-io-tuning-ab-matrix.md new file mode 100644 index 000000000..baf0b906b --- /dev/null +++ b/docs/operations/object-io-tuning-ab-matrix.md @@ -0,0 +1,218 @@ +# Object I/O (GET/PUT) tuning A/B matrix runbook + +> Scope: **parameter tuning** — measuring the effect of changing one +> `RUSTFS_*` runtime knob at a time, against a fixed binary. This is +> deliberately different from the code-change A/B gate in +> [`hotpath-warp-ab-runbook.md`](hotpath-warp-ab-runbook.md) and the formal +> ABBA validation in +> [`hotpath-warp-abba-runbook.md`](hotpath-warp-abba-runbook.md), which compare +> a baseline binary against a candidate binary. +> +> When a knob change turns out to need a code change, use those two runbooks +> for the code-level validation and come back here for the knob-level sweep. + +## 1. What this runbook answers + +For each tuning knob it answers three questions: + +1. Which stage is actually slow — `set_disk_encode`, `set_disk_rename`, + `metadata_fanout`, `bitrot_verify`, etc.? +2. Is the knob the real bottleneck, or is the stage slow for another reason? +3. Does widening/loosening the knob buy throughput without an unacceptable + memory (RSS) or tail-latency regression? + +The core discipline is **one variable per A/B cell**. Never change two knobs in +the same cell, or the result is unexplainable. + +## 2. Prerequisites + +- Linux bench host (or an ansible-managed cluster); a laptop smoke run is too + noisy to decide anything. +- `warp` on `PATH` (or pass `--warp-bin` to the driver). +- The observability metrics runtime **enabled**. The stage histograms below are + not emitted when `RUSTFS_OBS_METRICS_EXPORT_ENABLED=false` or the runtime is + otherwise off — see + [`hotpath-warp-ab-runbook.md`](hotpath-warp-ab-runbook.md) for the no-log / + no-monitor baseline env. +- A warm, disposable data set. Recreate the bucket per run; do not bench against + production data. + +Load driver and gate are reused, not reimplemented: + +- `scripts/run_object_batch_bench_enhanced.sh` — warp driver with rounds, + median aggregation, `baseline_compare.csv`, and Prometheus service-metric + capture. +- `scripts/hotpath_warp_ab_gate.sh` — relative budget gate over the deltas. +- `scripts/run_hotpath_warp_ab.sh` — optional orchestrator when a knob needs + the full baseline-vs-candidate treatment (e.g. two different defaults). + +## 3. Fixed test conditions (lock before you start) + +Record these per run; a result without them is not reproducible: + +```text +nodes, disks_per_node, total_disks, cpu_per_node, mem_per_node, +network, erasure_set_drive_count, endpoint_mode (direct|lb), +rustfs_commit_sha, warp --version, durability mode +``` + +Workload matrix (the same shapes the hotpath gate uses, expanded for the +stage-breakdown object sizes): + +| Workload | mode | sizes | +| --- | --- | --- | +| small-fixed | put / get | 4KiB, 100KiB | +| ec-boundary | put / get | 1MiB, 4MiB | +| large-stream | put / get | 10MiB, 16MiB, 32MiB | +| mixed | mixed | 256KiB | + +Concurrency ladder: `8, 16, 32, 64` (add `96, 128` on a bigger rig). Duration +`120s`, `--rounds >= 3`, cooldown `>= 30s`. + +Isolate background noise before the sweep: scanner deep-verify, heal, +replication, lifecycle transition, periodic capacity refresh — record whether +each is on rather than silently assuming it is off. + +## 4. Measurement stack + +The code already instruments every stage below. Drive each A/B cell with these +histograms (names verified against `crates/io-metrics/src/lib.rs`): + +- PUT stages: `rustfs_s3_put_object_stage_duration_ms{stage=...}` — compute + P50/P95/P99 per stage. Stages: `app_bucket_validate`, `app_sse_config_lookup`, + `app_object_lock_config_lookup`, `app_put_opts_build`, `app_prelookup`, + `ingress_prepare`, `app_encryption_prepare`, `app_replication_decision`, + `app_store_put`, `app_post_store_bookkeeping`, `app_capacity_update`, + `set_disk_writer_setup`, `set_disk_encode`, `set_disk_rename`, + `set_disk_old_data_cleanup`. +- GET stages: `rustfs_io_get_object_stage_duration_seconds{path=..., stage=...}` — + the `path` label separates the read paths: `legacy_duplex`, `codec_streaming`, + `direct_memory`, `body_cache`, `inline_direct`, `internal_meta`, + `remote_transition`, `set_disk`, `empty`. Stages: `metadata`, + `metadata_cache_lookup`, `metadata_fanout`, `metadata_resolve`, `object_info`, + `path_decision`, `quorum_reached`, `range`, `reader_setup`, + `stripe_read`, `stripe_read_first_shard`, `stripe_read_quorum`, `decode`, + `reconstruct`, `emit`, `fill`, `output_poll`, `output_lock_wait`, + `bitrot_verify`, `first_byte`, `full_body`, `response_handoff`, + `lock_acquire`. +- EC memory pressure: `rustfs_ec_encode_inflight_bytes_current` and the + allocator reclaim gauge; plus node RSS and CPU. + +Host telemetry (collect alongside every cell): + +```bash +pidstat -durh 5 > telemetry/pidstat.txt & +mpstat 5 > telemetry/mpstat.txt & +iostat -xz 5 > telemetry/iostat.txt & +``` + +## 5. Tuning knob catalog + +Defaults are verified against `crates/config/src/constants/object.rs` and +`crates/ecstore/src/erasure/coding/encode.rs`. + +### 5.1 PUT + +| Knob | Default | Controls | Validating stage | Risk if widened | +| --- | --- | --- | --- | --- | +| `RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES` | 32MiB | EC encode producer/consumer memory budget (blocks queued between encode and shard write) | `set_disk_encode` P95 + `rustfs_ec_encode_inflight_bytes_current` | RSS growth under high concurrency | +| `RUSTFS_OBJECT_IO_BUFFER_SIZE` | 128KiB | Streaming read-in / write-out block size | `ingress_prepare`, `set_disk_encode` | Larger buffers = fewer polls, more resident memory | +| `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` | 4MiB | duplex pipe capacity (shared, but PUT path uses it less than GET) | `set_disk_encode` feed smoothness | Memory per in-flight request | +| `RUSTFS_DURABILITY_MODE` / `RUSTFS_DRIVE_SYNC_ENABLE` | mode-dependent | per-shard fsync/sync discipline on commit | `set_disk_rename` P99 | Weakening it changes the durability contract — treat as a deliberate tradeoff, not a free win | +| `RUSTFS_RUNTIME_WORKER_THREADS` / `RUSTFS_RUNTIME_MAX_BLOCKING_THREADS` | Tokio defaults | async workers + `spawn_blocking` pool feeding per-block encode | `set_disk_encode` P95 + mpstat | Oversubscription | + +### 5.2 GET + +| Knob | Default | Controls | Validating stage | Risk if enabled | +| --- | --- | --- | --- | --- | +| `RUSTFS_GET_CODEC_STREAMING_ROLLOUT` | `off` | switches the read path from `legacy_duplex` to the pull-based `ErasureDecodeReader` (`codec_streaming`) | compare `path="legacy_duplex"` vs `path="codec_streaming"` for `decode`/`emit`/`output_lock_wait`/`stripe_read` | behavioral change to the read path; rollout is `off` by default for a reason | +| `RUSTFS_GET_CODEC_STREAMING_ENGINE` | `legacy` | `legacy` vs `rustfs` decode engine under the streaming reader | `reconstruct`/`decode` per `path` | engine swap on a correctness-critical path | +| `RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE` | `false` | multipart objects on the streaming reader | same, multipart cells | wider format coverage | +| `RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE` (+ `_MAX_SIZE`, `_FIRST_READER_SETUP`) | `false` / 512KiB | prefer data-shard readers before parity | `stripe_read_first_shard`/`stripe_read_quorum` | shard-selection order change | +| `RUSTFS_OBJECT_GET_SKIP_BITROT_VERIFY` | `false` | skip per-shard HighwayHash verify | `bitrot_verify` | **do not default on** — measures the theoretical ceiling only | +| `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` | 4MiB | legacy GET in-process pipe capacity | `output_lock_wait`/`output_poll` | memory per in-flight GET | +| `RUSTFS_GET_SEEK_BUFFER_ENABLE` | `false` | in-memory seek buffer for small GET | `first_byte` | experimental, startup-latched — see [`get-path-experimental-switches.md`](get-path-experimental-switches.md) | +| `RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE` | `false` | adds `response_handoff` attribution (metrics only) | `response_handoff` | small per-request bookkeeping cost | + +## 6. A/B matrix + +Run each row as an independent cell. Baseline is the shipped default; candidate +is one knob moved. Everything else (topology, sizes, concurrency, rounds, +durability) stays fixed. + +### 6.1 PUT + +| # | Knob | Baseline | Candidates | Signal metric | Judgement | +| --- | --- | --- | --- | --- | --- | +| P1 | encode in-flight | 32MiB | 48MiB, 64MiB, 96MiB | `set_disk_encode` P95 + throughput + RSS | throughput up AND `set_disk_encode` down AND RSS tolerable → budget was the binding constraint | +| P2 | object I/O buffer | 128KiB | 256KiB, 512KiB, 1MiB | `ingress_prepare` + `set_disk_encode` | encode smooths with bounded RSS → upstream feed was too small | +| P3 | duplex buffer | 4MiB | 8MiB, 16MiB | `set_disk_encode` feed variance | lower priority than P1/P2 | +| P4 | blocking threads | default | 512, 768, 1024 | `set_disk_encode` P95 + mpstat | per-block `spawn_blocking` is scheduler-bound if P95 falls | +| P5 | durability | current mode | weaker/stronger mode | `set_disk_rename` P99 | only as a **deliberate durability tradeoff**, never a silent default change | +| P6 | set drive count | current | other valid widths | all PUT stages | highest cost — only after P1–P5, and only on a rebuildable topology | + +### 6.2 GET + +| # | Knob | Baseline | Candidates | Signal metric | Judgement | +| --- | --- | --- | --- | --- | --- | +| G1 | codec streaming rollout | `off` | `on` (pct ramp 10/50/100) | `path="legacy_duplex"` vs `path="codec_streaming"` for `decode`/`emit`/`output_lock_wait`/`stripe_read` + throughput | streaming beats duplex on `decode`+`output_lock_wait` and byte-for-byte output matches → candidate for default | +| G2 | codec streaming engine | `legacy` | `rustfs` | `reconstruct`/`decode` per `path` | engine swap is neutral-or-better on CPU with identical bytes | +| G3 | multipart streaming | `false` | `true` | multipart GET cells | only after G1 is stable on single-part | +| G4 | data-blocks-first | `false` | `true` | `stripe_read_first_shard`/`stripe_read_quorum` | fewer shard reads without a correctness regression | +| G5 | duplex buffer | 4MiB | 8MiB, 16MiB | `output_lock_wait`/`output_poll` (legacy path) | only if still on `legacy_duplex` | +| G6 | skip bitrot verify | `false` | `true` | `bitrot_verify` | **ceiling measurement only**; do not carry into production | + +## 7. Execution sequence + +1. Freeze the conditions in §3 and record the provenance block. +2. Run the baseline cell (all defaults) and capture stage histograms + host + telemetry. +3. Pick the **one** most-likely knob from the analysis. For large-object PUT + that is almost always `set_disk_encode` → P1; for GET it is G1 (the + `legacy_duplex` → `codec_streaming` switch). +4. Sweep that knob's candidate column one value at a time, same workload. +5. Read the decision table in §8; if the stage did not move, the knob is not + the bottleneck — stop widening it and pick the next stage. + +Driver invocation for one PUT cell: + +```bash +scripts/run_object_batch_bench_enhanced.sh \ + --tool warp --endpoint http://127.0.0.1:9000 \ + --access-key "$RUSTFS_ACCESS_KEY" --secret-key "$RUSTFS_SECRET_KEY" \ + --bucket rustfs-put-tuning --warp-mode put \ + --sizes 16MiB,32MiB --concurrency 32 --duration 120s --rounds 3 \ + --out-dir target/bench/put-tuning-p1-64mib +``` + +## 8. Interpretation / decision table + +| Stage high | Most likely cause | Next action | +| --- | --- | --- | +| `set_disk_encode` | per-block EC encode scheduling + in-flight budget | P1 → P4 → P2, in that order | +| `set_disk_rename` | commit tail (rename fan-out / RPC / fsync) | P5 (durability) and cluster tail analysis, not encode | +| `set_disk_writer_setup` | per-disk `BitrotWriter` + temp-file create | disk/filesystem metadata; per-disk fan-out cost | +| `set_disk_old_data_cleanup` | overwrite / versioned-object directory delete | confirm overwrite-vs-new-write; defer cleanup further | +| `metadata_fanout` / `metadata_resolve` | cross-disk `xl.meta` read + quorum | metadata cache hit rate; small-object fixed cost | +| `bitrot_verify` | HighwayHash verify on the read path | G6 ceiling only; do not default on | +| `output_lock_wait` / `output_poll` | legacy duplex backpressure | G1 (move off duplex) or G5 | +| `stripe_read*` | shard concurrency / selection | G4 shard-selection, disk/network tail | + +## 9. Guardrails + +- **Never weaken correctness for throughput**: read/write quorum, bitrot verify, + `xl.meta` validation, and durability (`RUSTFS_DURABILITY_MODE`) are integrity + contracts, not knobs. P5 and G6 are ceiling measurements and must be labelled + as such; do not carry their values into production without an explicit + durability/correctness decision. +- **One variable per cell.** A cell that changes two knobs is thrown away. +- **Memory is part of the result.** A throughput win with unbounded RSS growth + is a regression; record RSS and the EC in-flight gauge for every PUT cell. +- **Startup-latched knobs** (`RUSTFS_GET_SEEK_BUFFER_ENABLE`, + `RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE`, and the codec-streaming + switches) require a process restart to change — see + [`get-path-experimental-switches.md`](get-path-experimental-switches.md). +- **Archive the raw data.** Keep the `baseline_compare.csv`, `median_summary.csv`, + stage histograms, and host telemetry per cell; the conclusion must trace back + to them. Do not commit benchmark result snapshots to the repo — record them in + the issue tracker. diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index 1a3c2b383..c73d4c487 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -465,6 +465,16 @@ impl Operation for ImportBucketMetadata { file_contents.push((file_path, content)); } + let durable_quota_import = imported_quota_requires_fleet_proof(&file_contents)?; + let quota_fleet_proof = + if durable_quota_import { + Some(crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| { + s3_error!(ServiceUnavailable, "durable quota capability is not confirmed across the cluster") + })?) + } else { + None + }; + // Extract bucket names let mut bucket_names = Vec::new(); for (file_path, _) in &file_contents { @@ -707,21 +717,6 @@ impl Operation for ImportBucketMetadata { } BUCKET_QUOTA_CONFIG_FILE => { - if let Err(e) = serde_json::from_slice::(&content) { - warn!( - event = EVENT_ADMIN_BUCKET_META_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_BUCKET_META, - action = "import_bucket_metadata", - result = "config_deserialize_failed", - bucket = %bucket_name, - config_name = %conf_name, - error = %e, - "admin bucket meta state" - ); - continue; - } - let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, @@ -830,10 +825,6 @@ impl Operation for ImportBucketMetadata { } } - // Persist the assembled metadata to disk. Prior to this, the import only mutated the - // in-memory `bucket_metadatas` map and returned 200, silently dropping every imported - // config. `metadata_sys::update` loads the on-disk metadata, overwrites the given config - // field and saves it, preserving any configs not present in the import archive. for (bucket_name, metadata) in &bucket_metadatas { for (config_file, data) in imported_configs_to_persist(metadata) { let site_replication_item = imported_config_to_site_replication_item(bucket_name, metadata, config_file, &data)?; @@ -842,7 +833,21 @@ impl Operation for ImportBucketMetadata { } else { None }; - if let Err(e) = metadata_sys::update(bucket_name, config_file, data).await { + let persist_result = if config_file == BUCKET_QUOTA_CONFIG_FILE { + let quota: BucketQuota = + serde_json::from_slice(&data).map_err(|e| s3_error!(InvalidRequest, "invalid bucket quota: {e}"))?; + if quota.uses_durable_reservations() { + let proof = quota_fleet_proof.as_ref().ok_or_else(|| { + s3_error!(ServiceUnavailable, "durable quota capability is not confirmed across the cluster") + })?; + metadata_sys::update_quota_if_incarnation(bucket_name, data, metadata.bucket_incarnation_id, proof).await + } else { + metadata_sys::update_if_incarnation(bucket_name, config_file, data, metadata.bucket_incarnation_id).await + } + } else { + metadata_sys::update_if_incarnation(bucket_name, config_file, data, metadata.bucket_incarnation_id).await + }; + if let Err(e) = persist_result { warn!( event = EVENT_ADMIN_BUCKET_META_STATE, component = LOG_COMPONENT_ADMIN, @@ -885,6 +890,26 @@ impl Operation for ImportBucketMetadata { } } +fn imported_quota_requires_fleet_proof(file_contents: &[(String, Vec)]) -> S3Result { + let mut durable = false; + for (file_path, content) in file_contents { + let mut parts = file_path.split(SLASH_SEPARATOR); + let Some(_bucket) = parts.next() else { + continue; + }; + if parts.next() != Some(BUCKET_QUOTA_CONFIG_FILE) { + continue; + } + let quota: BucketQuota = + serde_json::from_slice(content).map_err(|e| s3_error!(InvalidRequest, "invalid bucket quota: {e}"))?; + if quota.has_unsupported_reservation_protocol() { + return Err(s3_error!(InvalidRequest, "unsupported bucket quota reservation protocol")); + } + durable |= quota.uses_durable_reservations(); + } + Ok(durable) +} + /// The `(config_file, data)` pairs to persist for an imported bucket's metadata: every non-empty /// config field keyed by its on-disk config-file name, as owned data ready for /// `metadata_sys::update`. Empty fields are skipped so an import never overwrites an existing @@ -1076,4 +1101,31 @@ mod import_persist_tests { assert!(!has_site_replication_item); } + + #[test] + fn quota_import_preflight_rejects_invalid_and_unknown_protocols() { + let missing_limit = vec![( + format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"), + br#"{"quota":0,"reservation_protocol":1}"#.to_vec(), + )]; + assert!(imported_quota_requires_fleet_proof(&missing_limit).is_err()); + + let unknown_protocol = vec![( + format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"), + br#"{"quota":0,"reservation_protocol":2,"reservation_quota":1024}"#.to_vec(), + )]; + assert!(imported_quota_requires_fleet_proof(&unknown_protocol).is_err()); + } + + #[test] + fn quota_import_preflight_requires_proof_only_for_durable_quota() { + let legacy = vec![(format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"), br#"{"quota":1024}"#.to_vec())]; + assert!(!imported_quota_requires_fleet_proof(&legacy).expect("legacy quota should remain compatible")); + + let durable = vec![( + format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"), + serde_json::to_vec(&BucketQuota::new(Some(1024))).expect("durable quota should encode"), + )]; + assert!(imported_quota_requires_fleet_proof(&durable).expect("durable quota should pass preflight")); + } } diff --git a/rustfs/src/admin/handlers/quota.rs b/rustfs/src/admin/handlers/quota.rs index 35124b038..7dc2621be 100644 --- a/rustfs/src/admin/handlers/quota.rs +++ b/rustfs/src/admin/handlers/quota.rs @@ -22,6 +22,7 @@ use crate::admin::storage_api::bucket::metadata_sys::{self, BucketMetadataSys}; use crate::admin::storage_api::bucket::quota::checker::QuotaChecker; use crate::admin::storage_api::bucket::quota::{BucketQuota, QuotaError, QuotaOperation}; use crate::auth::{check_key_valid, get_session_token}; +use crate::error::ApiError; use crate::server::ADMIN_PREFIX; use hyper::{Method, StatusCode}; use matchit::Params; @@ -293,16 +294,36 @@ impl Operation for SetBucketQuotaHandler { return Err(s3_error!(InvalidArgument, "{}", rustfs_config::QUOTA_INVALID_TYPE_ERROR_MSG)); } + let fleet_proof = if request.quota.is_some() { + Some(crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| { + S3Error::with_message( + s3s::S3ErrorCode::ServiceUnavailable, + "durable quota capability is not confirmed across the cluster".to_string(), + ) + })?) + } else { + None + }; + let quota = BucketQuota::new(request.quota); let metadata_sys_lock = bucket_metadata_from_context() .ok_or_else(|| s3_error!(InternalError, "{}", rustfs_config::QUOTA_METADATA_SYSTEM_ERROR_MSG))?; let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); - let updated_at = quota_checker - .set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id) - .await - .map_err(|e| s3_error!(InternalError, "failed to set quota: {}", e))?; + let updated_at = match fleet_proof.as_ref() { + Some(fleet_proof) => { + quota_checker + .set_durable_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id, fleet_proof) + .await + } + None => { + quota_checker + .set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id) + .await + } + } + .map_err(ApiError::from)?; if let Err(err) = site_replication_bucket_meta_hook(SRBucketMeta { bucket: bucket.clone(), diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 85d529836..bf946c0c4 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -29,6 +29,7 @@ use crate::admin::storage_api::bucket::metadata::{ BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG, }; use crate::admin::storage_api::bucket::metadata_sys; +use crate::admin::storage_api::bucket::quota::BucketQuota; use crate::admin::storage_api::bucket::replication; use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; use crate::admin::storage_api::bucket::target_sys::BucketTargetSys; @@ -7895,9 +7896,35 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { if !skip_config_write { if let Some(data) = data { - metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) - .await - .map_err(ApiError::from)?; + if item.r#type == "quota-config" { + let quota: BucketQuota = serde_json::from_slice(&data) + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid bucket quota: {e}")))?; + if quota.has_unsupported_reservation_protocol() { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "unsupported bucket quota reservation protocol".to_string(), + )); + } + if quota.uses_durable_reservations() { + let proof = crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| { + S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "durable quota capability is not confirmed across the cluster".to_string(), + ) + })?; + metadata_sys::update_quota_if_incarnation(&item.bucket, data, expected_incarnation_id, &proof) + .await + .map_err(ApiError::from)?; + } else { + metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + } + } else { + metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + } } else { metadata_sys::delete_if_incarnation(&item.bucket, config_file, expected_incarnation_id) .await diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 13a45fd35..1f3ac8856 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -2901,7 +2901,7 @@ async fn handle_misc_extension_request(req: &mut S3Request, route: &MiscEx MiscExtRoute::ObjectLambda { bucket, object } => { let get_req = build_object_lambda_get_request(req, bucket, object)?; let usecase = default_object_usecase(); - let get_resp = Box::pin(usecase.execute_get_object(get_req)).await?; + let get_resp = usecase.execute_get_object(get_req).await?; invoke_object_lambda_target(req, bucket, object, get_resp).await } MiscExtRoute::ListenNotification { bucket } => { diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index fd01de999..17841d1de 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -64,7 +64,9 @@ mod ecstore_metrics { } mod ecstore_notification { - pub(crate) use crate::storage::storage_api::ecstore_notification::NotificationSys; + pub(crate) use crate::storage::storage_api::ecstore_notification::{ + CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof, + }; } #[allow(unused_imports)] @@ -112,6 +114,10 @@ pub(crate) type TierCreds = ecstore_tier::tier_admin::TierCreds; pub(crate) type TierType = ecstore_tier::tier_config::TierType; pub(crate) type TierConfigUpdateError = crate::storage::storage_api::TierConfigUpdateError; +pub(crate) fn acquire_cross_pool_fence_fleet_proof() -> Option { + ecstore_notification::acquire_cross_pool_fence_fleet_proof() +} + pub(crate) mod runtime_sources { pub(crate) type DailyAllTierStats = super::DailyAllTierStats; pub(crate) type ECStore = super::ECStore; @@ -296,6 +302,15 @@ pub(crate) mod metadata_sys { super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await } + pub(crate) async fn update_quota_if_incarnation( + bucket: &str, + data: Vec, + expected_incarnation_id: uuid::Uuid, + proof: &super::ecstore_notification::CrossPoolFenceFleetProofToken, + ) -> Result { + super::ecstore_bucket::metadata_sys::update_quota_if_incarnation(bucket, data, expected_incarnation_id, proof).await + } + pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result { super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await } diff --git a/rustfs/src/app/gating_test_env.rs b/rustfs/src/app/gating_test_env.rs index e25a4471e..f8d5bfd9b 100644 --- a/rustfs/src/app/gating_test_env.rs +++ b/rustfs/src/app/gating_test_env.rs @@ -23,6 +23,9 @@ //! `ECStore` and one metadata-sys initialization exist per test binary. use super::storage_api::test::bucket::metadata_sys; +use super::storage_api::test::bucket::quota::BucketQuota; +use super::storage_api::test::bucket::quota::checker::QuotaChecker; +use super::storage_api::test::contract::bucket::MakeBucketOptions; use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions}; use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints}; use super::{context::AppContext, object_traffic_health::ObjectTrafficHealth}; @@ -87,6 +90,17 @@ pub(crate) async fn shared_gating_ecstore() -> Arc { crate::storage::storage_api::new_global_notification_sys(endpoint_pools.clone()) .await .expect("initialize notification system for gating test env"); + let topology_fingerprint = + crate::storage::storage_api::heal_control_startup_consumer::heal_topology_fingerprint(&endpoint_pools) + .expect("single-node gating topology should hash"); + crate::storage::storage_api::start_remote_version_state_fleet_probe(topology_fingerprint); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while crate::storage::storage_api::ecstore_notification::acquire_cross_pool_fence_fleet_proof().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .expect("single-node cross-pool fence capability proof should publish"); let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) @@ -107,6 +121,24 @@ pub(crate) async fn shared_gating_ecstore() -> Arc { ecstore } +pub(crate) async fn durable_quota_test_bucket(prefix: &str, limit: u64) -> (Arc, String) { + let store = shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let bucket = format!("{prefix:.30}-{}", uuid::Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create durable quota test bucket"); + super::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 0).await; + let metadata_sys = + crate::app::storage_api::test::get_global_bucket_metadata_sys().expect("test app context should expose bucket metadata"); + QuotaChecker::new(metadata_sys) + .set_quota_config(&bucket, BucketQuota::new(Some(limit))) + .await + .expect("configure durable quota test bucket"); + (store, bucket) +} + pub(crate) async fn shared_gating_ambient() -> Arc { let store = shared_gating_ecstore().await; if let Some(ambient) = crate::runtime_sources::current_app_context() { diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 8dd561da1..66ffa26cc 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -33,7 +33,7 @@ use super::storage_api::multipart_usecase::contract::multipart::{CompletePart, M use super::storage_api::multipart_usecase::contract::object::{ObjectIO as _, ObjectOperations as _}; use super::storage_api::multipart_usecase::contract::range::HTTPRangeSpec; use super::storage_api::multipart_usecase::data_usage::{ - record_bucket_object_version_write_memory, record_bucket_object_write_memory, + quota_object_size, record_bucket_object_version_write_memory, record_bucket_object_write_memory, }; use super::storage_api::multipart_usecase::error::{StorageError, is_err_object_not_found, is_err_version_not_found}; use super::storage_api::multipart_usecase::helper::OperationHelper; @@ -64,10 +64,10 @@ use super::storage_api::multipart_usecase::{ }; use crate::app::object_data_cache::{ ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success, - invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation, + invalidate_object_data_cache_before_mutation, }; use crate::app::object_usecase::{ - acquire_copy_bucket_lifecycle_locks, build_put_like_object_lock_metadata, map_quota_check_outcome, + acquire_copy_bucket_lifecycle_locks, apply_quota_admission, build_put_like_object_lock_metadata, map_quota_check_outcome, validate_existing_object_lock_for_write, }; use crate::app::runtime_sources::{ @@ -83,6 +83,8 @@ use rustfs_io_metrics::record_s3_op; use rustfs_s3_ops::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; +#[cfg(test)] +use rustfs_utils::http::insert_header; use rustfs_utils::http::{ SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_header, get_source_scheme, @@ -226,12 +228,8 @@ fn internal_object_info_lookup_opts(mut opts: ObjectOptions) -> ObjectOptions { opts } -fn logical_object_size(info: &ObjectInfo) -> Result { - u64::try_from(info.get_actual_size()?).map_err(|_| StorageError::PartMissingOrCorrupt) -} - fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result { - match logical_object_size(info) { + match quota_object_size(info) { Ok(size) => Ok(size), Err(err) if fail_closed => Err(ApiError::from(err).into()), Err(_) => Ok(info.size.max(0) as u64), @@ -507,11 +505,7 @@ impl DefaultMultipartUsecase { Ok(existing_obj_info) => { validate_existing_object_lock_for_write(&existing_obj_info, ¤t_opts)?; let physical_size = existing_obj_info.size.max(0) as u64; - let logical_size = if opts.replication_request { - Ok(physical_size) - } else { - logical_object_size(&existing_obj_info) - }; + let logical_size = quota_object_size(&existing_obj_info); Some((physical_size, logical_size)) } Err(err) => { @@ -560,32 +554,20 @@ impl DefaultMultipartUsecase { }; let quota_metadata_sys = self.bucket_metadata_sys(); + let quota_tracking = quota_metadata_sys.is_some(); + let mut quota_enabled = false; if let Some(metadata_sys) = quota_metadata_sys.as_ref() { let quota_checker = QuotaChecker::new(metadata_sys.clone()); let check_result = map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?; - // Ciphertext-passthrough replication parts use a different size basis and retain - // the existing post-commit accounting path until they carry a trusted logical-size proof. - if !opts.replication_request - && let Some(quota_limit) = check_result.quota_limit - { - let installed = check_result - .current_usage - .is_some_and(|current_usage| opts.set_quota_admission(current_usage, quota_limit)); - if !installed { - return Err(S3Error::with_message( - S3ErrorCode::ServiceUnavailable, - "Bucket quota check temporarily unavailable, please retry".to_string(), - )); - } - } + quota_enabled = check_result.quota_limit.is_some(); + apply_quota_admission(&mut opts, &check_result)?; } let previous_current_size = match previous_current_sizes { - Some((physical_size, _)) if opts.replication_request => Some(physical_size), - Some((_, Ok(logical_size))) => Some(logical_size), - Some((_, Err(err))) if opts.quota_admission.is_some() => return Err(ApiError::from(err).into()), - Some((physical_size, Err(_))) => Some(physical_size), + Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size), + Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err).into()), + Some((physical_size, _)) => Some(physical_size), None => None, }; @@ -595,7 +577,6 @@ impl DefaultMultipartUsecase { let key = key.clone(); let upload_id = upload_id.clone(); let opts = opts.clone(); - let quota_metadata_sys = quota_metadata_sys.clone(); async move { let obj_info = store .clone() @@ -605,37 +586,9 @@ impl DefaultMultipartUsecase { let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await; record_capacity_write(Some(capacity_scope_token)).await; - if let Some(metadata_sys) = quota_metadata_sys.as_ref() { - if opts.replication_request { - let quota_checker = QuotaChecker::new(metadata_sys.clone()); - match quota_checker - .check_quota(&bucket, QuotaOperation::PutObject, obj_info.size.max(0) as u64) - .await - { - Ok(check_result) if !check_result.allowed => { - let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await; - let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await; - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!( - "Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes", - check_result.current_usage.unwrap_or(0), - check_result.quota_limit.unwrap_or(0) - ), - )); - } - Err(err) => { - warn!("Quota check failed for bucket {} after multipart completion: {}", bucket, err); - } - Ok(_) => {} - } - } + if quota_tracking { + let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?; - let committed_size = if opts.replication_request { - obj_info.size.max(0) as u64 - } else { - quota_accounting_object_size(&obj_info, opts.quota_admission.is_some())? - }; if versioned { record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { @@ -845,6 +798,20 @@ impl DefaultMultipartUsecase { let ciphertext_passthrough = replication_authorized && get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") && rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some(); + if ciphertext_passthrough && let Some(metadata_sys) = self.bucket_metadata_sys() { + let check_result = map_quota_check_outcome( + &bucket, + QuotaChecker::new(metadata_sys) + .check_quota(&bucket, QuotaOperation::PutObject, 0) + .await, + )?; + if check_result.quota_limit.is_some() { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(), + )); + } + } if ciphertext_passthrough { insert_str(&mut metadata, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, "true".to_string()); } @@ -1678,6 +1645,24 @@ mod tests { assert_eq!(quota_accounting_object_size(&info, true).expect("logical size should resolve"), 8192); assert_eq!(quota_accounting_object_size(&info, false).expect("logical size should resolve"), 8192); + + let mut poisoned_metadata = HashMap::new(); + insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string()); + insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let poisoned = ObjectInfo { + size: 17, + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 4096, + actual_size: 4096, + ..Default::default() + }]), + user_defined: Arc::new(poisoned_metadata), + ..Default::default() + }; + assert_eq!( + quota_accounting_object_size(&poisoned, true).expect("persisted part size must be charged"), + 4096 + ); } #[test] @@ -2086,30 +2071,14 @@ mod tests { #[tokio::test] #[serial_test::serial] async fn compressed_complete_records_logical_quota_usage_and_overwrite_delta() { - use crate::app::storage_api::multipart_usecase::bucket::quota::BucketQuota; - use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; - use crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test; - - let store = crate::app::gating_test_env::shared_gating_ecstore().await; - crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; - - let bucket = format!("compressed-complete-quota-{}", Uuid::new_v4()); + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("compressed-complete-quota", 16_384).await; let object = "object"; - store - .make_bucket(&bucket, &MakeBucketOptions::default()) - .await - .expect("create compressed quota bucket"); - seed_bucket_usage_memory_for_test(&bucket, 0).await; let usecase = DefaultMultipartUsecase::from_global(); let metadata_sys = usecase .bucket_metadata_sys() .expect("test app context should expose bucket metadata"); - let mut quota_checker = QuotaChecker::new(metadata_sys); - quota_checker - .set_quota_config(&bucket, BucketQuota::new(Some(16_384))) - .await - .expect("configure bucket quota"); + let quota_checker = QuotaChecker::new(metadata_sys); for (actual_size, payload_byte) in [(8192_i64, 0x61), (4096_i64, 0x62)] { let mut create_opts = ObjectOptions::default(); @@ -2154,6 +2123,173 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial] + async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() { + let (_store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await; + let usecase = DefaultMultipartUsecase::from_global(); + let input = CreateMultipartUploadInput::builder() + .bucket(bucket) + .key("object".to_string()) + .build() + .expect("create multipart request should build"); + let mut request = build_request(input, Method::POST); + insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + request + .headers + .insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256")); + request.extensions.insert(crate::storage::access::ReqInfo { + replication_request_authorized: true, + ..Default::default() + }); + + let err = usecase + .execute_create_multipart_upload(request) + .await + .expect_err("quota-enabled ciphertext multipart replication should fail before upload creation"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[tokio::test] + #[serial_test::serial] + async fn concurrent_completions_share_durable_bucket_quota_reservations() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-complete-quota", 6000).await; + + let usecase = DefaultMultipartUsecase::from_global(); + + let mut inputs = Vec::new(); + for object in ["first", "second"] { + let upload = store + .new_multipart_upload(&bucket, object, &ObjectOptions::default()) + .await + .expect("create concurrent multipart upload"); + let mut reader = PutObjReader::from_vec(vec![0x71; 4096]); + let part = store + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .expect("stage concurrent multipart part"); + inputs.push( + CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .upload_id(upload.upload_id) + .multipart_upload(Some(CompletedMultipartUpload { + parts: Some(vec![CompletedPart { + part_number: Some(1), + e_tag: part.etag.map(|etag| to_s3s_etag(&etag)), + ..Default::default() + }]), + })) + .build() + .expect("build concurrent completion input"), + ); + } + + let first_usecase = usecase.clone(); + let first = first_usecase.execute_complete_multipart_upload(build_request(inputs.remove(0), Method::POST)); + let second = usecase.execute_complete_multipart_upload(build_request(inputs.remove(0), Method::POST)); + let (first, second) = tokio::join!(first, second); + + assert_eq!(usize::from(first.is_ok()) + usize::from(second.is_ok()), 1); + let denied = first.err().or_else(|| second.err()).expect("one completion must be denied"); + assert_eq!(denied.code(), &S3ErrorCode::InvalidRequest); + } + + #[tokio::test] + #[serial_test::serial] + async fn multipart_completion_rejects_rotated_quota_capability_before_rename() { + use crate::app::storage_api::test::set_disk::{MultipartCommitBarrier, MultipartCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("rotated-proof-mpu-quota", 4096).await; + let object = "object"; + let upload = store + .new_multipart_upload(&bucket, object, &ObjectOptions::default()) + .await + .expect("create multipart upload"); + let mut reader = PutObjReader::from_vec(vec![0x78; 4096]); + let part = store + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .expect("stage multipart part"); + let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::BeforeQuotaRename); + let complete_store = Arc::clone(&store); + let complete_bucket = bucket.clone(); + let upload_id = upload.upload_id.clone(); + let complete = tokio::spawn(async move { + complete_store + .complete_multipart_upload( + &complete_bucket, + object, + &upload_id, + vec![CompletePart { + part_num: 1, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + }); + barrier.wait_until_paused().await; + assert!( + crate::storage::storage_api::ecstore_notification::rotate_cross_pool_fence_fleet_proof_for_test(), + "the gating environment must have a current fleet proof" + ); + barrier.release(); + + let err = complete + .await + .expect("completion task should not panic") + .expect_err("a replaced fleet proof must fence multipart rename"); + assert!(matches!( + err, + StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + .. + } + )); + store + .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default()) + .await + .expect("proof rotation must preserve the multipart upload for retry"); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_multipart_completion_has_zero_quota_growth() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("data-movement-mpu-quota", 0).await; + let object = "object"; + let mut movement_opts = ObjectOptions { + data_movement: true, + ..Default::default() + }; + let upload = store + .new_multipart_upload(&bucket, object, &movement_opts) + .await + .expect("create data-movement multipart upload"); + let mut reader = PutObjReader::from_vec(vec![0x7a; 4096]); + let part = store + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &movement_opts) + .await + .expect("stage data-movement multipart part"); + movement_opts.preserve_etag = Some("movement-etag".to_string()); + let completed = store + .complete_multipart_upload( + &bucket, + object, + &upload.upload_id, + vec![CompletePart { + part_num: 1, + etag: part.etag, + ..Default::default() + }], + &movement_opts, + ) + .await + .expect("moving an already-accounted multipart object between pools must have zero quota growth"); + assert_eq!(completed.size, 4096); + } + #[tokio::test] #[serial_test::serial] async fn rejected_empty_parts_preserve_existing_object_and_staging() { diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index b48430d28..d404add26 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -24,6 +24,8 @@ use super::storage_api::object_usecase::access::{ has_bypass_governance_header, load_bucket_generation_from_store, recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref, }; +#[cfg(test)] +use super::storage_api::object_usecase::bucket::quota::BucketQuota; use super::storage_api::object_usecase::bucket::quota::checker::QuotaChecker; #[cfg(test)] use super::storage_api::object_usecase::bucket::replication::{ReplicationState, replication_statuses_map}; @@ -63,8 +65,9 @@ use super::storage_api::object_usecase::contract::namespace::NamespaceLocking; use super::storage_api::object_usecase::contract::object::{ObjectIO as _, ObjectOperations as _}; use super::storage_api::object_usecase::contract::range::HTTPRangeSpec; use super::storage_api::object_usecase::data_usage::{ - record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, - record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, + quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, + record_bucket_object_version_write_memory, record_bucket_object_write_memory, + record_bucket_object_write_unknown_previous_memory, }; use super::storage_api::object_usecase::deadlock_detector; use super::storage_api::object_usecase::ecfs::FS; @@ -131,11 +134,12 @@ use rustfs_notify::EventArgsBuilder; use rustfs_object_capacity::capacity_manager::get_capacity_manager; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object}; -use rustfs_s3select_api::object_store::bytes_stream; use rustfs_targets::{EventName, get_request_host, get_request_port, get_request_user_agent}; use rustfs_utils::CompressionAlgorithm; #[cfg(test)] use rustfs_utils::http::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; +#[cfg(test)] +use rustfs_utils::http::insert_header; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, @@ -538,6 +542,69 @@ pub(super) fn map_quota_check_outcome(bucket: &str, outcome: Result S3Result<()> { + if result.uses_durable_reservations { + return Ok(()); + } + let Some(quota_limit) = result.quota_limit else { + return Ok(()); + }; + let Some(current_usage) = result.current_usage else { + return Err(S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "Bucket quota check temporarily unavailable, please retry".to_string(), + )); + }; + if current_usage > quota_limit { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"), + )); + } + let _ = opts.set_quota_admission(current_usage, quota_limit); + Ok(()) +} + +fn ensure_object_size_within_quota(result: &QuotaCheckResult, new_size: u64) -> S3Result<()> { + let (Some(current_usage), Some(quota_limit)) = (result.current_usage, result.quota_limit) else { + return Ok(()); + }; + if new_size > quota_limit { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"), + )); + } + Ok(()) +} + +fn ensure_legacy_archive_size_within_quota(result: &QuotaCheckResult, total_unpacked_size: u64) -> S3Result<()> { + if result.uses_durable_reservations { + return Ok(()); + } + let (Some(current_usage), Some(quota_limit)) = (result.current_usage, result.quota_limit) else { + return Ok(()); + }; + let expected_usage = current_usage + .checked_add(total_unpacked_size) + .ok_or_else(|| s3_error!(InvalidArgument, "Archive total size overflowed quota accounting"))?; + if expected_usage > quota_limit { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"), + )); + } + Ok(()) +} + +fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result { + match quota_object_size(info) { + Ok(size) => Ok(size), + Err(err) if fail_closed => Err(ApiError::from(err).into()), + Err(_) => Ok(info.size.max(0) as u64), + } +} + fn request_uses_aws_chunked(headers: &HeaderMap) -> bool { let has_aws_chunked = |header_name: &str| { headers @@ -850,7 +917,7 @@ pin_project! { } struct MemoryTrackedBytesStream { - bytes: Bytes, + bytes: Option, emitted: bool, completed: bool, expected: usize, @@ -964,7 +1031,7 @@ impl MemoryTrackedBytesStream { ) -> Self { let length_mismatch = bytes.len() != expected; Self { - bytes, + bytes: Some(bytes), emitted: false, completed: !length_mismatch && expected == 0, expected, @@ -1032,35 +1099,36 @@ impl futures::Stream for MemoryTrackedBytesStream { // differently sized body. This is a defense-in-depth backstop; the // buffered/cache callers reject the mismatch before headers are sent. if this.length_mismatch { + let actual = this.bytes.as_ref().map_or(0, Bytes::len); this.emitted = true; this.finish_err(); return Poll::Ready(Some(Err(std::io::Error::new( std::io::ErrorKind::InvalidData, - format!( - "materialized GET body length mismatch: expected {}, got {}", - this.expected, - this.bytes.len() - ), + format!("materialized GET body length mismatch: expected {}, got {}", this.expected, actual), )))); } - let first_byte_elapsed = (!this.bytes.is_empty()).then(|| this.started.elapsed()); + let Some(bytes) = this.bytes.take() else { + return Poll::Ready(None); + }; + let bytes_len = bytes.len(); + let first_byte_elapsed = (!bytes.is_empty()).then(|| this.started.elapsed()); this.emitted = true; if let Some(elapsed) = first_byte_elapsed { rustfs_io_metrics::record_get_object_first_byte_latency(GET_OBJECT_STAGE_PATH_S3_HANDLER, elapsed.as_secs_f64()); } - if this.bytes.len() >= this.expected { + if bytes_len >= this.expected { this.finish_ok(); } if let Some(poll_start) = poll_start { rustfs_io_metrics::record_get_object_memory_body_stream_poll( this.source, GET_READER_STREAM_POLL_READY_DATA, - this.bytes.len(), + bytes_len, poll_start.elapsed().as_secs_f64(), ); } - Poll::Ready(Some(Ok(this.bytes.clone()))) + Poll::Ready(Some(Ok(bytes))) } } @@ -3838,13 +3906,6 @@ fn put_object_extract_limits() -> ArchiveLimits { ArchiveLimits::default() } -fn put_object_extract_quota_exceeded(current_usage: u64, quota_limit: u64) -> S3Error { - S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"), - ) -} - fn validate_put_object_extract_entry_count(count: usize, limits: ArchiveLimits) -> S3Result<()> { if count > limits.max_entries { return Err(s3_error!( @@ -4067,12 +4128,12 @@ impl DefaultObjectUsecase { .unwrap_or_else(|| RustFSBufferConfig::default().base_config.default_unknown) } - async fn check_bucket_quota(&self, bucket: &str, op: QuotaOperation, size: u64) -> S3Result<()> { + async fn check_bucket_quota(&self, bucket: &str, op: QuotaOperation, size: u64) -> S3Result> { let Some(metadata_sys) = self.bucket_metadata_sys() else { - return Ok(()); + return Ok(None); }; let quota_checker = QuotaChecker::new(metadata_sys); - map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(|_| ()) + map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(Some) } fn build_memory_bytes_blob( @@ -4087,10 +4148,7 @@ impl DefaultObjectUsecase { let bytes_len = bytes.len(); let guard = rustfs_io_metrics::track_get_object_buffered_bytes(bytes_len); let remaining = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX); - let blob = StreamingBlob::wrap(bytes_stream( - MemoryTrackedBytesStream::new(bytes, remaining, source, guard, lifecycle), - remaining, - )); + let blob = StreamingBlob::wrap(MemoryTrackedBytesStream::new(bytes, remaining, source, guard, lifecycle)); if let Some(handoff_start) = handoff_start { rustfs_io_metrics::record_get_object_response_handoff( "single_chunk", @@ -5468,9 +5526,24 @@ impl DefaultObjectUsecase { } } - #[instrument(level = "info", skip(self, _fs, req))] - #[hotpath::measure(impl_type = "DefaultObjectUsecase")] + #[instrument(name = "execute_put_object", level = "info", skip(self, _fs, req))] pub async fn execute_put_object(&self, _fs: &FS, req: S3Request) -> S3Result> { + self.execute_put_object_boxed(_fs, req).await + } + + fn execute_put_object_boxed<'a>( + &'a self, + _fs: &'a FS, + req: S3Request, + ) -> impl std::future::Future>> + Send + 'a { + Box::pin(self.execute_put_object_inner(_fs, req)) + } + + #[hotpath::measure( + label = "rustfs::app::object_usecase::DefaultObjectUsecase::execute_put_object", + impl_type = "DefaultObjectUsecase" + )] + async fn execute_put_object_inner(&self, _fs: &FS, req: S3Request) -> S3Result> { let start_time = std::time::Instant::now(); let mut req = req; @@ -5572,8 +5645,22 @@ impl DefaultObjectUsecase { // Resolve the authoritative decoded/plain object length (rejecting negative/unknown) before anything else consumes it. let mut size = resolve_put_object_authoritative_size(&req.headers, content_length)?; - // Bucket-quota admission runs exactly once, and only now that the authoritative object length is known. `size` is the same basis the settle phase records via ObjectInfo.size (actual, pre-compression/pre-encryption logical size), NOT the aws-chunked wire Content-Length. When no quota is configured this stays a zero-extra-I/O fast path; once a hard quota is set, checker/config/usage faults fail closed with a retryable error. - self.check_bucket_quota(&bucket, quota_operation, size as u64).await?; + // The app check preserves the existing S3 error contract; the storage + // commit path reserves the exact net logical growth under its locks. + let quota_check = self + .check_bucket_quota( + &bucket, + quota_operation, + u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + ) + .await?; + let quota_enabled = quota_check.as_ref().is_some_and(|result| result.quota_limit.is_some()); + if quota_enabled && ciphertext_passthrough { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(), + )); + } let put_stage_metrics_enabled = rustfs_io_metrics::put_stage_metrics_enabled(); let ingress_stage_start = put_stage_metrics_enabled.then(Instant::now); @@ -5747,6 +5834,9 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; + if let Some(quota_check) = quota_check.as_ref() { + apply_quota_admission(&mut opts, quota_check)?; + } rustfs_io_metrics::record_put_object_stage_duration_from("app_put_opts_build", put_opts_stage_start); apply_bucket_generation_guard(&req, &bucket, &mut opts)?; apply_put_request_object_lock_opts( @@ -5784,7 +5874,11 @@ impl DefaultObjectUsecase { Some(match previous_current_info { Ok(existing_obj_info) => { validate_existing_object_lock_for_write(&existing_obj_info, &opts)?; - Some(existing_obj_info.size.max(0) as u64) + Some(if quota_enabled { + quota_object_size(&existing_obj_info).map_err(ApiError::from)? + } else { + existing_obj_info.size.max(0) as u64 + }) } Err(err) => { if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { @@ -5799,6 +5893,12 @@ impl DefaultObjectUsecase { rustfs_io_metrics::record_put_object_stage_duration_from("app_prelookup", prelookup_stage_start); let actual_size = size; + if !ciphertext_passthrough && let Some(quota_check) = quota_check.as_ref() { + ensure_object_size_within_quota( + quota_check, + u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + )?; + } let mut md5hex = if let Some(base64_md5) = content_md5 { let md5 = base64_simd::STANDARD @@ -6084,18 +6184,14 @@ impl DefaultObjectUsecase { // backfill reproduces the lookup's observation bit for bit (latest // version's ObjectInfo.size — 0 for a delete-marker latest — or // not-found → None). + let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?; match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size)) { Some(previous_current_size) => { if put_versioned { - record_bucket_object_version_write_memory( - &bucket, - previous_current_size, - obj_info.size.max(0) as u64, - ) - .await; + record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { - record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; + record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; } } None => { @@ -6111,8 +6207,7 @@ impl DefaultObjectUsecase { put_versioned, "put_object old-size backfill unknown; recording degraded usage delta" ); - record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, put_versioned) - .await; + record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, put_versioned).await; } } @@ -6489,9 +6584,23 @@ impl DefaultObjectUsecase { }) } - #[instrument(level = "trace", skip(self, req))] - #[hotpath::measure(impl_type = "DefaultObjectUsecase")] + #[instrument(name = "execute_get_object", level = "trace", skip(self, req))] pub async fn execute_get_object(&self, req: S3Request) -> S3Result> { + self.execute_get_object_boxed(req).await + } + + fn execute_get_object_boxed( + &self, + req: S3Request, + ) -> impl std::future::Future>> + Send + '_ { + Box::pin(self.execute_get_object_inner(req)) + } + + #[hotpath::measure( + label = "rustfs::app::object_usecase::DefaultObjectUsecase::execute_get_object", + impl_type = "DefaultObjectUsecase" + )] + async fn execute_get_object_inner(&self, req: S3Request) -> S3Result> { if let Some(context) = &self.context { let _ = context.object_store(); } @@ -6945,8 +7054,15 @@ impl DefaultObjectUsecase { result } - #[instrument(level = "debug", skip(self, req))] - pub async fn execute_copy_object(&self, req: S3Request) -> S3Result> { + pub fn execute_copy_object( + &self, + req: S3Request, + ) -> impl std::future::Future>> + Send + '_ { + Box::pin(self.execute_copy_object_inner(req)) + } + + #[instrument(name = "execute_copy_object", level = "debug", skip(self, req))] + async fn execute_copy_object_inner(&self, req: S3Request) -> S3Result> { if let Some(context) = &self.context { let _ = context.object_store(); } @@ -7205,7 +7321,7 @@ impl DefaultObjectUsecase { if _self_copy_lock_guard.is_some() { current_opts.no_lock = true; } - let previous_current_size = match store.get_object_info(&bucket, &key, ¤t_opts).await { + let previous_current_sizes = match store.get_object_info(&bucket, &key, ¤t_opts).await { Ok(existing_obj_info) => { validate_existing_object_lock_for_write(&existing_obj_info, &dst_opts)?; if let Some(expected) = expected_current_version_id.as_deref() @@ -7213,7 +7329,7 @@ impl DefaultObjectUsecase { { return Err(s3_error!(PreconditionFailed)); } - Some(existing_obj_info.size.max(0) as u64) + Some((existing_obj_info.size.max(0) as u64, quota_object_size(&existing_obj_info))) } Err(err) => { if expected_current_version_id.is_some() { @@ -7534,8 +7650,29 @@ impl DefaultObjectUsecase { src_info.user_defined = Arc::new(user_defined); - self.check_bucket_quota(&bucket, QuotaOperation::CopyObject, src_info.size as u64) + let quota_check = self + .check_bucket_quota( + &bucket, + QuotaOperation::CopyObject, + u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + ) .await?; + let quota_enabled = quota_check.as_ref().is_some_and(|result| result.quota_limit.is_some()); + if let Some(quota_check) = quota_check.as_ref() { + apply_quota_admission(&mut dst_opts, quota_check)?; + } + let previous_current_size = match previous_current_sizes { + Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size), + Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err).into()), + Some((physical_size, _)) => Some(physical_size), + None => None, + }; + if let Some(quota_check) = quota_check.as_ref() { + ensure_object_size_within_quota( + quota_check, + u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + )?; + } let has_bucket_metadata = self.bucket_metadata_sys().is_some(); let cache_adapter = self.object_data_cache(); let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; @@ -7570,10 +7707,11 @@ impl DefaultObjectUsecase { let dest_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; if has_bucket_metadata { + let committed_size = quota_accounting_object_size(&oi, quota_enabled)?; if dest_versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; + record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { - record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; + record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; } } @@ -9053,6 +9191,17 @@ impl DefaultObjectUsecase { #[instrument(level = "debug", skip(self, req))] #[hotpath::measure(impl_type = "DefaultObjectUsecase")] pub async fn execute_put_object_extract(&self, req: S3Request) -> S3Result> { + self.execute_put_object_extract_boxed(req).await + } + + fn execute_put_object_extract_boxed( + &self, + req: S3Request, + ) -> impl std::future::Future>> + Send + '_ { + Box::pin(self.execute_put_object_extract_inner(req)) + } + + async fn execute_put_object_extract_inner(&self, req: S3Request) -> S3Result> { let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, S3Operation::PutObject).suppress_event(); let request_context = helper.request_context_or_from_request(&req); let auth_method = req.method.clone(); @@ -9168,7 +9317,12 @@ impl DefaultObjectUsecase { } validate_object_key(&key, "PUT")?; validate_table_catalog_object_mutation(&bucket, &key).await?; - self.check_bucket_quota(&bucket, QuotaOperation::PutObject, size as u64) + let _ = self + .check_bucket_quota( + &bucket, + QuotaOperation::PutObject, + u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + ) .await?; // Apply adaptive buffer sizing based on file size for optimal streaming performance. @@ -9225,14 +9379,17 @@ impl DefaultObjectUsecase { let extract_options = resolve_put_object_extract_options(&req.headers)?; let extract_limits = put_object_extract_limits(); - let extract_quota_snapshot = if let Some(metadata_sys) = self.bucket_metadata_sys() { + let extract_quota_check = if let Some(metadata_sys) = self.bucket_metadata_sys() { let quota_checker = QuotaChecker::new(metadata_sys); let check_result = map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?; - check_result.current_usage.zip(check_result.quota_limit) + Some(check_result) } else { None }; + let extract_quota_enabled = extract_quota_check + .as_ref() + .is_some_and(|result| result.quota_limit.is_some()); let version_id = match event_version_id { Some(v) => v.to_string(), None => String::new(), @@ -9312,10 +9469,8 @@ impl DefaultObjectUsecase { .checked_add(entry_size) .ok_or_else(|| s3_error!(InvalidArgument, "Archive total unpacked size overflowed while processing entries"))?; validate_put_object_extract_total_size(total_unpacked_size, extract_limits)?; - if let Some((current_usage, quota_limit)) = extract_quota_snapshot - && current_usage.saturating_add(total_unpacked_size) > quota_limit - { - return Err(put_object_extract_quota_exceeded(current_usage, quota_limit)); + if let Some(quota_check) = extract_quota_check.as_ref() { + ensure_legacy_archive_size_within_quota(quota_check, total_unpacked_size)?; } let mut size = i64::try_from(entry_size).map_err(|_| s3_error!(InvalidArgument, "Archive entry size does not fit into i64"))?; @@ -9360,6 +9515,9 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; + if let Some(quota_check) = extract_quota_check.as_ref() { + apply_quota_admission(&mut opts, quota_check)?; + } opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id; opts.object_lock_config_snapshot = Some(Arc::clone(&object_lock_config_snapshot)); let pax_authorization = @@ -9503,19 +9661,18 @@ impl DefaultObjectUsecase { return Err(ApiError::from(e).into()); } }; + let committed_size = quota_accounting_object_size(&obj_info, extract_quota_enabled)?; let extract_versioned = BucketVersioningSys::prefix_enabled(&bucket, &fpath).await; match previous_current_size_from_backfill(backfilled_old_current_size) { Some(previous_current_size) => { if extract_versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64) - .await; + record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { - record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; + record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; } } None => { - record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, extract_versioned) - .await; + record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, extract_versioned).await; } } let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await; @@ -15931,13 +16088,6 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); } - #[test] - fn put_object_extract_quota_exceeded_matches_existing_error_shape() { - let err = put_object_extract_quota_exceeded(10, 8); - assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); - assert_eq!(err.message(), Some("Bucket quota exceeded. Current usage: 10 bytes, limit: 8 bytes")); - } - #[tokio::test] async fn execute_put_object_rejects_post_object_sse_kms_from_input() { let input = PutObjectInput::builder() @@ -16422,8 +16572,12 @@ mod tests { .expect("create self-copy test bucket"); let payload = b"object whose key equals its bucket".to_vec(); let mut reader = PutObjReader::from_vec(payload.clone()); + let setup_opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; store - .put_object(&bucket, &bucket, &mut reader, &ObjectOptions::default()) + .put_object(&bucket, &bucket, &mut reader, &setup_opts) .await .expect("put object whose key equals its bucket"); @@ -17660,9 +17814,99 @@ mod tests { quota_limit: Some(2048), operation_size: 512, remaining: Some(512), + uses_durable_reservations: true, } } + #[tokio::test] + #[serial_test::serial] + async fn quota_rejects_ciphertext_replication_before_polling_the_body() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let (_store, bucket) = + crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-replication-early-reject", 4096).await; + let body_polled = Arc::new(AtomicBool::new(false)); + let body_polled_in_stream = Arc::clone(&body_polled); + let body = StreamingBlob::wrap(futures::stream::once(async move { + body_polled_in_stream.store(true, Ordering::Release); + Ok::(Bytes::from_static(b"ciphertext")) + })); + let input = PutObjectInput::builder() + .bucket(bucket) + .key("object".to_string()) + .body(Some(body)) + .content_length(Some(10)) + .build() + .expect("ciphertext replication PUT input should build"); + let mut request = build_request(input, Method::PUT); + insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + request + .headers + .insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256")); + request.extensions.insert(crate::storage::access::ReqInfo { + replication_request_authorized: true, + ..Default::default() + }); + + let err = DefaultObjectUsecase::from_global() + .execute_put_object(&FS::new(), request) + .await + .expect_err("quota-enabled ciphertext replication should fail at ingress"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(!body_polled.load(Ordering::Acquire), "rejected ciphertext body must not be consumed"); + } + + #[tokio::test] + #[serial_test::serial] + async fn legacy_quota_rejects_full_put_before_polling_the_body() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use std::sync::atomic::{AtomicBool, Ordering}; + + const GI_B: u64 = 1024 * 1024 * 1024; + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let bucket = format!("legacy-quota-{}", Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create legacy quota test bucket"); + crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 4 * GI_B).await; + let metadata_sys = DefaultObjectUsecase::from_global() + .bucket_metadata_sys() + .expect("test app context should expose bucket metadata"); + QuotaChecker::new(metadata_sys) + .set_quota_config( + &bucket, + BucketQuota { + quota: Some(5 * GI_B), + ..Default::default() + }, + ) + .await + .expect("configure legacy quota"); + + let body_polled = Arc::new(AtomicBool::new(false)); + let body_polled_in_stream = Arc::clone(&body_polled); + let body = StreamingBlob::wrap(futures::stream::once(async move { + body_polled_in_stream.store(true, Ordering::Release); + Ok::(Bytes::new()) + })); + let input = PutObjectInput::builder() + .bucket(bucket) + .key("object".to_string()) + .body(Some(body)) + .content_length(Some(i64::try_from(2 * GI_B).expect("test size should fit i64"))) + .build() + .expect("legacy quota PUT input should build"); + + let err = DefaultObjectUsecase::from_global() + .execute_put_object(&FS::new(), build_request(input, Method::PUT)) + .await + .expect_err("4 GiB used plus a 2 GiB PUT must exceed a 5 GiB legacy quota"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(!body_polled.load(Ordering::Acquire), "legacy quota rejection must not consume the body"); + } + #[test] fn quota_admission_allows_within_limit() { let result = map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write"); @@ -17673,12 +17917,421 @@ mod tests { assert_eq!(result.remaining, Some(512)); } + #[tokio::test] + #[serial_test::serial] + async fn concurrent_puts_share_durable_bucket_quota_reservations() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-put-quota", 6000).await; + + let first_opts = ObjectOptions::default(); + let second_opts = ObjectOptions::default(); + let first_store = Arc::clone(&store); + let first_bucket = bucket.clone(); + let first = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x73; 4096]); + first_store.put_object(&first_bucket, "first", &mut reader, &first_opts).await + }); + let second = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x74; 4096]); + store.put_object(&bucket, "second", &mut reader, &second_opts).await + }); + let (first, second) = tokio::join!(first, second); + let first = first.expect("first PUT task should not panic"); + let second = second.expect("second PUT task should not panic"); + + assert_eq!(usize::from(first.is_ok()) + usize::from(second.is_ok()), 1); + let denied = first.err().or_else(|| second.err()).expect("one PUT must be denied"); + assert!(matches!( + denied, + StorageError::QuotaExceeded { + current: 4096, + limit: 6000 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn concurrent_within_limit_puts_keep_independent_mutation_fences() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-fence-quota", 8192).await; + let first_barrier = PutObjectCommitBarrier::install(&bucket, "first", PutObjectCommitPause::BeforeQuotaRename); + let second_barrier = PutObjectCommitBarrier::install(&bucket, "second", PutObjectCommitPause::BeforeQuotaRename); + + let first_store = Arc::clone(&store); + let first_bucket = bucket.clone(); + let first = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x75; 4096]); + first_store + .put_object(&first_bucket, "first", &mut reader, &ObjectOptions::default()) + .await + }); + let second_store = Arc::clone(&store); + let second_bucket = bucket.clone(); + let second = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x76; 4096]); + second_store + .put_object(&second_bucket, "second", &mut reader, &ObjectOptions::default()) + .await + }); + + first_barrier.wait_until_paused().await; + second_barrier.wait_until_paused().await; + first_barrier.release(); + second_barrier.release(); + + first + .await + .expect("first PUT task should not panic") + .expect("first within-limit PUT should commit"); + second + .await + .expect("second PUT task should not panic") + .expect("second within-limit PUT should commit"); + } + + #[tokio::test] + #[serial_test::serial] + async fn put_rejects_rotated_quota_capability_before_rename() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("rotated-proof-put-quota", 4096).await; + let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename); + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x77; 4096]); + put_store + .put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + assert!( + crate::storage::storage_api::ecstore_notification::rotate_cross_pool_fence_fleet_proof_for_test(), + "the gating environment must have a current fleet proof" + ); + barrier.release(); + + let err = put + .await + .expect("PUT task should not panic") + .expect_err("a replaced fleet proof must fence the authoritative rename"); + assert!(matches!( + err, + StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + .. + } + )); + store + .get_object_info(&bucket, "object", &ObjectOptions::default()) + .await + .expect_err("proof rotation before rename must leave no committed object"); + } + + #[tokio::test] + #[serial_test::serial] + async fn durable_quota_reclaims_overwrites_and_deleted_bytes() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("quota-delta-reconcile", 4096).await; + + for byte in [0x41, 0x42] { + let mut reader = PutObjReader::from_vec(vec![byte; 4096]); + store + .put_object(&bucket, "object", &mut reader, &ObjectOptions::default()) + .await + .expect("same-size overwrite must consume no additional quota"); + } + + store + .delete_object(&bucket, "object", ObjectOptions::default()) + .await + .expect("delete quota-tracked object"); + let mut replacement = PutObjReader::from_vec(vec![0x43; 4096]); + store + .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) + .await + .expect("deleted bytes must be reclaimed before rejecting a replacement"); + + let mut excess = PutObjReader::from_vec(vec![0x44]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("one byte beyond the reclaimed exact quota must be denied"); + assert!(matches!( + err, + StorageError::QuotaExceeded { + current: 4096, + limit: 4096 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_put_has_zero_quota_growth() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("data-movement-put-quota", 0).await; + let mut reader = PutObjReader::from_vec(vec![0x79; 4096]); + let stored = store + .put_object( + &bucket, + "object", + &mut reader, + &ObjectOptions { + data_movement: true, + ..Default::default() + }, + ) + .await + .expect("moving an already-accounted object between pools must have zero quota growth"); + assert_eq!(stored.size, 4096); + } + + #[tokio::test] + #[serial_test::serial] + async fn cancelled_put_releases_durable_quota_reservation() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("cancelled-put-quota", 4096).await; + + let barrier = PutObjectCommitBarrier::install(&bucket, "cancelled", PutObjectCommitPause::AfterQuotaReservation); + let cancelled_store = Arc::clone(&store); + let cancelled_bucket = bucket.clone(); + let cancelled = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x51; 4096]); + cancelled_store + .put_object(&cancelled_bucket, "cancelled", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + cancelled.abort(); + let cancelled_result = cancelled.await; + assert!(cancelled_result.is_err(), "the paused request must be cancelled"); + drop(barrier); + + let mut replacement = PutObjReader::from_vec(vec![0x52; 4096]); + store + .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) + .await + .expect("cancelling before commit must release the complete reservation"); + } + + #[tokio::test] + #[serial_test::serial] + async fn cancelled_put_after_commit_marker_is_reconciled() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("cancelled-spawned-put-quota", 4096).await; + let commit_barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename); + let first_store = Arc::clone(&store); + let first_bucket = bucket.clone(); + let first = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x53; 4096]); + first_store + .put_object(&first_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + commit_barrier.wait_until_paused().await; + first.abort(); + assert!(first.await.is_err(), "the outer request task must be cancelled"); + drop(commit_barrier); + + store + .get_object_info(&bucket, "object", &ObjectOptions::default()) + .await + .expect_err("cancelling before rename must not commit the object"); + let mut replacement = PutObjReader::from_vec(vec![0x54; 4096]); + store + .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) + .await + .expect("the next admission must reap the abandoned commit marker"); + } + + #[tokio::test] + #[serial_test::serial] + async fn committed_put_survives_quota_ledger_settlement_failure() { + use crate::app::storage_api::test::set_disk::{ + PutObjectCommitBarrier, PutObjectCommitPause, fail_next_quota_ledger_save_for_test, + }; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("settlement-failure-quota", 4096).await; + let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename); + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x59; 4096]); + put_store + .put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + fail_next_quota_ledger_save_for_test(); + barrier.release(); + put.await + .expect("PUT task should not panic") + .expect("a post-commit ledger failure must not change the successful write result"); + let stored = store + .get_object_info(&bucket, "object", &ObjectOptions::default()) + .await + .expect("the committed object must remain visible"); + assert_eq!(stored.size, 4096); + } + + #[tokio::test] + #[serial_test::serial] + async fn suspended_null_version_overwrite_uses_exact_quota_delta() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("suspended-version-quota", 6200).await; + let mut versioned_reader = PutObjReader::from_vec(vec![0x61; 4096]); + store + .put_object( + &bucket, + "object", + &mut versioned_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("write UUID version"); + + for (size, byte) in [(1024, 0x62), (2048, 0x63)] { + let mut reader = PutObjReader::from_vec(vec![byte; size]); + store + .put_object( + &bucket, + "object", + &mut reader, + &ObjectOptions { + version_suspended: true, + ..Default::default() + }, + ) + .await + .expect("suspended write should replace only the exact null version"); + } + + let mut excess = PutObjReader::from_vec(vec![0x64; 57]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("UUID plus replacement null version must consume 6144 bytes"); + assert!(matches!( + err, + StorageError::QuotaExceeded { + current: 6144, + limit: 6200 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn durable_quota_reservation_observes_lowered_config_revision() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("lowered-quota-revision", 8192).await; + let mut initial = PutObjReader::from_vec(vec![0x71; 4096]); + store + .put_object(&bucket, "initial", &mut initial, &ObjectOptions::default()) + .await + .expect("write under original quota"); + + let metadata_sys = DefaultObjectUsecase::from_global() + .bucket_metadata_sys() + .expect("test app context should expose bucket metadata"); + QuotaChecker::new(metadata_sys) + .set_quota_config(&bucket, BucketQuota::new(Some(4096))) + .await + .expect("lower bucket quota"); + let mut excess = PutObjReader::from_vec(vec![0x72]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("reservation must not use the stale larger quota revision"); + assert!(matches!( + err, + StorageError::QuotaExceeded { + current: 4096, + limit: 4096 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn quota_enable_waits_for_unlimited_commit() { + use crate::app::storage_api::test::metadata_sys::ConfigWriteLockProbe; + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("quota-config-fence", 8192).await; + let metadata_sys = DefaultObjectUsecase::from_global() + .bucket_metadata_sys() + .expect("test app context should expose bucket metadata"); + QuotaChecker::new(Arc::clone(&metadata_sys)) + .set_quota_config(&bucket, BucketQuota::new(None)) + .await + .expect("clear quota before the fenced write"); + let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::AfterQuotaReservation); + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x73; 4096]); + put_store + .put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + + let update_probe = ConfigWriteLockProbe::install(&bucket); + let update_bucket = bucket.clone(); + let update = tokio::spawn(async move { + QuotaChecker::new(metadata_sys) + .set_quota_config(&update_bucket, BucketQuota::new(Some(0))) + .await + }); + update_probe.wait_until_attempted().await; + assert!( + !update.is_finished(), + "quota mutation must wait for the reservation's metadata transaction guard" + ); + + barrier.release(); + put.await + .expect("PUT task should not panic") + .expect("the write linearized before the quota update must commit"); + update + .await + .expect("quota update task should not panic") + .expect("quota update should proceed after commit"); + + let mut excess = PutObjReader::from_vec(vec![0x74]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("writes after the zero-byte quota update must be denied"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 4096, limit: 0 })); + } + #[test] fn quota_admission_rejects_over_limit() { let err = map_quota_check_outcome("bucket", Ok(quota_result(false))).expect_err("an over-limit result rejects the write"); assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); } + #[test] + fn legacy_quota_admission_rejects_already_over_limit() { + let result = QuotaCheckResult { + allowed: true, + current_usage: Some(6), + quota_limit: Some(5), + operation_size: 0, + remaining: Some(0), + uses_durable_reservations: false, + }; + let mut opts = ObjectOptions::default(); + let err = + apply_quota_admission(&mut opts, &result).expect_err("legacy completion must not bypass an already exceeded quota"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + #[test] fn quota_admission_fails_closed_on_checker_error() { // A configured hard quota must never be bypassed by an internal fault: a checker error becomes a retryable ServiceUnavailable, not a silent allow. @@ -17692,6 +18345,45 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); } + #[test] + fn legacy_archive_quota_rejects_cumulative_size_and_overflow() { + let legacy = QuotaCheckResult { + allowed: true, + current_usage: Some(4), + quota_limit: Some(5), + operation_size: 0, + remaining: Some(1), + uses_durable_reservations: false, + }; + assert!(ensure_legacy_archive_size_within_quota(&legacy, 2).is_err()); + assert!(ensure_legacy_archive_size_within_quota(&legacy, 1).is_ok()); + + let maxed = QuotaCheckResult { + current_usage: Some(u64::MAX), + quota_limit: Some(u64::MAX), + ..legacy + }; + assert!(ensure_legacy_archive_size_within_quota(&maxed, 1).is_err()); + } + + #[test] + fn early_quota_filter_rejects_only_an_individually_impossible_object() { + let stale_full_usage = QuotaCheckResult { + allowed: true, + current_usage: Some(4096), + quota_limit: Some(4096), + operation_size: 0, + remaining: Some(0), + uses_durable_reservations: true, + }; + + ensure_object_size_within_quota(&stale_full_usage, 4096) + .expect("commit-time ledger must decide whether stale usage was reclaimed"); + let err = ensure_object_size_within_quota(&stale_full_usage, 4097) + .expect_err("an object larger than the whole quota can never fit"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + #[test] fn quota_admission_fails_closed_on_unknown_authoritative_usage() { let err = map_quota_check_outcome( diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 39d7d7685..20e937790 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -47,6 +47,8 @@ pub(crate) mod capacity { pub(crate) mod data_usage { use std::sync::Arc; + pub(crate) use crate::storage::storage_api::ecstore_data_usage::quota_object_size; + pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) { crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await; } @@ -1214,4 +1216,14 @@ pub(crate) mod test { pub(crate) use crate::storage::storage_api::{ ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, }; + pub(crate) mod set_disk { + pub(crate) use crate::storage::storage_api::ecstore_set_disk::{ + MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause, + fail_next_quota_ledger_save_for_test, + }; + } + + pub(crate) mod metadata_sys { + pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::ConfigWriteLockProbe; + } } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index e27c6dc29..ce4be49d5 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -301,7 +301,7 @@ impl S3 for FS { #[instrument(level = "debug", skip(self, req))] async fn copy_object(&self, req: S3Request) -> S3Result> { let usecase = s3_api::object_usecase_for(self); - Box::pin(usecase.execute_copy_object(req)).await + usecase.execute_copy_object(req).await } #[instrument( @@ -704,7 +704,7 @@ impl S3 for FS { async fn get_object(&self, req: S3Request) -> S3Result> { crate::hp_guard!("S3::get_object"); let usecase = s3_api::object_usecase_for(self); - Box::pin(usecase.execute_get_object(req)).await + usecase.execute_get_object(req).await } async fn get_object_acl(&self, req: S3Request) -> S3Result> { @@ -1261,7 +1261,7 @@ impl S3 for FS { async fn put_object(&self, req: S3Request) -> S3Result> { crate::hp_guard!("S3::put_object"); let usecase = s3_api::object_usecase_for(self); - Box::pin(usecase.execute_put_object(self, req)).await + usecase.execute_put_object(self, req).await } async fn put_object_acl(&self, req: S3Request) -> S3Result> { diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 131f3ce1f..d06e0892f 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -153,9 +153,7 @@ fn remove_heal_control_replay( static HEAL_CONTROL_REPLAY_CACHE: OnceLock>>> = OnceLock::new(); static NODE_CAPABILITY_SERVER_EPOCH: LazyLock = LazyLock::new(Uuid::new_v4); -// RUSTFS_COMPAT_TODO(cross-pool-fence-v1): advertise unsupported during predeployment. Remove after composite acquisition, -// activation fencing, fleet proof, commit-time proof revalidation, and fail-closed revocation ship together. -const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 0; +const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1; fn admit_heal_control_replay( replay_cache: &mut HashMap>, @@ -2945,7 +2943,7 @@ mod tests { } #[tokio::test] - async fn snapshot_lease_acquire_and_renew_handlers_fail_closed() { + async fn snapshot_lease_acquire_and_renew_handlers_fail_closed_for_missing_disk() { let service = make_server(); let disk = "http://node-a:9000/data/rustfs0".to_string(); @@ -2962,7 +2960,7 @@ mod tests { let acquire = service .acquire_snapshot_lease(acquire) .await - .expect("disabled acquire should return a protocol response") + .expect("missing-disk acquire should return a protocol response") .into_inner(); let mut renew = Request::new(SnapshotLeaseRenewRequest { @@ -2979,14 +2977,14 @@ mod tests { let renew = service .renew_snapshot_lease(renew) .await - .expect("disabled renew should return a protocol response") + .expect("missing-disk renew should return a protocol response") .into_inner(); for response in [acquire, renew] { assert!(!response.success); assert!(response.token.is_empty()); assert_eq!(response.protocol_version, 1); - assert_eq!(response.error, Some(DiskError::UnsupportedDisk.into())); + assert_eq!(response.error, Some(DiskError::other("cannot find disk").into())); } } @@ -3342,7 +3340,7 @@ mod tests { } #[tokio::test] - async fn cross_pool_fence_probe_authenticates_unsupported_rollout_state() { + async fn cross_pool_fence_probe_authenticates_supported_v1_state() { let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string()); let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true); assert!( @@ -3407,7 +3405,7 @@ mod tests { assert!(response.success); assert_eq!(response.error_info, None); - assert_eq!(&response.result[..4], &0_u32.to_be_bytes()); + assert_eq!(&response.result[..4], &1_u32.to_be_bytes()); let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..]) .expect("capability identity should decode"); assert_eq!(topology_member, "node-a:9000"); diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index 0e0f2defe..ce0ba42e5 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -37,19 +37,28 @@ const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512; const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024; const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1; +fn snapshot_lease_response(result: Result) -> Response { + match result { + Ok(token) => Response::new(SnapshotLeaseResponse { + success: true, + token: token.as_bytes().to_vec().into(), + protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION, + error: None, + }), + Err(err) => Response::new(SnapshotLeaseResponse { + success: false, + token: Bytes::new(), + protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION, + error: Some(err.into()), + }), + } +} + struct DecodedRpcPayload { value: T, from_msgpack: bool, } -fn snapshot_lease_disabled_response() -> SnapshotLeaseResponse { - SnapshotLeaseResponse { - success: false, - token: Bytes::new(), - protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION, - error: Some(DiskError::UnsupportedDisk.into()), - } -} fn decode_msgpack_or_json( binary: &[u8], json: &str, @@ -241,7 +250,12 @@ impl NodeService { rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref()), "acquire_snapshot_lease", )?; - Ok(Response::new(snapshot_lease_disabled_response())) + let request = request.into_inner(); + let result = match self.find_disk(&request.disk).await { + Some(disk) => disk.acquire_snapshot_lease(&request.volume, &request.path).await, + None => Err(DiskError::other("cannot find disk")), + }; + Ok(snapshot_lease_response(result)) } pub(super) async fn handle_renew_snapshot_lease( @@ -253,7 +267,14 @@ impl NodeService { rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref()), "renew_snapshot_lease", )?; - Ok(Response::new(snapshot_lease_disabled_response())) + let request = request.into_inner(); + let token = + SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?; + let result = match self.find_disk(&request.disk).await { + Some(disk) => disk.renew_snapshot_lease(&request.volume, &request.path, token).await, + None => Err(DiskError::other("cannot find disk")), + }; + Ok(snapshot_lease_response(result)) } pub(super) async fn handle_release_snapshot_lease( @@ -266,8 +287,11 @@ impl NodeService { "release_snapshot_lease", )?; let request = request.into_inner(); - let token = - SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?; + let token = if request.token.as_ref() == SnapshotLeaseToken::revoke_all().as_bytes() { + SnapshotLeaseToken::revoke_all() + } else { + SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))? + }; let Some(disk) = self.find_disk(&request.disk).await else { return Ok(Response::new(SnapshotLeaseMutationResponse { success: false, @@ -1025,7 +1049,7 @@ impl NodeService { .rename_data( &request.src_volume, &request.src_path, - decoded_file_info.value, + &decoded_file_info.value, &request.dst_volume, &request.dst_path, ) @@ -1494,11 +1518,11 @@ mod tests { use super::{ compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, encode_batch_read_version_response_payloads, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, - encode_read_multiple_response_payloads, encode_rename_data_response_payloads, snapshot_lease_disabled_response, + encode_read_multiple_response_payloads, encode_rename_data_response_payloads, }; use crate::storage::storage_api::ReadMultipleResp; + use crate::storage::storage_api::RenameDataResp; use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp; - use crate::storage::storage_api::{DiskError, RenameDataResp}; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::global_internode_metrics; use serde::{Deserialize, Serialize}; @@ -1509,17 +1533,6 @@ mod tests { count: u32, } - #[test] - fn snapshot_lease_acquire_and_renew_fail_closed() { - let response = snapshot_lease_disabled_response(); - let expected_error = DiskError::UnsupportedDisk.into(); - - assert!(!response.success); - assert!(response.token.is_empty()); - assert_eq!(response.protocol_version, 1); - assert_eq!(response.error, Some(expected_error)); - } - #[test] fn decode_msgpack_or_json_prefers_binary_payload() { let payload = SamplePayload { diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index b12169c71..aacd91f8a 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -430,7 +430,7 @@ pub(crate) mod ecstore_config { pub(crate) mod ecstore_data_usage { pub(crate) use rustfs_ecstore::api::data_usage::{ apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached, - load_data_usage_from_backend, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, + load_data_usage_from_backend, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend, }; @@ -487,8 +487,12 @@ pub(crate) mod ecstore_metrics { #[allow(unused_imports)] pub(crate) mod ecstore_notification { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::notification::rotate_cross_pool_fence_fleet_proof_for_test; pub(crate) use rustfs_ecstore::api::notification::{ - NotificationSys, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe, + CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof, + cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys, + start_remote_version_state_fleet_probe, }; } @@ -546,6 +550,11 @@ pub(crate) mod ecstore_test_support { } pub(crate) mod ecstore_set_disk { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::set_disk::test_util::{ + MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause, + fail_next_quota_ledger_save_for_test, + }; pub(crate) use rustfs_ecstore::api::set_disk::{ DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class, }; @@ -1130,7 +1139,9 @@ pub(crate) trait StorageDiskRpcExt { ) -> DiskResult<()>; async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult; async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>; + async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult; async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()>; + async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult; async fn stat_volume(&self, volume: &str) -> DiskResult; async fn list_volumes(&self) -> DiskResult>; async fn make_volume(&self, volume: &str) -> DiskResult<()>; @@ -1139,7 +1150,7 @@ pub(crate) trait StorageDiskRpcExt { &self, src_volume: &str, src_path: &str, - file_info: rustfs_filemeta::FileInfo, + file_info: &rustfs_filemeta::FileInfo, dst_volume: &str, dst_path: &str, ) -> DiskResult; @@ -1258,10 +1269,18 @@ where ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await } + async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult { + ecstore_disk::DiskAPI::acquire_snapshot_lease(self, volume, path).await + } + async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()> { ecstore_disk::DiskAPI::release_snapshot_lease(self, volume, path, token).await } + async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult { + ecstore_disk::DiskAPI::renew_snapshot_lease(self, volume, path, token).await + } + async fn stat_volume(&self, volume: &str) -> DiskResult { ecstore_disk::DiskAPI::stat_volume(self, volume).await } @@ -1282,11 +1301,11 @@ where &self, src_volume: &str, src_path: &str, - file_info: rustfs_filemeta::FileInfo, + file_info: &rustfs_filemeta::FileInfo, dst_volume: &str, dst_path: &str, ) -> DiskResult { - ecstore_disk::DiskAPI::rename_data(self, src_volume, src_path, file_info, dst_volume, dst_path).await + ecstore_disk::DiskAPI::rename_data(self, src_volume, src_path, file_info.clone(), dst_volume, dst_path).await } async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult> { diff --git a/scripts/check_s3s_footprint.sh b/scripts/check_s3s_footprint.sh index 30cdcb3f0..56fb60d35 100755 --- a/scripts/check_s3s_footprint.sh +++ b/scripts/check_s3s_footprint.sh @@ -26,7 +26,7 @@ cd "$(dirname "$0")/.." # Excludes crates/e2e_test/ — test infrastructure legitimately uses s3s # to verify S3 behavior and does not widen the production s3s surface. S3S_IMPORT_FILES_BASELINE=213 -S3_ERROR_LINES_BASELINE=1621 +S3_ERROR_LINES_BASELINE=1620 S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::' E2E_TEST_GLOB='--glob=!crates/e2e_test/**'