mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-16 18:08:21 +00:00
fix(quota): enforce durable hard quota reservations
This commit is contained in:
@@ -131,6 +131,8 @@ pub mod bucket {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub mod metadata_sys {
|
pub mod metadata_sys {
|
||||||
|
#[cfg(feature = "test-util")]
|
||||||
|
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||||
pub use crate::bucket::metadata_sys::{
|
pub use crate::bucket::metadata_sys::{
|
||||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
|
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
|
||||||
@@ -140,7 +142,7 @@ pub mod bucket {
|
|||||||
get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config,
|
get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config,
|
||||||
get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
|
get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
|
||||||
set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
||||||
update_under_transaction_lock,
|
update_quota_if_incarnation, update_under_transaction_lock,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -316,7 +318,7 @@ pub mod data_usage {
|
|||||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||||
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
|
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
|
||||||
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
|
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size,
|
||||||
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
|
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
|
||||||
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
||||||
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
|
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
|
||||||
@@ -404,7 +406,8 @@ pub mod metrics {
|
|||||||
|
|
||||||
pub mod notification {
|
pub mod notification {
|
||||||
pub use crate::services::notification_sys::{
|
pub use crate::services::notification_sys::{
|
||||||
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
|
||||||
|
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
|
||||||
start_remote_version_state_fleet_probe,
|
start_remote_version_state_fleet_probe,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -464,6 +467,7 @@ pub mod set_disk {
|
|||||||
|
|
||||||
#[cfg(feature = "test-util")]
|
#[cfg(feature = "test-util")]
|
||||||
pub mod test_util {
|
pub mod test_util {
|
||||||
|
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||||
pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,6 +50,72 @@ use uuid::Uuid;
|
|||||||
|
|
||||||
const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
|
const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
|
||||||
|
|
||||||
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
struct ConfigWriteLockProbeState {
|
||||||
|
bucket: String,
|
||||||
|
arrived: tokio::sync::Notify,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
|
||||||
|
|
||||||
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
pub struct ConfigWriteLockProbe {
|
||||||
|
state: Arc<ConfigWriteLockProbeState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
impl ConfigWriteLockProbe {
|
||||||
|
pub fn install(bucket: &str) -> Self {
|
||||||
|
let state = Arc::new(ConfigWriteLockProbeState {
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
arrived: tokio::sync::Notify::new(),
|
||||||
|
});
|
||||||
|
let mut probes = CONFIG_WRITE_LOCK_PROBES
|
||||||
|
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||||
|
.lock()
|
||||||
|
.expect("config write lock probe mutex should not poison");
|
||||||
|
assert!(
|
||||||
|
!probes.iter().any(|current| current.bucket == state.bucket),
|
||||||
|
"config write lock probe must be unique for a bucket"
|
||||||
|
);
|
||||||
|
probes.push(Arc::clone(&state));
|
||||||
|
drop(probes);
|
||||||
|
Self { state }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn wait_until_attempted(&self) {
|
||||||
|
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||||
|
.await
|
||||||
|
.expect("bucket config update should attempt the transaction lock");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
impl Drop for ConfigWriteLockProbe {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let mut probes = CONFIG_WRITE_LOCK_PROBES
|
||||||
|
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||||
|
.lock()
|
||||||
|
.expect("config write lock probe mutex should not poison");
|
||||||
|
probes.retain(|state| !Arc::ptr_eq(state, &self.state));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
fn notify_config_write_lock_attempt(bucket: &str) {
|
||||||
|
let probe = CONFIG_WRITE_LOCK_PROBES
|
||||||
|
.get_or_init(|| StdMutex::new(Vec::new()))
|
||||||
|
.lock()
|
||||||
|
.expect("config write lock probe mutex should not poison")
|
||||||
|
.iter()
|
||||||
|
.find(|probe| probe.bucket == bucket)
|
||||||
|
.cloned();
|
||||||
|
if let Some(probe) = probe {
|
||||||
|
probe.arrived.notify_one();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
enum MetadataLoadMode {
|
enum MetadataLoadMode {
|
||||||
Initial,
|
Initial,
|
||||||
@@ -590,6 +656,31 @@ pub async fn update_under_transaction_lock(
|
|||||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn update_quota_if_incarnation(
|
||||||
|
bucket: &str,
|
||||||
|
data: Vec<u8>,
|
||||||
|
expected_incarnation_id: Uuid,
|
||||||
|
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||||
|
) -> Result<OffsetDateTime> {
|
||||||
|
let sys = get_bucket_metadata_sys()?;
|
||||||
|
let guard = Box::pin(acquire_config_write_guard_for_incarnation(
|
||||||
|
sys.clone(),
|
||||||
|
bucket,
|
||||||
|
Some(expected_incarnation_id),
|
||||||
|
))
|
||||||
|
.await?;
|
||||||
|
if !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof) {
|
||||||
|
return Err(Error::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "quota_capability",
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
object: rustfs_config::QUOTA_CONFIG_FILE.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn update_bucket_targets_under_transaction_lock(
|
pub async fn update_bucket_targets_under_transaction_lock(
|
||||||
guard: &BucketMetadataMutationGuard,
|
guard: &BucketMetadataMutationGuard,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
@@ -734,7 +825,26 @@ async fn acquire_transaction_lock_with_sys(
|
|||||||
let lock = api
|
let lock = api
|
||||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||||
.await?;
|
.await?;
|
||||||
Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
|
let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout());
|
||||||
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
{
|
||||||
|
tokio::pin!(acquire);
|
||||||
|
let mut notified = false;
|
||||||
|
let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
|
||||||
|
std::task::Poll::Pending => {
|
||||||
|
if !notified {
|
||||||
|
notify_config_write_lock_attempt(bucket);
|
||||||
|
notified = true;
|
||||||
|
}
|
||||||
|
std::task::Poll::Pending
|
||||||
|
}
|
||||||
|
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
Ok(guard)
|
||||||
|
}
|
||||||
|
#[cfg(not(any(test, feature = "test-util")))]
|
||||||
|
Ok(acquire.await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The lock resource name is deliberately still the `bucket-targets` one it
|
/// The lock resource name is deliberately still the `bucket-targets` one it
|
||||||
@@ -889,6 +999,37 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Re-read the quota configuration and bucket incarnation from the same
|
||||||
|
/// authoritative metadata blob while the caller holds the bucket metadata
|
||||||
|
/// transaction read lock.
|
||||||
|
pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
|
||||||
|
ctx: &crate::runtime::instance::InstanceContext,
|
||||||
|
bucket: &str,
|
||||||
|
) -> Result<(Option<BucketQuota>, Uuid, OffsetDateTime)> {
|
||||||
|
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||||
|
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||||
|
|
||||||
|
match bucket_meta_sys
|
||||||
|
.read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
BucketMetadataAuthority::Authoritative(metadata)
|
||||||
|
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||||
|
{
|
||||||
|
Ok((
|
||||||
|
metadata.quota_config.clone(),
|
||||||
|
metadata.bucket_incarnation_id,
|
||||||
|
metadata.quota_config_updated_at,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
BucketMetadataAuthority::Authoritative(_) => {
|
||||||
|
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
|
||||||
|
}
|
||||||
|
BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
|
||||||
|
BucketMetadataAuthority::Fabricated => Err(Error::other(format!("bucket quota metadata is not authoritative: {bucket}"))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||||
|
|||||||
@@ -158,6 +158,26 @@ impl QuotaChecker {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn set_durable_quota_config_if_incarnation(
|
||||||
|
&mut self,
|
||||||
|
bucket: &str,
|
||||||
|
quota: BucketQuota,
|
||||||
|
expected_incarnation_id: uuid::Uuid,
|
||||||
|
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||||
|
) -> Result<OffsetDateTime, QuotaError> {
|
||||||
|
let json_data = serde_json::to_vec("a).map_err(|e| QuotaError::InvalidConfig {
|
||||||
|
reason: format!("Failed to serialize quota config: {}", e),
|
||||||
|
})?;
|
||||||
|
let start_time = Instant::now();
|
||||||
|
let updated_at =
|
||||||
|
crate::bucket::metadata_sys::update_quota_if_incarnation(bucket, json_data, expected_incarnation_id, proof)
|
||||||
|
.await
|
||||||
|
.map_err(QuotaError::StorageError)?;
|
||||||
|
|
||||||
|
rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
|
||||||
|
Ok(updated_at)
|
||||||
|
}
|
||||||
|
|
||||||
async fn set_quota_config_for_incarnation(
|
async fn set_quota_config_for_incarnation(
|
||||||
&mut self,
|
&mut self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -13,40 +13,97 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub mod checker;
|
pub mod checker;
|
||||||
|
pub(crate) mod reservation;
|
||||||
|
|
||||||
use crate::error::Result;
|
use crate::error::Result;
|
||||||
use rustfs_config::{
|
use rustfs_config::{
|
||||||
QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE,
|
QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE,
|
||||||
QUOTA_NOT_FOUND_ERROR_CODE,
|
QUOTA_NOT_FOUND_ERROR_CODE,
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
pub enum QuotaType {
|
pub enum QuotaType {
|
||||||
/// Hard quota: reject immediately when exceeded
|
/// Hard quota accounting.
|
||||||
#[default]
|
#[default]
|
||||||
#[serde(alias = "HARD", alias = "hard")]
|
#[serde(alias = "HARD", alias = "hard")]
|
||||||
Hard,
|
Hard,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
|
||||||
|
|
||||||
/// Bucket quota configuration. quota_type defaults to Hard when omitted.
|
/// Bucket quota configuration. quota_type defaults to Hard when omitted.
|
||||||
#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)]
|
#[derive(Debug, Default, Clone, PartialEq)]
|
||||||
pub struct BucketQuota {
|
pub struct BucketQuota {
|
||||||
#[serde(default)]
|
|
||||||
pub quota: Option<u64>,
|
pub quota: Option<u64>,
|
||||||
/// Defaults to Hard when missing.
|
/// Defaults to Hard when missing.
|
||||||
#[serde(default)]
|
|
||||||
pub quota_type: QuotaType,
|
pub quota_type: QuotaType,
|
||||||
|
/// Optional durable reservation protocol. The wire format gives older
|
||||||
|
/// nodes a zero hard quota so a mixed-version fleet fails closed.
|
||||||
|
pub reservation_protocol: Option<u32>,
|
||||||
/// Timestamp when this quota configuration was set (for audit purposes)
|
/// Timestamp when this quota configuration was set (for audit purposes)
|
||||||
#[serde(default, with = "time::serde::rfc3339::option")]
|
|
||||||
pub created_at: Option<OffsetDateTime>,
|
pub created_at: Option<OffsetDateTime>,
|
||||||
/// Accept updated_at for compatibility; not used.
|
/// Accept updated_at for compatibility; not used.
|
||||||
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
|
|
||||||
pub updated_at: Option<OffsetDateTime>,
|
pub updated_at: Option<OffsetDateTime>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Serialize)]
|
||||||
|
struct BucketQuotaWire {
|
||||||
|
#[serde(default)]
|
||||||
|
quota: Option<u64>,
|
||||||
|
#[serde(default)]
|
||||||
|
quota_type: QuotaType,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
reservation_protocol: Option<u32>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
reservation_quota: Option<u64>,
|
||||||
|
#[serde(default, with = "time::serde::rfc3339::option")]
|
||||||
|
created_at: Option<OffsetDateTime>,
|
||||||
|
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
|
||||||
|
updated_at: Option<OffsetDateTime>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Serialize for BucketQuota {
|
||||||
|
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||||
|
where
|
||||||
|
S: Serializer,
|
||||||
|
{
|
||||||
|
let durable = self.uses_durable_reservations();
|
||||||
|
BucketQuotaWire {
|
||||||
|
quota: if durable { Some(0) } else { self.quota },
|
||||||
|
quota_type: self.quota_type.clone(),
|
||||||
|
reservation_protocol: self.reservation_protocol,
|
||||||
|
reservation_quota: if durable { self.quota } else { None },
|
||||||
|
created_at: self.created_at,
|
||||||
|
updated_at: self.updated_at,
|
||||||
|
}
|
||||||
|
.serialize(serializer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'de> Deserialize<'de> for BucketQuota {
|
||||||
|
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||||
|
where
|
||||||
|
D: Deserializer<'de>,
|
||||||
|
{
|
||||||
|
let wire = BucketQuotaWire::deserialize(deserializer)?;
|
||||||
|
let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) {
|
||||||
|
wire.reservation_quota
|
||||||
|
} else {
|
||||||
|
wire.quota
|
||||||
|
};
|
||||||
|
Ok(Self {
|
||||||
|
quota,
|
||||||
|
quota_type: wire.quota_type,
|
||||||
|
reservation_protocol: wire.reservation_protocol,
|
||||||
|
created_at: wire.created_at,
|
||||||
|
updated_at: wire.updated_at,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl BucketQuota {
|
impl BucketQuota {
|
||||||
/// Serialize to JSON bytes. Same format as parse_all_configs.
|
/// Serialize to JSON bytes. Same format as parse_all_configs.
|
||||||
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
|
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
|
||||||
@@ -63,6 +120,7 @@ impl BucketQuota {
|
|||||||
Self {
|
Self {
|
||||||
quota,
|
quota,
|
||||||
quota_type: QuotaType::Hard,
|
quota_type: QuotaType::Hard,
|
||||||
|
reservation_protocol: quota.map(|_| QUOTA_RESERVATION_PROTOCOL_V1),
|
||||||
created_at: Some(now),
|
created_at: Some(now),
|
||||||
updated_at: None,
|
updated_at: None,
|
||||||
}
|
}
|
||||||
@@ -72,7 +130,19 @@ impl BucketQuota {
|
|||||||
self.quota
|
self.quota
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn uses_durable_reservations(&self) -> bool {
|
||||||
|
self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn has_unsupported_reservation_protocol(&self) -> bool {
|
||||||
|
self.reservation_protocol
|
||||||
|
.is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool {
|
pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool {
|
||||||
|
if operation_size == 0 {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
if let Some(quota_limit) = self.quota {
|
if let Some(quota_limit) = self.quota {
|
||||||
current_usage.saturating_add(operation_size) <= quota_limit
|
current_usage.saturating_add(operation_size) <= quota_limit
|
||||||
} else {
|
} else {
|
||||||
@@ -210,7 +280,51 @@ mod tests {
|
|||||||
let buf = q.marshal_msg().expect("marshal");
|
let buf = q.marshal_msg().expect("marshal");
|
||||||
let restored = BucketQuota::unmarshal(&buf).expect("unmarshal");
|
let restored = BucketQuota::unmarshal(&buf).expect("unmarshal");
|
||||||
assert_eq!(q.quota, restored.quota);
|
assert_eq!(q.quota, restored.quota);
|
||||||
assert_eq!(q.quota_type, restored.quota_type);
|
assert_eq!(restored.quota_type, QuotaType::Hard);
|
||||||
|
assert_eq!(restored.reservation_protocol, Some(QUOTA_RESERVATION_PROTOCOL_V1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn clearing_quota_keeps_the_legacy_compatible_type() {
|
||||||
|
let quota = BucketQuota::new(None);
|
||||||
|
|
||||||
|
assert_eq!(quota.quota_type, QuotaType::Hard);
|
||||||
|
assert_eq!(quota.reservation_protocol, None);
|
||||||
|
assert!(!quota.uses_durable_reservations());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn durable_quota_makes_legacy_nodes_fail_closed() {
|
||||||
|
let json = serde_json::to_vec(&BucketQuota::new(Some(2048))).expect("durable quota should serialize");
|
||||||
|
let quota: BucketQuota = serde_json::from_slice(&json).expect("current quota version should parse");
|
||||||
|
assert!(quota.uses_durable_reservations());
|
||||||
|
assert_eq!(quota.quota, Some(2048));
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
enum LegacyQuotaType {
|
||||||
|
Hard,
|
||||||
|
}
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct LegacyBucketQuota {
|
||||||
|
#[allow(dead_code)]
|
||||||
|
quota: Option<u64>,
|
||||||
|
#[allow(dead_code)]
|
||||||
|
quota_type: LegacyQuotaType,
|
||||||
|
}
|
||||||
|
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
||||||
|
.expect("legacy readers should ignore the reservation protocol field");
|
||||||
|
assert_eq!(legacy.quota, Some(0));
|
||||||
|
assert!(matches!(legacy.quota_type, LegacyQuotaType::Hard));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_reservation_protocol_does_not_activate_v1() {
|
||||||
|
let quota: BucketQuota =
|
||||||
|
serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#)
|
||||||
|
.expect("future protocol should remain parseable");
|
||||||
|
|
||||||
|
assert!(!quota.uses_durable_reservations());
|
||||||
|
assert!(quota.has_unsupported_reservation_protocol());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// unmarshal accepts format without quota_type
|
/// unmarshal accepts format without quota_type
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -248,6 +248,16 @@ fn decode_remote_version_state_capability(expected_member: &str, result: &[u8])
|
|||||||
Ok(server_epoch)
|
Ok(server_epoch)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn decode_cross_pool_fence_capability(expected_member: &str, result: &[u8]) -> Result<(u32, Uuid)> {
|
||||||
|
let version = result
|
||||||
|
.get(..4)
|
||||||
|
.and_then(|value| value.try_into().ok())
|
||||||
|
.map(u32::from_be_bytes)
|
||||||
|
.ok_or_else(|| Error::other("peer returned an invalid cross-pool fence capability version"))?;
|
||||||
|
let epoch = decode_remote_version_state_capability(expected_member, &result[4..])?;
|
||||||
|
Ok((version, epoch))
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct PeerLiveEventsBatch {
|
pub struct PeerLiveEventsBatch {
|
||||||
pub events: Vec<u8>,
|
pub events: Vec<u8>,
|
||||||
@@ -1288,6 +1298,16 @@ impl PeerRestClient {
|
|||||||
Ok((self.topology_member.clone(), epoch))
|
Ok((self.topology_member.clone(), epoch))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn probe_cross_pool_fence(&self, topology_fingerprint: String) -> Result<(String, u32, Uuid)> {
|
||||||
|
let mut probe = rustfs_protos::CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX.to_vec();
|
||||||
|
probe.extend_from_slice(Uuid::new_v4().as_bytes());
|
||||||
|
let result = self
|
||||||
|
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||||
|
.await?;
|
||||||
|
let (supported_version, epoch) = decode_cross_pool_fence_capability(&self.topology_member, &result)?;
|
||||||
|
Ok((self.topology_member.clone(), supported_version, epoch))
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||||
self.finalize_result(
|
self.finalize_result(
|
||||||
async {
|
async {
|
||||||
@@ -2738,6 +2758,24 @@ mod tests {
|
|||||||
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
|
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cross_pool_fence_capability_decoder_fails_closed() {
|
||||||
|
let epoch = Uuid::new_v4();
|
||||||
|
let result = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", epoch.as_bytes())
|
||||||
|
.expect("small capability response should encode");
|
||||||
|
assert_eq!(
|
||||||
|
decode_cross_pool_fence_capability("node-a:9000", &result).expect("valid capability should decode"),
|
||||||
|
(1, epoch)
|
||||||
|
);
|
||||||
|
for malformed in [&[][..], &[0, 0, 0][..], &result[..result.len() - 1]] {
|
||||||
|
assert!(decode_cross_pool_fence_capability("node-a:9000", malformed).is_err());
|
||||||
|
}
|
||||||
|
assert!(decode_cross_pool_fence_capability("node-b:9000", &result).is_err());
|
||||||
|
let nil = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", Uuid::nil().as_bytes())
|
||||||
|
.expect("small capability response should encode");
|
||||||
|
assert!(decode_cross_pool_fence_capability("node-a:9000", &nil).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
struct TierMutationResponseFixture<'a> {
|
struct TierMutationResponseFixture<'a> {
|
||||||
version: u32,
|
version: u32,
|
||||||
phase: TierMutationRpcPhase,
|
phase: TierMutationRpcPhase,
|
||||||
|
|||||||
@@ -1355,7 +1355,7 @@ impl BucketUsageAccumulator {
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let object_size = object.size.max(0) as u64;
|
let object_size = quota_object_size(object)?;
|
||||||
self.current_live_versions = self.current_live_versions.saturating_add(1);
|
self.current_live_versions = self.current_live_versions.saturating_add(1);
|
||||||
self.size_histogram.add(object_size);
|
self.size_histogram.add(object_size);
|
||||||
self.total_size = self.total_size.saturating_add(object_size);
|
self.total_size = self.total_size.saturating_add(object_size);
|
||||||
@@ -1385,6 +1385,20 @@ impl BucketUsageAccumulator {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
|
||||||
|
let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||||
|
let persisted_part_size = if object.parts.is_empty() {
|
||||||
|
u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
|
||||||
|
} else {
|
||||||
|
object.parts.iter().try_fold(0_u64, |total, part| {
|
||||||
|
let actual_size = u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||||
|
let part_size = actual_size.max(u64::try_from(part.size).map_err(|_| Error::PartMissingOrCorrupt)?);
|
||||||
|
total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
|
||||||
|
})?
|
||||||
|
};
|
||||||
|
Ok(logical_size.max(persisted_part_size))
|
||||||
|
}
|
||||||
|
|
||||||
type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
|
type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||||
|
|
||||||
pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Result<BucketUsageInfo, Error> {
|
pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Result<BucketUsageInfo, Error> {
|
||||||
@@ -3124,6 +3138,80 @@ mod tests {
|
|||||||
assert_eq!(usage.object_versions_histogram.get("BETWEEN_1000_AND_10000"), Some(&1));
|
assert_eq!(usage.object_versions_histogram.get("BETWEEN_1000_AND_10000"), Some(&1));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bucket_usage_uses_the_larger_of_logical_and_physical_size() {
|
||||||
|
let mut metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||||
|
"klauspost/compress/s2".to_string(),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "4096".to_string());
|
||||||
|
let object = ObjectInfo {
|
||||||
|
name: "compressed".to_string(),
|
||||||
|
size: 128,
|
||||||
|
user_defined: Arc::new(metadata),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut usage = BucketUsageAccumulator::default();
|
||||||
|
usage
|
||||||
|
.record("bucket", &object)
|
||||||
|
.expect("valid compressed metadata should be counted");
|
||||||
|
assert_eq!(usage.finish().size, 4096);
|
||||||
|
|
||||||
|
let mut framed_metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut framed_metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||||
|
"klauspost/compress/s2".to_string(),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(&mut framed_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string());
|
||||||
|
let framed = ObjectInfo {
|
||||||
|
name: "framed".to_string(),
|
||||||
|
size: 17,
|
||||||
|
user_defined: Arc::new(framed_metadata),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(quota_object_size(&framed).expect("physical framing must remain quota-accounted"), 17);
|
||||||
|
|
||||||
|
let mut corrupt_metadata = (*object.user_defined).clone();
|
||||||
|
rustfs_utils::http::insert_str(&mut corrupt_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "-1".to_string());
|
||||||
|
let corrupt = ObjectInfo {
|
||||||
|
user_defined: Arc::new(corrupt_metadata),
|
||||||
|
..object
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
BucketUsageAccumulator::default().record("bucket", &corrupt),
|
||||||
|
Err(Error::PartMissingOrCorrupt)
|
||||||
|
),
|
||||||
|
"negative logical metadata must not become a smaller quota baseline"
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut poisoned_metadata = HashMap::new();
|
||||||
|
rustfs_utils::http::insert_str(
|
||||||
|
&mut poisoned_metadata,
|
||||||
|
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||||
|
"klauspost/compress/s2".to_string(),
|
||||||
|
);
|
||||||
|
rustfs_utils::http::insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string());
|
||||||
|
let poisoned = ObjectInfo {
|
||||||
|
name: "legacy-swift-metadata".to_string(),
|
||||||
|
size: 4096,
|
||||||
|
user_defined: Arc::new(poisoned_metadata),
|
||||||
|
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||||
|
size: 4096,
|
||||||
|
actual_size: 4096,
|
||||||
|
..Default::default()
|
||||||
|
}]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
quota_object_size(&poisoned).expect("persisted part accounting must bound legacy user metadata"),
|
||||||
|
4096
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {
|
async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {
|
||||||
|
|||||||
@@ -22,17 +22,18 @@ use crate::disk::{
|
|||||||
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
|
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
|
||||||
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo,
|
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo,
|
||||||
DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize,
|
DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize,
|
||||||
PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET,
|
PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
|
||||||
RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET,
|
||||||
STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP,
|
||||||
conv_part_err_to_int,
|
SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int,
|
||||||
endpoint::Endpoint,
|
endpoint::Endpoint,
|
||||||
error::{DiskError, Error, FileAccessDeniedWithContext, Result},
|
error::{DiskError, Error, FileAccessDeniedWithContext, Result},
|
||||||
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
|
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
|
||||||
format::FormatV3,
|
format::FormatV3,
|
||||||
fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename},
|
fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename},
|
||||||
os,
|
is_quota_mutation_fence_path, os,
|
||||||
os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source},
|
os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source},
|
||||||
|
quota_mutation_fence_path,
|
||||||
};
|
};
|
||||||
use crate::erasure::coding::{self, bitrot_verify};
|
use crate::erasure::coding::{self, bitrot_verify};
|
||||||
use crate::runtime::sources as runtime_sources;
|
use crate::runtime::sources as runtime_sources;
|
||||||
@@ -55,9 +56,7 @@ use std::collections::HashMap;
|
|||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::fmt::Debug;
|
use std::fmt::Debug;
|
||||||
use std::io::{Error as IoError, SeekFrom};
|
use std::io::{Error as IoError, SeekFrom};
|
||||||
#[cfg(target_os = "linux")]
|
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering};
|
||||||
use std::sync::atomic::AtomicBool;
|
|
||||||
use std::sync::atomic::{AtomicU32, Ordering};
|
|
||||||
use std::sync::{Arc, OnceLock};
|
use std::sync::{Arc, OnceLock};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use std::{
|
use std::{
|
||||||
@@ -4751,6 +4750,25 @@ struct SnapshotLeaseEntry {
|
|||||||
tokens: HashSet<SnapshotLeaseToken>,
|
tokens: HashSet<SnapshotLeaseToken>,
|
||||||
pending_delete: Option<DeleteOptions>,
|
pending_delete: Option<DeleteOptions>,
|
||||||
deleting: bool,
|
deleting: bool,
|
||||||
|
mutation_fence: Option<Arc<QuotaMutationFenceState>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct QuotaMutationFenceState {
|
||||||
|
revoked: AtomicBool,
|
||||||
|
running: AtomicUsize,
|
||||||
|
notify: Notify,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct QuotaMutationFenceClaim {
|
||||||
|
state: Arc<QuotaMutationFenceState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for QuotaMutationFenceClaim {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.state.running.fetch_sub(1, Ordering::AcqRel);
|
||||||
|
self.state.notify.notify_waiters();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
@@ -7315,6 +7333,34 @@ fn normalize_path_components(path: impl AsRef<Path>) -> PathBuf {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl LocalDisk {
|
impl LocalDisk {
|
||||||
|
async fn claim_quota_mutation_fence(
|
||||||
|
&self,
|
||||||
|
volume: &str,
|
||||||
|
path: &str,
|
||||||
|
token: SnapshotLeaseToken,
|
||||||
|
) -> Result<Arc<QuotaMutationFenceClaim>> {
|
||||||
|
let key = SnapshotLeaseKey {
|
||||||
|
volume: RUSTFS_META_BUCKET.to_string(),
|
||||||
|
path: quota_mutation_fence_path(volume, path),
|
||||||
|
};
|
||||||
|
let state = {
|
||||||
|
let registry = self.snapshot_leases.lock().await;
|
||||||
|
let entry = registry.entries.get(&key).ok_or(DiskError::FileNotFound)?;
|
||||||
|
let state = entry.mutation_fence.as_ref().ok_or(DiskError::FileNotFound)?;
|
||||||
|
if !entry.tokens.contains(&token) || state.revoked.load(Ordering::Acquire) {
|
||||||
|
return Err(DiskError::FileNotFound);
|
||||||
|
}
|
||||||
|
state.running.fetch_add(1, Ordering::AcqRel);
|
||||||
|
Arc::clone(state)
|
||||||
|
};
|
||||||
|
if state.revoked.load(Ordering::Acquire) {
|
||||||
|
state.running.fetch_sub(1, Ordering::AcqRel);
|
||||||
|
state.notify.notify_waiters();
|
||||||
|
return Err(DiskError::FileNotFound);
|
||||||
|
}
|
||||||
|
Ok(Arc::new(QuotaMutationFenceClaim { state }))
|
||||||
|
}
|
||||||
|
|
||||||
async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result<bool> {
|
async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result<bool> {
|
||||||
let path = format!("{object}/{data_dir}");
|
let path = format!("{object}/{data_dir}");
|
||||||
let data_path = self.io_get_object_path(volume, &path)?;
|
let data_path = self.io_get_object_path(volume, &path)?;
|
||||||
@@ -8648,7 +8694,30 @@ impl DiskAPI for LocalDisk {
|
|||||||
// optimistic; this lease establishes the local commit/delete order and
|
// optimistic; this lease establishes the local commit/delete order and
|
||||||
// remains owned by any blocking syscall that outlives async cancellation.
|
// remains owned by any blocking syscall that outlives async cancellation.
|
||||||
let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?;
|
let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?;
|
||||||
|
let quota_fence_token =
|
||||||
|
match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) {
|
||||||
|
Some(value) => {
|
||||||
|
let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?;
|
||||||
|
Some(SnapshotLeaseToken::from_slice(token.as_bytes())?)
|
||||||
|
}
|
||||||
|
None if rustfs_utils::http::metadata_compat::contains_key_str(
|
||||||
|
&fi.metadata,
|
||||||
|
QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||||
|
) =>
|
||||||
|
{
|
||||||
|
return Err(DiskError::FileCorrupt);
|
||||||
|
}
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX);
|
||||||
|
let quota_fence_claim = match quota_fence_token {
|
||||||
|
Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await;
|
let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await;
|
||||||
|
if let Some(claim) = quota_fence_claim {
|
||||||
|
mutation_lease.attach_external_guard(claim);
|
||||||
|
}
|
||||||
if fi.is_legacy_indexed_delete_marker() {
|
if fi.is_legacy_indexed_delete_marker() {
|
||||||
fi.erasure.index = 0;
|
fi.erasure.index = 0;
|
||||||
}
|
}
|
||||||
@@ -9638,11 +9707,26 @@ impl DiskAPI for LocalDisk {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
|
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
|
||||||
let file_path = self.io_get_object_path(volume, path)?;
|
|
||||||
let key = SnapshotLeaseKey {
|
let key = SnapshotLeaseKey {
|
||||||
volume: volume.to_string(),
|
volume: volume.to_string(),
|
||||||
path: path.to_string(),
|
path: path.to_string(),
|
||||||
};
|
};
|
||||||
|
if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) {
|
||||||
|
let mut registry = self.snapshot_leases.lock().await;
|
||||||
|
let entry = registry.entries.entry(key).or_default();
|
||||||
|
let state = entry
|
||||||
|
.mutation_fence
|
||||||
|
.get_or_insert_with(|| Arc::new(QuotaMutationFenceState::default()));
|
||||||
|
if state.revoked.load(Ordering::Acquire) {
|
||||||
|
return Err(DiskError::FileNotFound);
|
||||||
|
}
|
||||||
|
let token = SnapshotLeaseToken::new();
|
||||||
|
entry.tokens.insert(token);
|
||||||
|
return Ok(token);
|
||||||
|
}
|
||||||
|
|
||||||
|
let file_path = self.io_get_object_path(volume, path)?;
|
||||||
|
let _mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, volume, &file_path).await;
|
||||||
let token = {
|
let token = {
|
||||||
let mut registry = self.snapshot_leases.lock().await;
|
let mut registry = self.snapshot_leases.lock().await;
|
||||||
if registry.entries.get(&key).is_some_and(|entry| entry.deleting) {
|
if registry.entries.get(&key).is_some_and(|entry| entry.deleting) {
|
||||||
@@ -9670,6 +9754,48 @@ impl DiskAPI for LocalDisk {
|
|||||||
volume: volume.to_string(),
|
volume: volume.to_string(),
|
||||||
path: path.to_string(),
|
path: path.to_string(),
|
||||||
};
|
};
|
||||||
|
if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) {
|
||||||
|
if !token.is_revoke_all() {
|
||||||
|
let mut registry = self.snapshot_leases.lock().await;
|
||||||
|
let Some(entry) = registry.entries.get_mut(&key) else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
entry.tokens.remove(&token);
|
||||||
|
let removable = entry.tokens.is_empty()
|
||||||
|
&& entry
|
||||||
|
.mutation_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_none_or(|state| state.running.load(Ordering::Acquire) == 0);
|
||||||
|
if removable {
|
||||||
|
registry.entries.remove(&key);
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let state = {
|
||||||
|
let mut registry = self.snapshot_leases.lock().await;
|
||||||
|
let Some(entry) = registry.entries.get_mut(&key) else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
let Some(state) = entry.mutation_fence.as_ref().cloned() else {
|
||||||
|
registry.entries.remove(&key);
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
state.revoked.store(true, Ordering::Release);
|
||||||
|
entry.tokens.clear();
|
||||||
|
state
|
||||||
|
};
|
||||||
|
loop {
|
||||||
|
let notified = state.notify.notified();
|
||||||
|
tokio::pin!(notified);
|
||||||
|
notified.as_mut().enable();
|
||||||
|
if state.running.load(Ordering::Acquire) == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
notified.await;
|
||||||
|
}
|
||||||
|
self.snapshot_leases.lock().await.entries.remove(&key);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let opts = {
|
let opts = {
|
||||||
let mut registry = self.snapshot_leases.lock().await;
|
let mut registry = self.snapshot_leases.lock().await;
|
||||||
let Some(entry) = registry.entries.get_mut(&key) else {
|
let Some(entry) = registry.entries.get_mut(&key) else {
|
||||||
@@ -18723,6 +18849,48 @@ mod test {
|
|||||||
assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound)));
|
assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn quota_mutation_fence_revoke_waits_for_active_claim_and_rejects_late_claims() {
|
||||||
|
use tempfile::tempdir;
|
||||||
|
|
||||||
|
let root_dir = tempdir().expect("temp dir should be created");
|
||||||
|
let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
|
||||||
|
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||||
|
let bucket = "quota-fence-volume";
|
||||||
|
let object = "object";
|
||||||
|
let fence_path = quota_mutation_fence_path(bucket, object);
|
||||||
|
let token = disk
|
||||||
|
.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path)
|
||||||
|
.await
|
||||||
|
.expect("quota mutation token should be prepared");
|
||||||
|
let claim = disk
|
||||||
|
.claim_quota_mutation_fence(bucket, object, token)
|
||||||
|
.await
|
||||||
|
.expect("prepared token should be claimable");
|
||||||
|
|
||||||
|
let release_disk = Arc::clone(&disk);
|
||||||
|
let mut release = tokio::spawn(async move {
|
||||||
|
release_disk
|
||||||
|
.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, SnapshotLeaseToken::revoke_all())
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(50), &mut release).await.is_err(),
|
||||||
|
"revoke must wait until an already claimed mutation has finished"
|
||||||
|
);
|
||||||
|
|
||||||
|
drop(claim);
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), release)
|
||||||
|
.await
|
||||||
|
.expect("revoke should wake after the final claim drops")
|
||||||
|
.expect("revoke task should not panic")
|
||||||
|
.expect("revoke should succeed");
|
||||||
|
assert!(matches!(
|
||||||
|
disk.claim_quota_mutation_fence(bucket, object, token).await,
|
||||||
|
Err(DiskError::FileNotFound)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn delete_version_keeps_later_part_until_snapshot_release() {
|
async fn delete_version_keeps_later_part_until_snapshot_release() {
|
||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|||||||
@@ -71,6 +71,28 @@ use time::OffsetDateTime;
|
|||||||
use tokio::io::{AsyncRead, AsyncWrite};
|
use tokio::io::{AsyncRead, AsyncWrite};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
|
||||||
|
pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
|
||||||
|
|
||||||
|
pub(crate) fn quota_mutation_fence_path(bucket: &str, object: &str) -> String {
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
let mut input = Vec::with_capacity(bucket.len() + object.len() + 1);
|
||||||
|
input.extend_from_slice(bucket.as_bytes());
|
||||||
|
input.push(0);
|
||||||
|
input.extend_from_slice(object.as_bytes());
|
||||||
|
let digest = Sha256::digest(input);
|
||||||
|
format!(
|
||||||
|
"{QUOTA_MUTATION_FENCE_PREFIX}{}",
|
||||||
|
hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn is_quota_mutation_fence_path(path: &str) -> bool {
|
||||||
|
path.strip_prefix(QUOTA_MUTATION_FENCE_PREFIX)
|
||||||
|
.is_some_and(|digest| digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||||
|
}
|
||||||
|
|
||||||
pub type DiskStore = Arc<Disk>;
|
pub type DiskStore = Arc<Disk>;
|
||||||
|
|
||||||
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
|
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
|
||||||
@@ -95,6 +117,20 @@ impl SnapshotLeaseToken {
|
|||||||
pub fn as_bytes(&self) -> &[u8; 16] {
|
pub fn as_bytes(&self) -> &[u8; 16] {
|
||||||
self.0.as_bytes()
|
self.0.as_bytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn as_uuid(self) -> Uuid {
|
||||||
|
self.0
|
||||||
|
}
|
||||||
|
|
||||||
|
#[doc(hidden)]
|
||||||
|
pub fn revoke_all() -> Self {
|
||||||
|
Self(Uuid::nil())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[doc(hidden)]
|
||||||
|
pub fn is_revoke_all(self) -> bool {
|
||||||
|
self.0.is_nil()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for SnapshotLeaseToken {
|
impl Default for SnapshotLeaseToken {
|
||||||
|
|||||||
@@ -306,12 +306,20 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc<NamespaceMutationLock> {
|
|||||||
pub(crate) struct NamespaceMutationLease {
|
pub(crate) struct NamespaceMutationLease {
|
||||||
_namespace_guard: OwnedMutexGuard<()>,
|
_namespace_guard: OwnedMutexGuard<()>,
|
||||||
_volume_guard: Option<OwnedRwLockReadGuard<()>>,
|
_volume_guard: Option<OwnedRwLockReadGuard<()>>,
|
||||||
|
external_guard: Mutex<Option<Arc<dyn Send + Sync>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NamespaceMutationLease {
|
||||||
|
pub(crate) fn attach_external_guard(&self, guard: Arc<dyn Send + Sync>) {
|
||||||
|
*self.external_guard.lock() = Some(guard);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationLease> {
|
async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationLease> {
|
||||||
Arc::new(NamespaceMutationLease {
|
Arc::new(NamespaceMutationLease {
|
||||||
_namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
|
_namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
|
||||||
_volume_guard: None,
|
_volume_guard: None,
|
||||||
|
external_guard: Mutex::new(None),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -327,6 +335,7 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
|
|||||||
Arc::new(NamespaceMutationLease {
|
Arc::new(NamespaceMutationLease {
|
||||||
_namespace_guard: namespace_guard,
|
_namespace_guard: namespace_guard,
|
||||||
_volume_guard: Some(volume_guard),
|
_volume_guard: Some(volume_guard),
|
||||||
|
external_guard: Mutex::new(None),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -44,12 +44,14 @@ const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3;
|
|||||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||||
const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification";
|
const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification";
|
||||||
const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation";
|
const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation";
|
||||||
|
const EVENT_NOTIFICATION_CAPABILITY_PROBE: &str = "notification_capability_probe";
|
||||||
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||||
const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
|
const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
|
||||||
const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
|
const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
|
||||||
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
|
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
|
||||||
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||||
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||||
|
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1;
|
||||||
|
|
||||||
/// Cached result from the last successful admin call to a peer.
|
/// Cached result from the last successful admin call to a peer.
|
||||||
struct PeerAdminCache {
|
struct PeerAdminCache {
|
||||||
@@ -95,15 +97,15 @@ lazy_static! {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct RemoteVersionStateFleetProof {
|
struct FleetCapabilityProof {
|
||||||
topology_fingerprint: String,
|
topology_fingerprint: String,
|
||||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||||
expires_at: Instant,
|
expires_at: Instant,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RemoteVersionStateFleetProof {
|
impl FleetCapabilityProof {
|
||||||
fn token(&self) -> RemoteVersionStateFleetProofToken {
|
fn token(&self) -> FleetCapabilityProofToken {
|
||||||
RemoteVersionStateFleetProofToken {
|
FleetCapabilityProofToken {
|
||||||
topology_fingerprint: self.topology_fingerprint.clone(),
|
topology_fingerprint: self.topology_fingerprint.clone(),
|
||||||
peer_epochs: self.peer_epochs.clone(),
|
peer_epochs: self.peer_epochs.clone(),
|
||||||
}
|
}
|
||||||
@@ -111,37 +113,41 @@ impl RemoteVersionStateFleetProof {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, PartialEq, Eq)]
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
pub(crate) struct RemoteVersionStateFleetProofToken {
|
struct FleetCapabilityProofToken {
|
||||||
topology_fingerprint: String,
|
topology_fingerprint: String,
|
||||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct RemoteVersionStateFleetProofState {
|
struct FleetCapabilityProofState {
|
||||||
proof: Option<RemoteVersionStateFleetProof>,
|
proof: Option<FleetCapabilityProof>,
|
||||||
topology_conflict: bool,
|
topology_conflict: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
|
pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken);
|
||||||
|
|
||||||
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
|
pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken);
|
||||||
|
|
||||||
|
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||||
|
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||||
|
|
||||||
fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
|
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||||
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
|
CROSS_POOL_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
|
fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||||
replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
|
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn replace_remote_version_state_fleet_proof_in(
|
fn replace_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>, proof: Option<FleetCapabilityProof>) {
|
||||||
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
|
|
||||||
proof: Option<RemoteVersionStateFleetProof>,
|
|
||||||
) {
|
|
||||||
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
|
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn publish_remote_version_state_probe_result(
|
fn publish_fleet_capability_probe_result(
|
||||||
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
|
slot: &std::sync::RwLock<FleetCapabilityProofState>,
|
||||||
topology_fingerprint: &str,
|
topology_fingerprint: &str,
|
||||||
result: Result<BTreeMap<String, Uuid>>,
|
result: Result<BTreeMap<String, Uuid>>,
|
||||||
observed_at: Instant,
|
observed_at: Instant,
|
||||||
@@ -155,7 +161,7 @@ fn publish_remote_version_state_probe_result(
|
|||||||
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
|
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
|
||||||
.map(|proof| Arc::clone(&proof.peer_epochs))
|
.map(|proof| Arc::clone(&proof.peer_epochs))
|
||||||
.unwrap_or_else(|| Arc::new(peer_epochs));
|
.unwrap_or_else(|| Arc::new(peer_epochs));
|
||||||
state.proof = Some(RemoteVersionStateFleetProof {
|
state.proof = Some(FleetCapabilityProof {
|
||||||
topology_fingerprint: topology_fingerprint.to_string(),
|
topology_fingerprint: topology_fingerprint.to_string(),
|
||||||
peer_epochs,
|
peer_epochs,
|
||||||
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
|
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
|
||||||
@@ -163,7 +169,7 @@ fn publish_remote_version_state_probe_result(
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
replace_remote_version_state_fleet_proof_in(slot, None);
|
replace_fleet_capability_proof(slot, None);
|
||||||
Some(err)
|
Some(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -174,27 +180,44 @@ pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersion
|
|||||||
let state = remote_version_state_fleet_proof_slot()
|
let state = remote_version_state_fleet_proof_slot()
|
||||||
.read()
|
.read()
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
|
acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(RemoteVersionStateFleetProofToken)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn acquire_remote_version_state_fleet_proof_from(
|
fn acquire_fleet_capability_proof_from(
|
||||||
state: &RemoteVersionStateFleetProofState,
|
state: &FleetCapabilityProofState,
|
||||||
expected_topology: &str,
|
expected_topology: &str,
|
||||||
now: Instant,
|
now: Instant,
|
||||||
) -> Option<RemoteVersionStateFleetProofToken> {
|
) -> Option<FleetCapabilityProofToken> {
|
||||||
if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
|
if state.topology_conflict || !fleet_capability_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
|
state.proof.as_ref().map(FleetCapabilityProof::token)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
|
pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
|
||||||
|
fleet_capability_proof_matches(remote_version_state_fleet_proof_slot(), &proof.0)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn acquire_cross_pool_fence_fleet_proof() -> Option<CrossPoolFenceFleetProofToken> {
|
||||||
|
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||||
|
let state = cross_pool_fence_fleet_proof_slot()
|
||||||
|
.read()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(CrossPoolFenceFleetProofToken)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToken) -> bool {
|
||||||
|
fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fleet_capability_proof_matches(
|
||||||
|
slot: &std::sync::RwLock<FleetCapabilityProofState>,
|
||||||
|
proof: &FleetCapabilityProofToken,
|
||||||
|
) -> bool {
|
||||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
let state = remote_version_state_fleet_proof_slot()
|
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
.read()
|
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
||||||
if state.topology_conflict {
|
if state.topology_conflict {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -206,11 +229,7 @@ pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStat
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn remote_version_state_fleet_proof_valid_at(
|
fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool {
|
||||||
proof: Option<&RemoteVersionStateFleetProof>,
|
|
||||||
expected_topology: &str,
|
|
||||||
now: Instant,
|
|
||||||
) -> bool {
|
|
||||||
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
|
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -224,11 +243,11 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
|
|||||||
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
|
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
|
||||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
|
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
|
||||||
let mut state = remote_version_state_fleet_proof_slot()
|
for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] {
|
||||||
.write()
|
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
state.topology_conflict = true;
|
||||||
state.topology_conflict = true;
|
state.proof = None;
|
||||||
state.proof = None;
|
}
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -249,13 +268,23 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
|||||||
}
|
}
|
||||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||||
};
|
};
|
||||||
|
let fence_result = match get_global_notification_sys() {
|
||||||
|
Some(notification_sys) => timeout(
|
||||||
|
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||||
|
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||||
|
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||||
|
};
|
||||||
let topology_conflict = remote_version_state_fleet_proof_slot()
|
let topology_conflict = remote_version_state_fleet_proof_slot()
|
||||||
.read()
|
.read()
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
.topology_conflict;
|
.topology_conflict;
|
||||||
if topology_conflict {
|
if topology_conflict {
|
||||||
replace_remote_version_state_fleet_proof(None);
|
replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
|
||||||
} else if let Some(err) = publish_remote_version_state_probe_result(
|
replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None);
|
||||||
|
} else if let Some(err) = publish_fleet_capability_probe_result(
|
||||||
remote_version_state_fleet_proof_slot(),
|
remote_version_state_fleet_proof_slot(),
|
||||||
&topology_fingerprint,
|
&topology_fingerprint,
|
||||||
result,
|
result,
|
||||||
@@ -263,6 +292,24 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
|||||||
) {
|
) {
|
||||||
debug!(error = %err, "remote version state fleet capability probe failed closed");
|
debug!(error = %err, "remote version state fleet capability probe failed closed");
|
||||||
}
|
}
|
||||||
|
if !topology_conflict
|
||||||
|
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||||
|
cross_pool_fence_fleet_proof_slot(),
|
||||||
|
&topology_fingerprint,
|
||||||
|
fence_result,
|
||||||
|
Instant::now(),
|
||||||
|
)
|
||||||
|
{
|
||||||
|
debug!(
|
||||||
|
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||||
|
capability = "cross_pool_fence_v1",
|
||||||
|
state = "failed_closed",
|
||||||
|
error = %err,
|
||||||
|
"notification capability probe"
|
||||||
|
);
|
||||||
|
}
|
||||||
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
|
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -330,6 +377,27 @@ impl NotificationSys {
|
|||||||
}
|
}
|
||||||
Ok(peer_epochs)
|
Ok(peer_epochs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||||
|
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||||
|
return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
|
||||||
|
}
|
||||||
|
let probes = self.peer_clients.iter().map(|client| async {
|
||||||
|
let client = client
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| Error::other("cross-pool fence capability peer is unreachable"))?;
|
||||||
|
client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
|
||||||
|
});
|
||||||
|
let mut peer_epochs = BTreeMap::new();
|
||||||
|
for result in join_all(probes).await {
|
||||||
|
let (peer, version, epoch) = result?;
|
||||||
|
if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
|
||||||
|
return Err(Error::other("cross-pool fence capability version is unsupported"));
|
||||||
|
}
|
||||||
|
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||||
|
}
|
||||||
|
Ok(peer_epochs)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct NotificationPeerErr {
|
pub struct NotificationPeerErr {
|
||||||
@@ -2145,16 +2213,16 @@ mod tests {
|
|||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let mut peer_epochs = BTreeMap::new();
|
let mut peer_epochs = BTreeMap::new();
|
||||||
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
|
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
|
||||||
let proof = RemoteVersionStateFleetProof {
|
let proof = FleetCapabilityProof {
|
||||||
topology_fingerprint: "topology-a".to_string(),
|
topology_fingerprint: "topology-a".to_string(),
|
||||||
peer_epochs: Arc::new(peer_epochs),
|
peer_epochs: Arc::new(peer_epochs),
|
||||||
expires_at: now + Duration::from_secs(1),
|
expires_at: now + Duration::from_secs(1),
|
||||||
};
|
};
|
||||||
|
|
||||||
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
|
assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
|
||||||
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
|
assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now));
|
||||||
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
|
assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
|
||||||
assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
|
assert!(!fleet_capability_proof_valid_at(None, "topology-a", now));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2168,25 +2236,25 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
|
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let proof = RemoteVersionStateFleetProof {
|
let proof = FleetCapabilityProof {
|
||||||
topology_fingerprint: "topology-a".to_string(),
|
topology_fingerprint: "topology-a".to_string(),
|
||||||
peer_epochs: Arc::new(BTreeMap::new()),
|
peer_epochs: Arc::new(BTreeMap::new()),
|
||||||
expires_at: now + Duration::from_secs(1),
|
expires_at: now + Duration::from_secs(1),
|
||||||
};
|
};
|
||||||
|
|
||||||
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
|
assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
|
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let proof = RemoteVersionStateFleetProof {
|
let proof = FleetCapabilityProof {
|
||||||
topology_fingerprint: "topology-a".to_string(),
|
topology_fingerprint: "topology-a".to_string(),
|
||||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||||
expires_at: now + Duration::from_secs(1),
|
expires_at: now + Duration::from_secs(1),
|
||||||
};
|
};
|
||||||
let captured = proof.token();
|
let captured = proof.token();
|
||||||
let restarted = RemoteVersionStateFleetProof {
|
let restarted = FleetCapabilityProof {
|
||||||
topology_fingerprint: proof.topology_fingerprint.clone(),
|
topology_fingerprint: proof.topology_fingerprint.clone(),
|
||||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||||
expires_at: proof.expires_at,
|
expires_at: proof.expires_at,
|
||||||
@@ -2197,11 +2265,11 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
|
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
|
||||||
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
|
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let epoch = Uuid::new_v4();
|
let epoch = Uuid::new_v4();
|
||||||
let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
|
let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
|
||||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
|
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
|
||||||
let original = slot
|
let original = slot
|
||||||
.read()
|
.read()
|
||||||
.expect("proof slot should not poison")
|
.expect("proof slot should not poison")
|
||||||
@@ -2210,9 +2278,7 @@ mod tests {
|
|||||||
.expect("successful probe should publish proof")
|
.expect("successful probe should publish proof")
|
||||||
.token();
|
.token();
|
||||||
|
|
||||||
assert!(
|
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none());
|
||||||
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
|
|
||||||
);
|
|
||||||
let renewed = slot
|
let renewed = slot
|
||||||
.read()
|
.read()
|
||||||
.expect("proof slot should not poison")
|
.expect("proof slot should not poison")
|
||||||
@@ -2224,8 +2290,7 @@ mod tests {
|
|||||||
|
|
||||||
let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||||
assert!(
|
assert!(
|
||||||
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
|
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2)).is_none()
|
||||||
.is_none()
|
|
||||||
);
|
);
|
||||||
let replaced = slot
|
let replaced = slot
|
||||||
.read()
|
.read()
|
||||||
@@ -2240,18 +2305,18 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
|
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let mut state = RemoteVersionStateFleetProofState {
|
let mut state = FleetCapabilityProofState {
|
||||||
proof: Some(RemoteVersionStateFleetProof {
|
proof: Some(FleetCapabilityProof {
|
||||||
topology_fingerprint: "topology-a".to_string(),
|
topology_fingerprint: "topology-a".to_string(),
|
||||||
peer_epochs: Arc::new(BTreeMap::new()),
|
peer_epochs: Arc::new(BTreeMap::new()),
|
||||||
expires_at: now + Duration::from_secs(1),
|
expires_at: now + Duration::from_secs(1),
|
||||||
}),
|
}),
|
||||||
topology_conflict: false,
|
topology_conflict: false,
|
||||||
};
|
};
|
||||||
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
|
assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some());
|
||||||
|
|
||||||
state.topology_conflict = true;
|
state.topology_conflict = true;
|
||||||
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
|
assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2267,19 +2332,19 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
|
fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
|
||||||
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
|
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
||||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
||||||
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
|
publish_fleet_capability_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
|
||||||
);
|
);
|
||||||
assert!(slot.read().expect("proof slot should not poison").proof.is_none());
|
assert!(slot.read().expect("proof slot should not poison").proof.is_none());
|
||||||
|
|
||||||
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
||||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
||||||
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -538,7 +538,7 @@ impl MetadataQuorumAccumulator {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(in crate::set_disk) fn default_write_quorum(&self) -> usize {
|
pub(crate) fn default_write_quorum(&self) -> usize {
|
||||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||||
return self.total_disks;
|
return self.total_disks;
|
||||||
}
|
}
|
||||||
@@ -2816,7 +2816,7 @@ impl SetDisks {
|
|||||||
self.set_drive_count - self.default_parity_count
|
self.set_drive_count - self.default_parity_count
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(in crate::set_disk) fn default_write_quorum(&self) -> usize {
|
pub(crate) fn default_write_quorum(&self) -> usize {
|
||||||
let mut data_count = self.set_drive_count - self.default_parity_count;
|
let mut data_count = self.set_drive_count - self.default_parity_count;
|
||||||
if data_count == self.default_parity_count {
|
if data_count == self.default_parity_count {
|
||||||
data_count += 1
|
data_count += 1
|
||||||
@@ -2825,6 +2825,89 @@ impl SetDisks {
|
|||||||
data_count
|
data_count
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(in crate::set_disk) async fn prepare_quota_mutation_fences(
|
||||||
|
disks: &[Option<DiskStore>],
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
write_quorum: usize,
|
||||||
|
) -> crate::error::Result<(Vec<Option<DiskStore>>, Vec<Option<SnapshotLeaseToken>>)> {
|
||||||
|
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
|
||||||
|
let results = join_all(disks.iter().map(|disk| {
|
||||||
|
let disk = disk.clone();
|
||||||
|
let fence_path = fence_path.clone();
|
||||||
|
async move {
|
||||||
|
let disk = disk?;
|
||||||
|
match disk.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path).await {
|
||||||
|
Ok(token) => Some((disk, token)),
|
||||||
|
Err(_) => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.await;
|
||||||
|
if results.iter().flatten().count() < write_quorum {
|
||||||
|
for (disk, token) in results.iter().flatten() {
|
||||||
|
let _ = disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, *token).await;
|
||||||
|
}
|
||||||
|
return Err(StorageError::ErasureWriteQuorum);
|
||||||
|
}
|
||||||
|
let mut fenced_disks = Vec::with_capacity(results.len());
|
||||||
|
let mut tokens = Vec::with_capacity(results.len());
|
||||||
|
for result in results {
|
||||||
|
match result {
|
||||||
|
Some((disk, token)) => {
|
||||||
|
fenced_disks.push(Some(disk));
|
||||||
|
tokens.push(Some(token));
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
fenced_disks.push(None);
|
||||||
|
tokens.push(None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok((fenced_disks, tokens))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in crate::set_disk) async fn release_quota_mutation_fences(
|
||||||
|
disks: &[Option<DiskStore>],
|
||||||
|
tokens: &[Option<SnapshotLeaseToken>],
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
write_quorum: usize,
|
||||||
|
) -> crate::error::Result<()> {
|
||||||
|
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
|
||||||
|
let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| {
|
||||||
|
let disk = disk.as_ref()?.clone();
|
||||||
|
let token = (*token)?;
|
||||||
|
let fence_path = fence_path.clone();
|
||||||
|
Some(async move { disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, token).await })
|
||||||
|
}))
|
||||||
|
.await;
|
||||||
|
if results.iter().filter(|result| result.is_ok()).count() < write_quorum {
|
||||||
|
return Err(StorageError::ErasureWriteQuorum);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in crate::set_disk) async fn abort_quota_reservation_after_fence(
|
||||||
|
reservation: crate::bucket::quota::reservation::QuotaReservation,
|
||||||
|
disks: &[Option<DiskStore>],
|
||||||
|
tokens: &[Option<SnapshotLeaseToken>],
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
write_quorum: usize,
|
||||||
|
fenced: bool,
|
||||||
|
) {
|
||||||
|
let safe_to_abort = !fenced
|
||||||
|
|| Self::release_quota_mutation_fences(disks, tokens, bucket, object, write_quorum)
|
||||||
|
.await
|
||||||
|
.is_ok();
|
||||||
|
if safe_to_abort {
|
||||||
|
reservation.abort().await;
|
||||||
|
} else {
|
||||||
|
reservation.defer_after_fence();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
|
||||||
#[allow(clippy::type_complexity)]
|
#[allow(clippy::type_complexity)]
|
||||||
pub(in crate::set_disk) async fn rename_data(
|
pub(in crate::set_disk) async fn rename_data(
|
||||||
|
|||||||
@@ -22,6 +22,7 @@
|
|||||||
|
|
||||||
use super::super::*;
|
use super::super::*;
|
||||||
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
|
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
|
||||||
|
use crate::bucket::quota::reservation;
|
||||||
use crate::crash_inject::{self, CrashPoint};
|
use crate::crash_inject::{self, CrashPoint};
|
||||||
use crate::multipart_listing::paginate_multipart_listing;
|
use crate::multipart_listing::paginate_multipart_listing;
|
||||||
use futures::{StreamExt, stream};
|
use futures::{StreamExt, stream};
|
||||||
@@ -1760,6 +1761,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
|
|
||||||
fi.parts = Vec::with_capacity(uploaded_parts.len());
|
fi.parts = Vec::with_capacity(uploaded_parts.len());
|
||||||
|
|
||||||
|
let quota_context =
|
||||||
|
reservation::begin(&self.ctx, bucket, object, opts.quota_admission, self.pool_index, self.set_index).await?;
|
||||||
|
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
|
||||||
|
let preserve_replication_ciphertext = opts.replication_request
|
||||||
|
&& contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
|
||||||
|
if quota_context.is_enforced() && preserve_replication_ciphertext {
|
||||||
|
return Err(Error::PartMissingOrCorrupt);
|
||||||
|
}
|
||||||
|
let transformed_object = fi.is_compressed() || should_persist_encryption_original_size(&fi.metadata);
|
||||||
|
|
||||||
let mut object_size: usize = 0;
|
let mut object_size: usize = 0;
|
||||||
let mut object_actual_size: i64 = 0;
|
let mut object_actual_size: i64 = 0;
|
||||||
|
|
||||||
@@ -1883,15 +1894,23 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
checksum_combined.extend_from_slice(cs.raw.as_slice());
|
checksum_combined.extend_from_slice(cs.raw.as_slice());
|
||||||
}
|
}
|
||||||
|
|
||||||
object_size += ext_part.size;
|
object_size = object_size.checked_add(ext_part.size).ok_or(Error::PartMissingOrCorrupt)?;
|
||||||
if opts.quota_admission.is_some() && ext_part.actual_size < 0 {
|
if ext_part.actual_size < 0 && (!opts.replication_request || quota_context.is_enforced()) {
|
||||||
return Err(Error::PartMissingOrCorrupt);
|
return Err(Error::PartMissingOrCorrupt);
|
||||||
}
|
}
|
||||||
|
let normalized_actual_size = if ext_part.actual_size >= 0 && !transformed_object {
|
||||||
|
ext_part
|
||||||
|
.actual_size
|
||||||
|
.max(i64::try_from(ext_part.size).map_err(|_| Error::PartMissingOrCorrupt)?)
|
||||||
|
} else {
|
||||||
|
ext_part.actual_size
|
||||||
|
};
|
||||||
object_actual_size = object_actual_size
|
object_actual_size = object_actual_size
|
||||||
.checked_add(ext_part.actual_size)
|
.checked_add(normalized_actual_size)
|
||||||
.ok_or(Error::PartMissingOrCorrupt)?;
|
.ok_or(Error::PartMissingOrCorrupt)?;
|
||||||
|
let mut completed_part = completed_multipart_object_part(p.part_num, ext_part);
|
||||||
fi.parts.push(completed_multipart_object_part(p.part_num, ext_part));
|
completed_part.actual_size = normalized_actual_size;
|
||||||
|
fi.parts.push(completed_part);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(wtcs) = opts.want_checksum.as_ref() {
|
if let Some(wtcs) = opts.want_checksum.as_ref() {
|
||||||
@@ -1918,15 +1937,34 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(admission) = opts.quota_admission {
|
let declared_replication_actual_size = opts
|
||||||
let quota_operation_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
.replication_request
|
||||||
if quota_operation_size > admission.remaining() {
|
.then(|| get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP))
|
||||||
return Err(Error::QuotaExceeded {
|
.flatten();
|
||||||
current: admission.current_usage(),
|
let replication_actual_size = if opts.replication_request && quota_context.is_enforced() {
|
||||||
limit: admission.quota_limit(),
|
let observed_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||||
});
|
let declared_cap = declared_replication_actual_size
|
||||||
}
|
.as_deref()
|
||||||
}
|
.map(|value| value.parse::<u64>().map_err(|_| Error::PartMissingOrCorrupt))
|
||||||
|
.transpose()?
|
||||||
|
.unwrap_or(0);
|
||||||
|
let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(&fi.metadata)
|
||||||
|
.map_err(Error::other)?
|
||||||
|
.map(u64::try_from)
|
||||||
|
.transpose()
|
||||||
|
.map_err(|_| Error::PartMissingOrCorrupt)?
|
||||||
|
.unwrap_or(0);
|
||||||
|
Some(observed_size.max(declared_cap).max(declared_encryption_size))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let quota_new_size = match replication_actual_size {
|
||||||
|
Some(size) => size.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?),
|
||||||
|
None if quota_context.is_enforced() => u64::try_from(object_actual_size)
|
||||||
|
.map_err(|_| Error::PartMissingOrCorrupt)?
|
||||||
|
.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?),
|
||||||
|
None => 0,
|
||||||
|
};
|
||||||
if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) {
|
if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) {
|
||||||
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) {
|
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) {
|
||||||
fi.checksum = Some(Bytes::from(rc_crc_bytes));
|
fi.checksum = Some(Bytes::from(rc_crc_bytes));
|
||||||
@@ -1979,7 +2017,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if opts.replication_request {
|
if opts.replication_request {
|
||||||
if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) {
|
if let Some(actual_size) = replication_actual_size {
|
||||||
|
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
|
||||||
|
if persist_encryption_original_size {
|
||||||
|
fi.metadata
|
||||||
|
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
|
||||||
|
}
|
||||||
|
} else if let Some(actual_size) = declared_replication_actual_size {
|
||||||
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone());
|
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone());
|
||||||
if persist_encryption_original_size {
|
if persist_encryption_original_size {
|
||||||
fi.metadata
|
fi.metadata
|
||||||
@@ -2080,11 +2124,142 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
return Err(StorageError::Unexpected);
|
return Err(StorageError::Unexpected);
|
||||||
}
|
}
|
||||||
|
|
||||||
// The trailing `_` drops the rename_data old-size backfill
|
let quota_old_size = if quota_context.is_enforced() {
|
||||||
// (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit
|
reservation::replaced_logical_size(&self, bucket, object, opts).await?
|
||||||
// `get_object_info` lookup, so the backfill has no consumer here yet.
|
} else {
|
||||||
let (online_disks, convergence, op_old_dir, cleanup_disks, _) = Self::rename_data(
|
0
|
||||||
&shuffle_disks,
|
};
|
||||||
|
let mut quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?;
|
||||||
|
let (commit_disks, quota_fence_tokens) = if quota_mutation_fence {
|
||||||
|
match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await {
|
||||||
|
Ok((disks, tokens)) => {
|
||||||
|
for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) {
|
||||||
|
if let Some(token) = token {
|
||||||
|
insert_str(
|
||||||
|
&mut metadata.metadata,
|
||||||
|
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||||
|
token.as_uuid().to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(disks, tokens)
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
quota_reservation.abort().await;
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
(shuffle_disks.clone(), vec![None; shuffle_disks.len()])
|
||||||
|
};
|
||||||
|
if quota_reservation.is_lock_lost()
|
||||||
|
|| !quota_reservation.capability_proof_matches()
|
||||||
|
|| object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||||
|
|| upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||||
|
|| opts
|
||||||
|
.namespace_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
{
|
||||||
|
Self::abort_quota_reservation_after_fence(
|
||||||
|
quota_reservation,
|
||||||
|
&commit_disks,
|
||||||
|
"a_fence_tokens,
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
write_quorum,
|
||||||
|
quota_mutation_fence,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "quota_reservation",
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
object: object.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Err(err) = ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts) {
|
||||||
|
Self::abort_quota_reservation_after_fence(
|
||||||
|
quota_reservation,
|
||||||
|
&commit_disks,
|
||||||
|
"a_fence_tokens,
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
write_quorum,
|
||||||
|
quota_mutation_fence,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
if let Err(err) = self
|
||||||
|
.require_current_restore_operation_id(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
opts,
|
||||||
|
expected_restore_operation_id,
|
||||||
|
"complete_multipart_upload_quota_reservation",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Self::abort_quota_reservation_after_fence(
|
||||||
|
quota_reservation,
|
||||||
|
&commit_disks,
|
||||||
|
"a_fence_tokens,
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
write_quorum,
|
||||||
|
quota_mutation_fence,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(err) = quota_reservation.mark_commit_started().await {
|
||||||
|
Self::abort_quota_reservation_after_fence(
|
||||||
|
quota_reservation,
|
||||||
|
&commit_disks,
|
||||||
|
"a_fence_tokens,
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
write_quorum,
|
||||||
|
quota_mutation_fence,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
if quota_reservation.is_lock_lost()
|
||||||
|
|| object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||||
|
|| upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||||
|
|| opts
|
||||||
|
.namespace_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
|| opts
|
||||||
|
.bucket_lifecycle_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
{
|
||||||
|
Self::abort_quota_reservation_after_fence(
|
||||||
|
quota_reservation,
|
||||||
|
&commit_disks,
|
||||||
|
"a_fence_tokens,
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
write_quorum,
|
||||||
|
quota_mutation_fence,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "quota_reservation",
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
object: object.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let rename_result = Self::rename_data(
|
||||||
|
&commit_disks,
|
||||||
RUSTFS_META_MULTIPART_BUCKET,
|
RUSTFS_META_MULTIPART_BUCKET,
|
||||||
&upload_id_path,
|
&upload_id_path,
|
||||||
&parts_metadatas,
|
&parts_metadatas,
|
||||||
@@ -2092,7 +2267,17 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
|||||||
object,
|
object,
|
||||||
write_quorum,
|
write_quorum,
|
||||||
)
|
)
|
||||||
.await?;
|
.await;
|
||||||
|
if quota_mutation_fence {
|
||||||
|
let _ = Self::release_quota_mutation_fences(&commit_disks, "a_fence_tokens, bucket, object, write_quorum).await;
|
||||||
|
}
|
||||||
|
if rename_result.is_ok() {
|
||||||
|
quota_reservation.commit().await;
|
||||||
|
}
|
||||||
|
let (online_disks, convergence, op_old_dir, cleanup_disks, _) = match rename_result {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(err) => return Err(err.into()),
|
||||||
|
};
|
||||||
|
|
||||||
// Detach admission before any post-commit await: client cancellation
|
// Detach admission before any post-commit await: client cancellation
|
||||||
// must not couple durable convergence repair to cleanup work.
|
// must not couple durable convergence repair to cleanup work.
|
||||||
@@ -2635,7 +2820,7 @@ mod tests {
|
|||||||
let (upload_id, parts) =
|
let (upload_id, parts) =
|
||||||
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
|
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
|
||||||
let mut denied_opts = ObjectOptions::default();
|
let mut denied_opts = ObjectOptions::default();
|
||||||
assert!(denied_opts.set_quota_admission(100, 4195));
|
assert!(denied_opts.set_quota_admission(100, 4180));
|
||||||
|
|
||||||
let err = set_disks
|
let err = set_disks
|
||||||
.clone()
|
.clone()
|
||||||
@@ -2646,7 +2831,7 @@ mod tests {
|
|||||||
err,
|
err,
|
||||||
StorageError::QuotaExceeded {
|
StorageError::QuotaExceeded {
|
||||||
current: 100,
|
current: 100,
|
||||||
limit: 4195
|
limit: 4180
|
||||||
}
|
}
|
||||||
));
|
));
|
||||||
|
|
||||||
@@ -2664,7 +2849,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
let mut allowed_opts = ObjectOptions::default();
|
let mut allowed_opts = ObjectOptions::default();
|
||||||
assert!(allowed_opts.set_quota_admission(100, 4196));
|
assert!(allowed_opts.set_quota_admission(100, 4181));
|
||||||
let completed = set_disks
|
let completed = set_disks
|
||||||
.clone()
|
.clone()
|
||||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts)
|
.complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts)
|
||||||
@@ -2705,6 +2890,120 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn replication_quota_uses_server_observed_part_size_as_lower_bound() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "multipart-replication-quota-bucket";
|
||||||
|
let object = "object";
|
||||||
|
make_bucket_on_all(&disk_stores, bucket).await;
|
||||||
|
|
||||||
|
let mut create_opts = ObjectOptions::default();
|
||||||
|
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
|
||||||
|
let upload = set_disks
|
||||||
|
.new_multipart_upload(bucket, object, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("replication multipart upload should be created");
|
||||||
|
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x72; 4096], 1).await;
|
||||||
|
let mut complete_opts = ObjectOptions {
|
||||||
|
replication_request: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(complete_opts.set_quota_admission(0, 4095));
|
||||||
|
|
||||||
|
let err = set_disks
|
||||||
|
.clone()
|
||||||
|
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part.clone()], &complete_opts)
|
||||||
|
.await
|
||||||
|
.expect_err("a forged tiny replication logical size must not reduce quota admission");
|
||||||
|
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
|
||||||
|
assert!(
|
||||||
|
set_disks
|
||||||
|
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
|
||||||
|
.await
|
||||||
|
.is_ok(),
|
||||||
|
"quota rejection must leave replicated multipart parts retryable"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(complete_opts.set_quota_admission(0, 4096));
|
||||||
|
let completed = set_disks
|
||||||
|
.clone()
|
||||||
|
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
|
||||||
|
.await
|
||||||
|
.expect("the physical safety boundary should admit the transformed replica");
|
||||||
|
assert_eq!(completed.get_actual_size().expect("replica logical size should parse"), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn direct_multipart_quota_uses_server_observed_part_size_as_lower_bound() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "multipart-direct-quota-bucket";
|
||||||
|
let object = "object";
|
||||||
|
make_bucket_on_all(&disk_stores, bucket).await;
|
||||||
|
|
||||||
|
let upload = set_disks
|
||||||
|
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("multipart upload should be created");
|
||||||
|
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x73; 4096], 1).await;
|
||||||
|
let mut complete_opts = ObjectOptions::default();
|
||||||
|
assert!(complete_opts.set_quota_admission(0, 4095));
|
||||||
|
|
||||||
|
let err = set_disks
|
||||||
|
.clone()
|
||||||
|
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
|
||||||
|
.await
|
||||||
|
.expect_err("a forged tiny direct logical size must not reduce quota admission");
|
||||||
|
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
|
||||||
|
assert!(
|
||||||
|
set_disks
|
||||||
|
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
|
||||||
|
.await
|
||||||
|
.is_ok(),
|
||||||
|
"quota rejection must leave direct multipart parts retryable"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() {
|
||||||
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "multipart-compressed-ciphertext-quota-bucket";
|
||||||
|
let object = "object";
|
||||||
|
make_bucket_on_all(&disk_stores, bucket).await;
|
||||||
|
|
||||||
|
let mut create_opts = ObjectOptions::default();
|
||||||
|
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
|
||||||
|
insert_str(
|
||||||
|
&mut create_opts.user_defined,
|
||||||
|
rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT,
|
||||||
|
"true".to_string(),
|
||||||
|
);
|
||||||
|
let upload = set_disks
|
||||||
|
.new_multipart_upload(bucket, object, &create_opts)
|
||||||
|
.await
|
||||||
|
.expect("ciphertext multipart upload should be created");
|
||||||
|
let payload = vec![0x74; 4096];
|
||||||
|
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &payload, 1).await;
|
||||||
|
let mut complete_opts = ObjectOptions {
|
||||||
|
replication_request: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(complete_opts.set_quota_admission(0, u64::MAX));
|
||||||
|
|
||||||
|
let err = set_disks
|
||||||
|
.clone()
|
||||||
|
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
|
||||||
|
.await
|
||||||
|
.expect_err("ciphertext replication has no server-observed logical quota size");
|
||||||
|
assert!(matches!(err, StorageError::PartMissingOrCorrupt));
|
||||||
|
assert!(
|
||||||
|
set_disks
|
||||||
|
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
|
||||||
|
.await
|
||||||
|
.is_ok(),
|
||||||
|
"rejection must leave ciphertext multipart parts retryable"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn complete_multipart_quota_rejects_invalid_logical_sizes() {
|
async fn complete_multipart_quota_rejects_invalid_logical_sizes() {
|
||||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ use crate::bucket::lifecycle::{
|
|||||||
save_transition_transaction_record,
|
save_transition_transaction_record,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
use crate::bucket::quota::reservation;
|
||||||
use crate::bucket::replication::{
|
use crate::bucket::replication::{
|
||||||
DeleteReplicationConfigSnapshot, VersionPurgeStatusType, replication_state_to_filemeta, version_purge_status_to_filemeta,
|
DeleteReplicationConfigSnapshot, VersionPurgeStatusType, replication_state_to_filemeta, version_purge_status_to_filemeta,
|
||||||
};
|
};
|
||||||
@@ -1495,9 +1496,215 @@ impl SetDisks {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let quota_context =
|
||||||
|
reservation::begin(&self.ctx, bucket, object, opts.quota_admission, self.pool_index, self.set_index).await?;
|
||||||
|
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
|
||||||
|
let mut replication_quota_size = None;
|
||||||
|
|
||||||
|
if opts.replication_request {
|
||||||
|
if quota_context.is_enforced() && opts.preserve_ciphertext {
|
||||||
|
return Err(Error::PartMissingOrCorrupt);
|
||||||
|
}
|
||||||
|
if quota_context.is_enforced() {
|
||||||
|
let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||||
|
let physical_size = u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||||
|
let transformed = contains_key_str(&user_defined, SUFFIX_COMPRESSION)
|
||||||
|
|| parts_metadatas
|
||||||
|
.first()
|
||||||
|
.is_some_and(|metadata| should_persist_encryption_original_size(&metadata.metadata));
|
||||||
|
let declared_size = get_str(&user_defined, SUFFIX_ACTUAL_SIZE)
|
||||||
|
.map(|value| value.parse::<u64>().map_err(|_| Error::PartMissingOrCorrupt))
|
||||||
|
.transpose()?
|
||||||
|
.unwrap_or(0);
|
||||||
|
let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(&user_defined)
|
||||||
|
.map_err(Error::other)?
|
||||||
|
.map(u64::try_from)
|
||||||
|
.transpose()
|
||||||
|
.map_err(|_| Error::PartMissingOrCorrupt)?
|
||||||
|
.unwrap_or(0);
|
||||||
|
let logical_size = observed_size.max(declared_size).max(declared_encryption_size);
|
||||||
|
let persisted_size = if transformed {
|
||||||
|
logical_size
|
||||||
|
} else {
|
||||||
|
logical_size.max(physical_size)
|
||||||
|
};
|
||||||
|
replication_quota_size = Some(logical_size.max(physical_size));
|
||||||
|
actual_size = i64::try_from(persisted_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||||
|
for metadata in &mut parts_metadatas {
|
||||||
|
insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, persisted_size.to_string());
|
||||||
|
if should_persist_encryption_original_size(&metadata.metadata) {
|
||||||
|
metadata
|
||||||
|
.metadata
|
||||||
|
.insert("x-rustfs-encryption-original-size".to_string(), persisted_size.to_string());
|
||||||
|
}
|
||||||
|
if let Some(part) = metadata.parts.first_mut() {
|
||||||
|
part.actual_size = actual_size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if actual_size >= 0 {
|
||||||
|
let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||||
|
let transformed = contains_key_str(&user_defined, SUFFIX_COMPRESSION)
|
||||||
|
|| parts_metadatas
|
||||||
|
.first()
|
||||||
|
.is_some_and(|metadata| should_persist_encryption_original_size(&metadata.metadata));
|
||||||
|
let server_observed_size = if transformed {
|
||||||
|
observed_size
|
||||||
|
} else {
|
||||||
|
observed_size.max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?)
|
||||||
|
};
|
||||||
|
actual_size = i64::try_from(server_observed_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||||
|
for metadata in &mut parts_metadatas {
|
||||||
|
insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, server_observed_size.to_string());
|
||||||
|
if should_persist_encryption_original_size(&metadata.metadata) {
|
||||||
|
metadata
|
||||||
|
.metadata
|
||||||
|
.insert("x-rustfs-encryption-original-size".to_string(), server_observed_size.to_string());
|
||||||
|
}
|
||||||
|
if let Some(part) = metadata.parts.first_mut() {
|
||||||
|
part.actual_size = actual_size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let (quota_old_size, quota_new_size) = if quota_context.is_enforced() {
|
||||||
|
let new_size = match replication_quota_size {
|
||||||
|
Some(size) => size,
|
||||||
|
None => u64::try_from(actual_size)
|
||||||
|
.map_err(|_| Error::PartMissingOrCorrupt)?
|
||||||
|
.max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?),
|
||||||
|
};
|
||||||
|
(reservation::replaced_logical_size(self, bucket, object, opts).await?, new_size)
|
||||||
|
} else {
|
||||||
|
(0, 0)
|
||||||
|
};
|
||||||
|
let mut quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?;
|
||||||
|
let (commit_disks, quota_fence_tokens) = if quota_mutation_fence {
|
||||||
|
match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await {
|
||||||
|
Ok((disks, tokens)) => {
|
||||||
|
for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) {
|
||||||
|
if let Some(token) = token {
|
||||||
|
insert_str(
|
||||||
|
&mut metadata.metadata,
|
||||||
|
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||||
|
token.as_uuid().to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(disks, tokens)
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
quota_reservation.abort().await;
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
(shuffle_disks.clone(), vec![None; shuffle_disks.len()])
|
||||||
|
};
|
||||||
|
if quota_reservation.is_lock_lost()
|
||||||
|
|| !quota_reservation.capability_proof_matches()
|
||||||
|
|| object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||||
|
|| opts
|
||||||
|
.namespace_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
|| opts
|
||||||
|
.bucket_lifecycle_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
|| bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||||
|
{
|
||||||
|
Self::abort_quota_reservation_after_fence(
|
||||||
|
quota_reservation,
|
||||||
|
&commit_disks,
|
||||||
|
"a_fence_tokens,
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
write_quorum,
|
||||||
|
quota_mutation_fence,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "quota_reservation",
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
object: object.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Err(err) = self
|
||||||
|
.require_current_restore_operation_id(
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
opts,
|
||||||
|
expected_restore_operation_id,
|
||||||
|
"put_object_quota_reservation",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Self::abort_quota_reservation_after_fence(
|
||||||
|
quota_reservation,
|
||||||
|
&commit_disks,
|
||||||
|
"a_fence_tokens,
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
write_quorum,
|
||||||
|
quota_mutation_fence,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
|
||||||
let rename_stage_start = Instant::now();
|
let rename_stage_start = Instant::now();
|
||||||
let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = Self::rename_data(
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
&shuffle_disks,
|
pause_put_object_commit(bucket, object, PutObjectCommitPause::AfterQuotaReservation).await;
|
||||||
|
if let Err(err) = quota_reservation.mark_commit_started().await {
|
||||||
|
Self::abort_quota_reservation_after_fence(
|
||||||
|
quota_reservation,
|
||||||
|
&commit_disks,
|
||||||
|
"a_fence_tokens,
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
write_quorum,
|
||||||
|
quota_mutation_fence,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeQuotaRename).await;
|
||||||
|
if quota_reservation.is_lock_lost()
|
||||||
|
|| object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||||
|
|| opts
|
||||||
|
.namespace_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
|| opts
|
||||||
|
.bucket_lifecycle_lock_fence
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||||
|
|| bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||||
|
{
|
||||||
|
Self::abort_quota_reservation_after_fence(
|
||||||
|
quota_reservation,
|
||||||
|
&commit_disks,
|
||||||
|
"a_fence_tokens,
|
||||||
|
bucket,
|
||||||
|
object,
|
||||||
|
write_quorum,
|
||||||
|
quota_mutation_fence,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "quota_reservation",
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
object: object.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let rename_result = Self::rename_data(
|
||||||
|
&commit_disks,
|
||||||
RUSTFS_META_TMP_BUCKET,
|
RUSTFS_META_TMP_BUCKET,
|
||||||
tmp_dir.as_str(),
|
tmp_dir.as_str(),
|
||||||
&parts_metadatas,
|
&parts_metadatas,
|
||||||
@@ -1505,7 +1712,18 @@ impl SetDisks {
|
|||||||
object,
|
object,
|
||||||
write_quorum,
|
write_quorum,
|
||||||
)
|
)
|
||||||
.await?;
|
.await;
|
||||||
|
if quota_mutation_fence {
|
||||||
|
let _ =
|
||||||
|
Self::release_quota_mutation_fences(&commit_disks, "a_fence_tokens, bucket, object, write_quorum).await;
|
||||||
|
}
|
||||||
|
if rename_result.is_ok() {
|
||||||
|
quota_reservation.commit().await;
|
||||||
|
}
|
||||||
|
let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = match rename_result {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(err) => return Err(err.into()),
|
||||||
|
};
|
||||||
// Do this before any post-commit await so request cancellation cannot
|
// Do this before any post-commit await so request cancellation cannot
|
||||||
// bypass best-effort admission. A process crash before admission
|
// bypass best-effort admission. A process crash before admission
|
||||||
// remains subject to the existing scanner reconciliation path.
|
// remains subject to the existing scanner reconciliation path.
|
||||||
@@ -2636,6 +2854,8 @@ fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: boo
|
|||||||
pub enum PutObjectCommitPause {
|
pub enum PutObjectCommitPause {
|
||||||
BeforeNamespace,
|
BeforeNamespace,
|
||||||
AfterNamespace,
|
AfterNamespace,
|
||||||
|
AfterQuotaReservation,
|
||||||
|
BeforeQuotaRename,
|
||||||
BeforeMetadata,
|
BeforeMetadata,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5568,6 +5788,124 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod replication_quota_safety_tests {
|
||||||
|
use super::hermetic_set_disks_support::hermetic_set_disks;
|
||||||
|
use super::*;
|
||||||
|
use std::io::Cursor;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn replication_put_quota_uses_physical_bytes_as_a_safety_floor() {
|
||||||
|
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "replication-put-quota-safety";
|
||||||
|
for disk in &disks {
|
||||||
|
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||||
|
}
|
||||||
|
let mut user_defined = HashMap::new();
|
||||||
|
insert_str(
|
||||||
|
&mut user_defined,
|
||||||
|
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||||
|
"klauspost/compress/s2".to_string(),
|
||||||
|
);
|
||||||
|
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string());
|
||||||
|
let payload = vec![0x61; 4096];
|
||||||
|
|
||||||
|
let mut denied_opts = ObjectOptions {
|
||||||
|
replication_request: true,
|
||||||
|
user_defined: user_defined.clone(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(denied_opts.set_quota_admission(0, 4095));
|
||||||
|
let mut denied_reader = PutObjReader::new(
|
||||||
|
HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false)
|
||||||
|
.expect("construct forged replication reader"),
|
||||||
|
);
|
||||||
|
let err = set_disks
|
||||||
|
.put_object(bucket, "object", &mut denied_reader, &denied_opts)
|
||||||
|
.await
|
||||||
|
.expect_err("server-observed bytes must prevent a tiny replication quota claim");
|
||||||
|
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
|
||||||
|
|
||||||
|
let mut allowed_opts = ObjectOptions {
|
||||||
|
replication_request: true,
|
||||||
|
user_defined,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(allowed_opts.set_quota_admission(0, 4096));
|
||||||
|
let mut allowed_reader = PutObjReader::new(
|
||||||
|
HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false)
|
||||||
|
.expect("construct exact-boundary replication reader"),
|
||||||
|
);
|
||||||
|
let stored = set_disks
|
||||||
|
.put_object(bucket, "object", &mut allowed_reader, &allowed_opts)
|
||||||
|
.await
|
||||||
|
.expect("server-observed exact quota boundary should succeed");
|
||||||
|
assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn direct_put_cannot_persist_a_tiny_logical_size() {
|
||||||
|
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "direct-put-quota-safety";
|
||||||
|
for disk in &disks {
|
||||||
|
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||||
|
}
|
||||||
|
let payload = vec![0x62; 4096];
|
||||||
|
let mut denied_opts = ObjectOptions::default();
|
||||||
|
assert!(denied_opts.set_quota_admission(0, 4095));
|
||||||
|
let mut denied_reader = PutObjReader::new(
|
||||||
|
HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false)
|
||||||
|
.expect("construct forged direct reader"),
|
||||||
|
);
|
||||||
|
let err = set_disks
|
||||||
|
.put_object(bucket, "object", &mut denied_reader, &denied_opts)
|
||||||
|
.await
|
||||||
|
.expect_err("server-observed bytes must prevent a tiny direct quota claim");
|
||||||
|
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 }));
|
||||||
|
|
||||||
|
let mut allowed_opts = ObjectOptions::default();
|
||||||
|
assert!(allowed_opts.set_quota_admission(0, 4096));
|
||||||
|
let mut allowed_reader = PutObjReader::new(
|
||||||
|
HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false)
|
||||||
|
.expect("construct exact-boundary direct reader"),
|
||||||
|
);
|
||||||
|
let stored = set_disks
|
||||||
|
.put_object(bucket, "object", &mut allowed_reader, &allowed_opts)
|
||||||
|
.await
|
||||||
|
.expect("server-observed exact quota boundary should succeed");
|
||||||
|
assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 4096);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() {
|
||||||
|
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||||
|
let bucket = "ciphertext-replication-quota-safety";
|
||||||
|
for disk in &disks {
|
||||||
|
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||||
|
}
|
||||||
|
let mut user_defined = HashMap::new();
|
||||||
|
user_defined.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
|
||||||
|
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string());
|
||||||
|
let mut opts = ObjectOptions {
|
||||||
|
replication_request: true,
|
||||||
|
preserve_ciphertext: true,
|
||||||
|
user_defined,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(opts.set_quota_admission(0, u64::MAX));
|
||||||
|
let payload = vec![0x63; 4096];
|
||||||
|
let mut reader = PutObjReader::new(
|
||||||
|
HashReader::from_stream(Cursor::new(payload), 4096, 4096, None, None, false)
|
||||||
|
.expect("construct ciphertext replication reader"),
|
||||||
|
);
|
||||||
|
let err = set_disks
|
||||||
|
.put_object(bucket, "object", &mut reader, &opts)
|
||||||
|
.await
|
||||||
|
.expect_err("ciphertext replication without a server-observed logical size must fail closed");
|
||||||
|
assert!(matches!(err, StorageError::PartMissingOrCorrupt));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod get_object_downstream_close_accounting_tests {
|
mod get_object_downstream_close_accounting_tests {
|
||||||
use super::hermetic_set_disks_support::hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks;
|
||||||
|
|||||||
@@ -1175,6 +1175,32 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial(storage_class_env)]
|
||||||
|
async fn quota_object_fence_ignores_an_unrelated_offline_pool() {
|
||||||
|
let temp_dir = tempfile::tempdir().expect("create quota fence store dir");
|
||||||
|
let (_ctx, store, shutdown) =
|
||||||
|
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "quota-object-fence", &[4, 4])).await;
|
||||||
|
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||||
|
let bucket = format!("quota-object-fence-{}", uuid::Uuid::new_v4());
|
||||||
|
let object = "object.bin";
|
||||||
|
store
|
||||||
|
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("create quota fence bucket");
|
||||||
|
store.pools[1].disk_set[0].disks.write().await.fill(None);
|
||||||
|
|
||||||
|
crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, Some((0, 0)))
|
||||||
|
.await
|
||||||
|
.expect("the selected pool fence should ignore an unrelated offline pool");
|
||||||
|
let err = crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, None)
|
||||||
|
.await
|
||||||
|
.expect_err("legacy reservations must conservatively fence every pool");
|
||||||
|
assert!(matches!(err, StorageError::ErasureWriteQuorum));
|
||||||
|
|
||||||
|
shutdown.cancel();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial_test::serial(storage_class_env)]
|
#[serial_test::serial(storage_class_env)]
|
||||||
async fn tag_updates_skip_active_rebalance_source_pool() {
|
async fn tag_updates_skip_active_rebalance_source_pool() {
|
||||||
|
|||||||
@@ -641,7 +641,7 @@ pub(crate) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) {
|
|||||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta)));
|
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta)));
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 {
|
pub(crate) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 {
|
||||||
observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default()
|
observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -67,10 +67,39 @@ const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
|
|||||||
const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object";
|
const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object";
|
||||||
const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state";
|
const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state";
|
||||||
const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at";
|
const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at";
|
||||||
|
const USER_METADATA_PREFIX: &str = "x-amz-meta-";
|
||||||
|
|
||||||
/// Maximum object size in bytes (5GB - Swift default)
|
/// Maximum object size in bytes (5GB - Swift default)
|
||||||
const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024;
|
const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
fn stored_swift_user_metadata_key(key: &str) -> String {
|
||||||
|
if rustfs_utils::http::is_internal_key(key)
|
||||||
|
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-amz-")
|
||||||
|
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-")
|
||||||
|
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-")
|
||||||
|
{
|
||||||
|
format!("{USER_METADATA_PREFIX}{key}")
|
||||||
|
} else {
|
||||||
|
key.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn swift_user_metadata(headers: &HeaderMap) -> Option<HashMap<String, String>> {
|
||||||
|
let mut metadata = HashMap::new();
|
||||||
|
let mut present = false;
|
||||||
|
for (header_name, header_value) in headers.iter() {
|
||||||
|
let header_name = header_name.as_str().to_lowercase();
|
||||||
|
let Some(key) = header_name.strip_prefix("x-object-meta-") else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
present = true;
|
||||||
|
if let Ok(value) = header_value.to_str() {
|
||||||
|
metadata.insert(stored_swift_user_metadata_key(key), value.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
present.then_some(metadata)
|
||||||
|
}
|
||||||
|
|
||||||
/// Object key translator for Swift object names
|
/// Object key translator for Swift object names
|
||||||
///
|
///
|
||||||
/// Handles URL encoding/decoding and path normalization for Swift object keys.
|
/// Handles URL encoding/decoding and path normalization for Swift object keys.
|
||||||
@@ -303,15 +332,7 @@ where
|
|||||||
let bucket = mapper.swift_to_s3_bucket(container, &project_id);
|
let bucket = mapper.swift_to_s3_bucket(container, &project_id);
|
||||||
|
|
||||||
// 5. Extract Swift metadata from X-Object-Meta-* headers
|
// 5. Extract Swift metadata from X-Object-Meta-* headers
|
||||||
let mut user_metadata = HashMap::new();
|
let mut user_metadata = swift_user_metadata(headers).unwrap_or_default();
|
||||||
for (header_name, header_value) in headers.iter() {
|
|
||||||
let header_str = header_name.as_str().to_lowercase();
|
|
||||||
if let Some(meta_key) = header_str.strip_prefix("x-object-meta-")
|
|
||||||
&& let Ok(value_str) = header_value.to_str()
|
|
||||||
{
|
|
||||||
user_metadata.insert(meta_key.to_string(), value_str.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 6. Extract Content-Type if provided
|
// 6. Extract Content-Type if provided
|
||||||
if let Some(content_type) = headers.get("content-type")
|
if let Some(content_type) = headers.get("content-type")
|
||||||
@@ -739,15 +760,7 @@ pub async fn update_object_metadata(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 8. Extract new metadata from X-Object-Meta-* headers
|
// 8. Extract new metadata from X-Object-Meta-* headers
|
||||||
let mut new_metadata = HashMap::new();
|
let mut new_metadata = swift_user_metadata(headers).unwrap_or_default();
|
||||||
for (header_name, header_value) in headers.iter() {
|
|
||||||
let header_str = header_name.as_str().to_lowercase();
|
|
||||||
if let Some(meta_key) = header_str.strip_prefix("x-object-meta-")
|
|
||||||
&& let Ok(value_str) = header_value.to_str()
|
|
||||||
{
|
|
||||||
new_metadata.insert(meta_key.to_string(), value_str.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 9. Also update Content-Type if provided
|
// 9. Also update Content-Type if provided
|
||||||
if let Some(content_type) = headers.get("content-type")
|
if let Some(content_type) = headers.get("content-type")
|
||||||
@@ -889,19 +902,8 @@ pub async fn copy_object(
|
|||||||
let mut new_metadata = (*src_info.user_defined).clone();
|
let mut new_metadata = (*src_info.user_defined).clone();
|
||||||
|
|
||||||
// 11. If custom metadata headers provided, use those instead (Swift behavior)
|
// 11. If custom metadata headers provided, use those instead (Swift behavior)
|
||||||
let mut has_custom_meta = false;
|
if let Some(custom_metadata) = swift_user_metadata(headers) {
|
||||||
for (header_name, header_value) in headers.iter() {
|
new_metadata = custom_metadata;
|
||||||
let header_str = header_name.as_str().to_lowercase();
|
|
||||||
if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") {
|
|
||||||
if !has_custom_meta {
|
|
||||||
// First custom meta header - clear source metadata
|
|
||||||
new_metadata.clear();
|
|
||||||
has_custom_meta = true;
|
|
||||||
}
|
|
||||||
if let Ok(value_str) = header_value.to_str() {
|
|
||||||
new_metadata.insert(meta_key.to_string(), value_str.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 12. Also check for Content-Type override
|
// 12. Also check for Content-Type override
|
||||||
@@ -1123,6 +1125,23 @@ mod tests {
|
|||||||
assert!(ObjectKeyMapper::validate_object_name("unicode-文件.txt").is_ok());
|
assert!(ObjectKeyMapper::validate_object_name("unicode-文件.txt").is_ok());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn swift_user_metadata_cannot_materialize_internal_storage_keys() {
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert("x-object-meta-x-rustfs-internal-actual-size", "1".parse().expect("valid metadata value"));
|
||||||
|
headers.insert("x-object-meta-description", "safe".parse().expect("valid metadata value"));
|
||||||
|
let metadata = swift_user_metadata(&headers).expect("custom metadata should be detected");
|
||||||
|
|
||||||
|
assert_eq!(metadata.get("x-amz-meta-x-rustfs-internal-actual-size").map(String::as_str), Some("1"));
|
||||||
|
assert_eq!(metadata.get("description").map(String::as_str), Some("safe"));
|
||||||
|
assert!(!metadata.contains_key("x-rustfs-internal-actual-size"));
|
||||||
|
assert_eq!(
|
||||||
|
stored_swift_user_metadata_key("x-minio-encryption-original-size"),
|
||||||
|
"x-amz-meta-x-minio-encryption-original-size"
|
||||||
|
);
|
||||||
|
assert_eq!(stored_swift_user_metadata_key("description"), "description");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_validate_object_name_empty() {
|
fn test_validate_object_name_empty() {
|
||||||
let result = ObjectKeyMapper::validate_object_name("");
|
let result = ObjectKeyMapper::validate_object_name("");
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ for later deletion.
|
|||||||
- `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: mixed-version deployments continue writing version 1 snapshots until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2.
|
- `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: mixed-version deployments continue writing version 1 snapshots until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2.
|
||||||
- `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration.
|
- `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration.
|
||||||
- `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract.
|
- `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract.
|
||||||
- `cross-pool-fence-v1` authenticated unsupported advertisement: predeployment servers recognize the versioned cross-pool fence capability probe but report support version 0, allowing a later all-peer probe to distinguish predeployment nodes without activating a second lock domain. Replace the unsupported advertisement only when composite lock acquisition, a cluster-wide activation fence, complete fleet proof, commit-time proof revalidation, and fail-closed revocation ship together.
|
|
||||||
- `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration.
|
- `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration.
|
||||||
- `rustfs-5509` FileInfo positional MessagePack decoding: beta.11 serialized 28 fields, while beta.12 inserted transition-version fields in the middle and serialized an incompatible 30-field array. New releases write named maps and retain readers for both shipped array layouts so direct and rolling upgrades can read either release. Remove the positional-array readers after every supported direct-upgrade release writes named maps and no retained RPC payload can contain a pre-map FileInfo array.
|
- `rustfs-5509` FileInfo positional MessagePack decoding: beta.11 serialized 28 fields, while beta.12 inserted transition-version fields in the middle and serialized an incompatible 30-field array. New releases write named maps and retain readers for both shipped array layouts so direct and rolling upgrades can read either release. Remove the positional-array readers after every supported direct-upgrade release writes named maps and no retained RPC payload can contain a pre-map FileInfo array.
|
||||||
- `rustfs-5416` Helm distributed startup wait setting: charts that predate explicit local endpoint identity expose startupWaitTimeoutSeconds for their peer DNS/TCP init gate. The new chart keeps the value accepted but ignores it after moving startup convergence into RustFS. Remove the value and its documentation after the minimum supported direct-upgrade chart includes localEndpointHost.autoInject and no longer renders the peer gate.
|
- `rustfs-5416` Helm distributed startup wait setting: charts that predate explicit local endpoint identity expose startupWaitTimeoutSeconds for their peer DNS/TCP init gate. The new chart keeps the value accepted but ignores it after moving startup convergence into RustFS. Remove the value and its documentation after the minimum supported direct-upgrade chart includes localEndpointHost.autoInject and no longer renders the peer gate.
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ use crate::admin::storage_api::bucket::metadata_sys::{self, BucketMetadataSys};
|
|||||||
use crate::admin::storage_api::bucket::quota::checker::QuotaChecker;
|
use crate::admin::storage_api::bucket::quota::checker::QuotaChecker;
|
||||||
use crate::admin::storage_api::bucket::quota::{BucketQuota, QuotaError, QuotaOperation};
|
use crate::admin::storage_api::bucket::quota::{BucketQuota, QuotaError, QuotaOperation};
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
|
use crate::error::ApiError;
|
||||||
use crate::server::ADMIN_PREFIX;
|
use crate::server::ADMIN_PREFIX;
|
||||||
use hyper::{Method, StatusCode};
|
use hyper::{Method, StatusCode};
|
||||||
use matchit::Params;
|
use matchit::Params;
|
||||||
@@ -293,16 +294,36 @@ impl Operation for SetBucketQuotaHandler {
|
|||||||
return Err(s3_error!(InvalidArgument, "{}", rustfs_config::QUOTA_INVALID_TYPE_ERROR_MSG));
|
return Err(s3_error!(InvalidArgument, "{}", rustfs_config::QUOTA_INVALID_TYPE_ERROR_MSG));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let fleet_proof = if request.quota.is_some() {
|
||||||
|
Some(crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| {
|
||||||
|
S3Error::with_message(
|
||||||
|
s3s::S3ErrorCode::ServiceUnavailable,
|
||||||
|
"durable quota capability is not confirmed across the cluster".to_string(),
|
||||||
|
)
|
||||||
|
})?)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
let quota = BucketQuota::new(request.quota);
|
let quota = BucketQuota::new(request.quota);
|
||||||
|
|
||||||
let metadata_sys_lock = bucket_metadata_from_context()
|
let metadata_sys_lock = bucket_metadata_from_context()
|
||||||
.ok_or_else(|| s3_error!(InternalError, "{}", rustfs_config::QUOTA_METADATA_SYSTEM_ERROR_MSG))?;
|
.ok_or_else(|| s3_error!(InternalError, "{}", rustfs_config::QUOTA_METADATA_SYSTEM_ERROR_MSG))?;
|
||||||
let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
|
let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone());
|
||||||
|
|
||||||
let updated_at = quota_checker
|
let updated_at = match fleet_proof.as_ref() {
|
||||||
.set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id)
|
Some(fleet_proof) => {
|
||||||
.await
|
quota_checker
|
||||||
.map_err(|e| s3_error!(InternalError, "failed to set quota: {}", e))?;
|
.set_durable_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id, fleet_proof)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
quota_checker
|
||||||
|
.set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.map_err(ApiError::from)?;
|
||||||
|
|
||||||
if let Err(err) = site_replication_bucket_meta_hook(SRBucketMeta {
|
if let Err(err) = site_replication_bucket_meta_hook(SRBucketMeta {
|
||||||
bucket: bucket.clone(),
|
bucket: bucket.clone(),
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ use crate::admin::storage_api::bucket::metadata::{
|
|||||||
BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG,
|
BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG,
|
||||||
};
|
};
|
||||||
use crate::admin::storage_api::bucket::metadata_sys;
|
use crate::admin::storage_api::bucket::metadata_sys;
|
||||||
|
use crate::admin::storage_api::bucket::quota::BucketQuota;
|
||||||
use crate::admin::storage_api::bucket::replication;
|
use crate::admin::storage_api::bucket::replication;
|
||||||
use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
|
use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
|
||||||
use crate::admin::storage_api::bucket::target_sys::BucketTargetSys;
|
use crate::admin::storage_api::bucket::target_sys::BucketTargetSys;
|
||||||
@@ -7895,9 +7896,35 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> {
|
|||||||
|
|
||||||
if !skip_config_write {
|
if !skip_config_write {
|
||||||
if let Some(data) = data {
|
if let Some(data) = data {
|
||||||
metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id)
|
if item.r#type == "quota-config" {
|
||||||
.await
|
let quota: BucketQuota = serde_json::from_slice(&data)
|
||||||
.map_err(ApiError::from)?;
|
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid bucket quota: {e}")))?;
|
||||||
|
if quota.has_unsupported_reservation_protocol() {
|
||||||
|
return Err(S3Error::with_message(
|
||||||
|
S3ErrorCode::InvalidRequest,
|
||||||
|
"unsupported bucket quota reservation protocol".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if quota.uses_durable_reservations() {
|
||||||
|
let proof = crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| {
|
||||||
|
S3Error::with_message(
|
||||||
|
S3ErrorCode::ServiceUnavailable,
|
||||||
|
"durable quota capability is not confirmed across the cluster".to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
metadata_sys::update_quota_if_incarnation(&item.bucket, data, expected_incarnation_id, &proof)
|
||||||
|
.await
|
||||||
|
.map_err(ApiError::from)?;
|
||||||
|
} else {
|
||||||
|
metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id)
|
||||||
|
.await
|
||||||
|
.map_err(ApiError::from)?;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id)
|
||||||
|
.await
|
||||||
|
.map_err(ApiError::from)?;
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
metadata_sys::delete_if_incarnation(&item.bucket, config_file, expected_incarnation_id)
|
metadata_sys::delete_if_incarnation(&item.bucket, config_file, expected_incarnation_id)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -64,7 +64,9 @@ mod ecstore_metrics {
|
|||||||
}
|
}
|
||||||
|
|
||||||
mod ecstore_notification {
|
mod ecstore_notification {
|
||||||
pub(crate) use crate::storage::storage_api::ecstore_notification::NotificationSys;
|
pub(crate) use crate::storage::storage_api::ecstore_notification::{
|
||||||
|
CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
@@ -112,6 +114,10 @@ pub(crate) type TierCreds = ecstore_tier::tier_admin::TierCreds;
|
|||||||
pub(crate) type TierType = ecstore_tier::tier_config::TierType;
|
pub(crate) type TierType = ecstore_tier::tier_config::TierType;
|
||||||
pub(crate) type TierConfigUpdateError = crate::storage::storage_api::TierConfigUpdateError;
|
pub(crate) type TierConfigUpdateError = crate::storage::storage_api::TierConfigUpdateError;
|
||||||
|
|
||||||
|
pub(crate) fn acquire_cross_pool_fence_fleet_proof() -> Option<ecstore_notification::CrossPoolFenceFleetProofToken> {
|
||||||
|
ecstore_notification::acquire_cross_pool_fence_fleet_proof()
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) mod runtime_sources {
|
pub(crate) mod runtime_sources {
|
||||||
pub(crate) type DailyAllTierStats = super::DailyAllTierStats;
|
pub(crate) type DailyAllTierStats = super::DailyAllTierStats;
|
||||||
pub(crate) type ECStore = super::ECStore;
|
pub(crate) type ECStore = super::ECStore;
|
||||||
@@ -296,6 +302,15 @@ pub(crate) mod metadata_sys {
|
|||||||
super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await
|
super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn update_quota_if_incarnation(
|
||||||
|
bucket: &str,
|
||||||
|
data: Vec<u8>,
|
||||||
|
expected_incarnation_id: uuid::Uuid,
|
||||||
|
proof: &super::ecstore_notification::CrossPoolFenceFleetProofToken,
|
||||||
|
) -> Result<OffsetDateTime> {
|
||||||
|
super::ecstore_bucket::metadata_sys::update_quota_if_incarnation(bucket, data, expected_incarnation_id, proof).await
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result<uuid::Uuid> {
|
pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result<uuid::Uuid> {
|
||||||
super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await
|
super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,6 +23,9 @@
|
|||||||
//! `ECStore` and one metadata-sys initialization exist per test binary.
|
//! `ECStore` and one metadata-sys initialization exist per test binary.
|
||||||
|
|
||||||
use super::storage_api::test::bucket::metadata_sys;
|
use super::storage_api::test::bucket::metadata_sys;
|
||||||
|
use super::storage_api::test::bucket::quota::BucketQuota;
|
||||||
|
use super::storage_api::test::bucket::quota::checker::QuotaChecker;
|
||||||
|
use super::storage_api::test::contract::bucket::MakeBucketOptions;
|
||||||
use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions};
|
use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions};
|
||||||
use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
|
use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
|
||||||
use super::{context::AppContext, object_traffic_health::ObjectTrafficHealth};
|
use super::{context::AppContext, object_traffic_health::ObjectTrafficHealth};
|
||||||
@@ -87,6 +90,17 @@ pub(crate) async fn shared_gating_ecstore() -> Arc<ECStore> {
|
|||||||
crate::storage::storage_api::new_global_notification_sys(endpoint_pools.clone())
|
crate::storage::storage_api::new_global_notification_sys(endpoint_pools.clone())
|
||||||
.await
|
.await
|
||||||
.expect("initialize notification system for gating test env");
|
.expect("initialize notification system for gating test env");
|
||||||
|
let topology_fingerprint =
|
||||||
|
crate::storage::storage_api::heal_control_startup_consumer::heal_topology_fingerprint(&endpoint_pools)
|
||||||
|
.expect("single-node gating topology should hash");
|
||||||
|
crate::storage::storage_api::start_remote_version_state_fleet_probe(topology_fingerprint);
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||||
|
while crate::storage::storage_api::ecstore_notification::acquire_cross_pool_fence_fleet_proof().is_none() {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("single-node cross-pool fence capability proof should publish");
|
||||||
|
|
||||||
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
|
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
||||||
@@ -107,6 +121,24 @@ pub(crate) async fn shared_gating_ecstore() -> Arc<ECStore> {
|
|||||||
ecstore
|
ecstore
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn durable_quota_test_bucket(prefix: &str, limit: u64) -> (Arc<ECStore>, String) {
|
||||||
|
let store = shared_gating_ecstore().await;
|
||||||
|
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
||||||
|
let bucket = format!("{prefix:.30}-{}", uuid::Uuid::new_v4().simple());
|
||||||
|
store
|
||||||
|
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("create durable quota test bucket");
|
||||||
|
super::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 0).await;
|
||||||
|
let metadata_sys =
|
||||||
|
crate::app::storage_api::test::get_global_bucket_metadata_sys().expect("test app context should expose bucket metadata");
|
||||||
|
QuotaChecker::new(metadata_sys)
|
||||||
|
.set_quota_config(&bucket, BucketQuota::new(Some(limit)))
|
||||||
|
.await
|
||||||
|
.expect("configure durable quota test bucket");
|
||||||
|
(store, bucket)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn shared_gating_ambient() -> Arc<AppContext> {
|
pub(crate) async fn shared_gating_ambient() -> Arc<AppContext> {
|
||||||
let store = shared_gating_ecstore().await;
|
let store = shared_gating_ecstore().await;
|
||||||
if let Some(ambient) = crate::runtime_sources::current_app_context() {
|
if let Some(ambient) = crate::runtime_sources::current_app_context() {
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ use super::storage_api::multipart_usecase::contract::multipart::{CompletePart, M
|
|||||||
use super::storage_api::multipart_usecase::contract::object::{ObjectIO as _, ObjectOperations as _};
|
use super::storage_api::multipart_usecase::contract::object::{ObjectIO as _, ObjectOperations as _};
|
||||||
use super::storage_api::multipart_usecase::contract::range::HTTPRangeSpec;
|
use super::storage_api::multipart_usecase::contract::range::HTTPRangeSpec;
|
||||||
use super::storage_api::multipart_usecase::data_usage::{
|
use super::storage_api::multipart_usecase::data_usage::{
|
||||||
record_bucket_object_version_write_memory, record_bucket_object_write_memory,
|
quota_object_size, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
|
||||||
};
|
};
|
||||||
use super::storage_api::multipart_usecase::error::{StorageError, is_err_object_not_found, is_err_version_not_found};
|
use super::storage_api::multipart_usecase::error::{StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||||
use super::storage_api::multipart_usecase::helper::OperationHelper;
|
use super::storage_api::multipart_usecase::helper::OperationHelper;
|
||||||
@@ -63,10 +63,10 @@ use super::storage_api::multipart_usecase::{
|
|||||||
};
|
};
|
||||||
use crate::app::object_data_cache::{
|
use crate::app::object_data_cache::{
|
||||||
ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success,
|
ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success,
|
||||||
invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation,
|
invalidate_object_data_cache_before_mutation,
|
||||||
};
|
};
|
||||||
use crate::app::object_usecase::{
|
use crate::app::object_usecase::{
|
||||||
acquire_copy_bucket_lifecycle_locks, build_put_like_object_lock_metadata, map_quota_check_outcome,
|
acquire_copy_bucket_lifecycle_locks, apply_quota_admission, build_put_like_object_lock_metadata, map_quota_check_outcome,
|
||||||
validate_existing_object_lock_for_write,
|
validate_existing_object_lock_for_write,
|
||||||
};
|
};
|
||||||
use crate::app::runtime_sources::{
|
use crate::app::runtime_sources::{
|
||||||
@@ -82,6 +82,8 @@ use rustfs_io_metrics::record_s3_op;
|
|||||||
use rustfs_s3_ops::S3Operation;
|
use rustfs_s3_ops::S3Operation;
|
||||||
use rustfs_targets::EventName;
|
use rustfs_targets::EventName;
|
||||||
use rustfs_utils::CompressionAlgorithm;
|
use rustfs_utils::CompressionAlgorithm;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_utils::http::insert_header;
|
||||||
use rustfs_utils::http::{
|
use rustfs_utils::http::{
|
||||||
SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
|
SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
|
||||||
SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_header, get_source_scheme,
|
SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_header, get_source_scheme,
|
||||||
@@ -225,12 +227,8 @@ fn internal_object_info_lookup_opts(mut opts: ObjectOptions) -> ObjectOptions {
|
|||||||
opts
|
opts
|
||||||
}
|
}
|
||||||
|
|
||||||
fn logical_object_size(info: &ObjectInfo) -> Result<u64, StorageError> {
|
|
||||||
u64::try_from(info.get_actual_size()?).map_err(|_| StorageError::PartMissingOrCorrupt)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result<u64> {
|
fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result<u64> {
|
||||||
match logical_object_size(info) {
|
match quota_object_size(info) {
|
||||||
Ok(size) => Ok(size),
|
Ok(size) => Ok(size),
|
||||||
Err(err) if fail_closed => Err(ApiError::from(err).into()),
|
Err(err) if fail_closed => Err(ApiError::from(err).into()),
|
||||||
Err(_) => Ok(info.size.max(0) as u64),
|
Err(_) => Ok(info.size.max(0) as u64),
|
||||||
@@ -506,11 +504,7 @@ impl DefaultMultipartUsecase {
|
|||||||
Ok(existing_obj_info) => {
|
Ok(existing_obj_info) => {
|
||||||
validate_existing_object_lock_for_write(&existing_obj_info, ¤t_opts)?;
|
validate_existing_object_lock_for_write(&existing_obj_info, ¤t_opts)?;
|
||||||
let physical_size = existing_obj_info.size.max(0) as u64;
|
let physical_size = existing_obj_info.size.max(0) as u64;
|
||||||
let logical_size = if opts.replication_request {
|
let logical_size = quota_object_size(&existing_obj_info);
|
||||||
Ok(physical_size)
|
|
||||||
} else {
|
|
||||||
logical_object_size(&existing_obj_info)
|
|
||||||
};
|
|
||||||
Some((physical_size, logical_size))
|
Some((physical_size, logical_size))
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
@@ -559,32 +553,19 @@ impl DefaultMultipartUsecase {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let quota_metadata_sys = self.bucket_metadata_sys();
|
let quota_metadata_sys = self.bucket_metadata_sys();
|
||||||
|
let mut quota_enabled = false;
|
||||||
if let Some(metadata_sys) = quota_metadata_sys.as_ref() {
|
if let Some(metadata_sys) = quota_metadata_sys.as_ref() {
|
||||||
let quota_checker = QuotaChecker::new(metadata_sys.clone());
|
let quota_checker = QuotaChecker::new(metadata_sys.clone());
|
||||||
let check_result =
|
let check_result =
|
||||||
map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
|
map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
|
||||||
// Ciphertext-passthrough replication parts use a different size basis and retain
|
quota_enabled = check_result.quota_limit.is_some();
|
||||||
// the existing post-commit accounting path until they carry a trusted logical-size proof.
|
apply_quota_admission(&mut opts, &check_result)?;
|
||||||
if !opts.replication_request
|
|
||||||
&& let Some(quota_limit) = check_result.quota_limit
|
|
||||||
{
|
|
||||||
let installed = check_result
|
|
||||||
.current_usage
|
|
||||||
.is_some_and(|current_usage| opts.set_quota_admission(current_usage, quota_limit));
|
|
||||||
if !installed {
|
|
||||||
return Err(S3Error::with_message(
|
|
||||||
S3ErrorCode::ServiceUnavailable,
|
|
||||||
"Bucket quota check temporarily unavailable, please retry".to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let previous_current_size = match previous_current_sizes {
|
let previous_current_size = match previous_current_sizes {
|
||||||
Some((physical_size, _)) if opts.replication_request => Some(physical_size),
|
Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size),
|
||||||
Some((_, Ok(logical_size))) => Some(logical_size),
|
Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err).into()),
|
||||||
Some((_, Err(err))) if opts.quota_admission.is_some() => return Err(ApiError::from(err).into()),
|
Some((physical_size, _)) => Some(physical_size),
|
||||||
Some((physical_size, Err(_))) => Some(physical_size),
|
|
||||||
None => None,
|
None => None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -596,37 +577,8 @@ impl DefaultMultipartUsecase {
|
|||||||
let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await;
|
let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await;
|
||||||
record_capacity_write(Some(capacity_scope_token)).await;
|
record_capacity_write(Some(capacity_scope_token)).await;
|
||||||
|
|
||||||
if let Some(metadata_sys) = quota_metadata_sys.as_ref() {
|
if quota_metadata_sys.is_some() {
|
||||||
if opts.replication_request {
|
let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?;
|
||||||
let quota_checker = QuotaChecker::new(metadata_sys.clone());
|
|
||||||
match quota_checker
|
|
||||||
.check_quota(&bucket, QuotaOperation::PutObject, obj_info.size.max(0) as u64)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(check_result) if !check_result.allowed => {
|
|
||||||
let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await;
|
|
||||||
let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await;
|
|
||||||
return Err(S3Error::with_message(
|
|
||||||
S3ErrorCode::InvalidRequest,
|
|
||||||
format!(
|
|
||||||
"Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes",
|
|
||||||
check_result.current_usage.unwrap_or(0),
|
|
||||||
check_result.quota_limit.unwrap_or(0)
|
|
||||||
),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
warn!("Quota check failed for bucket {} after multipart completion: {}", bucket, err);
|
|
||||||
}
|
|
||||||
Ok(_) => {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let committed_size = if opts.replication_request {
|
|
||||||
obj_info.size.max(0) as u64
|
|
||||||
} else {
|
|
||||||
quota_accounting_object_size(&obj_info, opts.quota_admission.is_some())?
|
|
||||||
};
|
|
||||||
if versioned {
|
if versioned {
|
||||||
record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await;
|
record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await;
|
||||||
} else {
|
} else {
|
||||||
@@ -817,6 +769,20 @@ impl DefaultMultipartUsecase {
|
|||||||
let ciphertext_passthrough = replication_authorized
|
let ciphertext_passthrough = replication_authorized
|
||||||
&& get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true")
|
&& get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true")
|
||||||
&& rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some();
|
&& rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some();
|
||||||
|
if ciphertext_passthrough && let Some(metadata_sys) = self.bucket_metadata_sys() {
|
||||||
|
let check_result = map_quota_check_outcome(
|
||||||
|
&bucket,
|
||||||
|
QuotaChecker::new(metadata_sys)
|
||||||
|
.check_quota(&bucket, QuotaOperation::PutObject, 0)
|
||||||
|
.await,
|
||||||
|
)?;
|
||||||
|
if check_result.quota_limit.is_some() {
|
||||||
|
return Err(S3Error::with_message(
|
||||||
|
S3ErrorCode::InvalidRequest,
|
||||||
|
"SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
if ciphertext_passthrough {
|
if ciphertext_passthrough {
|
||||||
insert_str(&mut metadata, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, "true".to_string());
|
insert_str(&mut metadata, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, "true".to_string());
|
||||||
}
|
}
|
||||||
@@ -1650,6 +1616,24 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(quota_accounting_object_size(&info, true).expect("logical size should resolve"), 8192);
|
assert_eq!(quota_accounting_object_size(&info, true).expect("logical size should resolve"), 8192);
|
||||||
assert_eq!(quota_accounting_object_size(&info, false).expect("logical size should resolve"), 8192);
|
assert_eq!(quota_accounting_object_size(&info, false).expect("logical size should resolve"), 8192);
|
||||||
|
|
||||||
|
let mut poisoned_metadata = HashMap::new();
|
||||||
|
insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string());
|
||||||
|
insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string());
|
||||||
|
let poisoned = ObjectInfo {
|
||||||
|
size: 17,
|
||||||
|
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||||
|
size: 4096,
|
||||||
|
actual_size: 4096,
|
||||||
|
..Default::default()
|
||||||
|
}]),
|
||||||
|
user_defined: Arc::new(poisoned_metadata),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
quota_accounting_object_size(&poisoned, true).expect("persisted part size must be charged"),
|
||||||
|
4096
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2058,30 +2042,14 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial_test::serial]
|
#[serial_test::serial]
|
||||||
async fn compressed_complete_records_logical_quota_usage_and_overwrite_delta() {
|
async fn compressed_complete_records_logical_quota_usage_and_overwrite_delta() {
|
||||||
use crate::app::storage_api::multipart_usecase::bucket::quota::BucketQuota;
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("compressed-complete-quota", 16_384).await;
|
||||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
|
||||||
use crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test;
|
|
||||||
|
|
||||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
|
||||||
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
|
||||||
|
|
||||||
let bucket = format!("compressed-complete-quota-{}", Uuid::new_v4());
|
|
||||||
let object = "object";
|
let object = "object";
|
||||||
store
|
|
||||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
|
||||||
.await
|
|
||||||
.expect("create compressed quota bucket");
|
|
||||||
seed_bucket_usage_memory_for_test(&bucket, 0).await;
|
|
||||||
|
|
||||||
let usecase = DefaultMultipartUsecase::from_global();
|
let usecase = DefaultMultipartUsecase::from_global();
|
||||||
let metadata_sys = usecase
|
let metadata_sys = usecase
|
||||||
.bucket_metadata_sys()
|
.bucket_metadata_sys()
|
||||||
.expect("test app context should expose bucket metadata");
|
.expect("test app context should expose bucket metadata");
|
||||||
let mut quota_checker = QuotaChecker::new(metadata_sys);
|
let quota_checker = QuotaChecker::new(metadata_sys);
|
||||||
quota_checker
|
|
||||||
.set_quota_config(&bucket, BucketQuota::new(Some(16_384)))
|
|
||||||
.await
|
|
||||||
.expect("configure bucket quota");
|
|
||||||
|
|
||||||
for (actual_size, payload_byte) in [(8192_i64, 0x61), (4096_i64, 0x62)] {
|
for (actual_size, payload_byte) in [(8192_i64, 0x61), (4096_i64, 0x62)] {
|
||||||
let mut create_opts = ObjectOptions::default();
|
let mut create_opts = ObjectOptions::default();
|
||||||
@@ -2126,6 +2094,78 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() {
|
||||||
|
let (_store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await;
|
||||||
|
let usecase = DefaultMultipartUsecase::from_global();
|
||||||
|
let input = CreateMultipartUploadInput::builder()
|
||||||
|
.bucket(bucket)
|
||||||
|
.key("object".to_string())
|
||||||
|
.build()
|
||||||
|
.expect("create multipart request should build");
|
||||||
|
let mut request = build_request(input, Method::POST);
|
||||||
|
insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||||
|
request
|
||||||
|
.headers
|
||||||
|
.insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256"));
|
||||||
|
request.extensions.insert(crate::storage::access::ReqInfo {
|
||||||
|
replication_request_authorized: true,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
|
||||||
|
let err = usecase
|
||||||
|
.execute_create_multipart_upload(request)
|
||||||
|
.await
|
||||||
|
.expect_err("quota-enabled ciphertext multipart replication should fail before upload creation");
|
||||||
|
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn concurrent_completions_share_durable_bucket_quota_reservations() {
|
||||||
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-complete-quota", 6000).await;
|
||||||
|
|
||||||
|
let usecase = DefaultMultipartUsecase::from_global();
|
||||||
|
|
||||||
|
let mut inputs = Vec::new();
|
||||||
|
for object in ["first", "second"] {
|
||||||
|
let upload = store
|
||||||
|
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("create concurrent multipart upload");
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![0x71; 4096]);
|
||||||
|
let part = store
|
||||||
|
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("stage concurrent multipart part");
|
||||||
|
inputs.push(
|
||||||
|
CompleteMultipartUploadInput::builder()
|
||||||
|
.bucket(bucket.clone())
|
||||||
|
.key(object.to_string())
|
||||||
|
.upload_id(upload.upload_id)
|
||||||
|
.multipart_upload(Some(CompletedMultipartUpload {
|
||||||
|
parts: Some(vec![CompletedPart {
|
||||||
|
part_number: Some(1),
|
||||||
|
e_tag: part.etag.map(|etag| to_s3s_etag(&etag)),
|
||||||
|
..Default::default()
|
||||||
|
}]),
|
||||||
|
}))
|
||||||
|
.build()
|
||||||
|
.expect("build concurrent completion input"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let first_usecase = usecase.clone();
|
||||||
|
let first = first_usecase.execute_complete_multipart_upload(build_request(inputs.remove(0), Method::POST));
|
||||||
|
let second = usecase.execute_complete_multipart_upload(build_request(inputs.remove(0), Method::POST));
|
||||||
|
let (first, second) = tokio::join!(first, second);
|
||||||
|
|
||||||
|
assert_eq!(usize::from(first.is_ok()) + usize::from(second.is_ok()), 1);
|
||||||
|
let denied = first.err().or_else(|| second.err()).expect("one completion must be denied");
|
||||||
|
assert_eq!(denied.code(), &S3ErrorCode::InvalidRequest);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial_test::serial]
|
#[serial_test::serial]
|
||||||
async fn rejected_empty_parts_preserve_existing_object_and_staging() {
|
async fn rejected_empty_parts_preserve_existing_object_and_staging() {
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ use super::storage_api::object_usecase::access::{
|
|||||||
has_bypass_governance_header, load_bucket_generation_from_store, recursive_force_delete_is_authorized,
|
has_bypass_governance_header, load_bucket_generation_from_store, recursive_force_delete_is_authorized,
|
||||||
replication_request_authorized, req_info_mut, req_info_ref,
|
replication_request_authorized, req_info_mut, req_info_ref,
|
||||||
};
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::storage_api::object_usecase::bucket::quota::BucketQuota;
|
||||||
use super::storage_api::object_usecase::bucket::quota::checker::QuotaChecker;
|
use super::storage_api::object_usecase::bucket::quota::checker::QuotaChecker;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use super::storage_api::object_usecase::bucket::replication::{ReplicationState, replication_statuses_map};
|
use super::storage_api::object_usecase::bucket::replication::{ReplicationState, replication_statuses_map};
|
||||||
@@ -63,8 +65,9 @@ use super::storage_api::object_usecase::contract::namespace::NamespaceLocking;
|
|||||||
use super::storage_api::object_usecase::contract::object::{ObjectIO as _, ObjectOperations as _};
|
use super::storage_api::object_usecase::contract::object::{ObjectIO as _, ObjectOperations as _};
|
||||||
use super::storage_api::object_usecase::contract::range::HTTPRangeSpec;
|
use super::storage_api::object_usecase::contract::range::HTTPRangeSpec;
|
||||||
use super::storage_api::object_usecase::data_usage::{
|
use super::storage_api::object_usecase::data_usage::{
|
||||||
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
|
quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
|
||||||
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory,
|
record_bucket_object_version_write_memory, record_bucket_object_write_memory,
|
||||||
|
record_bucket_object_write_unknown_previous_memory,
|
||||||
};
|
};
|
||||||
use super::storage_api::object_usecase::deadlock_detector;
|
use super::storage_api::object_usecase::deadlock_detector;
|
||||||
use super::storage_api::object_usecase::ecfs::FS;
|
use super::storage_api::object_usecase::ecfs::FS;
|
||||||
@@ -134,6 +137,8 @@ use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_fo
|
|||||||
use rustfs_s3select_api::object_store::bytes_stream;
|
use rustfs_s3select_api::object_store::bytes_stream;
|
||||||
use rustfs_targets::{EventName, get_request_host, get_request_port, get_request_user_agent};
|
use rustfs_targets::{EventName, get_request_host, get_request_port, get_request_user_agent};
|
||||||
use rustfs_utils::CompressionAlgorithm;
|
use rustfs_utils::CompressionAlgorithm;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_utils::http::insert_header;
|
||||||
use rustfs_utils::http::{
|
use rustfs_utils::http::{
|
||||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE,
|
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE,
|
||||||
SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP,
|
SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP,
|
||||||
@@ -537,6 +542,43 @@ pub(super) fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckRe
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn apply_quota_admission(opts: &mut ObjectOptions, result: &QuotaCheckResult) -> S3Result<()> {
|
||||||
|
let Some(quota_limit) = result.quota_limit else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
let Some(current_usage) = result.current_usage else {
|
||||||
|
return Err(S3Error::with_message(
|
||||||
|
S3ErrorCode::ServiceUnavailable,
|
||||||
|
"Bucket quota check temporarily unavailable, please retry".to_string(),
|
||||||
|
));
|
||||||
|
};
|
||||||
|
if current_usage <= quota_limit {
|
||||||
|
let _ = opts.set_quota_admission(current_usage, quota_limit);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ensure_object_size_within_quota(result: &QuotaCheckResult, new_size: u64) -> S3Result<()> {
|
||||||
|
let (Some(current_usage), Some(quota_limit)) = (result.current_usage, result.quota_limit) else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
if new_size > quota_limit {
|
||||||
|
return Err(S3Error::with_message(
|
||||||
|
S3ErrorCode::InvalidRequest,
|
||||||
|
format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result<u64> {
|
||||||
|
match quota_object_size(info) {
|
||||||
|
Ok(size) => Ok(size),
|
||||||
|
Err(err) if fail_closed => Err(ApiError::from(err).into()),
|
||||||
|
Err(_) => Ok(info.size.max(0) as u64),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn request_uses_aws_chunked(headers: &HeaderMap) -> bool {
|
fn request_uses_aws_chunked(headers: &HeaderMap) -> bool {
|
||||||
let has_aws_chunked = |header_name: &str| {
|
let has_aws_chunked = |header_name: &str| {
|
||||||
headers
|
headers
|
||||||
@@ -3854,13 +3896,6 @@ fn put_object_extract_limits() -> ArchiveLimits {
|
|||||||
ArchiveLimits::default()
|
ArchiveLimits::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn put_object_extract_quota_exceeded(current_usage: u64, quota_limit: u64) -> S3Error {
|
|
||||||
S3Error::with_message(
|
|
||||||
S3ErrorCode::InvalidRequest,
|
|
||||||
format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn validate_put_object_extract_entry_count(count: usize, limits: ArchiveLimits) -> S3Result<()> {
|
fn validate_put_object_extract_entry_count(count: usize, limits: ArchiveLimits) -> S3Result<()> {
|
||||||
if count > limits.max_entries {
|
if count > limits.max_entries {
|
||||||
return Err(s3_error!(
|
return Err(s3_error!(
|
||||||
@@ -4083,12 +4118,12 @@ impl DefaultObjectUsecase {
|
|||||||
.unwrap_or_else(|| RustFSBufferConfig::default().base_config.default_unknown)
|
.unwrap_or_else(|| RustFSBufferConfig::default().base_config.default_unknown)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn check_bucket_quota(&self, bucket: &str, op: QuotaOperation, size: u64) -> S3Result<()> {
|
async fn check_bucket_quota(&self, bucket: &str, op: QuotaOperation, size: u64) -> S3Result<Option<QuotaCheckResult>> {
|
||||||
let Some(metadata_sys) = self.bucket_metadata_sys() else {
|
let Some(metadata_sys) = self.bucket_metadata_sys() else {
|
||||||
return Ok(());
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let quota_checker = QuotaChecker::new(metadata_sys);
|
let quota_checker = QuotaChecker::new(metadata_sys);
|
||||||
map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(|_| ())
|
map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(Some)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_memory_bytes_blob(
|
fn build_memory_bytes_blob(
|
||||||
@@ -5593,8 +5628,16 @@ impl DefaultObjectUsecase {
|
|||||||
// Resolve the authoritative decoded/plain object length (rejecting negative/unknown) before anything else consumes it.
|
// Resolve the authoritative decoded/plain object length (rejecting negative/unknown) before anything else consumes it.
|
||||||
let mut size = resolve_put_object_authoritative_size(&req.headers, content_length)?;
|
let mut size = resolve_put_object_authoritative_size(&req.headers, content_length)?;
|
||||||
|
|
||||||
// Bucket-quota admission runs exactly once, and only now that the authoritative object length is known. `size` is the same basis the settle phase records via ObjectInfo.size (actual, pre-compression/pre-encryption logical size), NOT the aws-chunked wire Content-Length. When no quota is configured this stays a zero-extra-I/O fast path; once a hard quota is set, checker/config/usage faults fail closed with a retryable error.
|
// The app check preserves the existing S3 error contract; the storage
|
||||||
self.check_bucket_quota(&bucket, quota_operation, size as u64).await?;
|
// commit path reserves the exact net logical growth under its locks.
|
||||||
|
let quota_check = self.check_bucket_quota(&bucket, quota_operation, 0).await?;
|
||||||
|
let quota_enabled = quota_check.as_ref().is_some_and(|result| result.quota_limit.is_some());
|
||||||
|
if quota_enabled && ciphertext_passthrough {
|
||||||
|
return Err(S3Error::with_message(
|
||||||
|
S3ErrorCode::InvalidRequest,
|
||||||
|
"SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
let ingress_stage_start = std::time::Instant::now();
|
let ingress_stage_start = std::time::Instant::now();
|
||||||
let should_compress =
|
let should_compress =
|
||||||
@@ -5760,6 +5803,9 @@ impl DefaultObjectUsecase {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(ApiError::from)?;
|
.map_err(ApiError::from)?;
|
||||||
|
if let Some(quota_check) = quota_check.as_ref() {
|
||||||
|
apply_quota_admission(&mut opts, quota_check)?;
|
||||||
|
}
|
||||||
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
|
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
|
||||||
apply_put_request_object_lock_opts(
|
apply_put_request_object_lock_opts(
|
||||||
&bucket,
|
&bucket,
|
||||||
@@ -5778,7 +5824,7 @@ impl DefaultObjectUsecase {
|
|||||||
// replication), the lookup is skipped and accounting is backfilled from
|
// replication), the lookup is skipped and accounting is backfilled from
|
||||||
// the dst xl.meta that rename_data already reads, saving a full-disk
|
// the dst xl.meta that rename_data already reads, saving a full-disk
|
||||||
// metadata fanout per PUT.
|
// metadata fanout per PUT.
|
||||||
let prelookup_required = version_id.is_some() || object_lock_checks_required(&bucket).await;
|
let prelookup_required = quota_enabled || version_id.is_some() || object_lock_checks_required(&bucket).await;
|
||||||
// Outer None = prelookup skipped (accounting comes from the commit
|
// Outer None = prelookup skipped (accounting comes from the commit
|
||||||
// backfill); Some(inner) = the previous current size as observed by the
|
// backfill); Some(inner) = the previous current size as observed by the
|
||||||
// lookup, with the pre-#1009 semantics kept bit-for-bit.
|
// lookup, with the pre-#1009 semantics kept bit-for-bit.
|
||||||
@@ -5795,7 +5841,11 @@ impl DefaultObjectUsecase {
|
|||||||
Some(match previous_current_info {
|
Some(match previous_current_info {
|
||||||
Ok(existing_obj_info) => {
|
Ok(existing_obj_info) => {
|
||||||
validate_existing_object_lock_for_write(&existing_obj_info, &opts)?;
|
validate_existing_object_lock_for_write(&existing_obj_info, &opts)?;
|
||||||
Some(existing_obj_info.size.max(0) as u64)
|
Some(if quota_enabled {
|
||||||
|
quota_object_size(&existing_obj_info).map_err(ApiError::from)?
|
||||||
|
} else {
|
||||||
|
existing_obj_info.size.max(0) as u64
|
||||||
|
})
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||||
@@ -5809,6 +5859,12 @@ impl DefaultObjectUsecase {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let actual_size = size;
|
let actual_size = size;
|
||||||
|
if !ciphertext_passthrough && let Some(quota_check) = quota_check.as_ref() {
|
||||||
|
ensure_object_size_within_quota(
|
||||||
|
quota_check,
|
||||||
|
u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
let mut md5hex = if let Some(base64_md5) = content_md5 {
|
let mut md5hex = if let Some(base64_md5) = content_md5 {
|
||||||
let md5 = base64_simd::STANDARD
|
let md5 = base64_simd::STANDARD
|
||||||
@@ -6077,12 +6133,13 @@ impl DefaultObjectUsecase {
|
|||||||
// backfill reproduces the lookup's observation bit for bit (latest
|
// backfill reproduces the lookup's observation bit for bit (latest
|
||||||
// version's ObjectInfo.size — 0 for a delete-marker latest — or
|
// version's ObjectInfo.size — 0 for a delete-marker latest — or
|
||||||
// not-found → None).
|
// not-found → None).
|
||||||
|
let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?;
|
||||||
match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size)) {
|
match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size)) {
|
||||||
Some(previous_current_size) => {
|
Some(previous_current_size) => {
|
||||||
if put_versioned {
|
if put_versioned {
|
||||||
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
|
record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await;
|
||||||
} else {
|
} else {
|
||||||
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
|
record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
@@ -6098,7 +6155,7 @@ impl DefaultObjectUsecase {
|
|||||||
put_versioned,
|
put_versioned,
|
||||||
"put_object old-size backfill unknown; recording degraded usage delta"
|
"put_object old-size backfill unknown; recording degraded usage delta"
|
||||||
);
|
);
|
||||||
record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, put_versioned).await;
|
record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, put_versioned).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -7172,7 +7229,7 @@ impl DefaultObjectUsecase {
|
|||||||
if _self_copy_lock_guard.is_some() {
|
if _self_copy_lock_guard.is_some() {
|
||||||
current_opts.no_lock = true;
|
current_opts.no_lock = true;
|
||||||
}
|
}
|
||||||
let previous_current_size = match store.get_object_info(&bucket, &key, ¤t_opts).await {
|
let previous_current_sizes = match store.get_object_info(&bucket, &key, ¤t_opts).await {
|
||||||
Ok(existing_obj_info) => {
|
Ok(existing_obj_info) => {
|
||||||
validate_existing_object_lock_for_write(&existing_obj_info, &dst_opts)?;
|
validate_existing_object_lock_for_write(&existing_obj_info, &dst_opts)?;
|
||||||
if let Some(expected) = expected_current_version_id.as_deref()
|
if let Some(expected) = expected_current_version_id.as_deref()
|
||||||
@@ -7180,7 +7237,7 @@ impl DefaultObjectUsecase {
|
|||||||
{
|
{
|
||||||
return Err(s3_error!(PreconditionFailed));
|
return Err(s3_error!(PreconditionFailed));
|
||||||
}
|
}
|
||||||
Some(existing_obj_info.size.max(0) as u64)
|
Some((existing_obj_info.size.max(0) as u64, quota_object_size(&existing_obj_info)))
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
if expected_current_version_id.is_some() {
|
if expected_current_version_id.is_some() {
|
||||||
@@ -7501,8 +7558,23 @@ impl DefaultObjectUsecase {
|
|||||||
|
|
||||||
src_info.user_defined = Arc::new(user_defined);
|
src_info.user_defined = Arc::new(user_defined);
|
||||||
|
|
||||||
self.check_bucket_quota(&bucket, QuotaOperation::CopyObject, src_info.size as u64)
|
let quota_check = self.check_bucket_quota(&bucket, QuotaOperation::CopyObject, 0).await?;
|
||||||
.await?;
|
let quota_enabled = quota_check.as_ref().is_some_and(|result| result.quota_limit.is_some());
|
||||||
|
if let Some(quota_check) = quota_check.as_ref() {
|
||||||
|
apply_quota_admission(&mut dst_opts, quota_check)?;
|
||||||
|
}
|
||||||
|
let previous_current_size = match previous_current_sizes {
|
||||||
|
Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size),
|
||||||
|
Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err).into()),
|
||||||
|
Some((physical_size, _)) => Some(physical_size),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
if let Some(quota_check) = quota_check.as_ref() {
|
||||||
|
ensure_object_size_within_quota(
|
||||||
|
quota_check,
|
||||||
|
u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
let has_bucket_metadata = self.bucket_metadata_sys().is_some();
|
let has_bucket_metadata = self.bucket_metadata_sys().is_some();
|
||||||
let cache_adapter = self.object_data_cache();
|
let cache_adapter = self.object_data_cache();
|
||||||
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
|
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
|
||||||
@@ -7526,10 +7598,11 @@ impl DefaultObjectUsecase {
|
|||||||
let dest_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
|
let dest_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
|
||||||
// Update quota tracking after successful copy
|
// Update quota tracking after successful copy
|
||||||
if has_bucket_metadata {
|
if has_bucket_metadata {
|
||||||
|
let committed_size = quota_accounting_object_size(&oi, quota_enabled)?;
|
||||||
if dest_versioned {
|
if dest_versioned {
|
||||||
record_bucket_object_version_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await;
|
record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await;
|
||||||
} else {
|
} else {
|
||||||
record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await;
|
record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -9117,8 +9190,7 @@ impl DefaultObjectUsecase {
|
|||||||
}
|
}
|
||||||
validate_object_key(&key, "PUT")?;
|
validate_object_key(&key, "PUT")?;
|
||||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||||
self.check_bucket_quota(&bucket, QuotaOperation::PutObject, size as u64)
|
let _ = self.check_bucket_quota(&bucket, QuotaOperation::PutObject, 0).await?;
|
||||||
.await?;
|
|
||||||
|
|
||||||
// Apply adaptive buffer sizing based on file size for optimal streaming performance.
|
// Apply adaptive buffer sizing based on file size for optimal streaming performance.
|
||||||
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
|
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
|
||||||
@@ -9174,14 +9246,17 @@ impl DefaultObjectUsecase {
|
|||||||
|
|
||||||
let extract_options = resolve_put_object_extract_options(&req.headers)?;
|
let extract_options = resolve_put_object_extract_options(&req.headers)?;
|
||||||
let extract_limits = put_object_extract_limits();
|
let extract_limits = put_object_extract_limits();
|
||||||
let extract_quota_snapshot = if let Some(metadata_sys) = self.bucket_metadata_sys() {
|
let extract_quota_check = if let Some(metadata_sys) = self.bucket_metadata_sys() {
|
||||||
let quota_checker = QuotaChecker::new(metadata_sys);
|
let quota_checker = QuotaChecker::new(metadata_sys);
|
||||||
let check_result =
|
let check_result =
|
||||||
map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
|
map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?;
|
||||||
check_result.current_usage.zip(check_result.quota_limit)
|
Some(check_result)
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
let extract_quota_enabled = extract_quota_check
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|result| result.quota_limit.is_some());
|
||||||
let version_id = match event_version_id {
|
let version_id = match event_version_id {
|
||||||
Some(v) => v.to_string(),
|
Some(v) => v.to_string(),
|
||||||
None => String::new(),
|
None => String::new(),
|
||||||
@@ -9261,11 +9336,6 @@ impl DefaultObjectUsecase {
|
|||||||
.checked_add(entry_size)
|
.checked_add(entry_size)
|
||||||
.ok_or_else(|| s3_error!(InvalidArgument, "Archive total unpacked size overflowed while processing entries"))?;
|
.ok_or_else(|| s3_error!(InvalidArgument, "Archive total unpacked size overflowed while processing entries"))?;
|
||||||
validate_put_object_extract_total_size(total_unpacked_size, extract_limits)?;
|
validate_put_object_extract_total_size(total_unpacked_size, extract_limits)?;
|
||||||
if let Some((current_usage, quota_limit)) = extract_quota_snapshot
|
|
||||||
&& current_usage.saturating_add(total_unpacked_size) > quota_limit
|
|
||||||
{
|
|
||||||
return Err(put_object_extract_quota_exceeded(current_usage, quota_limit));
|
|
||||||
}
|
|
||||||
let mut size =
|
let mut size =
|
||||||
i64::try_from(entry_size).map_err(|_| s3_error!(InvalidArgument, "Archive entry size does not fit into i64"))?;
|
i64::try_from(entry_size).map_err(|_| s3_error!(InvalidArgument, "Archive entry size does not fit into i64"))?;
|
||||||
// mtime 0 means "unset" in tar headers, and xl.meta cannot represent an
|
// mtime 0 means "unset" in tar headers, and xl.meta cannot represent an
|
||||||
@@ -9309,6 +9379,9 @@ impl DefaultObjectUsecase {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(ApiError::from)?;
|
.map_err(ApiError::from)?;
|
||||||
|
if let Some(quota_check) = extract_quota_check.as_ref() {
|
||||||
|
apply_quota_admission(&mut opts, quota_check)?;
|
||||||
|
}
|
||||||
opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id;
|
opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id;
|
||||||
opts.object_lock_config_snapshot = Some(Arc::clone(&object_lock_config_snapshot));
|
opts.object_lock_config_snapshot = Some(Arc::clone(&object_lock_config_snapshot));
|
||||||
let pax_authorization =
|
let pax_authorization =
|
||||||
@@ -9452,19 +9525,18 @@ impl DefaultObjectUsecase {
|
|||||||
return Err(ApiError::from(e).into());
|
return Err(ApiError::from(e).into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let committed_size = quota_accounting_object_size(&obj_info, extract_quota_enabled)?;
|
||||||
let extract_versioned = BucketVersioningSys::prefix_enabled(&bucket, &fpath).await;
|
let extract_versioned = BucketVersioningSys::prefix_enabled(&bucket, &fpath).await;
|
||||||
match previous_current_size_from_backfill(backfilled_old_current_size) {
|
match previous_current_size_from_backfill(backfilled_old_current_size) {
|
||||||
Some(previous_current_size) => {
|
Some(previous_current_size) => {
|
||||||
if extract_versioned {
|
if extract_versioned {
|
||||||
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64)
|
record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await;
|
||||||
.await;
|
|
||||||
} else {
|
} else {
|
||||||
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
|
record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, extract_versioned)
|
record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, extract_versioned).await;
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await;
|
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await;
|
||||||
@@ -15611,13 +15683,6 @@ mod tests {
|
|||||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn put_object_extract_quota_exceeded_matches_existing_error_shape() {
|
|
||||||
let err = put_object_extract_quota_exceeded(10, 8);
|
|
||||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
|
||||||
assert_eq!(err.message(), Some("Bucket quota exceeded. Current usage: 10 bytes, limit: 8 bytes"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn execute_put_object_rejects_post_object_sse_kms_from_input() {
|
async fn execute_put_object_rejects_post_object_sse_kms_from_input() {
|
||||||
let input = PutObjectInput::builder()
|
let input = PutObjectInput::builder()
|
||||||
@@ -17343,6 +17408,44 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn quota_rejects_ciphertext_replication_before_polling_the_body() {
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
|
let (_store, bucket) =
|
||||||
|
crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-replication-early-reject", 4096).await;
|
||||||
|
let body_polled = Arc::new(AtomicBool::new(false));
|
||||||
|
let body_polled_in_stream = Arc::clone(&body_polled);
|
||||||
|
let body = StreamingBlob::wrap(futures::stream::once(async move {
|
||||||
|
body_polled_in_stream.store(true, Ordering::Release);
|
||||||
|
Ok::<Bytes, std::io::Error>(Bytes::from_static(b"ciphertext"))
|
||||||
|
}));
|
||||||
|
let input = PutObjectInput::builder()
|
||||||
|
.bucket(bucket)
|
||||||
|
.key("object".to_string())
|
||||||
|
.body(Some(body))
|
||||||
|
.content_length(Some(10))
|
||||||
|
.build()
|
||||||
|
.expect("ciphertext replication PUT input should build");
|
||||||
|
let mut request = build_request(input, Method::PUT);
|
||||||
|
insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||||
|
request
|
||||||
|
.headers
|
||||||
|
.insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256"));
|
||||||
|
request.extensions.insert(crate::storage::access::ReqInfo {
|
||||||
|
replication_request_authorized: true,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
|
||||||
|
let err = DefaultObjectUsecase::from_global()
|
||||||
|
.execute_put_object(&FS::new(), request)
|
||||||
|
.await
|
||||||
|
.expect_err("quota-enabled ciphertext replication should fail at ingress");
|
||||||
|
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||||
|
assert!(!body_polled.load(Ordering::Acquire), "rejected ciphertext body must not be consumed");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn quota_admission_allows_within_limit() {
|
fn quota_admission_allows_within_limit() {
|
||||||
let result = map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write");
|
let result = map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write");
|
||||||
@@ -17353,6 +17456,340 @@ mod tests {
|
|||||||
assert_eq!(result.remaining, Some(512));
|
assert_eq!(result.remaining, Some(512));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn concurrent_puts_share_durable_bucket_quota_reservations() {
|
||||||
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-put-quota", 6000).await;
|
||||||
|
|
||||||
|
let first_opts = ObjectOptions::default();
|
||||||
|
let second_opts = ObjectOptions::default();
|
||||||
|
let first_store = Arc::clone(&store);
|
||||||
|
let first_bucket = bucket.clone();
|
||||||
|
let first = tokio::spawn(async move {
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![0x73; 4096]);
|
||||||
|
first_store.put_object(&first_bucket, "first", &mut reader, &first_opts).await
|
||||||
|
});
|
||||||
|
let second = tokio::spawn(async move {
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![0x74; 4096]);
|
||||||
|
store.put_object(&bucket, "second", &mut reader, &second_opts).await
|
||||||
|
});
|
||||||
|
let (first, second) = tokio::join!(first, second);
|
||||||
|
let first = first.expect("first PUT task should not panic");
|
||||||
|
let second = second.expect("second PUT task should not panic");
|
||||||
|
|
||||||
|
assert_eq!(usize::from(first.is_ok()) + usize::from(second.is_ok()), 1);
|
||||||
|
let denied = first.err().or_else(|| second.err()).expect("one PUT must be denied");
|
||||||
|
assert!(matches!(
|
||||||
|
denied,
|
||||||
|
StorageError::QuotaExceeded {
|
||||||
|
current: 4096,
|
||||||
|
limit: 6000
|
||||||
|
}
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn concurrent_within_limit_puts_keep_independent_mutation_fences() {
|
||||||
|
use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||||
|
|
||||||
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-fence-quota", 8192).await;
|
||||||
|
let first_barrier = PutObjectCommitBarrier::install(&bucket, "first", PutObjectCommitPause::BeforeQuotaRename);
|
||||||
|
let second_barrier = PutObjectCommitBarrier::install(&bucket, "second", PutObjectCommitPause::BeforeQuotaRename);
|
||||||
|
|
||||||
|
let first_store = Arc::clone(&store);
|
||||||
|
let first_bucket = bucket.clone();
|
||||||
|
let first = tokio::spawn(async move {
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![0x75; 4096]);
|
||||||
|
first_store
|
||||||
|
.put_object(&first_bucket, "first", &mut reader, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
let second_store = Arc::clone(&store);
|
||||||
|
let second_bucket = bucket.clone();
|
||||||
|
let second = tokio::spawn(async move {
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![0x76; 4096]);
|
||||||
|
second_store
|
||||||
|
.put_object(&second_bucket, "second", &mut reader, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
|
||||||
|
first_barrier.wait_until_paused().await;
|
||||||
|
second_barrier.wait_until_paused().await;
|
||||||
|
first_barrier.release();
|
||||||
|
second_barrier.release();
|
||||||
|
|
||||||
|
first
|
||||||
|
.await
|
||||||
|
.expect("first PUT task should not panic")
|
||||||
|
.expect("first within-limit PUT should commit");
|
||||||
|
second
|
||||||
|
.await
|
||||||
|
.expect("second PUT task should not panic")
|
||||||
|
.expect("second within-limit PUT should commit");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn durable_quota_reclaims_overwrites_and_deleted_bytes() {
|
||||||
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("quota-delta-reconcile", 4096).await;
|
||||||
|
|
||||||
|
for byte in [0x41, 0x42] {
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![byte; 4096]);
|
||||||
|
store
|
||||||
|
.put_object(&bucket, "object", &mut reader, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("same-size overwrite must consume no additional quota");
|
||||||
|
}
|
||||||
|
|
||||||
|
store
|
||||||
|
.delete_object(&bucket, "object", ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("delete quota-tracked object");
|
||||||
|
let mut replacement = PutObjReader::from_vec(vec![0x43; 4096]);
|
||||||
|
store
|
||||||
|
.put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("deleted bytes must be reclaimed before rejecting a replacement");
|
||||||
|
|
||||||
|
let mut excess = PutObjReader::from_vec(vec![0x44]);
|
||||||
|
let err = store
|
||||||
|
.put_object(&bucket, "excess", &mut excess, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("one byte beyond the reclaimed exact quota must be denied");
|
||||||
|
assert!(matches!(
|
||||||
|
err,
|
||||||
|
StorageError::QuotaExceeded {
|
||||||
|
current: 4096,
|
||||||
|
limit: 4096
|
||||||
|
}
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn cancelled_put_releases_durable_quota_reservation() {
|
||||||
|
use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||||
|
|
||||||
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("cancelled-put-quota", 4096).await;
|
||||||
|
|
||||||
|
let barrier = PutObjectCommitBarrier::install(&bucket, "cancelled", PutObjectCommitPause::AfterQuotaReservation);
|
||||||
|
let cancelled_store = Arc::clone(&store);
|
||||||
|
let cancelled_bucket = bucket.clone();
|
||||||
|
let cancelled = tokio::spawn(async move {
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![0x51; 4096]);
|
||||||
|
cancelled_store
|
||||||
|
.put_object(&cancelled_bucket, "cancelled", &mut reader, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
barrier.wait_until_paused().await;
|
||||||
|
cancelled.abort();
|
||||||
|
let cancelled_result = cancelled.await;
|
||||||
|
assert!(cancelled_result.is_err(), "the paused request must be cancelled");
|
||||||
|
drop(barrier);
|
||||||
|
|
||||||
|
let mut replacement = PutObjReader::from_vec(vec![0x52; 4096]);
|
||||||
|
store
|
||||||
|
.put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("cancelling before commit must release the complete reservation");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn cancelled_put_after_commit_marker_is_reconciled() {
|
||||||
|
use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||||
|
|
||||||
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("cancelled-spawned-put-quota", 4096).await;
|
||||||
|
let commit_barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename);
|
||||||
|
let first_store = Arc::clone(&store);
|
||||||
|
let first_bucket = bucket.clone();
|
||||||
|
let first = tokio::spawn(async move {
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![0x53; 4096]);
|
||||||
|
first_store
|
||||||
|
.put_object(&first_bucket, "object", &mut reader, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
commit_barrier.wait_until_paused().await;
|
||||||
|
first.abort();
|
||||||
|
assert!(first.await.is_err(), "the outer request task must be cancelled");
|
||||||
|
drop(commit_barrier);
|
||||||
|
|
||||||
|
store
|
||||||
|
.get_object_info(&bucket, "object", &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("cancelling before rename must not commit the object");
|
||||||
|
let mut replacement = PutObjReader::from_vec(vec![0x54; 4096]);
|
||||||
|
store
|
||||||
|
.put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("the next admission must reap the abandoned commit marker");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn committed_put_survives_quota_ledger_settlement_failure() {
|
||||||
|
use crate::app::storage_api::test::set_disk::{
|
||||||
|
PutObjectCommitBarrier, PutObjectCommitPause, fail_next_quota_ledger_save_for_test,
|
||||||
|
};
|
||||||
|
|
||||||
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("settlement-failure-quota", 4096).await;
|
||||||
|
let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename);
|
||||||
|
let put_store = Arc::clone(&store);
|
||||||
|
let put_bucket = bucket.clone();
|
||||||
|
let put = tokio::spawn(async move {
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![0x59; 4096]);
|
||||||
|
put_store
|
||||||
|
.put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
barrier.wait_until_paused().await;
|
||||||
|
fail_next_quota_ledger_save_for_test();
|
||||||
|
barrier.release();
|
||||||
|
put.await
|
||||||
|
.expect("PUT task should not panic")
|
||||||
|
.expect("a post-commit ledger failure must not change the successful write result");
|
||||||
|
let stored = store
|
||||||
|
.get_object_info(&bucket, "object", &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("the committed object must remain visible");
|
||||||
|
assert_eq!(stored.size, 4096);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn suspended_null_version_overwrite_uses_exact_quota_delta() {
|
||||||
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("suspended-version-quota", 6200).await;
|
||||||
|
let mut versioned_reader = PutObjReader::from_vec(vec![0x61; 4096]);
|
||||||
|
store
|
||||||
|
.put_object(
|
||||||
|
&bucket,
|
||||||
|
"object",
|
||||||
|
&mut versioned_reader,
|
||||||
|
&ObjectOptions {
|
||||||
|
versioned: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("write UUID version");
|
||||||
|
|
||||||
|
for (size, byte) in [(1024, 0x62), (2048, 0x63)] {
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![byte; size]);
|
||||||
|
store
|
||||||
|
.put_object(
|
||||||
|
&bucket,
|
||||||
|
"object",
|
||||||
|
&mut reader,
|
||||||
|
&ObjectOptions {
|
||||||
|
version_suspended: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("suspended write should replace only the exact null version");
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut excess = PutObjReader::from_vec(vec![0x64; 57]);
|
||||||
|
let err = store
|
||||||
|
.put_object(&bucket, "excess", &mut excess, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("UUID plus replacement null version must consume 6144 bytes");
|
||||||
|
assert!(matches!(
|
||||||
|
err,
|
||||||
|
StorageError::QuotaExceeded {
|
||||||
|
current: 6144,
|
||||||
|
limit: 6200
|
||||||
|
}
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn durable_quota_reservation_observes_lowered_config_revision() {
|
||||||
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("lowered-quota-revision", 8192).await;
|
||||||
|
let mut initial = PutObjReader::from_vec(vec![0x71; 4096]);
|
||||||
|
store
|
||||||
|
.put_object(&bucket, "initial", &mut initial, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect("write under original quota");
|
||||||
|
|
||||||
|
let metadata_sys = DefaultObjectUsecase::from_global()
|
||||||
|
.bucket_metadata_sys()
|
||||||
|
.expect("test app context should expose bucket metadata");
|
||||||
|
QuotaChecker::new(metadata_sys)
|
||||||
|
.set_quota_config(&bucket, BucketQuota::new(Some(4096)))
|
||||||
|
.await
|
||||||
|
.expect("lower bucket quota");
|
||||||
|
let mut excess = PutObjReader::from_vec(vec![0x72]);
|
||||||
|
let err = store
|
||||||
|
.put_object(&bucket, "excess", &mut excess, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("reservation must not use the stale larger quota revision");
|
||||||
|
assert!(matches!(
|
||||||
|
err,
|
||||||
|
StorageError::QuotaExceeded {
|
||||||
|
current: 4096,
|
||||||
|
limit: 4096
|
||||||
|
}
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn quota_enable_waits_for_unlimited_commit() {
|
||||||
|
use crate::app::storage_api::test::metadata_sys::ConfigWriteLockProbe;
|
||||||
|
use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||||
|
|
||||||
|
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("quota-config-fence", 8192).await;
|
||||||
|
let metadata_sys = DefaultObjectUsecase::from_global()
|
||||||
|
.bucket_metadata_sys()
|
||||||
|
.expect("test app context should expose bucket metadata");
|
||||||
|
QuotaChecker::new(Arc::clone(&metadata_sys))
|
||||||
|
.set_quota_config(&bucket, BucketQuota::new(None))
|
||||||
|
.await
|
||||||
|
.expect("clear quota before the fenced write");
|
||||||
|
let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::AfterQuotaReservation);
|
||||||
|
let put_store = Arc::clone(&store);
|
||||||
|
let put_bucket = bucket.clone();
|
||||||
|
let put = tokio::spawn(async move {
|
||||||
|
let mut reader = PutObjReader::from_vec(vec![0x73; 4096]);
|
||||||
|
put_store
|
||||||
|
.put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
barrier.wait_until_paused().await;
|
||||||
|
|
||||||
|
let update_probe = ConfigWriteLockProbe::install(&bucket);
|
||||||
|
let update_bucket = bucket.clone();
|
||||||
|
let update = tokio::spawn(async move {
|
||||||
|
QuotaChecker::new(metadata_sys)
|
||||||
|
.set_quota_config(&update_bucket, BucketQuota::new(Some(0)))
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
update_probe.wait_until_attempted().await;
|
||||||
|
assert!(
|
||||||
|
!update.is_finished(),
|
||||||
|
"quota mutation must wait for the reservation's metadata transaction guard"
|
||||||
|
);
|
||||||
|
|
||||||
|
barrier.release();
|
||||||
|
put.await
|
||||||
|
.expect("PUT task should not panic")
|
||||||
|
.expect("the write linearized before the quota update must commit");
|
||||||
|
update
|
||||||
|
.await
|
||||||
|
.expect("quota update task should not panic")
|
||||||
|
.expect("quota update should proceed after commit");
|
||||||
|
|
||||||
|
let mut excess = PutObjReader::from_vec(vec![0x74]);
|
||||||
|
let err = store
|
||||||
|
.put_object(&bucket, "excess", &mut excess, &ObjectOptions::default())
|
||||||
|
.await
|
||||||
|
.expect_err("writes after the zero-byte quota update must be denied");
|
||||||
|
assert!(matches!(err, StorageError::QuotaExceeded { current: 4096, limit: 0 }));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn quota_admission_rejects_over_limit() {
|
fn quota_admission_rejects_over_limit() {
|
||||||
let err = map_quota_check_outcome("bucket", Ok(quota_result(false))).expect_err("an over-limit result rejects the write");
|
let err = map_quota_check_outcome("bucket", Ok(quota_result(false))).expect_err("an over-limit result rejects the write");
|
||||||
@@ -17372,6 +17809,23 @@ mod tests {
|
|||||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
|
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn early_quota_filter_rejects_only_an_individually_impossible_object() {
|
||||||
|
let stale_full_usage = QuotaCheckResult {
|
||||||
|
allowed: true,
|
||||||
|
current_usage: Some(4096),
|
||||||
|
quota_limit: Some(4096),
|
||||||
|
operation_size: 0,
|
||||||
|
remaining: Some(0),
|
||||||
|
};
|
||||||
|
|
||||||
|
ensure_object_size_within_quota(&stale_full_usage, 4096)
|
||||||
|
.expect("commit-time ledger must decide whether stale usage was reclaimed");
|
||||||
|
let err = ensure_object_size_within_quota(&stale_full_usage, 4097)
|
||||||
|
.expect_err("an object larger than the whole quota can never fit");
|
||||||
|
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn quota_admission_fails_closed_on_unknown_authoritative_usage() {
|
fn quota_admission_fails_closed_on_unknown_authoritative_usage() {
|
||||||
let err = map_quota_check_outcome(
|
let err = map_quota_check_outcome(
|
||||||
|
|||||||
@@ -47,6 +47,8 @@ pub(crate) mod capacity {
|
|||||||
pub(crate) mod data_usage {
|
pub(crate) mod data_usage {
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
pub(crate) use crate::storage::storage_api::ecstore_data_usage::quota_object_size;
|
||||||
|
|
||||||
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
|
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
|
||||||
crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
|
crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
|
||||||
}
|
}
|
||||||
@@ -1212,4 +1214,13 @@ pub(crate) mod test {
|
|||||||
pub(crate) use crate::storage::storage_api::{
|
pub(crate) use crate::storage::storage_api::{
|
||||||
ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader,
|
ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader,
|
||||||
};
|
};
|
||||||
|
pub(crate) mod set_disk {
|
||||||
|
pub(crate) use crate::storage::storage_api::ecstore_set_disk::{
|
||||||
|
PutObjectCommitBarrier, PutObjectCommitPause, fail_next_quota_ledger_save_for_test,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) mod metadata_sys {
|
||||||
|
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::ConfigWriteLockProbe;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -398,6 +398,7 @@ impl From<QuotaError> for ApiError {
|
|||||||
QuotaError::ConfigNotFound { .. } => S3ErrorCode::NoSuchBucket,
|
QuotaError::ConfigNotFound { .. } => S3ErrorCode::NoSuchBucket,
|
||||||
QuotaError::UsageUnavailable { .. } => S3ErrorCode::ServiceUnavailable,
|
QuotaError::UsageUnavailable { .. } => S3ErrorCode::ServiceUnavailable,
|
||||||
QuotaError::InvalidConfig { .. } => S3ErrorCode::InvalidArgument,
|
QuotaError::InvalidConfig { .. } => S3ErrorCode::InvalidArgument,
|
||||||
|
QuotaError::StorageError(StorageError::NamespaceLockQuorumUnavailable { .. }) => S3ErrorCode::ServiceUnavailable,
|
||||||
QuotaError::StorageError(_) => S3ErrorCode::InternalError,
|
QuotaError::StorageError(_) => S3ErrorCode::InternalError,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -551,6 +552,20 @@ mod tests {
|
|||||||
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
|
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stale_quota_capability_maps_to_retryable_error() {
|
||||||
|
let api_error = ApiError::from(QuotaError::StorageError(StorageError::NamespaceLockQuorumUnavailable {
|
||||||
|
mode: "quota_capability",
|
||||||
|
bucket: "bucket".to_string(),
|
||||||
|
object: rustfs_config::QUOTA_CONFIG_FILE.to_string(),
|
||||||
|
required: 1,
|
||||||
|
achieved: 0,
|
||||||
|
}));
|
||||||
|
|
||||||
|
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
|
||||||
|
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_kms_cryptographic_error_is_not_retryable() {
|
fn test_kms_cryptographic_error_is_not_retryable() {
|
||||||
let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::cryptographic_error(
|
let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::cryptographic_error(
|
||||||
|
|||||||
@@ -153,9 +153,7 @@ fn remove_heal_control_replay(
|
|||||||
|
|
||||||
static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
|
static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
|
||||||
static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||||
// RUSTFS_COMPAT_TODO(cross-pool-fence-v1): advertise unsupported during predeployment. Remove after composite acquisition,
|
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1;
|
||||||
// activation fencing, fleet proof, commit-time proof revalidation, and fail-closed revocation ship together.
|
|
||||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 0;
|
|
||||||
|
|
||||||
fn admit_heal_control_replay(
|
fn admit_heal_control_replay(
|
||||||
replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
|
replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
|
||||||
@@ -2207,7 +2205,7 @@ mod tests {
|
|||||||
use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo, HealEndpoint};
|
use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo, HealEndpoint};
|
||||||
use crate::storage::storage_api::set_tonic_canonical_body_digest;
|
use crate::storage::storage_api::set_tonic_canonical_body_digest;
|
||||||
use crate::storage::storage_api::{
|
use crate::storage::storage_api::{
|
||||||
Endpoint,
|
Endpoint, RUSTFS_META_BUCKET, SnapshotLeaseToken,
|
||||||
ecstore_layout::{EndpointServerPools, Endpoints, PoolEndpoints},
|
ecstore_layout::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||||
};
|
};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
@@ -2945,14 +2943,22 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn snapshot_lease_acquire_and_renew_handlers_fail_closed() {
|
#[serial_test::serial]
|
||||||
|
async fn snapshot_lease_handlers_forward_to_local_disk() {
|
||||||
|
let temp_dir = tempfile::tempdir().expect("snapshot lease RPC test directory");
|
||||||
|
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||||
|
.base_dir(temp_dir.path())
|
||||||
|
.init_bucket_metadata(false)
|
||||||
|
.build()
|
||||||
|
.await;
|
||||||
let service = make_server();
|
let service = make_server();
|
||||||
let disk = "http://node-a:9000/data/rustfs0".to_string();
|
let disk = env.disk_paths[0].to_string_lossy().into_owned();
|
||||||
|
let fence_path = format!("tmp/quota-mutation-fences/{}", "0".repeat(64));
|
||||||
|
|
||||||
let mut acquire = Request::new(SnapshotLeaseRequest {
|
let mut acquire = Request::new(SnapshotLeaseRequest {
|
||||||
disk: disk.clone(),
|
disk: disk.clone(),
|
||||||
volume: "v".into(),
|
volume: RUSTFS_META_BUCKET.into(),
|
||||||
path: "p".into(),
|
path: fence_path.clone(),
|
||||||
ttl_ms: 60_000,
|
ttl_ms: 60_000,
|
||||||
});
|
});
|
||||||
let acquire_body =
|
let acquire_body =
|
||||||
@@ -2962,14 +2968,18 @@ mod tests {
|
|||||||
let acquire = service
|
let acquire = service
|
||||||
.acquire_snapshot_lease(acquire)
|
.acquire_snapshot_lease(acquire)
|
||||||
.await
|
.await
|
||||||
.expect("disabled acquire should return a protocol response")
|
.expect("acquire should return a protocol response")
|
||||||
.into_inner();
|
.into_inner();
|
||||||
|
assert!(acquire.success, "local disk should acquire the mutation fence");
|
||||||
|
assert_eq!(acquire.protocol_version, 1);
|
||||||
|
assert!(acquire.error.is_none());
|
||||||
|
let token = SnapshotLeaseToken::from_slice(&acquire.token).expect("acquire should return a valid token");
|
||||||
|
|
||||||
let mut renew = Request::new(SnapshotLeaseRenewRequest {
|
let mut renew = Request::new(SnapshotLeaseRenewRequest {
|
||||||
disk,
|
disk: disk.clone(),
|
||||||
volume: "v".into(),
|
volume: RUSTFS_META_BUCKET.into(),
|
||||||
path: "p".into(),
|
path: fence_path.clone(),
|
||||||
token: vec![1; 16].into(),
|
token: token.as_bytes().to_vec().into(),
|
||||||
ttl_ms: 60_000,
|
ttl_ms: 60_000,
|
||||||
});
|
});
|
||||||
let renew_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(renew.get_ref())
|
let renew_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(renew.get_ref())
|
||||||
@@ -2979,15 +2989,103 @@ mod tests {
|
|||||||
let renew = service
|
let renew = service
|
||||||
.renew_snapshot_lease(renew)
|
.renew_snapshot_lease(renew)
|
||||||
.await
|
.await
|
||||||
.expect("disabled renew should return a protocol response")
|
.expect("renew should return a protocol response")
|
||||||
.into_inner();
|
.into_inner();
|
||||||
|
assert!(renew.success, "local disk should renew the mutation fence");
|
||||||
|
assert_eq!(renew.protocol_version, 1);
|
||||||
|
assert!(renew.error.is_none());
|
||||||
|
let renewed = SnapshotLeaseToken::from_slice(&renew.token).expect("renew should return a valid token");
|
||||||
|
assert_ne!(renewed, token);
|
||||||
|
|
||||||
for response in [acquire, renew] {
|
let mut release = Request::new(SnapshotLeaseReleaseRequest {
|
||||||
assert!(!response.success);
|
disk: disk.clone(),
|
||||||
assert!(response.token.is_empty());
|
volume: RUSTFS_META_BUCKET.into(),
|
||||||
assert_eq!(response.protocol_version, 1);
|
path: fence_path.clone(),
|
||||||
assert_eq!(response.error, Some(DiskError::UnsupportedDisk.into()));
|
token: renewed.as_bytes().to_vec().into(),
|
||||||
}
|
});
|
||||||
|
let release_body = rustfs_protos::canonical_snapshot_lease_release_request_body(release.get_ref())
|
||||||
|
.expect("release request body should encode");
|
||||||
|
set_tonic_canonical_body_digest(&mut release, &release_body).expect("release digest metadata should encode");
|
||||||
|
mark_v2_authenticated(&mut release);
|
||||||
|
let release = service
|
||||||
|
.release_snapshot_lease(release)
|
||||||
|
.await
|
||||||
|
.expect("release should return a protocol response")
|
||||||
|
.into_inner();
|
||||||
|
assert!(release.success, "local disk should release the renewed token");
|
||||||
|
assert!(release.error.is_none());
|
||||||
|
|
||||||
|
let mut acquire = Request::new(SnapshotLeaseRequest {
|
||||||
|
disk: disk.clone(),
|
||||||
|
volume: RUSTFS_META_BUCKET.into(),
|
||||||
|
path: fence_path.clone(),
|
||||||
|
ttl_ms: 60_000,
|
||||||
|
});
|
||||||
|
let acquire_body =
|
||||||
|
rustfs_protos::canonical_snapshot_lease_request_body(acquire.get_ref()).expect("acquire request body should encode");
|
||||||
|
set_tonic_canonical_body_digest(&mut acquire, &acquire_body).expect("acquire digest metadata should encode");
|
||||||
|
mark_v2_authenticated(&mut acquire);
|
||||||
|
let active_token = service
|
||||||
|
.acquire_snapshot_lease(acquire)
|
||||||
|
.await
|
||||||
|
.expect("second acquire should return a protocol response")
|
||||||
|
.into_inner();
|
||||||
|
assert!(active_token.success);
|
||||||
|
|
||||||
|
let mut revoke = Request::new(SnapshotLeaseReleaseRequest {
|
||||||
|
disk: disk.clone(),
|
||||||
|
volume: RUSTFS_META_BUCKET.into(),
|
||||||
|
path: fence_path.clone(),
|
||||||
|
token: SnapshotLeaseToken::revoke_all().as_bytes().to_vec().into(),
|
||||||
|
});
|
||||||
|
let revoke_body = rustfs_protos::canonical_snapshot_lease_release_request_body(revoke.get_ref())
|
||||||
|
.expect("revoke-all request body should encode");
|
||||||
|
set_tonic_canonical_body_digest(&mut revoke, &revoke_body).expect("revoke-all digest metadata should encode");
|
||||||
|
mark_v2_authenticated(&mut revoke);
|
||||||
|
let revoke = service
|
||||||
|
.release_snapshot_lease(revoke)
|
||||||
|
.await
|
||||||
|
.expect("revoke-all should return a protocol response")
|
||||||
|
.into_inner();
|
||||||
|
assert!(revoke.success, "nil revoke-all sentinel must reach the local disk");
|
||||||
|
assert!(revoke.error.is_none());
|
||||||
|
|
||||||
|
let mut stale_renew = Request::new(SnapshotLeaseRenewRequest {
|
||||||
|
disk: disk.clone(),
|
||||||
|
volume: RUSTFS_META_BUCKET.into(),
|
||||||
|
path: fence_path.clone(),
|
||||||
|
token: active_token.token,
|
||||||
|
ttl_ms: 60_000,
|
||||||
|
});
|
||||||
|
let stale_renew_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(stale_renew.get_ref())
|
||||||
|
.expect("stale renew request body should encode");
|
||||||
|
set_tonic_canonical_body_digest(&mut stale_renew, &stale_renew_body).expect("stale renew digest metadata should encode");
|
||||||
|
mark_v2_authenticated(&mut stale_renew);
|
||||||
|
let stale_renew = service
|
||||||
|
.renew_snapshot_lease(stale_renew)
|
||||||
|
.await
|
||||||
|
.expect("stale renew should return a protocol response")
|
||||||
|
.into_inner();
|
||||||
|
assert!(!stale_renew.success, "revoke-all must invalidate active tokens");
|
||||||
|
assert!(stale_renew.token.is_empty());
|
||||||
|
assert!(stale_renew.error.is_some());
|
||||||
|
|
||||||
|
let mut malformed = Request::new(SnapshotLeaseRenewRequest {
|
||||||
|
disk,
|
||||||
|
volume: RUSTFS_META_BUCKET.into(),
|
||||||
|
path: fence_path,
|
||||||
|
token: vec![1; 15].into(),
|
||||||
|
ttl_ms: 60_000,
|
||||||
|
});
|
||||||
|
let malformed_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(malformed.get_ref())
|
||||||
|
.expect("malformed renew request body should encode");
|
||||||
|
set_tonic_canonical_body_digest(&mut malformed, &malformed_body).expect("malformed renew digest metadata should encode");
|
||||||
|
mark_v2_authenticated(&mut malformed);
|
||||||
|
let malformed = service
|
||||||
|
.renew_snapshot_lease(malformed)
|
||||||
|
.await
|
||||||
|
.expect_err("a malformed non-nil token must be rejected");
|
||||||
|
assert_eq!(malformed.code(), tonic::Code::InvalidArgument);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -3342,7 +3440,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn cross_pool_fence_probe_authenticates_unsupported_rollout_state() {
|
async fn cross_pool_fence_probe_authenticates_supported_v1_state() {
|
||||||
let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string());
|
let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string());
|
||||||
let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true);
|
let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true);
|
||||||
assert!(
|
assert!(
|
||||||
@@ -3407,7 +3505,7 @@ mod tests {
|
|||||||
|
|
||||||
assert!(response.success);
|
assert!(response.success);
|
||||||
assert_eq!(response.error_info, None);
|
assert_eq!(response.error_info, None);
|
||||||
assert_eq!(&response.result[..4], &0_u32.to_be_bytes());
|
assert_eq!(&response.result[..4], &super::CROSS_POOL_FENCE_SUPPORTED_VERSION.to_be_bytes());
|
||||||
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..])
|
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..])
|
||||||
.expect("capability identity should decode");
|
.expect("capability identity should decode");
|
||||||
assert_eq!(topology_member, "node-a:9000");
|
assert_eq!(topology_member, "node-a:9000");
|
||||||
|
|||||||
@@ -37,19 +37,28 @@ const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512;
|
|||||||
const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024;
|
const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024;
|
||||||
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
|
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
|
||||||
|
|
||||||
|
fn snapshot_lease_response(result: Result<SnapshotLeaseToken, DiskError>) -> Response<SnapshotLeaseResponse> {
|
||||||
|
match result {
|
||||||
|
Ok(token) => Response::new(SnapshotLeaseResponse {
|
||||||
|
success: true,
|
||||||
|
token: token.as_bytes().to_vec().into(),
|
||||||
|
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
|
||||||
|
error: None,
|
||||||
|
}),
|
||||||
|
Err(err) => Response::new(SnapshotLeaseResponse {
|
||||||
|
success: false,
|
||||||
|
token: Bytes::new(),
|
||||||
|
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
|
||||||
|
error: Some(err.into()),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
struct DecodedRpcPayload<T> {
|
struct DecodedRpcPayload<T> {
|
||||||
value: T,
|
value: T,
|
||||||
from_msgpack: bool,
|
from_msgpack: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn snapshot_lease_disabled_response() -> SnapshotLeaseResponse {
|
|
||||||
SnapshotLeaseResponse {
|
|
||||||
success: false,
|
|
||||||
token: Bytes::new(),
|
|
||||||
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
|
|
||||||
error: Some(DiskError::UnsupportedDisk.into()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fn decode_msgpack_or_json<T: DeserializeOwned>(
|
fn decode_msgpack_or_json<T: DeserializeOwned>(
|
||||||
binary: &[u8],
|
binary: &[u8],
|
||||||
json: &str,
|
json: &str,
|
||||||
@@ -241,7 +250,12 @@ impl NodeService {
|
|||||||
rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref()),
|
rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref()),
|
||||||
"acquire_snapshot_lease",
|
"acquire_snapshot_lease",
|
||||||
)?;
|
)?;
|
||||||
Ok(Response::new(snapshot_lease_disabled_response()))
|
let request = request.into_inner();
|
||||||
|
let result = match self.find_disk(&request.disk).await {
|
||||||
|
Some(disk) => disk.acquire_snapshot_lease(&request.volume, &request.path).await,
|
||||||
|
None => Err(DiskError::other("cannot find disk")),
|
||||||
|
};
|
||||||
|
Ok(snapshot_lease_response(result))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn handle_renew_snapshot_lease(
|
pub(super) async fn handle_renew_snapshot_lease(
|
||||||
@@ -253,7 +267,14 @@ impl NodeService {
|
|||||||
rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref()),
|
rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref()),
|
||||||
"renew_snapshot_lease",
|
"renew_snapshot_lease",
|
||||||
)?;
|
)?;
|
||||||
Ok(Response::new(snapshot_lease_disabled_response()))
|
let request = request.into_inner();
|
||||||
|
let token =
|
||||||
|
SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?;
|
||||||
|
let result = match self.find_disk(&request.disk).await {
|
||||||
|
Some(disk) => disk.renew_snapshot_lease(&request.volume, &request.path, token).await,
|
||||||
|
None => Err(DiskError::other("cannot find disk")),
|
||||||
|
};
|
||||||
|
Ok(snapshot_lease_response(result))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn handle_release_snapshot_lease(
|
pub(super) async fn handle_release_snapshot_lease(
|
||||||
@@ -266,8 +287,11 @@ impl NodeService {
|
|||||||
"release_snapshot_lease",
|
"release_snapshot_lease",
|
||||||
)?;
|
)?;
|
||||||
let request = request.into_inner();
|
let request = request.into_inner();
|
||||||
let token =
|
let token = if request.token.as_ref() == SnapshotLeaseToken::revoke_all().as_bytes() {
|
||||||
SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?;
|
SnapshotLeaseToken::revoke_all()
|
||||||
|
} else {
|
||||||
|
SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?
|
||||||
|
};
|
||||||
let Some(disk) = self.find_disk(&request.disk).await else {
|
let Some(disk) = self.find_disk(&request.disk).await else {
|
||||||
return Ok(Response::new(SnapshotLeaseMutationResponse {
|
return Ok(Response::new(SnapshotLeaseMutationResponse {
|
||||||
success: false,
|
success: false,
|
||||||
@@ -1494,7 +1518,7 @@ mod tests {
|
|||||||
use super::{
|
use super::{
|
||||||
compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info,
|
compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info,
|
||||||
encode_batch_read_version_response_payloads, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named,
|
encode_batch_read_version_response_payloads, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named,
|
||||||
encode_read_multiple_response_payloads, encode_rename_data_response_payloads, snapshot_lease_disabled_response,
|
encode_read_multiple_response_payloads, encode_rename_data_response_payloads,
|
||||||
};
|
};
|
||||||
use crate::storage::storage_api::ReadMultipleResp;
|
use crate::storage::storage_api::ReadMultipleResp;
|
||||||
use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp;
|
use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp;
|
||||||
@@ -1509,17 +1533,6 @@ mod tests {
|
|||||||
count: u32,
|
count: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn snapshot_lease_acquire_and_renew_fail_closed() {
|
|
||||||
let response = snapshot_lease_disabled_response();
|
|
||||||
let expected_error = DiskError::UnsupportedDisk.into();
|
|
||||||
|
|
||||||
assert!(!response.success);
|
|
||||||
assert!(response.token.is_empty());
|
|
||||||
assert_eq!(response.protocol_version, 1);
|
|
||||||
assert_eq!(response.error, Some(expected_error));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn decode_msgpack_or_json_prefers_binary_payload() {
|
fn decode_msgpack_or_json_prefers_binary_payload() {
|
||||||
let payload = SamplePayload {
|
let payload = SamplePayload {
|
||||||
|
|||||||
@@ -428,7 +428,7 @@ pub(crate) mod ecstore_config {
|
|||||||
pub(crate) mod ecstore_data_usage {
|
pub(crate) mod ecstore_data_usage {
|
||||||
pub(crate) use rustfs_ecstore::api::data_usage::{
|
pub(crate) use rustfs_ecstore::api::data_usage::{
|
||||||
apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached,
|
apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached,
|
||||||
load_data_usage_from_backend, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
|
load_data_usage_from_backend, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
|
||||||
record_bucket_object_version_write_memory, record_bucket_object_write_memory,
|
record_bucket_object_version_write_memory, record_bucket_object_write_memory,
|
||||||
record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend,
|
record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend,
|
||||||
};
|
};
|
||||||
@@ -486,7 +486,9 @@ pub(crate) mod ecstore_metrics {
|
|||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
pub(crate) mod ecstore_notification {
|
pub(crate) mod ecstore_notification {
|
||||||
pub(crate) use rustfs_ecstore::api::notification::{
|
pub(crate) use rustfs_ecstore::api::notification::{
|
||||||
NotificationSys, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe,
|
CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof,
|
||||||
|
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
|
||||||
|
start_remote_version_state_fleet_probe,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -544,6 +546,10 @@ pub(crate) mod ecstore_test_support {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) mod ecstore_set_disk {
|
pub(crate) mod ecstore_set_disk {
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{
|
||||||
|
PutObjectCommitBarrier, PutObjectCommitPause, fail_next_quota_ledger_save_for_test,
|
||||||
|
};
|
||||||
pub(crate) use rustfs_ecstore::api::set_disk::{
|
pub(crate) use rustfs_ecstore::api::set_disk::{
|
||||||
DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class,
|
DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class,
|
||||||
};
|
};
|
||||||
@@ -1128,7 +1134,9 @@ pub(crate) trait StorageDiskRpcExt {
|
|||||||
) -> DiskResult<()>;
|
) -> DiskResult<()>;
|
||||||
async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult<bytes::Bytes>;
|
async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult<bytes::Bytes>;
|
||||||
async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>;
|
async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>;
|
||||||
|
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult<SnapshotLeaseToken>;
|
||||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()>;
|
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()>;
|
||||||
|
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<SnapshotLeaseToken>;
|
||||||
async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo>;
|
async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo>;
|
||||||
async fn list_volumes(&self) -> DiskResult<Vec<VolumeInfo>>;
|
async fn list_volumes(&self) -> DiskResult<Vec<VolumeInfo>>;
|
||||||
async fn make_volume(&self, volume: &str) -> DiskResult<()>;
|
async fn make_volume(&self, volume: &str) -> DiskResult<()>;
|
||||||
@@ -1256,10 +1264,18 @@ where
|
|||||||
ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await
|
ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult<SnapshotLeaseToken> {
|
||||||
|
ecstore_disk::DiskAPI::acquire_snapshot_lease(self, volume, path).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()> {
|
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()> {
|
||||||
ecstore_disk::DiskAPI::release_snapshot_lease(self, volume, path, token).await
|
ecstore_disk::DiskAPI::release_snapshot_lease(self, volume, path, token).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<SnapshotLeaseToken> {
|
||||||
|
ecstore_disk::DiskAPI::renew_snapshot_lease(self, volume, path, token).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo> {
|
async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo> {
|
||||||
ecstore_disk::DiskAPI::stat_volume(self, volume).await
|
ecstore_disk::DiskAPI::stat_volume(self, volume).await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ cd "$(dirname "$0")/.."
|
|||||||
# Excludes crates/e2e_test/ — test infrastructure legitimately uses s3s
|
# Excludes crates/e2e_test/ — test infrastructure legitimately uses s3s
|
||||||
# to verify S3 behavior and does not widen the production s3s surface.
|
# to verify S3 behavior and does not widen the production s3s surface.
|
||||||
S3S_IMPORT_FILES_BASELINE=213
|
S3S_IMPORT_FILES_BASELINE=213
|
||||||
S3_ERROR_LINES_BASELINE=1621
|
S3_ERROR_LINES_BASELINE=1620
|
||||||
S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::'
|
S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::'
|
||||||
E2E_TEST_GLOB='--glob=!crates/e2e_test/**'
|
E2E_TEST_GLOB='--glob=!crates/e2e_test/**'
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user