fix(quota): close reservation bypasses

This commit is contained in:
cxymds
2026-08-13 13:59:03 +08:00
parent bd5685f8a2
commit c0d6b7827a
29 changed files with 2160 additions and 423 deletions
+5
View File
@@ -353,6 +353,11 @@ pub const DEFAULT_OBS_TRACES_EXPORT_ENABLED: bool = true;
/// Environment variable: RUSTFS_OBS_METRICS_EXPORT_ENABLED
pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true;
/// Default detailed PUT stage metrics enabled
/// Default value: false
/// Environment variable: RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED
pub const DEFAULT_OBS_PUT_STAGE_METRICS_ENABLED: bool = false;
/// Default logs export enabled
/// It is used to enable or disable exporting logs
/// Default value: true
+5
View File
@@ -44,6 +44,10 @@ pub const ENV_OBS_METRICS_EXPORT_ENABLED: &str = "RUSTFS_OBS_METRICS_EXPORT_ENAB
pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED";
pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ENABLED";
/// Enables detailed per-stage PUT metrics. Disabled by default because each
/// PUT records multiple timers and histograms when attribution is active.
pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED";
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
@@ -141,6 +145,7 @@ mod tests {
assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
assert_eq!(ENV_OBS_PUT_STAGE_METRICS_ENABLED, "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED");
// Test log cleanup related env keys
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
+3 -1
View File
@@ -405,6 +405,8 @@ pub mod metrics {
}
pub mod notification {
#[cfg(any(test, feature = "test-util"))]
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
pub use crate::services::notification_sys::{
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof,
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
@@ -468,7 +470,7 @@ pub mod set_disk {
#[cfg(feature = "test-util")]
pub mod test_util {
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
}
}
+18 -2
View File
@@ -52,6 +52,7 @@ impl QuotaChecker {
) -> Result<QuotaCheckResult, QuotaError> {
let start_time = Instant::now();
let quota_config = self.get_quota_config(bucket).await?;
let uses_durable_reservations = quota_config.uses_durable_reservations();
// If no quota limit is set, allow operation
let quota_limit = match quota_config.quota {
@@ -67,6 +68,7 @@ impl QuotaChecker {
quota_limit: None,
operation_size,
remaining: None,
uses_durable_reservations,
});
}
Some(q) => q,
@@ -74,14 +76,17 @@ impl QuotaChecker {
let current_usage = self.get_real_time_usage(bucket).await?;
let admission_size = if uses_durable_reservations { 0 } else { operation_size };
let expected_usage = match operation {
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => current_usage + operation_size,
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
current_usage.saturating_add(admission_size)
}
QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size),
};
let allowed = match operation {
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
quota_config.check_operation_allowed(current_usage, operation_size)
quota_config.check_operation_allowed(current_usage, admission_size)
}
QuotaOperation::DeleteObject => true,
};
@@ -105,6 +110,7 @@ impl QuotaChecker {
quota_limit: Some(quota_limit),
operation_size,
remaining,
uses_durable_reservations,
};
let duration = start_time.elapsed();
@@ -375,6 +381,7 @@ mod tests {
quota_limit: None,
operation_size: 1024,
remaining: None,
uses_durable_reservations: false,
};
assert!(result.allowed);
@@ -398,4 +405,13 @@ mod tests {
let allowed = quota.check_operation_allowed(512, 1024);
assert!(!allowed);
}
#[test]
fn legacy_quota_rejects_full_operation_while_v1_defers_net_growth() {
let legacy: BucketQuota = serde_json::from_str(r#"{"quota":5}"#).expect("legacy quota should parse");
let durable = BucketQuota::new(Some(5));
assert!(!legacy.check_operation_allowed(4, 2));
assert!(durable.uses_durable_reservations());
}
}
+14 -2
View File
@@ -20,7 +20,7 @@ use rustfs_config::{
QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE,
QUOTA_NOT_FOUND_ERROR_CODE,
};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
use thiserror::Error;
use time::OffsetDateTime;
@@ -90,7 +90,10 @@ impl<'de> Deserialize<'de> for BucketQuota {
{
let wire = BucketQuotaWire::deserialize(deserializer)?;
let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) {
wire.reservation_quota
Some(
wire.reservation_quota
.ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?,
)
} else {
wire.quota
};
@@ -164,6 +167,7 @@ pub struct QuotaCheckResult {
pub quota_limit: Option<u64>,
pub operation_size: u64,
pub remaining: Option<u64>,
pub uses_durable_reservations: bool,
}
#[derive(Debug)]
@@ -327,6 +331,14 @@ mod tests {
assert!(quota.has_unsupported_reservation_protocol());
}
#[test]
fn reservation_protocol_v1_requires_reservation_quota() {
let err = serde_json::from_str::<BucketQuota>(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":1}"#)
.expect_err("v1 without its authoritative quota must fail closed");
assert!(err.to_string().contains("reservation_quota is required"));
}
/// unmarshal accepts format without quota_type
#[test]
fn unmarshal_format_without_quota_type() {
+39 -8
View File
@@ -209,6 +209,7 @@ pub(crate) struct QuotaContext {
quota_limit: Option<u64>,
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
snapshot_admission: Option<QuotaAdmission>,
legacy_data_movement: bool,
metadata_guard: Option<NamespaceLockGuard>,
pool_index: Option<usize>,
set_index: Option<usize>,
@@ -233,6 +234,12 @@ impl QuotaContext {
}
return Ok(QuotaReservation::unlimited(self.metadata_guard));
}
if self.legacy_data_movement {
if new_size > old_size {
return Err(StorageError::PartMissingOrCorrupt);
}
return Ok(QuotaReservation::unlimited(self.metadata_guard));
}
let store = self.store.ok_or(StorageError::PartMissingOrCorrupt)?;
let bucket_incarnation = self.bucket_incarnation.ok_or(StorageError::PartMissingOrCorrupt)?;
let quota_revision = self.quota_revision.ok_or(StorageError::PartMissingOrCorrupt)?;
@@ -431,7 +438,8 @@ pub(crate) async fn begin(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
object: &str,
_snapshot_admission: Option<QuotaAdmission>,
snapshot_admission: Option<QuotaAdmission>,
data_movement: bool,
pool_index: usize,
set_index: usize,
) -> Result<QuotaContext> {
@@ -446,13 +454,14 @@ pub(crate) async fn begin(
quota_limit: None,
capability_proof: None,
snapshot_admission: None,
legacy_data_movement: false,
metadata_guard: None,
pool_index: None,
set_index: None,
});
}
#[cfg(test)]
if let Some(snapshot_admission) = _snapshot_admission {
if let Some(snapshot_admission) = snapshot_admission {
return Ok(QuotaContext {
store: None,
bucket: bucket.to_string(),
@@ -463,6 +472,7 @@ pub(crate) async fn begin(
quota_limit: Some(snapshot_admission.quota_limit()),
capability_proof: None,
snapshot_admission: Some(snapshot_admission),
legacy_data_movement: false,
metadata_guard: None,
pool_index: Some(pool_index),
set_index: Some(set_index),
@@ -480,6 +490,7 @@ pub(crate) async fn begin(
quota_limit: None,
capability_proof: None,
snapshot_admission: None,
legacy_data_movement: false,
metadata_guard: None,
pool_index: None,
set_index: None,
@@ -504,7 +515,8 @@ pub(crate) async fn begin(
{
return Err(StorageError::PartMissingOrCorrupt);
}
let capability_proof = if quota.as_ref().is_some_and(|quota| quota.uses_durable_reservations()) {
let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations());
let capability_proof = if durable_quota.is_some() {
Some(
crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
.ok_or_else(|| quota_capability_error(bucket, &ledger_object(bucket)))?,
@@ -512,10 +524,28 @@ pub(crate) async fn begin(
} else {
None
};
let quota_limit = quota
.filter(crate::bucket::quota::BucketQuota::uses_durable_reservations)
.and_then(|quota| quota.quota);
let store = if quota_limit.is_some() {
let durable_quota_limit = durable_quota.and_then(|quota| quota.quota);
let snapshot_admission = match quota.as_ref().filter(|quota| !quota.uses_durable_reservations()) {
Some(quota) => match (quota.quota, snapshot_admission) {
(Some(limit), Some(admission)) if admission.quota_limit() == limit => Some(admission),
(Some(_), None) if data_movement => None,
(Some(_), _) => return Err(StorageError::PartMissingOrCorrupt),
(None, _) => None,
},
None => None,
};
let legacy_data_movement = durable_quota_limit.is_none()
&& quota.as_ref().and_then(|quota| quota.quota).is_some()
&& snapshot_admission.is_none()
&& data_movement;
let quota_limit = durable_quota_limit
.or_else(|| snapshot_admission.map(QuotaAdmission::quota_limit))
.or_else(|| {
legacy_data_movement
.then(|| quota.as_ref().and_then(|quota| quota.quota))
.flatten()
});
let store = if durable_quota_limit.is_some() {
Some(metadata_sys::object_store_in(ctx).await?)
} else {
None
@@ -529,7 +559,8 @@ pub(crate) async fn begin(
quota_revision: Some(quota_revision),
quota_limit,
capability_proof,
snapshot_admission: None,
snapshot_admission,
legacy_data_movement,
metadata_guard: Some(metadata_guard),
pool_index: Some(pool_index),
set_index: Some(set_index),
+543 -26
View File
@@ -846,31 +846,49 @@ impl RemoteDisk {
/// default to 1 (see [`internode_idempotent_read_retries`]). MUST NOT be used for write/lock
/// RPCs — those must never auto-retry (quorum/idempotency safety). The `operation` closure is
/// re-invoked per attempt, so it must be `Fn` (rebuild the request from borrowed inputs, do not
/// move captured state out).
/// move captured state out). Attempts and backoff share one total timeout budget.
async fn execute_read_with_retry<T, F, Fut>(&self, op: &'static str, operation: F, timeout_duration: Duration) -> Result<T>
where
F: Fn() -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
let deadline = (!timeout_duration.is_zero()).then(|| {
time::Instant::now()
.checked_add(timeout_duration)
.unwrap_or_else(|| time::sleep(timeout_duration).deadline())
});
let max_retries = internode_idempotent_read_retries();
let mut attempt = 0usize;
loop {
// Only the final attempt marks the disk faulty / evicts the channel. Earlier retries
// ignore the failure, so a transient error cannot flip the disk into a faulty
// short-circuit (which would defeat the retry) or over-count failures.
let attempt_timeout = deadline
.map(|deadline| deadline.saturating_duration_since(time::Instant::now()))
.unwrap_or(Duration::ZERO);
if deadline.is_some() && attempt_timeout.is_zero() {
self.record_timeout(op, timeout_duration);
return Err(DiskError::Timeout);
}
let health_action = if attempt >= max_retries {
FailureHealthAction::MarkFailure
} else {
FailureHealthAction::IgnoreFailure
};
match self
.execute_with_timeout_for_op_and_health_action(op, &operation, timeout_duration, health_action)
.execute_with_timeout_for_op_and_health_action(op, &operation, attempt_timeout, health_action)
.await
{
Err(err) if attempt < max_retries && is_network_like_disk_error(&err) => {
if matches!(err, DiskError::Timeout) && deadline.is_some_and(|deadline| time::Instant::now() >= deadline) {
self.mark_faulty("read_operation_deadline");
return Err(err);
}
attempt += 1;
let backoff = REMOTE_DISK_READ_RETRY_BASE_BACKOFF
.saturating_mul(1u32 << u32::try_from(attempt - 1).unwrap_or(4).min(4));
if deadline.is_some_and(|deadline| deadline.saturating_duration_since(time::Instant::now()) <= backoff) {
attempt = max_retries;
continue;
}
debug!(
endpoint = %self.endpoint,
addr = %self.addr,
@@ -878,7 +896,17 @@ impl RemoteDisk {
attempt,
"retrying idempotent read-only RPC after transient network error"
);
tokio::time::sleep(backoff).await;
if let Some(deadline) = deadline {
if time::timeout_at(deadline, time::sleep(backoff)).await.is_err() {
self.record_timeout(op, timeout_duration);
return Err(DiskError::Timeout);
}
} else {
time::sleep(backoff).await;
}
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
}
other => return other,
}
@@ -957,32 +985,35 @@ impl RemoteDisk {
operation_result
}
Err(_) => {
// Timeout occurred, mark disk as potentially faulty
counter!(
"rustfs_drive_op_timeout_total",
"endpoint" => self.endpoint.to_string(),
"op" => op.to_string()
)
.increment(1);
self.record_timeout(op, timeout_duration);
if failure_health_action == FailureHealthAction::MarkFailure {
self.mark_faulty_and_evict("operation_timeout").await;
}
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
op,
timeout_ms = timeout_duration.as_millis(),
state = "timeout",
"Remote disk operation timed out"
);
Err(DiskError::Timeout)
}
}
}
fn record_timeout(&self, op: &'static str, timeout_duration: Duration) {
counter!(
"rustfs_drive_op_timeout_total",
"endpoint" => self.endpoint.to_string(),
"op" => op.to_string()
)
.increment(1);
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
addr = %self.addr,
op,
timeout_ms = timeout_duration.as_millis(),
state = "timeout",
"Remote disk operation timed out"
);
}
async fn handle_network_like_error<T>(
&self,
op: &'static str,
@@ -1016,7 +1047,7 @@ impl RemoteDisk {
}
}
async fn mark_faulty_and_evict(&self, reason: &'static str) {
fn mark_faulty(&self, reason: &'static str) -> bool {
let previous_state = self.runtime_state();
let transitioned_to_offline = self.mark_suspect_or_offline(reason);
let state = self.runtime_state();
@@ -1053,6 +1084,12 @@ impl RemoteDisk {
"Remote disk marked suspect"
);
}
}
state != previous_state
}
async fn mark_faulty_and_evict(&self, reason: &'static str) {
if self.mark_faulty(reason) {
counter!(
"rustfs_drive_connection_evict_total",
"endpoint" => self.endpoint.to_string(),
@@ -2068,7 +2105,7 @@ impl DiskAPI for RemoteDisk {
Ok(file_info)
},
get_max_timeout_duration(),
get_drive_metadata_timeout(),
)
.await
}
@@ -5094,6 +5131,452 @@ mod tests {
);
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_reset_during_backoff_preserves_recovery() {
let remote_disk = Arc::new(new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await);
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let first_attempt = Arc::new(tokio::sync::Notify::new());
let started = time::Instant::now();
let task_disk = Arc::clone(&remote_disk);
let task_attempts = Arc::clone(&attempts);
let task_first_attempt = Arc::clone(&first_attempt);
let task = tokio::spawn(async move {
task_disk
.execute_read_with_retry(
"read_version",
move || {
let attempt = task_attempts.fetch_add(1, Ordering::SeqCst);
let first_attempt = Arc::clone(&task_first_attempt);
async move {
if attempt == 0 {
time::sleep(Duration::from_millis(20)).await;
first_attempt.notify_one();
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
std_io::ErrorKind::ConnectionRefused,
"connection refused",
)));
}
Ok(())
}
},
Duration::from_millis(100),
)
.await
});
first_attempt.notified().await;
tokio::task::yield_now().await;
remote_disk.health.reset_for_store_init_retry(&remote_disk.endpoint);
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
.expect("remote disk address should parse")
.connect_lazy();
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
task.await
.expect("retry task should finish")
.expect("the retry should succeed after the health reset");
assert_eq!(attempts.load(Ordering::SeqCst), 2);
assert_eq!(started.elapsed(), Duration::from_millis(70));
assert_eq!(
remote_disk.health.waiting_count(),
0,
"health reset must not underflow the waiting counter"
);
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Online);
assert!(
runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await,
"a recovered channel must survive the retry backoff"
);
remote_disk.cancel_token.cancel();
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_still_retries_within_shared_deadline() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
.expect("remote disk address should parse")
.connect_lazy();
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
remote_disk
.execute_read_with_retry(
"read_version",
|| {
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
async move {
if attempt == 0 {
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
std_io::ErrorKind::ConnectionReset,
"connection reset",
)));
}
Ok(())
}
},
Duration::from_millis(100),
)
.await
.expect("a retry that fits the shared deadline should succeed");
assert_eq!(attempts.load(Ordering::SeqCst), 2);
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Online);
assert!(runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await);
remote_disk.cancel_token.cancel();
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_uses_remaining_budget_for_final_attempt() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let started = time::Instant::now();
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
.expect("remote disk address should parse")
.connect_lazy();
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
let err = remote_disk
.execute_read_with_retry(
"read_version",
|| {
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
async move {
if attempt == 0 {
time::sleep(Duration::from_millis(20)).await;
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
std_io::ErrorKind::ConnectionRefused,
"connection refused",
)));
}
std::future::pending::<Result<()>>().await
}
},
Duration::from_millis(100),
)
.await
.expect_err("the final retry should consume only the remaining total budget");
assert_eq!(err, DiskError::Timeout);
assert_eq!(attempts.load(Ordering::SeqCst), 2);
assert_eq!(started.elapsed(), Duration::from_millis(100));
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
assert!(!runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await);
remote_disk.cancel_token.cancel();
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_uses_final_attempt_at_exact_backoff_boundary() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let started = time::Instant::now();
let err = remote_disk
.execute_read_with_retry(
"read_version",
|| {
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
async move {
if attempt == 0 {
time::sleep(Duration::from_millis(50)).await;
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
std_io::ErrorKind::ConnectionRefused,
"connection refused",
)));
}
std::future::pending::<Result<()>>().await
}
},
Duration::from_millis(100),
)
.await
.expect_err("the exact backoff boundary should be reserved for a final attempt");
assert_eq!(err, DiskError::Timeout);
assert_eq!(attempts.load(Ordering::SeqCst), 2);
assert_eq!(started.elapsed(), Duration::from_millis(100));
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
remote_disk.cancel_token.cancel();
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_uses_final_attempt_below_backoff_budget() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let started = time::Instant::now();
let err = remote_disk
.execute_read_with_retry(
"read_version",
|| {
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
async move {
if attempt == 0 {
time::sleep(Duration::from_millis(80)).await;
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
std_io::ErrorKind::ConnectionRefused,
"connection refused",
)));
}
std::future::pending::<Result<()>>().await
}
},
Duration::from_millis(100),
)
.await
.expect_err("remaining budget below backoff should be reserved for a final attempt");
assert_eq!(err, DiskError::Timeout);
assert_eq!(attempts.load(Ordering::SeqCst), 2);
assert_eq!(started.elapsed(), Duration::from_millis(100));
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
remote_disk.cancel_token.cancel();
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_zero_timeout_disables_the_deadline() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let started = time::Instant::now();
remote_disk
.execute_read_with_retry(
"read_version",
|| {
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
async move {
if attempt == 0 {
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
std_io::ErrorKind::ConnectionReset,
"connection reset",
)));
}
Ok(())
}
},
Duration::ZERO,
)
.await
.expect("zero timeout should allow a retry without a deadline");
assert_eq!(attempts.load(Ordering::SeqCst), 2);
assert_eq!(started.elapsed(), REMOTE_DISK_READ_RETRY_BASE_BACKOFF);
remote_disk.cancel_token.cancel();
}
#[tokio::test]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_accepts_max_metadata_timeout() {
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, Some(u64::MAX.to_string()))], async {
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
remote_disk
.execute_read_with_retry("read_version", || async { Ok::<(), Error>(()) }, get_drive_metadata_timeout())
.await
.expect("the maximum configured metadata timeout must not panic");
remote_disk.cancel_token.cancel();
})
.await;
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_zero_retries_runs_once() {
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_IDEMPOTENT_READ_RETRIES, Some("0"))], async {
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let started = time::Instant::now();
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
.expect("remote disk address should parse")
.connect_lazy();
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
let err = remote_disk
.execute_read_with_retry(
"read_version",
|| {
attempts.fetch_add(1, Ordering::SeqCst);
async {
Err::<(), Error>(DiskError::Io(std_io::Error::new(
std_io::ErrorKind::ConnectionReset,
"connection reset",
)))
}
},
Duration::from_secs(1),
)
.await
.expect_err("zero retries should return the first network error");
assert!(matches!(err, DiskError::Io(ref io_err) if io_err.kind() == std_io::ErrorKind::ConnectionReset));
assert_eq!(attempts.load(Ordering::SeqCst), 1);
assert_eq!(started.elapsed(), Duration::ZERO);
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
assert!(!runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await);
remote_disk.cancel_token.cancel();
})
.await;
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_attempt_timeout_marks_health_without_evicting() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
let recorder = crate::test_metrics::CapturingRecorder::default();
let _recorder_guard = metrics::set_default_local_recorder(&recorder);
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
.expect("remote disk address should parse")
.connect_lazy();
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
let err = remote_disk
.execute_read_with_retry(
"read_version",
|| {
attempts.fetch_add(1, Ordering::SeqCst);
std::future::pending::<Result<()>>()
},
Duration::from_millis(100),
)
.await
.expect_err("an in-flight attempt that consumes the deadline should time out");
assert_eq!(err, DiskError::Timeout);
assert_eq!(attempts.load(Ordering::SeqCst), 1);
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
assert!(runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await);
assert_eq!(
recorder.counter_value(
"rustfs_drive_op_timeout_total",
&[
("endpoint", remote_disk.endpoint.to_string().as_str()),
("op", "read_version")
]
),
1
);
remote_disk.cancel_token.cancel();
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_does_not_retry_business_errors() {
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let err = remote_disk
.execute_read_with_retry(
"read_version",
|| {
attempts.fetch_add(1, Ordering::SeqCst);
async { Err::<(), Error>(DiskError::FileNotFound) }
},
Duration::from_secs(1),
)
.await
.expect_err("business errors should be returned directly");
assert_eq!(err, DiskError::FileNotFound);
assert_eq!(attempts.load(Ordering::SeqCst), 1);
remote_disk.cancel_token.cancel();
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_honors_configured_retry_count() {
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_IDEMPOTENT_READ_RETRIES, Some("2"))], async {
let remote_disk = new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await;
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let started = time::Instant::now();
let channel = TonicEndpoint::from_shared(remote_disk.addr.clone())
.expect("remote disk address should parse")
.connect_lazy();
runtime_sources::cache_test_node_channel(remote_disk.addr.clone(), channel).await;
let err = remote_disk
.execute_read_with_retry(
"read_version",
|| {
attempts.fetch_add(1, Ordering::SeqCst);
async {
Err::<(), Error>(DiskError::Io(std_io::Error::new(
std_io::ErrorKind::ConnectionReset,
"connection reset",
)))
}
},
Duration::from_secs(1),
)
.await
.expect_err("exhausted retries should return the last network error");
assert!(matches!(err, DiskError::Io(ref io_err) if io_err.kind() == std_io::ErrorKind::ConnectionReset));
assert_eq!(attempts.load(Ordering::SeqCst), 3);
assert_eq!(started.elapsed(), Duration::from_millis(150));
assert_eq!(remote_disk.runtime_state(), RuntimeDriveHealthState::Suspect);
assert!(!runtime_sources::test_node_channel_is_cached(&remote_disk.addr).await);
remote_disk.cancel_token.cancel();
})
.await;
}
#[tokio::test(start_paused = true)]
#[serial(remote_disk_read_retry)]
async fn execute_read_with_retry_stops_when_disk_turns_offline_during_backoff() {
let remote_disk = Arc::new(new_remote_disk_with_transport(Arc::new(RecordingInternodeDataTransport::default())).await);
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let first_attempt = Arc::new(tokio::sync::Notify::new());
let task_disk = Arc::clone(&remote_disk);
let task_attempts = Arc::clone(&attempts);
let task_first_attempt = Arc::clone(&first_attempt);
let task = tokio::spawn(async move {
task_disk
.execute_read_with_retry(
"read_version",
move || {
let attempt = task_attempts.fetch_add(1, Ordering::SeqCst);
let first_attempt = Arc::clone(&task_first_attempt);
async move {
if attempt == 0 {
first_attempt.notify_one();
return Err::<(), Error>(DiskError::Io(std_io::Error::new(
std_io::ErrorKind::ConnectionReset,
"connection reset",
)));
}
Ok(())
}
},
Duration::from_secs(1),
)
.await
});
first_attempt.notified().await;
tokio::task::yield_now().await;
remote_disk
.health
.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
time::advance(REMOTE_DISK_READ_RETRY_BASE_BACKOFF).await;
let err = task
.await
.expect("retry task should finish")
.expect_err("an offline disk must stop before the next attempt");
assert_eq!(err, DiskError::FaultyDisk);
assert_eq!(attempts.load(Ordering::SeqCst), 1);
remote_disk.cancel_token.cancel();
}
#[tokio::test]
async fn test_execute_with_timeout_evicts_cached_connection() {
let addr = "http://127.0.0.1:59991".to_string();
@@ -5603,6 +6086,40 @@ mod tests {
accept_task.abort();
}
#[tokio::test]
async fn read_version_uses_the_metadata_timeout_on_a_stalled_peer() {
runtime_sources::ensure_test_rpc_secret();
let Some((base_addr, accept_task)) = spawn_stalled_grpc_peer().await else {
return;
};
let remote_disk = remote_disk_for_addr(&base_addr).await;
temp_env::async_with_vars(
[
(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, Some("1")),
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("10")),
],
async {
let started = time::Instant::now();
let err = tokio::time::timeout(
Duration::from_secs(5),
remote_disk.read_version("bucket", "bucket", "object", "", &ReadOptions::default()),
)
.await
.expect("read_version must use the shorter metadata deadline")
.expect_err("a stalled peer must fail read_version");
assert!(matches!(err, DiskError::Timeout), "expected the metadata deadline to fire, got {err:?}");
assert!(started.elapsed() >= Duration::from_millis(900));
assert!(started.elapsed() < Duration::from_secs(2));
},
)
.await;
remote_disk.cancel_token.cancel();
accept_task.abort();
}
#[tokio::test]
async fn delete_volume_bounds_the_wait_on_a_stalled_peer() {
runtime_sources::ensure_test_rpc_secret();
+46 -3
View File
@@ -9243,7 +9243,7 @@ impl DiskAPI for LocalDisk {
if let Some(src_file_path_parent) = src_file_path.parent() {
if src_volume != super::RUSTFS_META_MULTIPART_BUCKET {
let _ = remove_std(src_file_path_parent);
let _ = std::fs::remove_dir(src_file_path_parent);
} else {
let _ = self
.delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false)
@@ -9568,7 +9568,7 @@ impl DiskAPI for LocalDisk {
if let Some(ref cleanup) = cleanup_path {
let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await;
} else if let Some(parent) = src_file_path.parent() {
let _ = remove_std(parent);
let _ = std::fs::remove_dir(parent);
}
// Heal reuses a version's `data_dir` and lands the rebuilt shard on
@@ -12565,6 +12565,10 @@ mod test {
.join(RUSTFS_META_TMP_BUCKET)
.join(tmp_object)
.join(new_data_dir.to_string());
let tmp_parent = tmp_data_dir
.parent()
.expect("tmp data dir should have a parent")
.to_path_buf();
fs::create_dir_all(&tmp_data_dir)
.await
.expect("new tmp data dir should be created");
@@ -12576,6 +12580,10 @@ mod test {
disk.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, new_fi, bucket, object)
.await
.expect("rename_data should commit");
assert!(
!tmp_parent.exists(),
"successful non-inline commit should remove the empty staging parent"
);
// The tmp xl.meta write point uses SyncMode::FileOnly: its parent dir
// ({tmp}/{tmp_object}) must not be fsynced.
@@ -12780,6 +12788,9 @@ mod test {
let tmp_object = "tmp-new-inline";
ensure_test_volume(&disk, bucket).await;
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
let tmp_parent = disk
.get_object_path(RUSTFS_META_TMP_BUCKET, tmp_object)
.expect("tmp parent should resolve");
let _mode = durability_mode_override::set(DurabilityMode::Strict);
let version_id = Uuid::parse_str("99999999-9999-9999-9999-999999999999").expect("version id should parse");
@@ -12788,6 +12799,7 @@ mod test {
disk.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, new_fi, bucket, object)
.await
.expect("inline rename_data should commit the new object");
assert!(!tmp_parent.exists(), "successful inline commit should remove the empty staging parent");
let bucket_dir = disk.get_bucket_path(bucket).expect("bucket path should resolve");
let prefix_dir = disk.get_object_path(bucket, "prefix").expect("prefix path should resolve");
@@ -12811,6 +12823,34 @@ mod test {
);
}
#[tokio::test]
async fn rename_data_inline_preserves_non_empty_staging_parent() {
use tempfile::tempdir;
let dir = tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
let bucket = "inline-staging-sentinel-bucket";
let object = "inline-object";
let tmp_object = "inline-stage-with-sentinel";
ensure_test_volume(&disk, bucket).await;
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
let tmp_parent = disk
.get_object_path(RUSTFS_META_TMP_BUCKET, tmp_object)
.expect("tmp parent should resolve");
fs::create_dir_all(&tmp_parent).await.expect("tmp parent should be created");
let sentinel = tmp_parent.join("sentinel");
fs::write(&sentinel, b"keep").await.expect("sentinel should be written");
let fi = test_file_info(object, Uuid::new_v4(), None, Some(Bytes::from_static(b"inline-payload")));
disk.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, fi, bucket, object)
.await
.expect("non-empty staging cleanup must not negate the committed object");
assert_eq!(fs::read(&sentinel).await.expect("sentinel should remain"), b"keep");
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[allow(clippy::await_holding_lock)]
@@ -12985,7 +13025,10 @@ mod test {
.expect("non-inline rename_data should commit");
assert!(!replacement_dir.exists(), "the destination object directory must not be replaced");
assert!(staging_parent.exists(), "the guarded staging parent must retain its identity");
assert!(
!staging_parent.exists(),
"successful commit should remove the empty staging parent after releasing its guard"
);
assert!(
!replacement_staging_parent.exists(),
"the staging parent must not be replaced between data and metadata publication"
+155 -40
View File
@@ -18,10 +18,12 @@ use crate::disk::error_reduce::{
};
use crate::erasure::coding::BitrotWriterWrapper;
use crate::erasure::coding::Erasure;
use crate::erasure::coding::erasure::EncodedBlock;
use crate::runtime::sources as runtime_sources;
use bytes::{Bytes, BytesMut};
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use rustfs_utils::HashAlgorithm;
use std::sync::Arc;
use std::time::Instant;
use std::vec;
@@ -223,8 +225,8 @@ async fn send_queued<T>(
sender.send(InflightEntry::new(entry, bytes)).await
}
fn queued_batch_bytes(batch: &[Vec<Bytes>]) -> usize {
batch.iter().map(|block| queued_block_bytes(block)).sum()
fn queued_batch_bytes(batch: &[EncodedBlock]) -> usize {
batch.iter().map(EncodedBlock::queued_bytes).sum()
}
fn dominant_error_summary_label(summary: &WriteQuorumFailureSummary) -> &'static str {
@@ -336,7 +338,7 @@ impl<'a> MultiWriter<'a> {
}
}
async fn write_shard(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>, shard: &Bytes) {
async fn write_shard(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>, shard: &[u8]) {
match writer_opt {
Some(writer) => {
match writer.write(shard).await {
@@ -361,12 +363,20 @@ impl<'a> MultiWriter<'a> {
}
pub async fn write(&mut self, data: Vec<Bytes>) -> std::io::Result<()> {
assert_eq!(data.len(), self.writers.len());
self.write_shards(data.iter().map(Bytes::as_ref)).await
}
async fn write_block(&mut self, block: &EncodedBlock) -> std::io::Result<()> {
self.write_shards(block.shards()).await
}
async fn write_shards<'b>(&mut self, shards: impl ExactSizeIterator<Item = &'b [u8]>) -> std::io::Result<()> {
assert_eq!(shards.len(), self.writers.len());
let budget = self.next_progress_budget();
{
let mut futures = FuturesUnordered::new();
for ((writer_opt, err), shard) in self.writers.iter_mut().zip(self.errs.iter_mut()).zip(data.iter()) {
for ((writer_opt, err), shard) in self.writers.iter_mut().zip(self.errs.iter_mut()).zip(shards) {
if err.is_some() {
continue; // Skip if we already have an error for this writer
}
@@ -490,10 +500,10 @@ impl<'a> MultiWriter<'a> {
}
impl Erasure {
async fn encode_block(self: Arc<Self>, encode_buf: Vec<u8>, len: usize) -> std::io::Result<(Vec<Bytes>, Vec<u8>)> {
async fn encode_block(self: Arc<Self>, encode_buf: Vec<u8>, len: usize) -> std::io::Result<(EncodedBlock, Vec<u8>)> {
let encode_stage_start = stage_timer_if_enabled();
let encode_once = move || {
let res = self.encode_data(&encode_buf[..len]);
let res = self.encode_data_block(&encode_buf[..len]);
(res, encode_buf)
};
@@ -518,9 +528,9 @@ impl Erasure {
Ok((res?, returned_buf))
}
async fn encode_block_bytes_mut(self: Arc<Self>, encode_buf: BytesMut, len: usize) -> std::io::Result<Vec<Bytes>> {
async fn encode_block_bytes_mut(self: Arc<Self>, encode_buf: BytesMut, len: usize) -> std::io::Result<EncodedBlock> {
let encode_stage_start = stage_timer_if_enabled();
let encode_once = move || self.encode_data_bytes_mut(encode_buf, len);
let encode_once = move || self.encode_data_bytes_mut_block(encode_buf, len);
let res = match tokio::runtime::Handle::current().runtime_flavor() {
// Same rationale as encode_block: inline the short EC burst on the
@@ -583,6 +593,39 @@ impl Erasure {
Ok((reader, total))
}
/// Encode a small inline object directly into its per-disk bitrot payloads.
/// The returned bytes are the same `[hash][shard]` representation produced
/// by `BitrotWriter`, ready to be embedded in each disk's staged `xl.meta`.
#[hotpath::measure(impl_type = "Erasure")]
pub(crate) async fn encode_inline_shards_with_size_hint<R>(
self: Arc<Self>,
mut reader: R,
size_hint: usize,
) -> std::io::Result<(R, usize, Vec<Bytes>)>
where
R: AsyncRead + Send + Sync + Unpin,
{
use tokio::io::AsyncReadExt;
let mut buf = Vec::with_capacity(small_ingest_capacity(&self, size_hint));
let total = reader.read_to_end(&mut buf).await?;
if total == 0 {
return Ok((reader, 0, Vec::new()));
}
let shards = self.encode_data_owned(buf)?;
let mut inline_shards = Vec::with_capacity(shards.len());
for shard in shards {
let hash = HashAlgorithm::HighwayHash256S.hash_encode(&shard);
let mut encoded = BytesMut::with_capacity(hash.as_ref().len() + shard.len());
encoded.extend_from_slice(hash.as_ref());
encoded.extend_from_slice(&shard);
inline_shards.push(encoded.freeze());
}
Ok((reader, total, inline_shards))
}
#[hotpath::measure(impl_type = "Erasure")]
pub async fn encode<R>(
self: Arc<Self>,
@@ -624,7 +667,7 @@ impl Erasure {
let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
let max_inflight_bytes = erasure_encode_max_inflight_bytes();
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Bytes>>>(inflight_blocks);
let (tx, mut rx) = mpsc::channel::<InflightEntry<EncodedBlock>>(inflight_blocks);
let mut task = AbortOnDropTask::new(tokio::spawn(async move {
let block_size = self.block_size;
@@ -646,7 +689,7 @@ impl Erasure {
let encode_buf = buf;
let res = self.clone().encode_block_bytes_mut(encode_buf, n).await?;
buf = BytesMut::with_capacity(ingest_capacity);
let queued_bytes = queued_block_bytes(&res);
let queued_bytes = res.queued_bytes();
let _producer_stage = rustfs_io_metrics::track_ec_encode_producer_bytes(queued_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = send_queued(&tx, res, queued_bytes).await {
@@ -676,7 +719,7 @@ impl Erasure {
let encode_buf = std::mem::take(&mut buf);
let (res, returned_buf) = self.clone().encode_block(encode_buf, n).await?;
buf = returned_buf;
let queued_bytes = queued_block_bytes(&res);
let queued_bytes = res.queued_bytes();
let _producer_stage = rustfs_io_metrics::track_ec_encode_producer_bytes(queued_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = send_queued(&tx, res, queued_bytes).await {
@@ -720,9 +763,9 @@ impl Erasure {
if block.is_empty() {
break;
}
let _writer_stage = rustfs_io_metrics::track_ec_encode_writer_bytes(queued_block_bytes(&block));
let _writer_stage = rustfs_io_metrics::track_ec_encode_writer_bytes(block.queued_bytes());
let write_stage_start = stage_timer_if_enabled();
if let Err(err) = writers.write(block).await {
if let Err(err) = writers.write_block(&block).await {
write_err = Some(err);
break;
}
@@ -769,7 +812,7 @@ impl Erasure {
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let batch_blocks = encode_batch_block_count().min(inflight_blocks);
let channel_capacity = inflight_blocks.div_ceil(batch_blocks).max(1);
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Vec<Bytes>>>>(channel_capacity);
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<EncodedBlock>>>(channel_capacity);
let mut task = AbortOnDropTask::new(tokio::spawn(async move {
let block_size = self.block_size;
@@ -786,7 +829,7 @@ impl Erasure {
let encode_buf = std::mem::take(&mut buf);
let (res, returned_buf) = self.clone().encode_block(encode_buf, n).await?;
buf = returned_buf;
let queued_bytes = queued_block_bytes(&res);
let queued_bytes = res.queued_bytes();
pending_batch_bytes = pending_batch_bytes.saturating_add(queued_bytes);
pending_batch.push(res);
drop(pending_batch_stage.take());
@@ -845,7 +888,7 @@ impl Erasure {
let _writer_stage = rustfs_io_metrics::track_ec_encode_writer_bytes(queued_batch_bytes(&batch));
let write_stage_start = stage_timer_if_enabled();
for block in batch {
if let Err(err) = writers.write(block).await {
if let Err(err) = writers.write_block(&block).await {
write_err = Some(err);
break;
}
@@ -1895,7 +1938,11 @@ mod tests {
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let (tx, rx) = mpsc::channel(2);
let mut rx = rx;
let batch = vec![vec![Bytes::from_static(b"queued")], vec![Bytes::from_static(b"batch")]];
let erasure = Erasure::new(1, 0, 16);
let batch = vec![
erasure.encode_data_block(b"queued").expect("first block should encode"),
erasure.encode_data_block(b"batch").expect("second block should encode"),
];
let batch_bytes = queued_batch_bytes(&batch);
send_queued(&tx, batch, batch_bytes).await.expect("batch should be queued");
@@ -2236,11 +2283,11 @@ mod tests {
.expect("bytesmut encode should succeed on current-thread runtime");
let expected_shard_size = payload.len().div_ceil(erasure.data_shards);
assert_eq!(shards.len(), erasure.total_shard_count());
assert!(shards.iter().all(|shard| shard.len() == expected_shard_size));
assert_eq!(shards.shards().len(), erasure.total_shard_count());
assert!(shards.shards().all(|shard| shard.len() == expected_shard_size));
let mut restored = Vec::new();
for shard in shards.iter().take(erasure.data_shards) {
for shard in shards.shards().take(erasure.data_shards) {
restored.extend_from_slice(shard);
}
restored.truncate(payload.len());
@@ -2343,6 +2390,35 @@ mod tests {
assert!(committed.lock().unwrap().is_empty());
}
#[tokio::test]
async fn encode_inline_shards_matches_writer_bitrot_layout() {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let payload = b"inline commit payload".to_vec();
let checksum_algo = HashAlgorithm::HighwayHash256S;
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
let (_reader, total, inline_shards) = erasure
.clone()
.encode_inline_shards_with_size_hint(reader, payload.len())
.await
.expect("inline shards should encode");
let raw_shards = erasure
.encode_data_owned(payload.clone())
.expect("reference shards should encode");
assert_eq!(total, payload.len());
assert_eq!(inline_shards.len(), DATA_SHARDS + PARITY_SHARDS);
for (inline, raw) in inline_shards.iter().zip(raw_shards) {
let mut writer = BitrotWriterWrapper::new(CustomWriter::new_inline_buffer(), raw.len(), checksum_algo.clone());
writer.write(&raw).await.expect("reference writer should accept shard");
writer.shutdown().await.expect("reference writer should shutdown");
assert_eq!(inline.as_ref(), writer.into_inline_data().expect("reference writer should retain bytes"));
}
}
/// encode_inline_small: small payload is encoded into the correct number of shards
/// and each writer receives data after shutdown.
#[tokio::test]
@@ -2506,7 +2582,7 @@ mod tests {
assert_eq!(&next[..], &data[16..]);
}
async fn committed_shards_for_ingest_mode(use_bytesmut_ingest: bool, uses_legacy: bool, payload: &[u8]) -> Vec<Vec<u8>> {
async fn committed_shards_for_pipeline(pipeline: EncodePipeline, uses_legacy: bool, payload: &[u8]) -> Vec<Vec<u8>> {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const TOTAL_SHARDS: usize = DATA_SHARDS + PARITY_SHARDS;
@@ -2520,10 +2596,16 @@ mod tests {
let erasure = Arc::new(Erasure::new_with_options(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE, uses_legacy));
let reader = tokio::io::BufReader::new(Cursor::new(payload.to_vec()));
let (_reader, total) = erasure
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, use_bytesmut_ingest)
.await
.expect("encode should succeed");
let (_reader, total) = match pipeline {
EncodePipeline::Vec => {
erasure
.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false)
.await
}
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true).await,
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, DATA_SHARDS).await,
}
.expect("encode should succeed");
assert_eq!(total, payload.len());
committed
@@ -2532,31 +2614,64 @@ mod tests {
.collect()
}
/// HP-10 (rustfs/backlog#931) merge gate: the BytesMut ingest path must produce
/// byte-for-byte identical shard streams to the default Vec ingest path, for both
/// legacy-aware shard-size formulas, across empty, sub-block, exactly-full-block,
/// and multi-block-with-partial-tail payloads.
async fn expected_committed_shards(uses_legacy: bool, payload: &[u8]) -> Vec<Vec<u8>> {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const TOTAL_SHARDS: usize = DATA_SHARDS + PARITY_SHARDS;
const BLOCK_SIZE: usize = 64;
let committed: Vec<Arc<Mutex<Vec<u8>>>> = (0..TOTAL_SHARDS).map(|_| Arc::new(Mutex::new(Vec::new()))).collect();
let mut writers: Vec<BitrotWriterWrapper> = committed
.iter()
.map(|c| bitrot_writer(DeferredCommitWriter::new(c.clone()), BLOCK_SIZE / DATA_SHARDS))
.collect();
let erasure = Erasure::new_with_options(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE, uses_legacy);
for block in payload.chunks(BLOCK_SIZE) {
let shards = erasure.encode_data(block).expect("reference block should encode");
for (writer, shard) in writers.iter_mut().zip(shards) {
let written = writer.write(&shard).await.expect("reference shard should write");
assert_eq!(written, shard.len());
}
}
for writer in &mut writers {
writer.shutdown().await.expect("reference writer should commit");
}
committed
.iter()
.map(|c| c.lock().expect("committed buffer should be lockable").clone())
.collect()
}
/// The streaming and batched paths must produce the same bitrot-wrapped shard
/// bytes as the public block encoder for both shard-size formulas and all block
/// boundary shapes.
#[tokio::test]
async fn bytesmut_ingest_matches_vec_ingest_byte_for_byte() {
const BLOCK_SIZE: usize = 64;
let payloads: Vec<Vec<u8>> = vec![
Vec::new(),
b"tiny".to_vec(),
vec![1],
vec![2; BLOCK_SIZE - 1],
(0..BLOCK_SIZE as u32).map(|i| i as u8).collect(), // exactly one full block
vec![3u8; BLOCK_SIZE * 4], // whole number of blocks
vec![4; BLOCK_SIZE + 1],
vec![3u8; BLOCK_SIZE * 4], // whole number of blocks
(0..(BLOCK_SIZE * 3 + 7) as u32).map(|i| (i % 251) as u8).collect(), // partial tail
];
for uses_legacy in [false, true] {
for payload in &payloads {
let vec_path = committed_shards_for_ingest_mode(false, uses_legacy, payload).await;
let bytesmut_path = committed_shards_for_ingest_mode(true, uses_legacy, payload).await;
assert_eq!(
vec_path,
bytesmut_path,
"ingest paths must be byte-identical (legacy={uses_legacy}, payload_len={})",
payload.len()
);
let expected = expected_committed_shards(uses_legacy, payload).await;
for pipeline in [EncodePipeline::Vec, EncodePipeline::BytesMut, EncodePipeline::Batched] {
let actual = committed_shards_for_pipeline(pipeline, uses_legacy, payload).await;
assert_eq!(
actual,
expected,
"streaming shards must match the public block encoder (legacy={uses_legacy}, payload_len={})",
payload.len()
);
}
}
}
}
+103 -20
View File
@@ -29,6 +29,46 @@ use tokio::io::AsyncRead;
use tracing::warn;
use uuid::Uuid;
pub(crate) struct EncodedBlock {
data: Bytes,
shard_size: usize,
}
impl EncodedBlock {
fn empty() -> Self {
Self {
data: Bytes::new(),
shard_size: 0,
}
}
pub(crate) fn is_empty(&self) -> bool {
self.data.is_empty()
}
pub(crate) fn queued_bytes(&self) -> usize {
self.data.len()
}
pub(crate) fn shards(&self) -> impl ExactSizeIterator<Item = &[u8]> {
debug_assert!(self.shard_size > 0, "only non-empty encoded blocks reach shard writers");
debug_assert_eq!(self.data.len() % self.shard_size, 0);
self.data.chunks_exact(self.shard_size)
}
fn into_shards(mut self, shard_count: usize) -> Vec<Bytes> {
if self.shard_size == 0 {
return vec![Bytes::new(); shard_count];
}
let mut shards = Vec::with_capacity(shard_count);
for _ in 0..shard_count {
shards.push(self.data.split_to(self.shard_size));
}
shards
}
}
const MODERN_MAX_TOTAL_SHARDS: usize = <reed_solomon_erasure::galois_8::Field as reed_solomon_erasure::Field>::ORDER;
const MODERN_REED_SOLOMON_CACHE_MAX_ENTRIES: usize = 64;
@@ -675,6 +715,17 @@ impl Erasure {
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
#[hotpath::measure(impl_type = "Erasure")]
pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
self.encode_data_block_inner(data)
.map(|block| block.into_shards(self.total_shard_count()))
}
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
#[hotpath::measure(label = "Erasure::encode_data", impl_type = "Erasure")]
pub(crate) fn encode_data_block(&self, data: &[u8]) -> io::Result<EncodedBlock> {
self.encode_data_block_inner(data)
}
fn encode_data_block_inner(&self, data: &[u8]) -> io::Result<EncodedBlock> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
} else {
@@ -682,7 +733,7 @@ impl Erasure {
};
let per_shard_size = shard_size_fn(data.len(), self.data_shards);
if per_shard_size == 0 {
return Ok(vec![Bytes::new(); self.total_shard_count()]);
return Ok(EncodedBlock::empty());
}
let need_total_size = per_shard_size * self.total_shard_count();
@@ -708,15 +759,10 @@ impl Erasure {
}
}
// Zero-copy split, all shards reference data_buffer
let mut data_buffer = data_buffer.freeze();
let mut shards = Vec::with_capacity(self.total_shard_count());
for _ in 0..self.total_shard_count() {
let shard = data_buffer.split_to(per_shard_size);
shards.push(shard);
}
Ok(shards)
Ok(EncodedBlock {
data: data_buffer.freeze(),
shard_size: per_shard_size,
})
}
/// Encode owned data, avoiding a copy when the caller already has a heap buffer.
@@ -786,7 +832,17 @@ impl Erasure {
/// `data_len <= block_size` — both shard-size formulas are monotone in
/// `data_len` — so this function never reallocates the buffer.
#[hotpath::measure(impl_type = "Erasure")]
pub fn encode_data_bytes_mut(&self, mut data_buffer: BytesMut, data_len: usize) -> io::Result<Vec<Bytes>> {
pub fn encode_data_bytes_mut(&self, data_buffer: BytesMut, data_len: usize) -> io::Result<Vec<Bytes>> {
self.encode_data_bytes_mut_block_inner(data_buffer, data_len)
.map(|block| block.into_shards(self.total_shard_count()))
}
#[hotpath::measure(label = "Erasure::encode_data_bytes_mut", impl_type = "Erasure")]
pub(crate) fn encode_data_bytes_mut_block(&self, data_buffer: BytesMut, data_len: usize) -> io::Result<EncodedBlock> {
self.encode_data_bytes_mut_block_inner(data_buffer, data_len)
}
fn encode_data_bytes_mut_block_inner(&self, mut data_buffer: BytesMut, data_len: usize) -> io::Result<EncodedBlock> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
} else {
@@ -794,7 +850,7 @@ impl Erasure {
};
let per_shard_size = shard_size_fn(data_len, self.data_shards);
if per_shard_size == 0 {
return Ok(vec![Bytes::new(); self.total_shard_count()]);
return Ok(EncodedBlock::empty());
}
let need_total_size = per_shard_size * self.total_shard_count();
@@ -821,14 +877,10 @@ impl Erasure {
}
}
let mut data_buffer = data_buffer.freeze();
let mut shards = Vec::with_capacity(self.total_shard_count());
for _ in 0..self.total_shard_count() {
let shard = data_buffer.split_to(per_shard_size);
shards.push(shard);
}
Ok(shards)
Ok(EncodedBlock {
data: data_buffer.freeze(),
shard_size: per_shard_size,
})
}
/// Decode and reconstruct missing data shards in-place.
@@ -1547,6 +1599,37 @@ mod tests {
}
}
#[test]
fn streaming_encoded_block_uses_one_contiguous_backing_buffer() {
let erasure = Erasure::new(8, 8, 64);
for data_len in [1, 63, 64] {
let data = (0..data_len).map(|i| i as u8).collect::<Vec<_>>();
let expected = erasure.encode_data(&data).expect("public encode should succeed");
let borrowed = erasure
.encode_data_block(&data)
.expect("borrowed streaming encode should succeed");
let owned = erasure
.encode_data_bytes_mut_block(BytesMut::from(&data[..]), data.len())
.expect("BytesMut streaming encode should succeed");
assert!(borrowed.shards().eq(expected.iter().map(Bytes::as_ref)));
assert!(owned.shards().eq(expected.iter().map(Bytes::as_ref)));
assert_eq!(borrowed.shards().len(), 16);
assert_eq!(borrowed.queued_bytes(), owned.queued_bytes());
let first = borrowed.shards().next().expect("encoded block should have shards").as_ptr();
for (index, shard) in borrowed.shards().enumerate() {
assert_eq!(shard.as_ptr(), first.wrapping_add(index * shard.len()));
}
}
assert_eq!(
std::mem::size_of::<EncodedBlock>(),
std::mem::size_of::<Bytes>() + std::mem::size_of::<usize>(),
"queue entries must contain one backing buffer handle, not per-shard handles"
);
}
/// HP-10 capacity invariant: both shard-size formulas are monotone in `data_len`,
/// so pre-reserving `shard_size(block_size) * total_shard_count` covers the
/// `need_total_size` of every block-or-smaller payload and the ingest buffer
@@ -210,6 +210,22 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke
fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
}
#[cfg(any(test, feature = "test-util"))]
pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
let mut state = cross_pool_fence_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(current) = state.proof.as_ref() else {
return false;
};
state.proof = Some(FleetCapabilityProof {
topology_fingerprint: current.topology_fingerprint.clone(),
peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()),
expires_at: current.expires_at,
});
true
}
fn fleet_capability_proof_matches(
slot: &std::sync::RwLock<FleetCapabilityProofState>,
proof: &FleetCapabilityProofToken,
+52
View File
@@ -1079,6 +1079,25 @@ impl SetDisks {
shuffled_disks
}
pub(super) fn shuffle_disks_owned(mut disks: Vec<Option<DiskStore>>, distribution: &[usize]) -> Vec<Option<DiskStore>> {
if distribution.is_empty() {
return disks;
}
let mut shuffled_disks = vec![None; disks.len()];
for (index, disk) in disks.iter_mut().enumerate() {
let Some(slot) = distribution
.get(index)
.and_then(|block_index| block_index.checked_sub(1))
.filter(|slot| *slot < shuffled_disks.len())
else {
continue;
};
shuffled_disks[slot] = disk.take();
}
shuffled_disks
}
pub(super) fn shuffle_check_parts(parts_errs: &[usize], distribution: &[usize]) -> Vec<usize> {
if distribution.is_empty() {
return parts_errs.to_vec();
@@ -1390,6 +1409,23 @@ mod tests {
assert_eq!(owned_slots, expected_slots, "fallback disk slots must match the borrowing variant");
}
#[tokio::test]
async fn owned_shuffle_preserves_fresh_put_metadata() {
let tempdir = tempfile::tempdir().expect("tempdir should be created");
let fi = FileInfo::new("bucket/object", 2, 1);
let parts = vec![fi.clone(); fi.erasure.distribution.len()];
let disks = shuffle_test_disks(&tempdir, parts.len()).await;
let (owned_disks, owned_parts) = SetDisks::shuffle_disks_and_parts_metadata_by_index_owned(disks, parts, &fi);
assert!(owned_disks.iter().all(Option::is_some), "fresh PUT must retain every online disk");
assert_eq!(
owned_parts,
vec![fi; owned_disks.len()],
"fresh PUT metadata with pending shard indexes must survive init fallback"
);
}
// backlog#949: corrupt/adversarial distribution values (0 or > N) must not
// trigger a `usize` underflow / out-of-bounds panic in the shuffle helpers.
#[test]
@@ -1419,6 +1455,22 @@ mod tests {
assert_eq!(result.len(), disks.len(), "output length must be preserved");
}
#[tokio::test]
async fn owned_disk_shuffle_matches_borrowing_variant() {
let tempdir = tempfile::tempdir().expect("tempdir should be created");
let mut disks = shuffle_test_disks(&tempdir, 4).await;
disks[1] = None;
disks[3] = None;
let distribution = [3, 1, 4, 2];
let expected = SetDisks::shuffle_disks(&disks, &distribution);
let actual = SetDisks::shuffle_disks_owned(disks, &distribution);
let expected_slots = expected.iter().map(Option::is_some).collect::<Vec<_>>();
let actual_slots = actual.iter().map(Option::is_some).collect::<Vec<_>>();
assert_eq!(actual_slots, expected_slots, "owned shuffle must preserve disk placement");
}
#[tokio::test]
async fn shuffle_disks_and_parts_metadata_survives_corrupt_distribution() {
let tempdir = tempfile::tempdir().expect("tempdir should be created");
+2 -2
View File
@@ -718,8 +718,8 @@ pub(crate) use core::io_primitives::disk_call_counters;
mod ctx;
mod metadata;
mod ops;
#[cfg(test)]
pub(crate) use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(any(test, feature = "test-util"))]
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(feature = "test-util")]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
pub(crate) use ops::object::body_cache_plaintext_len;
+33 -22
View File
@@ -27,24 +27,25 @@ use crate::crash_inject::{self, CrashPoint};
use crate::multipart_listing::paginate_multipart_listing;
use futures::{StreamExt, stream};
use std::future::Future;
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tokio::task::JoinSet;
const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum MultipartCommitPause {
pub enum MultipartCommitPause {
PutPartBeforeLockAcquire,
PutPartBeforeLockLost,
PutPartAfterRename,
BeforeLockLost,
BeforeQuotaRename,
AfterRename,
}
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
struct MultipartCommitBarrierState {
bucket: String,
object: String,
@@ -55,27 +56,22 @@ struct MultipartCommitBarrierState {
release: tokio::sync::Semaphore,
}
#[cfg(test)]
pub(crate) struct MultipartCommitBarrier {
#[cfg(any(test, feature = "test-util"))]
pub struct MultipartCommitBarrier {
state: Arc<MultipartCommitBarrierState>,
}
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<MultipartCommitBarrierState>>>> =
std::sync::OnceLock::new();
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
impl MultipartCommitBarrier {
pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
pub fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
Self::install_for_arrivals(bucket, object, pause, 1)
}
pub(crate) fn install_for_arrivals(
bucket: &str,
object: &str,
pause: MultipartCommitPause,
expected_arrivals: usize,
) -> Self {
pub fn install_for_arrivals(bucket: &str, object: &str, pause: MultipartCommitPause, expected_arrivals: usize) -> Self {
assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival");
let state = Arc::new(MultipartCommitBarrierState {
bucket: bucket.to_string(),
@@ -96,7 +92,7 @@ impl MultipartCommitBarrier {
Self { state }
}
pub(crate) async fn wait_until_paused(&self) {
pub async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), async {
loop {
let arrived = self.state.arrived.notified();
@@ -110,12 +106,12 @@ impl MultipartCommitBarrier {
.expect("multipart completion should reach the deterministic commit barrier");
}
pub(crate) fn release(&self) {
pub fn release(&self) {
self.state.release.add_permits(self.state.expected_arrivals);
}
}
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
impl Drop for MultipartCommitBarrier {
fn drop(&mut self) {
self.release();
@@ -129,7 +125,7 @@ impl Drop for MultipartCommitBarrier {
}
}
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
let barrier = MULTIPART_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -1761,8 +1757,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
fi.parts = Vec::with_capacity(uploaded_parts.len());
let quota_context =
reservation::begin(&self.ctx, bucket, object, opts.quota_admission, self.pool_index, self.set_index).await?;
let quota_context = reservation::begin(
&self.ctx,
bucket,
object,
opts.quota_admission,
opts.data_movement,
self.pool_index,
self.set_index,
)
.await?;
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
let preserve_replication_ciphertext = opts.replication_request
&& contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
@@ -2125,7 +2129,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
}
let quota_old_size = if quota_context.is_enforced() {
reservation::replaced_logical_size(&self, bucket, object, opts).await?
if opts.data_movement {
quota_new_size
} else {
reservation::replaced_logical_size(&self, bucket, object, opts).await?
}
} else {
0
};
@@ -2228,7 +2236,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
.await;
return Err(err);
}
#[cfg(any(test, feature = "test-util"))]
pause_multipart_commit(bucket, object, MultipartCommitPause::BeforeQuotaRename).await;
if quota_reservation.is_lock_lost()
|| !quota_reservation.capability_proof_matches()
|| object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| opts
+463 -102
View File
@@ -62,6 +62,10 @@ fn duration_millis_f64(duration: std::time::Duration) -> f64 {
duration.as_secs_f64() * 1000.0
}
fn committed_response_metadata_slot<D>(committed_disks: &[Option<D>], fallback_slot: usize) -> usize {
committed_disks.iter().position(Option::is_some).unwrap_or(fallback_slot)
}
#[cfg(test)]
mod duration_metrics_tests {
use super::duration_millis_f64;
@@ -73,6 +77,38 @@ mod duration_metrics_tests {
}
}
#[cfg(test)]
mod put_metadata_tests {
use super::*;
#[test]
fn committed_file_info_follows_exact_quorum_success_slot() {
let mut first_success = FileInfo::new("bucket/object", 2, 2);
first_success.name = "first-success".to_string();
let mut second_success = first_success.clone();
second_success.name = "second-success".to_string();
let mut parts_metadata = [FileInfo::default(), first_success, second_success, FileInfo::default()];
let committed_disks = [None, Some(()), Some(()), None];
assert_eq!(
committed_disks.iter().filter(|disk| disk.is_some()).count(),
2,
"fixture must meet exact quorum"
);
let selected_slot = committed_response_metadata_slot(&committed_disks, 3);
let selected = std::mem::take(&mut parts_metadata[selected_slot]);
assert_eq!(selected.name, "first-success");
assert_eq!(parts_metadata[1], FileInfo::default(), "selected metadata should move without cloning");
assert_eq!(parts_metadata[2].name, "second-success", "other committed metadata must remain available");
assert_eq!(
committed_response_metadata_slot::<()>(&[None, None, None, None], 3),
3,
"a violated post-commit success-mask invariant must not turn a durable PUT into an error"
);
}
}
fn is_restore_control_metadata(key: &str) -> bool {
key.eq_ignore_ascii_case(X_AMZ_RESTORE.as_str())
|| key.eq_ignore_ascii_case(rustfs_utils::http::headers::AMZ_RESTORE_EXPIRY_DAYS)
@@ -1060,10 +1096,7 @@ impl SetDisks {
}
fi.data_dir = Some(Uuid::new_v4());
let parts_metadata = vec![fi.clone(); disks.len()];
let (mut shuffle_disks, mut parts_metadatas) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
let mut shuffle_disks = Self::shuffle_disks_owned(disks, &fi.erasure.distribution);
let tmp_dir = Uuid::new_v4().to_string();
@@ -1075,61 +1108,91 @@ impl SetDisks {
let put_object_size = known_put_object_storage_size(data.size());
let is_inline_buffer = storage_class_config.should_inline(erasure.shard_file_size(put_object_size), opts.versioned);
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
let shard_file_size = erasure.shard_file_size(put_object_size);
let shard_size = erasure.shard_size();
let writer_setup_stage_start = Instant::now();
let writer_futs: Vec<_> = shuffle_disks
.iter()
.map(|disk_op| {
let tmp_obj = tmp_object.clone();
async move {
if let Some(disk) = disk_op
&& disk.is_online().await
{
match create_bitrot_writer(
is_inline_buffer,
Some(disk),
RUSTFS_META_TMP_BUCKET,
&tmp_obj,
shard_file_size,
shard_size,
HashAlgorithm::HighwayHash256S,
)
.await
{
Ok(writer) => (Some(writer), None),
Err(err) => {
warn!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
disk = ?disk,
state = "bitrot_writer_skipped",
error = ?err,
"Set disk bitrot writer skipped"
);
(None, Some(err))
}
}
} else {
(None, Some(DiskError::DiskNotFound))
}
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
let direct_inline_commit = matches!(write_path, SmallWritePath::Inline);
rustfs_io_metrics::record_put_object_path(write_path.metric_label());
let writer_setup_stage_start = collect_stage_timing.then(Instant::now);
let (mut writers, errors) = if direct_inline_commit {
let online = join_all(shuffle_disks.iter().map(|disk| async move {
if let Some(disk) = disk {
disk.is_online().await
} else {
false
}
})
.collect();
let writer_results = join_all(writer_futs).await;
let mut writers = Vec::with_capacity(writer_results.len());
let mut errors = Vec::with_capacity(writer_results.len());
for (w, e) in writer_results {
writers.push(w);
errors.push(e);
}))
.await;
let mut errors = Vec::with_capacity(online.len());
for (disk, is_online) in shuffle_disks.iter_mut().zip(online) {
if is_online {
errors.push(None);
} else {
*disk = None;
errors.push(Some(DiskError::DiskNotFound));
}
}
(std::iter::repeat_with(|| None).take(shuffle_disks.len()).collect(), errors)
} else {
let writer_futs: Vec<_> = shuffle_disks
.iter()
.map(|disk_op| {
let tmp_obj = tmp_object.clone();
async move {
if let Some(disk) = disk_op
&& disk.is_online().await
{
match create_bitrot_writer(
is_inline_buffer,
Some(disk),
RUSTFS_META_TMP_BUCKET,
&tmp_obj,
shard_file_size,
shard_size,
HashAlgorithm::HighwayHash256S,
)
.await
{
Ok(writer) => (Some(writer), None),
Err(err) => {
warn!(
event = EVENT_SET_DISK_WRITE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
disk = ?disk,
state = "bitrot_writer_skipped",
error = ?err,
"Set disk bitrot writer skipped"
);
(None, Some(err))
}
}
} else {
(None, Some(DiskError::DiskNotFound))
}
}
})
.collect();
let writer_results = join_all(writer_futs).await;
let mut writers = Vec::with_capacity(writer_results.len());
let mut errors = Vec::with_capacity(writer_results.len());
for (writer, error) in writer_results {
writers.push(writer);
errors.push(error);
}
(writers, errors)
};
let writer_setup_elapsed = writer_setup_stage_start.map(|stage_start| stage_start.elapsed());
let writer_setup_ms = writer_setup_elapsed
.map(|elapsed| elapsed.as_millis() as u64)
.unwrap_or_default();
if let Some(writer_setup_elapsed) = writer_setup_elapsed {
rustfs_io_metrics::record_put_object_stage_duration(
"set_disk_writer_setup",
duration_millis_f64(writer_setup_elapsed),
);
}
let writer_setup_elapsed = writer_setup_stage_start.elapsed();
let writer_setup_ms = writer_setup_elapsed.as_millis() as u64;
rustfs_io_metrics::record_put_object_stage_duration(
"set_disk_writer_setup",
duration_millis_f64(writer_setup_elapsed),
);
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
if nil_count < write_quorum {
@@ -1157,21 +1220,23 @@ impl SetDisks {
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
);
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
rustfs_io_metrics::record_put_object_path(write_path.metric_label());
let small_size_hint = if matches!(write_path, SmallWritePath::Inline | SmallWritePath::SingleBlockNonInline) {
usize::try_from(put_object_size).map_err(Error::other)?
} else {
0
};
let encode_stage_start = Instant::now();
let encode_stage_start = collect_stage_timing.then(Instant::now);
let mut inline_shards = None;
let (reader, w_size) = match write_path {
SmallWritePath::Inline => match Arc::clone(&erasure)
.encode_inline_small_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
.encode_inline_shards_with_size_hint(stream, small_size_hint)
.await
{
Ok((r, w)) => (r, w),
Ok((r, w, shards)) => {
inline_shards = Some(shards);
(r, w)
}
Err(e) => {
error!("encode_inline_small err {:?}", e);
return Err(e.into());
@@ -1204,9 +1269,11 @@ impl SetDisks {
}
},
};
let encode_elapsed = encode_stage_start.elapsed();
let encode_ms = encode_elapsed.as_millis() as u64;
rustfs_io_metrics::record_put_object_stage_duration("set_disk_encode", duration_millis_f64(encode_elapsed));
let encode_elapsed = encode_stage_start.map(|stage_start| stage_start.elapsed());
let encode_ms = encode_elapsed.map(|elapsed| elapsed.as_millis() as u64).unwrap_or_default();
if let Some(encode_elapsed) = encode_elapsed {
rustfs_io_metrics::record_put_object_stage_duration("set_disk_encode", duration_millis_f64(encode_elapsed));
}
let _ = mem::replace(&mut data.stream, reader);
// if let Err(err) = close_bitrot_writers(&mut writers).await {
@@ -1292,37 +1359,67 @@ impl SetDisks {
// drop it below reconstructable quorum (backlog#852 / #799 B3).
// `rename_data` re-checks write quorum over the surviving disks and
// rolls back if too few remain.
let committed_shards = drop_failed_writer_disks(&mut shuffle_disks, &writers);
let committed_shards = if matches!(write_path, SmallWritePath::Inline) {
shuffle_disks.iter().filter(|disk| disk.is_some()).count()
} else {
drop_failed_writer_disks(&mut shuffle_disks, &writers)
};
if committed_shards < write_quorum {
return Err(Error::other(format!(
"put_object write quorum unavailable after encode: {committed_shards} shard(s) committed, need {write_quorum}"
)));
}
for (i, pfi) in parts_metadatas.iter_mut().enumerate() {
pfi.metadata = user_defined.clone();
fi.metadata = user_defined;
fi.mod_time = mod_time;
fi.size = w_size as i64;
fi.versioned = opts.versioned || opts.version_suspended;
fi.add_object_part(1, etag, w_size, mod_time, actual_size, index_op, None);
if opts.data_movement {
fi.set_data_moved();
}
let parity_blocks = fi.erasure.parity_blocks;
let response_metadata_slot = shuffle_disks
.iter()
.rposition(Option::is_some)
.ok_or_else(|| Error::other("put_object write quorum unavailable after encode"))?;
let mut base_file_info = fi;
let mut parts_metadatas = Vec::with_capacity(shuffle_disks.len());
for (i, disk) in shuffle_disks.iter().enumerate() {
if disk.is_none() {
parts_metadatas.push(FileInfo::default());
continue;
}
let mut pfi = if i == response_metadata_slot {
std::mem::take(&mut base_file_info)
} else {
base_file_info.clone()
};
if is_inline_buffer {
if let Some(writer) = writers[i].take() {
if let Some(shards) = inline_shards.as_ref() {
pfi.data = Some(
shards
.get(i)
.cloned()
.ok_or_else(|| Error::other(format!("inline encoder omitted disk shard {i}")))?,
);
} else if let Some(writer) = writers[i].take() {
pfi.data = Some(writer.into_inline_data().map(Bytes::from).unwrap_or_default());
}
pfi.set_inline_data();
}
pfi.mod_time = mod_time;
pfi.size = w_size as i64;
pfi.versioned = opts.versioned || opts.version_suspended;
pfi.add_object_part(1, etag.clone(), w_size, mod_time, actual_size, index_op.clone(), None);
pfi.checksum = fi.checksum.clone();
if opts.data_movement {
pfi.set_data_moved();
}
parts_metadatas.push(pfi);
}
let committed_version_id = parts_metadatas[response_metadata_slot].version_id;
let committed_data_dir = parts_metadatas[response_metadata_slot].data_dir;
let is_compressed = parts_metadatas[response_metadata_slot].is_compressed();
drop(writers); // drop writers to close all files, this is to prevent FileAccessDenied errors when renaming data
if fi.erasure.parity_blocks == 0 {
if parity_blocks == 0 {
let written_size = i64::try_from(w_size).map_err(|_| Error::other("put_object written size overflows i64"))?;
let logical_shard_size = usize::try_from(erasure.shard_file_size(written_size))
.map_err(|_| Error::other("put_object shard size overflows usize"))?;
@@ -1496,8 +1593,16 @@ impl SetDisks {
});
}
let quota_context =
reservation::begin(&self.ctx, bucket, object, opts.quota_admission, self.pool_index, self.set_index).await?;
let quota_context = reservation::begin(
&self.ctx,
bucket,
object,
opts.quota_admission,
opts.data_movement,
self.pool_index,
self.set_index,
)
.await?;
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
let mut replication_quota_size = None;
@@ -1506,17 +1611,16 @@ impl SetDisks {
return Err(Error::PartMissingOrCorrupt);
}
if quota_context.is_enforced() {
let persisted_metadata = &parts_metadatas[response_metadata_slot].metadata;
let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
let physical_size = u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?;
let transformed = contains_key_str(&user_defined, SUFFIX_COMPRESSION)
|| parts_metadatas
.first()
.is_some_and(|metadata| should_persist_encryption_original_size(&metadata.metadata));
let declared_size = get_str(&user_defined, SUFFIX_ACTUAL_SIZE)
let transformed = contains_key_str(persisted_metadata, SUFFIX_COMPRESSION)
|| should_persist_encryption_original_size(persisted_metadata);
let declared_size = get_str(persisted_metadata, SUFFIX_ACTUAL_SIZE)
.map(|value| value.parse::<u64>().map_err(|_| Error::PartMissingOrCorrupt))
.transpose()?
.unwrap_or(0);
let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(&user_defined)
let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(persisted_metadata)
.map_err(Error::other)?
.map(u64::try_from)
.transpose()
@@ -1544,10 +1648,9 @@ impl SetDisks {
}
} else if actual_size >= 0 {
let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
let transformed = contains_key_str(&user_defined, SUFFIX_COMPRESSION)
|| parts_metadatas
.first()
.is_some_and(|metadata| should_persist_encryption_original_size(&metadata.metadata));
let persisted_metadata = &parts_metadatas[response_metadata_slot].metadata;
let transformed = contains_key_str(persisted_metadata, SUFFIX_COMPRESSION)
|| should_persist_encryption_original_size(persisted_metadata);
let server_observed_size = if transformed {
observed_size
} else {
@@ -1574,7 +1677,12 @@ impl SetDisks {
.map_err(|_| Error::PartMissingOrCorrupt)?
.max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?),
};
(reservation::replaced_logical_size(self, bucket, object, opts).await?, new_size)
let old_size = if opts.data_movement {
new_size
} else {
reservation::replaced_logical_size(self, bucket, object, opts).await?
};
(old_size, new_size)
} else {
(0, 0)
};
@@ -1674,6 +1782,7 @@ impl SetDisks {
#[cfg(any(test, feature = "test-util"))]
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeQuotaRename).await;
if quota_reservation.is_lock_lost()
|| !quota_reservation.capability_proof_matches()
|| object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| opts
.namespace_lock_fence
@@ -1736,7 +1845,7 @@ impl SetDisks {
Some(self.pool_index),
Some(self.set_index),
);
request.object_version_id = fi.version_id.map(|version_id| version_id.to_string());
request.object_version_id = committed_version_id.map(|version_id| version_id.to_string());
tokio::spawn(async move {
let _ = rustfs_common::heal_channel::send_heal_request(request).await;
});
@@ -1771,7 +1880,7 @@ impl SetDisks {
let mut cleanup_stage_ms: Option<u64> = None;
if let Some(old_dir) = op_old_dir {
let committed_dir = fi.data_dir.unwrap_or_default().to_string();
let committed_dir = committed_data_dir.unwrap_or_default().to_string();
let cleanup_stage_start = Instant::now();
// backlog#898: reclaiming the dereferenced old data dir is
// best-effort and returns a receipt (never `Err`). A failed GC
@@ -1808,16 +1917,10 @@ impl SetDisks {
}
}
for (i, op_disk) in online_disks.iter().enumerate() {
if let Some(disk) = op_disk
&& disk.is_online().await
{
fi = parts_metadatas[i].clone();
break;
}
}
let committed_metadata_slot = committed_response_metadata_slot(&online_disks, response_metadata_slot);
let mut fi = std::mem::take(&mut parts_metadatas[committed_metadata_slot]);
if fi.is_compressed() {
if is_compressed {
record_compression_total_memory(actual_size as u64, w_size as u64).await;
}
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &online_disks);
@@ -5906,6 +6009,264 @@ mod replication_quota_safety_tests {
}
}
#[cfg(test)]
mod inline_put_commit_path_tests {
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
use super::*;
use crate::disk::{DiskAPI as _, ReadOptions};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use tokio::io::AsyncReadExt;
async fn make_bucket(disks: &[DiskStore], bucket: &str) {
for disk in disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
}
#[tokio::test]
async fn inline_put_direct_commit_round_trips_verified_bitrot_shards() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "inline-direct-commit";
let object = "object.bin";
let payload: Vec<u8> = (0..16 * 1024).map(|index| (index % 251) as u8).collect();
make_bucket(&disk_stores, bucket).await;
let mut reader = PutObjReader::from_vec(payload.clone());
set_disks
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("inline PUT should commit");
let read_data = ReadOptions {
read_data: true,
..Default::default()
};
for (disk_index, disk) in disk_stores.iter().enumerate() {
let file_info = disk
.read_version("", bucket, object, "", &read_data)
.await
.unwrap_or_else(|err| panic!("disk {disk_index} should persist inline metadata: {err}"));
assert!(file_info.inline_data(), "disk {disk_index} should mark the shard inline");
let inline_data = file_info
.data
.as_ref()
.unwrap_or_else(|| panic!("disk {disk_index} should persist inline bitrot bytes"));
let erasure = erasure_from_file_info(&file_info, false).expect("persisted erasure layout should be valid");
let logical_shard_size =
usize::try_from(erasure.shard_file_size(payload.len() as i64)).expect("logical shard size should fit usize");
coding::bitrot_verify(
Cursor::new(inline_data.clone()),
inline_data.len(),
logical_shard_size,
HashAlgorithm::HighwayHash256S,
erasure.shard_size(),
)
.await
.unwrap_or_else(|err| panic!("disk {disk_index} inline shard should pass bitrot verification: {err}"));
}
let mut object_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("committed inline object should be readable");
let mut restored = Vec::new();
object_reader
.stream
.read_to_end(&mut restored)
.await
.expect("inline object should stream");
assert_eq!(restored, payload);
}
#[tokio::test]
async fn inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "inline-direct-quorum";
let exact_quorum_object = "exact-quorum.bin";
let below_quorum_object = "below-quorum.bin";
make_bucket(&disk_stores, bucket).await;
{
let mut disks = set_disks.disks.write().await;
disks[3] = None;
}
let mut reader = PutObjReader::from_vec(vec![0x5a; 4 * 1024]);
set_disks
.put_object(bucket, exact_quorum_object, &mut reader, &ObjectOptions::default())
.await
.expect("three online disks should satisfy the four-disk write quorum");
for (disk_index, disk) in disk_stores.iter().enumerate() {
let persisted = disk
.read_version("", bucket, exact_quorum_object, "", &ReadOptions::default())
.await;
assert_eq!(
persisted.is_ok(),
disk_index < 3,
"exact-quorum commit should publish only on the three online disks"
);
}
set_disks.disks.write().await[2] = None;
let mut reader = PutObjReader::from_vec(vec![0xa5; 4 * 1024]);
set_disks
.put_object(bucket, below_quorum_object, &mut reader, &ObjectOptions::default())
.await
.expect_err("two online disks are one below the four-disk write quorum");
for (disk_index, disk) in disk_stores.iter().enumerate() {
assert!(
disk.read_version("", bucket, below_quorum_object, "", &ReadOptions::default())
.await
.is_err(),
"disk {disk_index} must not expose an object after pre-commit quorum failure"
);
}
}
#[tokio::test]
async fn inline_put_direct_commit_handles_post_encode_rename_failures() {
use crate::disk::health_state::RuntimeDriveHealthState;
let payload = vec![0x5a; 4 * 1024];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "inline-direct-post-encode-quorum";
let object = "exact-quorum.bin";
make_bucket(&disk_stores, bucket).await;
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
let put = {
let set_disks = Arc::clone(&set_disks);
let payload = payload.clone();
tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(payload);
set_disks
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
})
};
barrier.wait_until_paused().await;
disk_stores[3].force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
barrier.release();
put.await
.expect("exact-quorum PUT task should complete")
.expect("one post-encode rename failure should preserve write quorum");
disk_stores[3].force_runtime_state_for_test(RuntimeDriveHealthState::Online);
for (disk_index, disk) in disk_stores.iter().enumerate() {
let persisted = disk.read_version("", bucket, object, "", &ReadOptions::default()).await;
assert_eq!(
persisted.is_ok(),
disk_index < 3,
"only disks that completed rename_data may publish the exact-quorum object"
);
}
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "inline-direct-post-encode-rollback";
let object = "rollback.bin";
let old_payload = vec![0x31; 4 * 1024];
make_bucket(&disk_stores, bucket).await;
let mut old_reader = PutObjReader::from_vec(old_payload.clone());
set_disks
.put_object(bucket, object, &mut old_reader, &ObjectOptions::default())
.await
.expect("old inline object should commit");
let read_data = ReadOptions {
read_data: true,
..Default::default()
};
let mut old_disk_data = Vec::with_capacity(disk_stores.len());
for disk in &disk_stores {
old_disk_data.push(
disk.read_version("", bucket, object, "", &read_data)
.await
.expect("old inline shard should be readable before overwrite")
.data,
);
}
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
let put = {
let set_disks = Arc::clone(&set_disks);
let payload = payload.clone();
tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(payload);
set_disks
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
})
};
barrier.wait_until_paused().await;
for disk in &disk_stores[2..] {
disk.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
}
barrier.release();
put.await
.expect("quorum-minus-one PUT task should complete")
.expect_err("two post-encode rename failures must fail write quorum");
for disk in &disk_stores[2..] {
disk.force_runtime_state_for_test(RuntimeDriveHealthState::Online);
}
for (disk_index, disk) in disk_stores.iter().enumerate() {
let restored = disk
.read_version("", bucket, object, "", &read_data)
.await
.unwrap_or_else(|err| panic!("disk {disk_index} should retain the old inline object: {err}"));
assert_eq!(restored.data, old_disk_data[disk_index]);
}
let mut object_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("old object should remain readable after quorum rollback");
let mut restored = Vec::new();
object_reader
.stream
.read_to_end(&mut restored)
.await
.expect("old object should stream after quorum rollback");
assert_eq!(restored, old_payload);
}
#[tokio::test]
async fn zero_length_put_keeps_existing_pipeline_layout_and_round_trips() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "zero-length-put";
let object = "empty.bin";
make_bucket(&disk_stores, bucket).await;
let mut reader = PutObjReader::from_vec(Vec::new());
set_disks
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("zero-length PUT should commit through the existing pipeline");
let read_data = ReadOptions {
read_data: true,
..Default::default()
};
for (disk_index, disk) in disk_stores.iter().enumerate() {
let file_info = disk
.read_version("", bucket, object, "", &read_data)
.await
.unwrap_or_else(|err| panic!("disk {disk_index} should persist empty-object metadata: {err}"));
assert_eq!(file_info.size, 0);
assert_eq!(file_info.data.as_deref(), Some(&[][..]));
}
let mut object_reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("empty object should be readable");
let mut restored = Vec::new();
object_reader
.stream
.read_to_end(&mut restored)
.await
.expect("empty object should stream");
assert!(restored.is_empty());
}
}
#[cfg(test)]
mod get_object_downstream_close_accounting_tests {
use super::hermetic_set_disks_support::hermetic_set_disks;
+68 -5
View File
@@ -58,7 +58,7 @@ use std::sync::{
/// When `false`, `record_put_object_path` and `record_put_object_stage_duration`
/// become no-ops, and callers can skip the `Instant::now()` syscalls entirely.
///
/// Set to `true` during startup when OTEL metric export is enabled.
/// Enabled only through an explicit runtime opt-in.
static PUT_STAGE_METRICS_ENABLED: AtomicBool = AtomicBool::new(false);
static GET_STAGE_METRICS_ENABLED: AtomicBool = AtomicBool::new(false);
@@ -78,7 +78,7 @@ static METRICS_ENABLED: AtomicBool = AtomicBool::new(false);
/// Enable or disable detailed per-stage PUT metrics.
///
/// Called once during startup, typically gated by `rustfs_obs::observability_metric_enabled()`.
/// Called once during startup after applying the detailed PUT attribution opt-in.
pub fn set_put_stage_metrics_enabled(enabled: bool) {
PUT_STAGE_METRICS_ENABLED.store(enabled, Ordering::Relaxed);
}
@@ -103,6 +103,12 @@ pub fn put_stage_metrics_enabled() -> bool {
PUT_STAGE_METRICS_ENABLED.load(Ordering::Relaxed)
}
/// Start a PUT-stage timer only when detailed PUT attribution is enabled.
#[inline(always)]
pub fn put_stage_timer() -> Option<std::time::Instant> {
put_stage_metrics_enabled().then(std::time::Instant::now)
}
#[inline(always)]
pub fn get_stage_metrics_enabled() -> bool {
GET_STAGE_METRICS_ENABLED.load(Ordering::Relaxed)
@@ -434,7 +440,7 @@ pub fn record_get_object_request_result(status: &str, duration_secs: f64) {
/// Record PutObject request start.
#[inline(always)]
pub fn record_put_object_request_start(concurrent_requests: usize) {
if !put_stage_metrics_enabled() {
if !metrics_enabled() {
return;
}
counter!("rustfs_io_put_object_requests_total").increment(1);
@@ -444,7 +450,7 @@ pub fn record_put_object_request_start(concurrent_requests: usize) {
/// Record PutObject request result.
#[inline(always)]
pub fn record_put_object_request_result(status: &str, duration_secs: f64) {
if !put_stage_metrics_enabled() {
if !metrics_enabled() {
return;
}
counter!("rustfs_io_put_object_request_results_total", "status" => status.to_string()).increment(1);
@@ -1905,7 +1911,7 @@ pub fn record_get_object(duration_ms: f64, size_bytes: i64) {
/// * `zero_copy_eligible` - Whether the request was eligible for a zero-copy path
#[inline(always)]
pub fn record_put_object(duration_ms: f64, size_bytes: i64, zero_copy_eligible: bool) {
if !put_stage_metrics_enabled() {
if !metrics_enabled() {
return;
}
counter!("rustfs_s3_put_object_total").increment(1);
@@ -2004,6 +2010,13 @@ pub fn record_put_object_stage_duration(stage: &'static str, duration_ms: f64) {
histogram!("rustfs_s3_put_object_stage_duration_ms", "stage" => stage).record(duration_ms);
}
#[inline(always)]
pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Option<std::time::Instant>) {
if let Some(started_at) = started_at {
record_put_object_stage_duration(stage, started_at.elapsed().as_secs_f64() * 1000.0);
}
}
/// Record generic internal operation stage duration (non-PUT paths).
/// Use this for metacache walks, listing, lifecycle, and other background
/// operations that are NOT part of the PUT object hot path.
@@ -2819,6 +2832,56 @@ mod tests {
assert!(!put_stage_metrics_enabled());
}
#[test]
fn put_stage_gate_does_not_disable_basic_put_metrics() {
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
metrics::with_local_recorder(&recorder, || {
set_metrics_enabled(true);
set_put_stage_metrics_enabled(false);
record_put_object_request_start(1);
record_put_object_request_result("ok", 0.001);
record_put_object(1.0, 1024, false);
record_put_object_stage_duration("disabled_stage", 0.5);
set_put_stage_metrics_enabled(true);
record_put_object_stage_duration("enabled_stage", 0.5);
set_put_stage_metrics_enabled(false);
set_metrics_enabled(false);
});
let metrics = snapshotter.snapshot().into_vec();
assert!(metrics.iter().any(|(composite, _, _, _)| {
composite.kind() == MetricKind::Counter && composite.key().name() == "rustfs_s3_put_object_total"
}));
assert!(metrics.iter().any(|(composite, _, _, _)| {
composite.kind() == MetricKind::Counter && composite.key().name() == "rustfs_io_put_object_requests_total"
}));
let stages = metrics
.iter()
.filter(|(composite, _, _, _)| {
composite.kind() == MetricKind::Histogram && composite.key().name() == "rustfs_s3_put_object_stage_duration_ms"
})
.flat_map(|(composite, _, _, _)| composite.key().labels().map(|label| label.value().to_string()))
.collect::<Vec<_>>();
assert_eq!(stages, ["enabled_stage"]);
}
#[test]
fn test_put_stage_timer_follows_metrics_switch() {
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
set_put_stage_metrics_enabled(false);
assert!(put_stage_timer().is_none());
set_put_stage_metrics_enabled(true);
assert!(put_stage_timer().is_some());
set_put_stage_metrics_enabled(false);
}
#[test]
fn test_record_get_object_path_and_stage() {
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
+57
View File
@@ -208,6 +208,7 @@ impl<'de> Deserialize<'de> for VaultKeyData {
RotatedAt,
EncryptedKeyMaterial,
BaselineVersion,
WrapBudgetReserved,
Unknown(BoundedUnknownFieldName),
}
@@ -242,6 +243,7 @@ impl<'de> Deserialize<'de> for VaultKeyData {
"rotated_at" => Field::RotatedAt,
"encrypted_key_material" => Field::EncryptedKeyMaterial,
"baseline_version" => Field::BaselineVersion,
"wrap_budget_reserved" => Field::WrapBudgetReserved,
_ => Field::Unknown(BoundedUnknownFieldName::new(value)),
})
}
@@ -285,6 +287,7 @@ impl<'de> Deserialize<'de> for VaultKeyData {
let mut rotated_at = None;
let mut encrypted_key_material = None;
let mut baseline_version = None;
let mut wrap_budget_reserved = None;
let mut unknown_fields = UnknownFieldSummary::default();
while let Some(field) = map.next_key()? {
@@ -301,6 +304,7 @@ impl<'de> Deserialize<'de> for VaultKeyData {
Field::RotatedAt => read_field!(rotated_at, "rotated_at"),
Field::EncryptedKeyMaterial => read_field!(encrypted_key_material, "encrypted_key_material"),
Field::BaselineVersion => read_field!(baseline_version, "baseline_version"),
Field::WrapBudgetReserved => read_field!(wrap_budget_reserved, "wrap_budget_reserved"),
Field::Unknown(field) => {
let _: IgnoredAny = map.next_value()?;
unknown_fields.observe(field);
@@ -322,6 +326,10 @@ impl<'de> Deserialize<'de> for VaultKeyData {
encrypted_key_material: encrypted_key_material
.ok_or_else(|| de::Error::missing_field("encrypted_key_material"))?,
baseline_version: baseline_version.unwrap_or(None),
// Absent on records written before wrap accounting existed, and
// on records an older build rewrote; zero restarts the
// reservation rather than blocking a wrap.
wrap_budget_reserved: wrap_budget_reserved.unwrap_or(0),
};
unknown_fields.record_for_vault_kv2_key();
Ok(key_data)
@@ -341,6 +349,7 @@ impl<'de> Deserialize<'de> for VaultKeyData {
"rotated_at",
"encrypted_key_material",
"baseline_version",
"wrap_budget_reserved",
];
deserializer.deserialize_struct("VaultKeyData", FIELDS, VaultKeyDataVisitor)
}
@@ -3014,6 +3023,54 @@ mod tests {
assert_eq!(legacy.version, 1);
}
/// Every declared `VaultKeyData` field must survive a serialize/deserialize
/// round trip through the hand-written `Deserialize`.
///
/// The hand-written impl lists its fields three times (the `Field` enum, the
/// match arms, the struct literal), so a field added to the struct alone
/// compiles on its own branch and only breaks once both branches merge —
/// which is exactly how `wrap_budget_reserved` briefly broke the build.
/// Asserting against the serialized key set makes the deserializer's
/// coverage a test failure rather than a merge-order accident.
#[test]
fn vault_key_data_deserializer_covers_every_serialized_field() {
let mut key_data = healthy_key_data();
key_data.wrap_budget_reserved = 7_000_000;
key_data.baseline_version = Some(2);
key_data.rotated_at = Some(Zoned::now());
key_data.deletion_date = Some(Zoned::now());
key_data.description = Some("described".to_string());
let value = serde_json::to_value(&key_data).expect("serialize key data");
let serialized_fields: Vec<String> = value
.as_object()
.expect("key data serializes to an object")
.keys()
.cloned()
.collect();
// Every serialized field must be a known field: an unknown one would be
// counted by the unknown-field observer instead of being read back.
let recorder = metrics_util::debugging::DebuggingRecorder::new();
let restored: VaultKeyData =
metrics::with_local_recorder(&recorder, || serde_json::from_value(value).expect("round trip"));
assert_eq!(
crate::test_support::unknown_field_metric(&recorder, "vault-kv2-key"),
0,
"a serialized field was not recognized by the deserializer; fields: {serialized_fields:?}"
);
// And every value must survive, not just parse.
assert_eq!(restored.wrap_budget_reserved, key_data.wrap_budget_reserved);
assert_eq!(restored.baseline_version, key_data.baseline_version);
assert_eq!(restored.version, key_data.version);
assert_eq!(restored.status, key_data.status);
assert_eq!(restored.description, key_data.description);
assert_eq!(restored.encrypted_key_material, key_data.encrypted_key_material);
assert!(restored.rotated_at.is_some());
assert!(restored.deletion_date.is_some());
}
#[test]
fn vault_key_data_unknown_fields_remain_readable_and_are_observed() {
// A record written by a newer build carries fields this build does not
+24 -18
View File
@@ -748,9 +748,10 @@ async fn handle_authenticated_request(
}
for (key, value) in info.user_defined.iter() {
if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
@@ -817,9 +818,10 @@ async fn handle_authenticated_request(
// Add custom metadata headers (X-Object-Meta-*)
for (key, value) in info.user_defined.iter() {
if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
@@ -856,9 +858,10 @@ async fn handle_authenticated_request(
// Add custom metadata headers (X-Object-Meta-*)
for (key, value) in info.user_defined.iter() {
if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
@@ -1168,9 +1171,10 @@ async fn handle_object_get(
}
for (key, value) in info.user_defined.iter() {
if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
@@ -1237,9 +1241,10 @@ async fn handle_object_get(
if key == "x-delete-at" {
// Add X-Delete-At header directly (not as X-Object-Meta-*)
response = response.header("x-delete-at", value.as_str());
} else if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
} else if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
@@ -1293,9 +1298,10 @@ async fn handle_object_head(
if key == "x-delete-at" {
// Add X-Delete-At header directly (not as X-Object-Meta-*)
response = response.header("x-delete-at", value.as_str());
} else if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
} else if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
+31
View File
@@ -84,6 +84,24 @@ fn stored_swift_user_metadata_key(key: &str) -> String {
}
}
pub(super) fn swift_response_user_metadata_key(key: &str) -> Option<&str> {
if rustfs_utils::http::is_internal_key(key)
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-")
{
return None;
}
if let Some(unescaped) = key.strip_prefix(USER_METADATA_PREFIX)
&& (rustfs_utils::http::is_internal_key(unescaped)
|| rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-amz-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-rustfs-encryption-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-minio-encryption-"))
{
return Some(unescaped);
}
Some(key)
}
fn swift_user_metadata(headers: &HeaderMap) -> Option<HashMap<String, String>> {
let mut metadata = HashMap::new();
let mut present = false;
@@ -1142,6 +1160,19 @@ mod tests {
assert_eq!(stored_swift_user_metadata_key("description"), "description");
}
#[test]
fn swift_user_metadata_response_mapping_is_reversible_and_filters_internal_keys() {
assert_eq!(
swift_response_user_metadata_key("x-amz-meta-x-rustfs-internal-actual-size"),
Some("x-rustfs-internal-actual-size")
);
assert_eq!(swift_response_user_metadata_key("x-amz-meta-x-amz-checksum"), Some("x-amz-checksum"));
assert_eq!(swift_response_user_metadata_key("x-amz-meta-description"), Some("x-amz-meta-description"));
assert_eq!(swift_response_user_metadata_key("description"), Some("description"));
assert_eq!(swift_response_user_metadata_key("x-rustfs-internal-actual-size"), None);
assert_eq!(swift_response_user_metadata_key("x-minio-internal-actual-size"), None);
}
#[test]
fn test_validate_object_name_empty() {
let result = ObjectKeyMapper::validate_object_name("");
+72 -20
View File
@@ -465,6 +465,16 @@ impl Operation for ImportBucketMetadata {
file_contents.push((file_path, content));
}
let durable_quota_import = imported_quota_requires_fleet_proof(&file_contents)?;
let quota_fleet_proof =
if durable_quota_import {
Some(crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| {
s3_error!(ServiceUnavailable, "durable quota capability is not confirmed across the cluster")
})?)
} else {
None
};
// Extract bucket names
let mut bucket_names = Vec::new();
for (file_path, _) in &file_contents {
@@ -707,21 +717,6 @@ impl Operation for ImportBucketMetadata {
}
BUCKET_QUOTA_CONFIG_FILE => {
if let Err(e) = serde_json::from_slice::<BucketQuota>(&content) {
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_BUCKET_META,
action = "import_bucket_metadata",
result = "config_deserialize_failed",
bucket = %bucket_name,
config_name = %conf_name,
error = %e,
"admin bucket meta state"
);
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
@@ -830,10 +825,6 @@ impl Operation for ImportBucketMetadata {
}
}
// Persist the assembled metadata to disk. Prior to this, the import only mutated the
// in-memory `bucket_metadatas` map and returned 200, silently dropping every imported
// config. `metadata_sys::update` loads the on-disk metadata, overwrites the given config
// field and saves it, preserving any configs not present in the import archive.
for (bucket_name, metadata) in &bucket_metadatas {
for (config_file, data) in imported_configs_to_persist(metadata) {
let site_replication_item = imported_config_to_site_replication_item(bucket_name, metadata, config_file, &data)?;
@@ -842,7 +833,21 @@ impl Operation for ImportBucketMetadata {
} else {
None
};
if let Err(e) = metadata_sys::update(bucket_name, config_file, data).await {
let persist_result = if config_file == BUCKET_QUOTA_CONFIG_FILE {
let quota: BucketQuota =
serde_json::from_slice(&data).map_err(|e| s3_error!(InvalidRequest, "invalid bucket quota: {e}"))?;
if quota.uses_durable_reservations() {
let proof = quota_fleet_proof.as_ref().ok_or_else(|| {
s3_error!(ServiceUnavailable, "durable quota capability is not confirmed across the cluster")
})?;
metadata_sys::update_quota_if_incarnation(bucket_name, data, metadata.bucket_incarnation_id, proof).await
} else {
metadata_sys::update_if_incarnation(bucket_name, config_file, data, metadata.bucket_incarnation_id).await
}
} else {
metadata_sys::update_if_incarnation(bucket_name, config_file, data, metadata.bucket_incarnation_id).await
};
if let Err(e) = persist_result {
warn!(
event = EVENT_ADMIN_BUCKET_META_STATE,
component = LOG_COMPONENT_ADMIN,
@@ -885,6 +890,26 @@ impl Operation for ImportBucketMetadata {
}
}
fn imported_quota_requires_fleet_proof(file_contents: &[(String, Vec<u8>)]) -> S3Result<bool> {
let mut durable = false;
for (file_path, content) in file_contents {
let mut parts = file_path.split(SLASH_SEPARATOR);
let Some(_bucket) = parts.next() else {
continue;
};
if parts.next() != Some(BUCKET_QUOTA_CONFIG_FILE) {
continue;
}
let quota: BucketQuota =
serde_json::from_slice(content).map_err(|e| s3_error!(InvalidRequest, "invalid bucket quota: {e}"))?;
if quota.has_unsupported_reservation_protocol() {
return Err(s3_error!(InvalidRequest, "unsupported bucket quota reservation protocol"));
}
durable |= quota.uses_durable_reservations();
}
Ok(durable)
}
/// The `(config_file, data)` pairs to persist for an imported bucket's metadata: every non-empty
/// config field keyed by its on-disk config-file name, as owned data ready for
/// `metadata_sys::update`. Empty fields are skipped so an import never overwrites an existing
@@ -1076,4 +1101,31 @@ mod import_persist_tests {
assert!(!has_site_replication_item);
}
#[test]
fn quota_import_preflight_rejects_invalid_and_unknown_protocols() {
let missing_limit = vec![(
format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"),
br#"{"quota":0,"reservation_protocol":1}"#.to_vec(),
)];
assert!(imported_quota_requires_fleet_proof(&missing_limit).is_err());
let unknown_protocol = vec![(
format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"),
br#"{"quota":0,"reservation_protocol":2,"reservation_quota":1024}"#.to_vec(),
)];
assert!(imported_quota_requires_fleet_proof(&unknown_protocol).is_err());
}
#[test]
fn quota_import_preflight_requires_proof_only_for_durable_quota() {
let legacy = vec![(format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"), br#"{"quota":1024}"#.to_vec())];
assert!(!imported_quota_requires_fleet_proof(&legacy).expect("legacy quota should remain compatible"));
let durable = vec![(
format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"),
serde_json::to_vec(&BucketQuota::new(Some(1024))).expect("durable quota should encode"),
)];
assert!(imported_quota_requires_fleet_proof(&durable).expect("durable quota should pass preflight"));
}
}
+95
View File
@@ -2166,6 +2166,101 @@ mod tests {
assert_eq!(denied.code(), &S3ErrorCode::InvalidRequest);
}
#[tokio::test]
#[serial_test::serial]
async fn multipart_completion_rejects_rotated_quota_capability_before_rename() {
use crate::app::storage_api::test::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("rotated-proof-mpu-quota", 4096).await;
let object = "object";
let upload = store
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
.await
.expect("create multipart upload");
let mut reader = PutObjReader::from_vec(vec![0x78; 4096]);
let part = store
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
.await
.expect("stage multipart part");
let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::BeforeQuotaRename);
let complete_store = Arc::clone(&store);
let complete_bucket = bucket.clone();
let upload_id = upload.upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(
&complete_bucket,
object,
&upload_id,
vec![CompletePart {
part_num: 1,
etag: part.etag,
..Default::default()
}],
&ObjectOptions::default(),
)
.await
});
barrier.wait_until_paused().await;
assert!(
crate::storage::storage_api::ecstore_notification::rotate_cross_pool_fence_fleet_proof_for_test(),
"the gating environment must have a current fleet proof"
);
barrier.release();
let err = complete
.await
.expect("completion task should not panic")
.expect_err("a replaced fleet proof must fence multipart rename");
assert!(matches!(
err,
StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
..
}
));
store
.get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
.await
.expect("proof rotation must preserve the multipart upload for retry");
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_multipart_completion_has_zero_quota_growth() {
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("data-movement-mpu-quota", 0).await;
let object = "object";
let mut movement_opts = ObjectOptions {
data_movement: true,
..Default::default()
};
let upload = store
.new_multipart_upload(&bucket, object, &movement_opts)
.await
.expect("create data-movement multipart upload");
let mut reader = PutObjReader::from_vec(vec![0x7a; 4096]);
let part = store
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &movement_opts)
.await
.expect("stage data-movement multipart part");
movement_opts.preserve_etag = Some("movement-etag".to_string());
let completed = store
.complete_multipart_upload(
&bucket,
object,
&upload.upload_id,
vec![CompletePart {
part_num: 1,
etag: part.etag,
..Default::default()
}],
&movement_opts,
)
.await
.expect("moving an already-accounted multipart object between pools must have zero quota growth");
assert_eq!(completed.size, 4096);
}
#[tokio::test]
#[serial_test::serial]
async fn rejected_empty_parts_preserve_existing_object_and_staging() {
+234 -10
View File
@@ -199,7 +199,7 @@ use std::str::FromStr;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
use std::time::{Duration, Instant};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::{OwnedSemaphorePermit, RwLock};
@@ -543,6 +543,9 @@ pub(super) fn map_quota_check_outcome(bucket: &str, outcome: Result<QuotaCheckRe
}
pub(super) fn apply_quota_admission(opts: &mut ObjectOptions, result: &QuotaCheckResult) -> S3Result<()> {
if result.uses_durable_reservations {
return Ok(());
}
let Some(quota_limit) = result.quota_limit else {
return Ok(());
};
@@ -571,6 +574,25 @@ fn ensure_object_size_within_quota(result: &QuotaCheckResult, new_size: u64) ->
Ok(())
}
fn ensure_legacy_archive_size_within_quota(result: &QuotaCheckResult, total_unpacked_size: u64) -> S3Result<()> {
if result.uses_durable_reservations {
return Ok(());
}
let (Some(current_usage), Some(quota_limit)) = (result.current_usage, result.quota_limit) else {
return Ok(());
};
let expected_usage = current_usage
.checked_add(total_unpacked_size)
.ok_or_else(|| s3_error!(InvalidArgument, "Archive total size overflowed quota accounting"))?;
if expected_usage > quota_limit {
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"),
));
}
Ok(())
}
fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result<u64> {
match quota_object_size(info) {
Ok(size) => Ok(size),
@@ -5630,7 +5652,13 @@ impl DefaultObjectUsecase {
// The app check preserves the existing S3 error contract; the storage
// commit path reserves the exact net logical growth under its locks.
let quota_check = self.check_bucket_quota(&bucket, quota_operation, 0).await?;
let quota_check = self
.check_bucket_quota(
&bucket,
quota_operation,
u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?,
)
.await?;
let quota_enabled = quota_check.as_ref().is_some_and(|result| result.quota_limit.is_some());
if quota_enabled && ciphertext_passthrough {
return Err(S3Error::with_message(
@@ -5639,7 +5667,8 @@ impl DefaultObjectUsecase {
));
}
let ingress_stage_start = std::time::Instant::now();
let put_stage_metrics_enabled = rustfs_io_metrics::put_stage_metrics_enabled();
let ingress_stage_start = put_stage_metrics_enabled.then(Instant::now);
let should_compress =
is_disk_compressible(&req.headers, &key) && size > MIN_DISK_COMPRESSIBLE_SIZE as i64 && !ciphertext_passthrough;
let server_side_encryption_requested =
@@ -5703,9 +5732,13 @@ impl DefaultObjectUsecase {
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let bucket_validate_stage_start = put_stage_metrics_enabled.then(Instant::now);
validate_bucket_exists(&store, &bucket).await?;
rustfs_io_metrics::record_put_object_stage_duration_from("app_bucket_validate", bucket_validate_stage_start);
let sse_config_stage_start = put_stage_metrics_enabled.then(Instant::now);
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
rustfs_io_metrics::record_put_object_stage_duration_from("app_sse_config_lookup", sse_config_stage_start);
debug!(
target: "rustfs::app::object_usecase",
component = "app",
@@ -5771,7 +5804,9 @@ impl DefaultObjectUsecase {
let mut metadata = metadata.unwrap_or_default();
let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some();
let object_lock_config_stage_start = put_stage_metrics_enabled.then(Instant::now);
let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?;
rustfs_io_metrics::record_put_object_stage_duration_from("app_object_lock_config_lookup", object_lock_config_stage_start);
apply_put_request_metadata(
&mut metadata,
&req.headers,
@@ -5793,6 +5828,7 @@ impl DefaultObjectUsecase {
has_explicit_object_lock_retention,
)?;
let put_opts_stage_start = put_stage_metrics_enabled.then(Instant::now);
let mut opts: ObjectOptions = put_opts_with_replication_authorization(
&bucket,
&key,
@@ -5806,6 +5842,7 @@ impl DefaultObjectUsecase {
if let Some(quota_check) = quota_check.as_ref() {
apply_quota_admission(&mut opts, quota_check)?;
}
rustfs_io_metrics::record_put_object_stage_duration_from("app_put_opts_build", put_opts_stage_start);
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
apply_put_request_object_lock_opts(
&bucket,
@@ -5824,10 +5861,11 @@ impl DefaultObjectUsecase {
// replication), the lookup is skipped and accounting is backfilled from
// the dst xl.meta that rename_data already reads, saving a full-disk
// metadata fanout per PUT.
let prelookup_required = quota_enabled || version_id.is_some() || object_lock_checks_required(&bucket).await;
let prelookup_required = version_id.is_some() || object_lock_checks_required_for_state(&object_lock_config_state);
// Outer None = prelookup skipped (accounting comes from the commit
// backfill); Some(inner) = the previous current size as observed by the
// lookup, with the pre-#1009 semantics kept bit-for-bit.
let prelookup_stage_start = (prelookup_required && put_stage_metrics_enabled).then(Instant::now);
let prelookup_previous_current_size: Option<Option<u64>> = if prelookup_required {
let current_opts: ObjectOptions = internal_object_info_lookup_opts(
get_opts(&bucket, &key, version_id.clone(), None, &req.headers)
@@ -5857,6 +5895,7 @@ impl DefaultObjectUsecase {
} else {
None
};
rustfs_io_metrics::record_put_object_stage_duration_from("app_prelookup", prelookup_stage_start);
let actual_size = size;
if !ciphertext_passthrough && let Some(quota_check) = quota_check.as_ref() {
@@ -5951,15 +5990,13 @@ impl DefaultObjectUsecase {
put_extra_checksum_headers = additional_checksum_echo_pairs(&opts.want_checksum);
}
rustfs_io_metrics::record_put_object_path(put_path);
rustfs_io_metrics::record_put_object_stage_duration(
"ingress_prepare",
ingress_stage_start.elapsed().as_secs_f64() * 1000.0,
);
rustfs_io_metrics::record_put_object_stage_duration_from("ingress_prepare", ingress_stage_start);
let mut helper = OperationHelper::new(&req, event_name, S3Operation::PutObject);
let ssekms_context = extract_ssekms_context_from_headers(&req.headers)?;
// Apply encryption using unified SSE API.
let encryption_stage_start = put_stage_metrics_enabled.then(Instant::now);
let write_principal = SseKmsPrincipal::from_request(&req);
let encryption_request = EncryptionRequest {
bucket: &bucket,
@@ -6003,6 +6040,7 @@ impl DefaultObjectUsecase {
}
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
rustfs_io_metrics::record_put_object_stage_duration_from("app_encryption_prepare", encryption_stage_start);
let mut reader = PutObjReader::new(reader);
@@ -6019,9 +6057,11 @@ impl DefaultObjectUsecase {
// post-commit schedule (see the reuse site further down), so a
// replication-config hot update can no longer split the two phases
// (https://github.com/rustfs/backlog/issues/1320).
let replication_decision_stage_start = put_stage_metrics_enabled.then(Instant::now);
let dsc =
must_replicate_object(&bucket, &key, &mt2, "".to_string(), opts.delete_marker_replication_status(), opts.clone())
.await;
rustfs_io_metrics::record_put_object_stage_duration_from("app_replication_decision", replication_decision_stage_start);
if dsc.replicate_any() {
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
@@ -6033,7 +6073,12 @@ impl DefaultObjectUsecase {
}
let cache_adapter = self.object_data_cache();
let cache_invalidate_before_stage_start = put_stage_metrics_enabled.then(Instant::now);
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
"app_cache_invalidate_before",
cache_invalidate_before_stage_start,
);
let store_put_watchdog = tokio_util::sync::CancellationToken::new();
spawn_traced({
@@ -6073,6 +6118,7 @@ impl DefaultObjectUsecase {
let object_traffic_progress = object_traffic_health
.as_deref()
.and_then(ObjectTrafficHealth::track_write_storage);
let store_put_stage_start = put_stage_metrics_enabled.then(Instant::now);
let (obj_info, backfilled_old_current_size) = match store
.put_object_with_old_current_size(&bucket, &key, &mut reader, &opts)
.await
@@ -6098,6 +6144,7 @@ impl DefaultObjectUsecase {
}
Err(err) => {
store_put_watchdog.cancel();
rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start);
warn!(
target: "rustfs::app::object_usecase",
event = EVENT_PUT_OBJECT_STORE_RETURNED,
@@ -6119,10 +6166,12 @@ impl DefaultObjectUsecase {
return result;
}
};
rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start);
drop(object_traffic_progress);
#[cfg(test)]
wait_for_put_post_store_test_hook(&bucket).await;
let post_store_stage_start = put_stage_metrics_enabled.then(Instant::now);
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await;
@@ -6222,10 +6271,13 @@ impl DefaultObjectUsecase {
let result = Ok(response);
let _ = helper.complete(&result);
rustfs_scanner::record_dirty_usage_bucket(&bucket);
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start);
// Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead)
let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now);
let manager = get_capacity_manager();
manager.record_write_operation().await;
rustfs_io_metrics::record_put_object_stage_duration_from("app_capacity_update", capacity_update_stage_start);
// Record PutObject metrics via zero-copy-metrics
{
@@ -7558,7 +7610,13 @@ impl DefaultObjectUsecase {
src_info.user_defined = Arc::new(user_defined);
let quota_check = self.check_bucket_quota(&bucket, QuotaOperation::CopyObject, 0).await?;
let quota_check = self
.check_bucket_quota(
&bucket,
QuotaOperation::CopyObject,
u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?,
)
.await?;
let quota_enabled = quota_check.as_ref().is_some_and(|result| result.quota_limit.is_some());
if let Some(quota_check) = quota_check.as_ref() {
apply_quota_admission(&mut dst_opts, quota_check)?;
@@ -9190,7 +9248,13 @@ impl DefaultObjectUsecase {
}
validate_object_key(&key, "PUT")?;
validate_table_catalog_object_mutation(&bucket, &key).await?;
let _ = self.check_bucket_quota(&bucket, QuotaOperation::PutObject, 0).await?;
let _ = self
.check_bucket_quota(
&bucket,
QuotaOperation::PutObject,
u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?,
)
.await?;
// Apply adaptive buffer sizing based on file size for optimal streaming performance.
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
@@ -9336,6 +9400,9 @@ impl DefaultObjectUsecase {
.checked_add(entry_size)
.ok_or_else(|| s3_error!(InvalidArgument, "Archive total unpacked size overflowed while processing entries"))?;
validate_put_object_extract_total_size(total_unpacked_size, extract_limits)?;
if let Some(quota_check) = extract_quota_check.as_ref() {
ensure_legacy_archive_size_within_quota(quota_check, total_unpacked_size)?;
}
let mut size =
i64::try_from(entry_size).map_err(|_| s3_error!(InvalidArgument, "Archive entry size does not fit into i64"))?;
// mtime 0 means "unset" in tar headers, and xl.meta cannot represent an
@@ -9644,6 +9711,13 @@ pub(super) async fn object_lock_checks_required(bucket: &str) -> bool {
.map_or(true, |metadata| metadata.object_locking())
}
fn object_lock_checks_required_for_state(state: &metadata_sys::ObjectLockConfigState) -> bool {
match state {
metadata_sys::ObjectLockConfigState::Configured { .. } | metadata_sys::ObjectLockConfigState::Fabricated => true,
metadata_sys::ObjectLockConfigState::ConfirmedAbsent => false,
}
}
/// rustfs/backlog#1009: map the rename_data old-size backfill onto the
/// `previous_current_size` value the usage-accounting helpers expect. Outer
/// `None` = unknown (no quorum agreement, or a peer predates the field) — the
@@ -10270,6 +10344,23 @@ mod tests {
assert_eq!(err.message(), Some(ERR_OBJECT_LOCK_RETENTION_HEADERS_MUST_BE_PAIRED));
}
#[test]
fn object_lock_checks_required_reuses_authoritative_state() {
assert!(!object_lock_checks_required_for_state(
&metadata_sys::ObjectLockConfigState::ConfirmedAbsent
));
let configured = metadata_sys::ObjectLockConfigState::Configured {
config: ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: None,
},
updated_at: OffsetDateTime::now_utc(),
};
assert!(object_lock_checks_required_for_state(&configured));
assert!(object_lock_checks_required_for_state(&metadata_sys::ObjectLockConfigState::Fabricated));
}
#[test]
fn build_put_like_object_lock_metadata_rejects_retain_until_date_without_mode() {
let retain_until = Timestamp::from(OffsetDateTime::now_utc().add(time::Duration::days(1)));
@@ -17405,6 +17496,7 @@ mod tests {
quota_limit: Some(2048),
operation_size: 512,
remaining: Some(512),
uses_durable_reservations: true,
}
}
@@ -17446,6 +17538,57 @@ mod tests {
assert!(!body_polled.load(Ordering::Acquire), "rejected ciphertext body must not be consumed");
}
#[tokio::test]
#[serial_test::serial]
async fn legacy_quota_rejects_full_put_before_polling_the_body() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use std::sync::atomic::{AtomicBool, Ordering};
const GI_B: u64 = 1024 * 1024 * 1024;
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
let bucket = format!("legacy-quota-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create legacy quota test bucket");
crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 4 * GI_B).await;
let metadata_sys = DefaultObjectUsecase::from_global()
.bucket_metadata_sys()
.expect("test app context should expose bucket metadata");
QuotaChecker::new(metadata_sys)
.set_quota_config(
&bucket,
BucketQuota {
quota: Some(5 * GI_B),
..Default::default()
},
)
.await
.expect("configure legacy quota");
let body_polled = Arc::new(AtomicBool::new(false));
let body_polled_in_stream = Arc::clone(&body_polled);
let body = StreamingBlob::wrap(futures::stream::once(async move {
body_polled_in_stream.store(true, Ordering::Release);
Ok::<Bytes, std::io::Error>(Bytes::new())
}));
let input = PutObjectInput::builder()
.bucket(bucket)
.key("object".to_string())
.body(Some(body))
.content_length(Some(i64::try_from(2 * GI_B).expect("test size should fit i64")))
.build()
.expect("legacy quota PUT input should build");
let err = DefaultObjectUsecase::from_global()
.execute_put_object(&FS::new(), build_request(input, Method::PUT))
.await
.expect_err("4 GiB used plus a 2 GiB PUT must exceed a 5 GiB legacy quota");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
assert!(!body_polled.load(Ordering::Acquire), "legacy quota rejection must not consume the body");
}
#[test]
fn quota_admission_allows_within_limit() {
let result = map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write");
@@ -17529,6 +17672,45 @@ mod tests {
.expect("second within-limit PUT should commit");
}
#[tokio::test]
#[serial_test::serial]
async fn put_rejects_rotated_quota_capability_before_rename() {
use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("rotated-proof-put-quota", 4096).await;
let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename);
let put_store = Arc::clone(&store);
let put_bucket = bucket.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![0x77; 4096]);
put_store
.put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
assert!(
crate::storage::storage_api::ecstore_notification::rotate_cross_pool_fence_fleet_proof_for_test(),
"the gating environment must have a current fleet proof"
);
barrier.release();
let err = put
.await
.expect("PUT task should not panic")
.expect_err("a replaced fleet proof must fence the authoritative rename");
assert!(matches!(
err,
StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
..
}
));
store
.get_object_info(&bucket, "object", &ObjectOptions::default())
.await
.expect_err("proof rotation before rename must leave no committed object");
}
#[tokio::test]
#[serial_test::serial]
async fn durable_quota_reclaims_overwrites_and_deleted_bytes() {
@@ -17566,6 +17748,26 @@ mod tests {
));
}
#[tokio::test]
#[serial_test::serial]
async fn data_movement_put_has_zero_quota_growth() {
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("data-movement-put-quota", 0).await;
let mut reader = PutObjReader::from_vec(vec![0x79; 4096]);
let stored = store
.put_object(
&bucket,
"object",
&mut reader,
&ObjectOptions {
data_movement: true,
..Default::default()
},
)
.await
.expect("moving an already-accounted object between pools must have zero quota growth");
assert_eq!(stored.size, 4096);
}
#[tokio::test]
#[serial_test::serial]
async fn cancelled_put_releases_durable_quota_reservation() {
@@ -17809,6 +18011,27 @@ mod tests {
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
}
#[test]
fn legacy_archive_quota_rejects_cumulative_size_and_overflow() {
let legacy = QuotaCheckResult {
allowed: true,
current_usage: Some(4),
quota_limit: Some(5),
operation_size: 0,
remaining: Some(1),
uses_durable_reservations: false,
};
assert!(ensure_legacy_archive_size_within_quota(&legacy, 2).is_err());
assert!(ensure_legacy_archive_size_within_quota(&legacy, 1).is_ok());
let maxed = QuotaCheckResult {
current_usage: Some(u64::MAX),
quota_limit: Some(u64::MAX),
..legacy
};
assert!(ensure_legacy_archive_size_within_quota(&maxed, 1).is_err());
}
#[test]
fn early_quota_filter_rejects_only_an_individually_impossible_object() {
let stale_full_usage = QuotaCheckResult {
@@ -17817,6 +18040,7 @@ mod tests {
quota_limit: Some(4096),
operation_size: 0,
remaining: Some(0),
uses_durable_reservations: true,
};
ensure_object_size_within_quota(&stale_full_usage, 4096)
+2 -1
View File
@@ -1216,7 +1216,8 @@ pub(crate) mod test {
};
pub(crate) mod set_disk {
pub(crate) use crate::storage::storage_api::ecstore_set_disk::{
PutObjectCommitBarrier, PutObjectCommitPause, fail_next_quota_ledger_save_for_test,
MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause,
fail_next_quota_ledger_save_for_test,
};
}
-15
View File
@@ -398,7 +398,6 @@ impl From<QuotaError> for ApiError {
QuotaError::ConfigNotFound { .. } => S3ErrorCode::NoSuchBucket,
QuotaError::UsageUnavailable { .. } => S3ErrorCode::ServiceUnavailable,
QuotaError::InvalidConfig { .. } => S3ErrorCode::InvalidArgument,
QuotaError::StorageError(StorageError::NamespaceLockQuorumUnavailable { .. }) => S3ErrorCode::ServiceUnavailable,
QuotaError::StorageError(_) => S3ErrorCode::InternalError,
};
@@ -552,20 +551,6 @@ mod tests {
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
}
#[test]
fn stale_quota_capability_maps_to_retryable_error() {
let api_error = ApiError::from(QuotaError::StorageError(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_capability",
bucket: "bucket".to_string(),
object: rustfs_config::QUOTA_CONFIG_FILE.to_string(),
required: 1,
achieved: 0,
}));
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
assert_eq!(api_error.message, "The service is unavailable. Please retry.");
}
#[test]
fn test_kms_cryptographic_error_is_not_retryable() {
let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::cryptographic_error(
+53 -4
View File
@@ -24,14 +24,63 @@ pub(crate) async fn init_observability_runtime(store: Arc<ECStore>, ctx: Cancell
init_update_check();
crate::allocator_reclaim::init_allocator_reclaim(ctx.clone());
if startup_runtime_sources::observability_metric_enabled() {
let metrics_enabled = startup_runtime_sources::observability_metric_enabled();
configure_metric_gates(metrics_enabled);
if metrics_enabled {
// Load persisted compression stats into memory early, before any PUTs can occur.
init_compression_total_memory_from_backend(store).await;
startup_runtime_sources::set_put_stage_metrics_enabled(true);
startup_runtime_sources::set_get_stage_metrics_enabled(true);
startup_runtime_sources::set_metrics_enabled(true);
startup_runtime_sources::init_metrics_runtime(ctx.clone());
crate::memory_observability::init_memory_observability(ctx.clone());
init_auto_tuner(ctx).await;
}
}
fn configure_metric_gates(metrics_enabled: bool) {
let put_stage_metrics_enabled = metrics_enabled
&& rustfs_utils::get_env_bool(
rustfs_config::observability::ENV_OBS_PUT_STAGE_METRICS_ENABLED,
rustfs_config::DEFAULT_OBS_PUT_STAGE_METRICS_ENABLED,
);
startup_runtime_sources::set_put_stage_metrics_enabled(put_stage_metrics_enabled);
startup_runtime_sources::set_get_stage_metrics_enabled(metrics_enabled);
startup_runtime_sources::set_metrics_enabled(metrics_enabled);
}
#[cfg(test)]
mod tests {
use super::*;
const PUT_STAGE_ENV: &str = rustfs_config::observability::ENV_OBS_PUT_STAGE_METRICS_ENABLED;
#[test]
#[serial_test::serial]
fn put_stage_metrics_require_explicit_opt_in() {
let previous_metrics = rustfs_io_metrics::metrics_enabled();
let previous_get_stages = rustfs_io_metrics::get_stage_metrics_enabled();
let previous_put_stages = rustfs_io_metrics::put_stage_metrics_enabled();
temp_env::with_var(PUT_STAGE_ENV, None::<&str>, || {
configure_metric_gates(true);
assert!(rustfs_io_metrics::metrics_enabled());
assert!(rustfs_io_metrics::get_stage_metrics_enabled());
assert!(!rustfs_io_metrics::put_stage_metrics_enabled());
});
temp_env::with_var(PUT_STAGE_ENV, Some("true"), || {
configure_metric_gates(true);
assert!(rustfs_io_metrics::metrics_enabled());
assert!(rustfs_io_metrics::get_stage_metrics_enabled());
assert!(rustfs_io_metrics::put_stage_metrics_enabled());
configure_metric_gates(false);
assert!(!rustfs_io_metrics::metrics_enabled());
assert!(!rustfs_io_metrics::get_stage_metrics_enabled());
assert!(!rustfs_io_metrics::put_stage_metrics_enabled());
});
startup_runtime_sources::set_metrics_enabled(previous_metrics);
startup_runtime_sources::set_get_stage_metrics_enabled(previous_get_stages);
startup_runtime_sources::set_put_stage_metrics_enabled(previous_put_stages);
}
}
+22 -120
View File
@@ -153,7 +153,9 @@ fn remove_heal_control_replay(
static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1;
// RUSTFS_COMPAT_TODO(cross-pool-fence-v1): advertise unsupported during predeployment. Remove after composite acquisition,
// activation fencing, fleet proof, commit-time proof revalidation, and fail-closed revocation ship together.
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 0;
fn admit_heal_control_replay(
replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
@@ -2205,7 +2207,7 @@ mod tests {
use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo, HealEndpoint};
use crate::storage::storage_api::set_tonic_canonical_body_digest;
use crate::storage::storage_api::{
Endpoint, RUSTFS_META_BUCKET, SnapshotLeaseToken,
Endpoint,
ecstore_layout::{EndpointServerPools, Endpoints, PoolEndpoints},
};
use bytes::Bytes;
@@ -2943,22 +2945,14 @@ mod tests {
}
#[tokio::test]
#[serial_test::serial]
async fn snapshot_lease_handlers_forward_to_local_disk() {
let temp_dir = tempfile::tempdir().expect("snapshot lease RPC test directory");
let env = rustfs_test_utils::TestECStoreEnv::builder()
.base_dir(temp_dir.path())
.init_bucket_metadata(false)
.build()
.await;
async fn snapshot_lease_acquire_and_renew_handlers_fail_closed() {
let service = make_server();
let disk = env.disk_paths[0].to_string_lossy().into_owned();
let fence_path = format!("tmp/quota-mutation-fences/{}", "0".repeat(64));
let disk = "http://node-a:9000/data/rustfs0".to_string();
let mut acquire = Request::new(SnapshotLeaseRequest {
disk: disk.clone(),
volume: RUSTFS_META_BUCKET.into(),
path: fence_path.clone(),
volume: "v".into(),
path: "p".into(),
ttl_ms: 60_000,
});
let acquire_body =
@@ -2968,18 +2962,14 @@ mod tests {
let acquire = service
.acquire_snapshot_lease(acquire)
.await
.expect("acquire should return a protocol response")
.expect("disabled acquire should return a protocol response")
.into_inner();
assert!(acquire.success, "local disk should acquire the mutation fence");
assert_eq!(acquire.protocol_version, 1);
assert!(acquire.error.is_none());
let token = SnapshotLeaseToken::from_slice(&acquire.token).expect("acquire should return a valid token");
let mut renew = Request::new(SnapshotLeaseRenewRequest {
disk: disk.clone(),
volume: RUSTFS_META_BUCKET.into(),
path: fence_path.clone(),
token: token.as_bytes().to_vec().into(),
disk,
volume: "v".into(),
path: "p".into(),
token: vec![1; 16].into(),
ttl_ms: 60_000,
});
let renew_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(renew.get_ref())
@@ -2989,103 +2979,15 @@ mod tests {
let renew = service
.renew_snapshot_lease(renew)
.await
.expect("renew should return a protocol response")
.expect("disabled renew should return a protocol response")
.into_inner();
assert!(renew.success, "local disk should renew the mutation fence");
assert_eq!(renew.protocol_version, 1);
assert!(renew.error.is_none());
let renewed = SnapshotLeaseToken::from_slice(&renew.token).expect("renew should return a valid token");
assert_ne!(renewed, token);
let mut release = Request::new(SnapshotLeaseReleaseRequest {
disk: disk.clone(),
volume: RUSTFS_META_BUCKET.into(),
path: fence_path.clone(),
token: renewed.as_bytes().to_vec().into(),
});
let release_body = rustfs_protos::canonical_snapshot_lease_release_request_body(release.get_ref())
.expect("release request body should encode");
set_tonic_canonical_body_digest(&mut release, &release_body).expect("release digest metadata should encode");
mark_v2_authenticated(&mut release);
let release = service
.release_snapshot_lease(release)
.await
.expect("release should return a protocol response")
.into_inner();
assert!(release.success, "local disk should release the renewed token");
assert!(release.error.is_none());
let mut acquire = Request::new(SnapshotLeaseRequest {
disk: disk.clone(),
volume: RUSTFS_META_BUCKET.into(),
path: fence_path.clone(),
ttl_ms: 60_000,
});
let acquire_body =
rustfs_protos::canonical_snapshot_lease_request_body(acquire.get_ref()).expect("acquire request body should encode");
set_tonic_canonical_body_digest(&mut acquire, &acquire_body).expect("acquire digest metadata should encode");
mark_v2_authenticated(&mut acquire);
let active_token = service
.acquire_snapshot_lease(acquire)
.await
.expect("second acquire should return a protocol response")
.into_inner();
assert!(active_token.success);
let mut revoke = Request::new(SnapshotLeaseReleaseRequest {
disk: disk.clone(),
volume: RUSTFS_META_BUCKET.into(),
path: fence_path.clone(),
token: SnapshotLeaseToken::revoke_all().as_bytes().to_vec().into(),
});
let revoke_body = rustfs_protos::canonical_snapshot_lease_release_request_body(revoke.get_ref())
.expect("revoke-all request body should encode");
set_tonic_canonical_body_digest(&mut revoke, &revoke_body).expect("revoke-all digest metadata should encode");
mark_v2_authenticated(&mut revoke);
let revoke = service
.release_snapshot_lease(revoke)
.await
.expect("revoke-all should return a protocol response")
.into_inner();
assert!(revoke.success, "nil revoke-all sentinel must reach the local disk");
assert!(revoke.error.is_none());
let mut stale_renew = Request::new(SnapshotLeaseRenewRequest {
disk: disk.clone(),
volume: RUSTFS_META_BUCKET.into(),
path: fence_path.clone(),
token: active_token.token,
ttl_ms: 60_000,
});
let stale_renew_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(stale_renew.get_ref())
.expect("stale renew request body should encode");
set_tonic_canonical_body_digest(&mut stale_renew, &stale_renew_body).expect("stale renew digest metadata should encode");
mark_v2_authenticated(&mut stale_renew);
let stale_renew = service
.renew_snapshot_lease(stale_renew)
.await
.expect("stale renew should return a protocol response")
.into_inner();
assert!(!stale_renew.success, "revoke-all must invalidate active tokens");
assert!(stale_renew.token.is_empty());
assert!(stale_renew.error.is_some());
let mut malformed = Request::new(SnapshotLeaseRenewRequest {
disk,
volume: RUSTFS_META_BUCKET.into(),
path: fence_path,
token: vec![1; 15].into(),
ttl_ms: 60_000,
});
let malformed_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(malformed.get_ref())
.expect("malformed renew request body should encode");
set_tonic_canonical_body_digest(&mut malformed, &malformed_body).expect("malformed renew digest metadata should encode");
mark_v2_authenticated(&mut malformed);
let malformed = service
.renew_snapshot_lease(malformed)
.await
.expect_err("a malformed non-nil token must be rejected");
assert_eq!(malformed.code(), tonic::Code::InvalidArgument);
for response in [acquire, renew] {
assert!(!response.success);
assert!(response.token.is_empty());
assert_eq!(response.protocol_version, 1);
assert_eq!(response.error, Some(DiskError::UnsupportedDisk.into()));
}
}
#[tokio::test]
@@ -3440,7 +3342,7 @@ mod tests {
}
#[tokio::test]
async fn cross_pool_fence_probe_authenticates_supported_v1_state() {
async fn cross_pool_fence_probe_authenticates_unsupported_rollout_state() {
let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string());
let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true);
assert!(
@@ -3505,7 +3407,7 @@ mod tests {
assert!(response.success);
assert_eq!(response.error_info, None);
assert_eq!(&response.result[..4], &super::CROSS_POOL_FENCE_SUPPORTED_VERSION.to_be_bytes());
assert_eq!(&response.result[..4], &0_u32.to_be_bytes());
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..])
.expect("capability identity should decode");
assert_eq!(topology_member, "node-a:9000");
+1 -1
View File
@@ -1521,8 +1521,8 @@ mod tests {
encode_read_multiple_response_payloads, encode_rename_data_response_payloads,
};
use crate::storage::storage_api::ReadMultipleResp;
use crate::storage::storage_api::RenameDataResp;
use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp;
use crate::storage::storage_api::{DiskError, RenameDataResp};
use rustfs_filemeta::FileInfo;
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use serde::{Deserialize, Serialize};
+4 -1
View File
@@ -485,6 +485,8 @@ pub(crate) mod ecstore_metrics {
#[allow(unused_imports)]
pub(crate) mod ecstore_notification {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::notification::rotate_cross_pool_fence_fleet_proof_for_test;
pub(crate) use rustfs_ecstore::api::notification::{
CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof,
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys,
@@ -548,7 +550,8 @@ pub(crate) mod ecstore_test_support {
pub(crate) mod ecstore_set_disk {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{
PutObjectCommitBarrier, PutObjectCommitPause, fail_next_quota_ledger_save_for_test,
MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause,
fail_next_quota_ledger_save_for_test,
};
pub(crate) use rustfs_ecstore::api::set_disk::{
DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class,
Executable → Regular
View File