From d60a77b75047f7027a1e1d696b3f84848fbac814 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 14 Aug 2026 14:26:00 +0800 Subject: [PATCH 01/71] fix(quota): enforce durable hard quota reservations (#6058) * fix(quota): enforce durable hard quota reservations * fix(quota): close reservation bypasses * fix(quota): isolate tests and box object futures * fix(quota): close legacy and deferred settlement bypasses * fix(app): keep object futures off caller stacks * fix(metrics): preserve object operation labels * fix(logging): retain GET trace guard contract --- crates/ecstore/src/api/mod.rs | 14 +- crates/ecstore/src/bucket/metadata_sys.rs | 143 ++- crates/ecstore/src/bucket/quota/checker.rs | 40 +- crates/ecstore/src/bucket/quota/mod.rs | 142 ++- .../ecstore/src/bucket/quota/reservation.rs | 1113 +++++++++++++++++ .../src/cluster/rpc/peer_rest_client.rs | 38 + crates/ecstore/src/data_usage/mod.rs | 123 +- crates/ecstore/src/disk/local.rs | 186 ++- crates/ecstore/src/disk/mod.rs | 36 + crates/ecstore/src/disk/os.rs | 9 + .../ecstore/src/services/notification_sys.rs | 221 ++-- .../src/set_disk/core/io_primitives.rs | 87 +- crates/ecstore/src/set_disk/mod.rs | 4 +- crates/ecstore/src/set_disk/ops/multipart.rs | 414 +++++- crates/ecstore/src/set_disk/ops/object.rs | 426 ++++++- crates/ecstore/src/store/init.rs | 26 + crates/ecstore/src/store/list_objects.rs | 2 +- crates/ecstore/src/store/object.rs | 7 +- crates/protocols/src/swift/handler.rs | 42 +- crates/protocols/src/swift/object.rs | 112 +- docs/architecture/compat-cleanup-register.md | 1 - rustfs/src/admin/handlers/bucket_meta.rs | 92 +- rustfs/src/admin/handlers/quota.rs | 29 +- rustfs/src/admin/handlers/site_replication.rs | 33 +- rustfs/src/admin/router.rs | 2 +- rustfs/src/admin/storage_api.rs | 17 +- rustfs/src/app/gating_test_env.rs | 32 + rustfs/src/app/multipart_usecase.rs | 298 +++-- rustfs/src/app/object_usecase.rs | 805 +++++++++++- rustfs/src/app/storage_api.rs | 12 + rustfs/src/storage/ecfs.rs | 6 +- rustfs/src/storage/rpc/node_service.rs | 8 +- rustfs/src/storage/rpc/node_service/disk.rs | 63 +- rustfs/src/storage/storage_api.rs | 23 +- scripts/check_s3s_footprint.sh | 2 +- 35 files changed, 4183 insertions(+), 425 deletions(-) create mode 100644 crates/ecstore/src/bucket/quota/reservation.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 4b3162313..ffc175659 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -131,6 +131,8 @@ pub mod bucket { } pub mod metadata_sys { + #[cfg(feature = "test-util")] + pub use crate::bucket::metadata_sys::ConfigWriteLockProbe; pub use crate::bucket::metadata_sys::{ BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy, @@ -140,7 +142,7 @@ pub mod bucket { get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, - update_under_transaction_lock, + update_quota_if_incarnation, update_under_transaction_lock, }; } @@ -316,7 +318,7 @@ pub mod data_usage { DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage, init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached, - load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, + load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory, refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer, @@ -403,8 +405,11 @@ pub mod metrics { } pub mod notification { + #[cfg(any(test, feature = "test-util"))] + pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test; pub use crate::services::notification_sys::{ - NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys, + CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, acquire_cross_pool_fence_fleet_proof, + cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe, }; } @@ -464,7 +469,8 @@ pub mod set_disk { #[cfg(feature = "test-util")] pub mod test_util { - pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test; + pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause}; } } diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 59f5aae37..40aa0bd77 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -50,6 +50,72 @@ use uuid::Uuid; const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60); +#[cfg(any(test, feature = "test-util"))] +struct ConfigWriteLockProbeState { + bucket: String, + arrived: tokio::sync::Notify, +} + +#[cfg(any(test, feature = "test-util"))] +static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock>>> = std::sync::OnceLock::new(); + +#[cfg(any(test, feature = "test-util"))] +pub struct ConfigWriteLockProbe { + state: Arc, +} + +#[cfg(any(test, feature = "test-util"))] +impl ConfigWriteLockProbe { + pub fn install(bucket: &str) -> Self { + let state = Arc::new(ConfigWriteLockProbeState { + bucket: bucket.to_string(), + arrived: tokio::sync::Notify::new(), + }); + let mut probes = CONFIG_WRITE_LOCK_PROBES + .get_or_init(|| StdMutex::new(Vec::new())) + .lock() + .expect("config write lock probe mutex should not poison"); + assert!( + !probes.iter().any(|current| current.bucket == state.bucket), + "config write lock probe must be unique for a bucket" + ); + probes.push(Arc::clone(&state)); + drop(probes); + Self { state } + } + + pub async fn wait_until_attempted(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("bucket config update should attempt the transaction lock"); + } +} + +#[cfg(any(test, feature = "test-util"))] +impl Drop for ConfigWriteLockProbe { + fn drop(&mut self) { + let mut probes = CONFIG_WRITE_LOCK_PROBES + .get_or_init(|| StdMutex::new(Vec::new())) + .lock() + .expect("config write lock probe mutex should not poison"); + probes.retain(|state| !Arc::ptr_eq(state, &self.state)); + } +} + +#[cfg(any(test, feature = "test-util"))] +fn notify_config_write_lock_attempt(bucket: &str) { + let probe = CONFIG_WRITE_LOCK_PROBES + .get_or_init(|| StdMutex::new(Vec::new())) + .lock() + .expect("config write lock probe mutex should not poison") + .iter() + .find(|probe| probe.bucket == bucket) + .cloned(); + if let Some(probe) = probe { + probe.arrived.notify_one(); + } +} + #[derive(Clone, Copy)] enum MetadataLoadMode { Initial, @@ -590,6 +656,31 @@ pub async fn update_under_transaction_lock( update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await } +pub async fn update_quota_if_incarnation( + bucket: &str, + data: Vec, + expected_incarnation_id: Uuid, + proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken, +) -> Result { + let sys = get_bucket_metadata_sys()?; + let guard = Box::pin(acquire_config_write_guard_for_incarnation( + sys.clone(), + bucket, + Some(expected_incarnation_id), + )) + .await?; + if !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof) { + return Err(Error::NamespaceLockQuorumUnavailable { + mode: "quota_capability", + bucket: bucket.to_string(), + object: rustfs_config::QUOTA_CONFIG_FILE.to_string(), + required: 1, + achieved: 0, + }); + } + update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await +} + pub async fn update_bucket_targets_under_transaction_lock( guard: &BucketMetadataMutationGuard, bucket: &str, @@ -734,7 +825,26 @@ async fn acquire_transaction_lock_with_sys( let lock = api .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket)) .await?; - Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?) + let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()); + #[cfg(any(test, feature = "test-util"))] + { + tokio::pin!(acquire); + let mut notified = false; + let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) { + std::task::Poll::Pending => { + if !notified { + notify_config_write_lock_attempt(bucket); + notified = true; + } + std::task::Poll::Pending + } + std::task::Poll::Ready(result) => std::task::Poll::Ready(result), + }) + .await?; + Ok(guard) + } + #[cfg(not(any(test, feature = "test-util")))] + Ok(acquire.await?) } /// The lock resource name is deliberately still the `bucket-targets` one it @@ -889,6 +999,37 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in( } } +/// Re-read the quota configuration and bucket incarnation from the same +/// authoritative metadata blob while the caller holds the bucket metadata +/// transaction read lock. +pub(crate) async fn get_quota_config_and_incarnation_from_disk_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result<(Option, Uuid, OffsetDateTime)> { + let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); + + match bucket_meta_sys + .read_authoritative_metadata_from_disk_under_transaction_lock(bucket) + .await? + { + BucketMetadataAuthority::Authoritative(metadata) + if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() => + { + Ok(( + metadata.quota_config.clone(), + metadata.bucket_incarnation_id, + metadata.quota_config_updated_at, + )) + } + BucketMetadataAuthority::Authoritative(_) => { + Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}"))) + } + BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())), + BucketMetadataAuthority::Fabricated => Err(Error::other(format!("bucket quota metadata is not authoritative: {bucket}"))), + } +} + pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; diff --git a/crates/ecstore/src/bucket/quota/checker.rs b/crates/ecstore/src/bucket/quota/checker.rs index 38b4e1423..ebb1b4dbb 100644 --- a/crates/ecstore/src/bucket/quota/checker.rs +++ b/crates/ecstore/src/bucket/quota/checker.rs @@ -52,6 +52,7 @@ impl QuotaChecker { ) -> Result { let start_time = Instant::now(); let quota_config = self.get_quota_config(bucket).await?; + let uses_durable_reservations = quota_config.uses_durable_reservations(); // If no quota limit is set, allow operation let quota_limit = match quota_config.quota { @@ -67,6 +68,7 @@ impl QuotaChecker { quota_limit: None, operation_size, remaining: None, + uses_durable_reservations, }); } Some(q) => q, @@ -74,14 +76,17 @@ impl QuotaChecker { let current_usage = self.get_real_time_usage(bucket).await?; + let admission_size = if uses_durable_reservations { 0 } else { operation_size }; let expected_usage = match operation { - QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => current_usage + operation_size, + QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => { + current_usage.saturating_add(admission_size) + } QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size), }; let allowed = match operation { QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => { - quota_config.check_operation_allowed(current_usage, operation_size) + quota_config.check_operation_allowed(current_usage, admission_size) } QuotaOperation::DeleteObject => true, }; @@ -105,6 +110,7 @@ impl QuotaChecker { quota_limit: Some(quota_limit), operation_size, remaining, + uses_durable_reservations, }; let duration = start_time.elapsed(); @@ -158,6 +164,26 @@ impl QuotaChecker { .await } + pub async fn set_durable_quota_config_if_incarnation( + &mut self, + bucket: &str, + quota: BucketQuota, + expected_incarnation_id: uuid::Uuid, + proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken, + ) -> Result { + let json_data = serde_json::to_vec("a).map_err(|e| QuotaError::InvalidConfig { + reason: format!("Failed to serialize quota config: {}", e), + })?; + let start_time = Instant::now(); + let updated_at = + crate::bucket::metadata_sys::update_quota_if_incarnation(bucket, json_data, expected_incarnation_id, proof) + .await + .map_err(QuotaError::StorageError)?; + + rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed()); + Ok(updated_at) + } + async fn set_quota_config_for_incarnation( &mut self, bucket: &str, @@ -355,6 +381,7 @@ mod tests { quota_limit: None, operation_size: 1024, remaining: None, + uses_durable_reservations: false, }; assert!(result.allowed); @@ -378,4 +405,13 @@ mod tests { let allowed = quota.check_operation_allowed(512, 1024); assert!(!allowed); } + + #[test] + fn legacy_quota_rejects_full_operation_while_v1_defers_net_growth() { + let legacy: BucketQuota = serde_json::from_str(r#"{"quota":5}"#).expect("legacy quota should parse"); + let durable = BucketQuota::new(Some(5)); + + assert!(!legacy.check_operation_allowed(4, 2)); + assert!(durable.uses_durable_reservations()); + } } diff --git a/crates/ecstore/src/bucket/quota/mod.rs b/crates/ecstore/src/bucket/quota/mod.rs index 9172fd652..157c750e4 100644 --- a/crates/ecstore/src/bucket/quota/mod.rs +++ b/crates/ecstore/src/bucket/quota/mod.rs @@ -13,40 +13,100 @@ // limitations under the License. pub mod checker; +pub(crate) mod reservation; use crate::error::Result; use rustfs_config::{ QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE, QUOTA_NOT_FOUND_ERROR_CODE, }; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; use thiserror::Error; use time::OffsetDateTime; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] pub enum QuotaType { - /// Hard quota: reject immediately when exceeded + /// Hard quota accounting. #[default] #[serde(alias = "HARD", alias = "hard")] Hard, } +pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1; + /// Bucket quota configuration. quota_type defaults to Hard when omitted. -#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)] +#[derive(Debug, Default, Clone, PartialEq)] pub struct BucketQuota { - #[serde(default)] pub quota: Option, /// Defaults to Hard when missing. - #[serde(default)] pub quota_type: QuotaType, + /// Optional durable reservation protocol. The wire format gives older + /// nodes a zero hard quota so a mixed-version fleet fails closed. + pub reservation_protocol: Option, /// Timestamp when this quota configuration was set (for audit purposes) - #[serde(default, with = "time::serde::rfc3339::option")] pub created_at: Option, /// Accept updated_at for compatibility; not used. - #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] pub updated_at: Option, } +#[derive(Deserialize, Serialize)] +struct BucketQuotaWire { + #[serde(default)] + quota: Option, + #[serde(default)] + quota_type: QuotaType, + #[serde(default, skip_serializing_if = "Option::is_none")] + reservation_protocol: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + reservation_quota: Option, + #[serde(default, with = "time::serde::rfc3339::option")] + created_at: Option, + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + updated_at: Option, +} + +impl Serialize for BucketQuota { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + let durable = self.uses_durable_reservations(); + BucketQuotaWire { + quota: if durable { Some(0) } else { self.quota }, + quota_type: self.quota_type.clone(), + reservation_protocol: self.reservation_protocol, + reservation_quota: if durable { self.quota } else { None }, + created_at: self.created_at, + updated_at: self.updated_at, + } + .serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for BucketQuota { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let wire = BucketQuotaWire::deserialize(deserializer)?; + let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) { + Some( + wire.reservation_quota + .ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?, + ) + } else { + wire.quota + }; + Ok(Self { + quota, + quota_type: wire.quota_type, + reservation_protocol: wire.reservation_protocol, + created_at: wire.created_at, + updated_at: wire.updated_at, + }) + } +} + impl BucketQuota { /// Serialize to JSON bytes. Same format as parse_all_configs. pub fn marshal_msg(&self) -> Result> { @@ -63,6 +123,7 @@ impl BucketQuota { Self { quota, quota_type: QuotaType::Hard, + reservation_protocol: quota.map(|_| QUOTA_RESERVATION_PROTOCOL_V1), created_at: Some(now), updated_at: None, } @@ -72,7 +133,19 @@ impl BucketQuota { self.quota } + pub fn uses_durable_reservations(&self) -> bool { + self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) + } + + pub fn has_unsupported_reservation_protocol(&self) -> bool { + self.reservation_protocol + .is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1) + } + pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool { + if operation_size == 0 { + return true; + } if let Some(quota_limit) = self.quota { current_usage.saturating_add(operation_size) <= quota_limit } else { @@ -94,6 +167,7 @@ pub struct QuotaCheckResult { pub quota_limit: Option, pub operation_size: u64, pub remaining: Option, + pub uses_durable_reservations: bool, } #[derive(Debug)] @@ -210,7 +284,59 @@ mod tests { let buf = q.marshal_msg().expect("marshal"); let restored = BucketQuota::unmarshal(&buf).expect("unmarshal"); assert_eq!(q.quota, restored.quota); - assert_eq!(q.quota_type, restored.quota_type); + assert_eq!(restored.quota_type, QuotaType::Hard); + assert_eq!(restored.reservation_protocol, Some(QUOTA_RESERVATION_PROTOCOL_V1)); + } + + #[test] + fn clearing_quota_keeps_the_legacy_compatible_type() { + let quota = BucketQuota::new(None); + + assert_eq!(quota.quota_type, QuotaType::Hard); + assert_eq!(quota.reservation_protocol, None); + assert!(!quota.uses_durable_reservations()); + } + + #[test] + fn durable_quota_makes_legacy_nodes_fail_closed() { + let json = serde_json::to_vec(&BucketQuota::new(Some(2048))).expect("durable quota should serialize"); + let quota: BucketQuota = serde_json::from_slice(&json).expect("current quota version should parse"); + assert!(quota.uses_durable_reservations()); + assert_eq!(quota.quota, Some(2048)); + + #[derive(Deserialize)] + enum LegacyQuotaType { + Hard, + } + #[derive(Deserialize)] + struct LegacyBucketQuota { + #[allow(dead_code)] + quota: Option, + #[allow(dead_code)] + quota_type: LegacyQuotaType, + } + let legacy = serde_json::from_slice::(&json) + .expect("legacy readers should ignore the reservation protocol field"); + assert_eq!(legacy.quota, Some(0)); + assert!(matches!(legacy.quota_type, LegacyQuotaType::Hard)); + } + + #[test] + fn unknown_reservation_protocol_does_not_activate_v1() { + let quota: BucketQuota = + serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#) + .expect("future protocol should remain parseable"); + + assert!(!quota.uses_durable_reservations()); + assert!(quota.has_unsupported_reservation_protocol()); + } + + #[test] + fn reservation_protocol_v1_requires_reservation_quota() { + let err = serde_json::from_str::(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":1}"#) + .expect_err("v1 without its authoritative quota must fail closed"); + + assert!(err.to_string().contains("reservation_quota is required")); } /// unmarshal accepts format without quota_type diff --git a/crates/ecstore/src/bucket/quota/reservation.rs b/crates/ecstore/src/bucket/quota/reservation.rs new file mode 100644 index 000000000..175785487 --- /dev/null +++ b/crates/ecstore/src/bucket/quota/reservation.rs @@ -0,0 +1,1113 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::bucket::metadata_sys; +use crate::config::com::{CONFIG_PREFIX, read_config_no_lock, save_config_with_opts}; +use crate::data_usage::compute_bucket_usage; +use crate::disk::RUSTFS_META_BUCKET; +use crate::disk::{DiskAPI, error::DiskError}; +use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found}; +use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission}; +use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; +use crate::storage_api_contracts::namespace::NamespaceLocking; +use crate::storage_api_contracts::object::ObjectOperations; +use crate::store::ECStore; +use futures::{StreamExt, stream}; +use rustfs_lock::NamespaceLockGuard; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; +use time::OffsetDateTime; +use tracing::warn; +use uuid::Uuid; + +const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1; +const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64; +const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128; +const ORPHAN_PROBE_CONCURRENCY: usize = 32; +const EVENT_QUOTA_LEDGER_SETTLEMENT: &str = "quota_ledger_settlement"; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_QUOTA: &str = "quota"; + +#[cfg(any(test, feature = "test-util"))] +static FAIL_NEXT_LEDGER_SAVE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +// Lock order: caller-held destination object/upload, bucket metadata +// transaction (read), operation reservation, then quota ledger. + +#[cfg(not(any(test, feature = "test-util")))] +const ORPHAN_MIN_AGE_SECONDS: i64 = 30; +#[cfg(any(test, feature = "test-util"))] +const ORPHAN_MIN_AGE_SECONDS: i64 = 0; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct PersistedReservation { + object: String, + old_size: u64, + new_size: u64, + created_at: i64, + #[serde(default)] + pool_index: Option, + #[serde(default)] + set_index: Option, + #[serde(default)] + commit_started: bool, +} + +impl PersistedReservation { + fn growth(&self) -> u64 { + self.new_size.saturating_sub(self.old_size) + } + + fn target(&self) -> Option<(usize, usize)> { + self.pool_index.zip(self.set_index) + } + + fn matches_expected(&self, expected: &Self) -> bool { + self.object == expected.object && self.old_size == expected.old_size && self.new_size == expected.new_size + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct QuotaLedger { + version: u8, + bucket_incarnation: Uuid, + quota_revision_unix_nanos: i128, + accounted_usage: u64, + reservations: BTreeMap, + #[serde(default)] + reconcile_required: bool, + #[serde(default)] + reap_cursor: Option, +} + +impl QuotaLedger { + fn new(bucket_incarnation: Uuid, quota_revision: OffsetDateTime, accounted_usage: u64) -> Self { + Self { + version: QUOTA_LEDGER_FORMAT_VERSION, + bucket_incarnation, + quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(), + accounted_usage, + reservations: BTreeMap::new(), + reconcile_required: false, + reap_cursor: None, + } + } + + fn matches(&self, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> bool { + self.bucket_incarnation == bucket_incarnation && self.quota_revision_unix_nanos == quota_revision.unix_timestamp_nanos() + } + + fn admitted_usage(&self) -> Result { + let reserved_growth = self.reservations.values().try_fold(0_u64, |total, reservation| { + total + .checked_add(reservation.growth()) + .ok_or(StorageError::PartMissingOrCorrupt) + })?; + if reserved_growth > self.accounted_usage { + return Err(StorageError::PartMissingOrCorrupt); + } + Ok(self.accounted_usage) + } + + fn reserve(&mut self, operation_id: Uuid, reservation: PersistedReservation) -> Result<()> { + self.accounted_usage = self + .accounted_usage + .checked_add(reservation.growth()) + .ok_or(StorageError::PartMissingOrCorrupt)?; + self.reservations.insert(operation_id, reservation); + Ok(()) + } + + fn commit(&mut self, operation_id: Uuid, expected: &PersistedReservation) -> Result<()> { + let Some(reservation) = self.reservations.remove(&operation_id) else { + return Err(StorageError::PartMissingOrCorrupt); + }; + if !reservation.matches_expected(expected) { + return Err(StorageError::PartMissingOrCorrupt); + } + if reservation.new_size < reservation.old_size { + self.reconcile_required = true; + } + Ok(()) + } + + fn abort(&mut self, operation_id: Uuid, expected: &PersistedReservation) -> Result<()> { + let Some(reservation) = self.reservations.remove(&operation_id) else { + return Ok(()); + }; + if !reservation.matches_expected(expected) { + return Err(StorageError::PartMissingOrCorrupt); + } + self.accounted_usage = self + .accounted_usage + .checked_sub(reservation.growth()) + .ok_or(StorageError::PartMissingOrCorrupt)?; + Ok(()) + } + + fn mark_commit_started(&mut self, operation_id: Uuid, expected: &PersistedReservation) -> Result<()> { + let reservation = self + .reservations + .get_mut(&operation_id) + .ok_or(StorageError::PartMissingOrCorrupt)?; + if !reservation.matches_expected(expected) { + return Err(StorageError::PartMissingOrCorrupt); + } + reservation.commit_started = true; + Ok(()) + } + + fn should_reconcile_after_denial(&self) -> bool { + self.reservations.is_empty() + } + + fn reap_candidates(&self, now: i64) -> (Vec, Option) { + let aged = self + .reservations + .iter() + .filter(|(_, reservation)| { + reservation.created_at > now || now.saturating_sub(reservation.created_at) >= ORPHAN_MIN_AGE_SECONDS + }) + .map(|(operation_id, _)| *operation_id) + .collect::>(); + let start = self + .reap_cursor + .and_then(|cursor| aged.iter().position(|operation_id| *operation_id > cursor)) + .unwrap_or(0); + let candidates = aged + .iter() + .cycle() + .skip(start) + .take(aged.len().min(MAX_ORPHAN_PROBES_PER_WRITE)) + .copied() + .collect::>(); + let next_cursor = candidates.last().copied(); + (candidates, next_cursor) + } +} + +pub(crate) struct QuotaContext { + store: Option>, + bucket: String, + object: String, + ledger_object: String, + bucket_incarnation: Option, + quota_revision: Option, + quota_limit: Option, + capability_proof: Option, + snapshot_admission: Option, + legacy_data_movement: bool, + metadata_guard: Option, + pool_index: Option, + set_index: Option, +} + +impl QuotaContext { + pub(crate) fn is_enforced(&self) -> bool { + self.quota_limit.is_some() + } + + pub(crate) async fn reserve(self, old_size: u64, new_size: u64) -> Result { + let Some(quota_limit) = self.quota_limit else { + return Ok(QuotaReservation::unlimited(self.metadata_guard)); + }; + if let Some(admission) = self.snapshot_admission { + let growth = new_size.saturating_sub(old_size); + if growth > admission.remaining() { + return Err(StorageError::QuotaExceeded { + current: admission.current_usage(), + limit: admission.quota_limit(), + }); + } + return Ok(QuotaReservation::unlimited(self.metadata_guard)); + } + if self.legacy_data_movement { + if new_size > old_size { + return Err(StorageError::PartMissingOrCorrupt); + } + return Ok(QuotaReservation::unlimited(self.metadata_guard)); + } + let store = self.store.ok_or(StorageError::PartMissingOrCorrupt)?; + let bucket_incarnation = self.bucket_incarnation.ok_or(StorageError::PartMissingOrCorrupt)?; + let quota_revision = self.quota_revision.ok_or(StorageError::PartMissingOrCorrupt)?; + let operation_id = Uuid::new_v4(); + let operation_lock_object = operation_lock_object(&self.ledger_object, operation_id); + let operation_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &operation_lock_object).await?; + let operation_guard = operation_lock.get_write_lock(get_lock_acquire_timeout()).await?; + let reservation = PersistedReservation { + object: self.object, + old_size, + new_size, + created_at: OffsetDateTime::now_utc().unix_timestamp(), + pool_index: self.pool_index, + set_index: self.set_index, + commit_started: false, + }; + let ledger_data = LedgerReservationData { + store: Arc::clone(&store), + bucket: self.bucket, + ledger_object: self.ledger_object, + operation_id, + reservation: reservation.clone(), + }; + let metadata_guard = self.metadata_guard; + let capability_proof = self.capability_proof; + + tokio::spawn(async move { + reap_stale_reservations(Arc::clone(&store), &ledger_data.bucket, &ledger_data.ledger_object).await?; + + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?; + let mut ledger = load_current_ledger_locked( + Arc::clone(&store), + &ledger_data.bucket, + &ledger_data.ledger_object, + bucket_incarnation, + quota_revision, + ) + .await?; + + let growth = reservation.growth(); + let mut current_usage = ledger.admitted_usage()?; + let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?; + if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() { + reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?; + current_usage = ledger.admitted_usage()?; + expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?; + } + if growth > 0 && expected_usage > quota_limit { + return Err(StorageError::QuotaExceeded { + current: current_usage, + limit: quota_limit, + }); + } + if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: ledger_data.bucket.clone(), + object: ledger_data.ledger_object.clone(), + required: 1, + achieved: 0, + }); + } + ledger.reserve(operation_id, reservation)?; + save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?; + + Ok(QuotaReservation { + ledger: Some(ledger_data), + operation_guard: Some(operation_guard), + metadata_guard, + capability_proof, + state: ReservationState::Pending, + }) + }) + .await + .map_err(|err| StorageError::other(format!("quota ledger reservation task failed: {err}")))? + } +} + +#[derive(Clone)] +struct LedgerReservationData { + store: Arc, + bucket: String, + ledger_object: String, + operation_id: Uuid, + reservation: PersistedReservation, +} + +pub(crate) struct QuotaReservation { + ledger: Option, + operation_guard: Option, + metadata_guard: Option, + capability_proof: Option, + state: ReservationState, +} + +#[derive(Clone, Copy)] +enum ReservationState { + Pending, + CommitStarted, + Committed, + FenceReleaseUncertain, +} + +impl QuotaReservation { + fn unlimited(metadata_guard: Option) -> Self { + Self { + ledger: None, + operation_guard: None, + metadata_guard, + capability_proof: None, + state: ReservationState::Pending, + } + } + + pub(crate) fn is_lock_lost(&self) -> bool { + self.operation_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) + || self.metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) + } + + pub(crate) fn capability_proof_matches(&self) -> bool { + self.capability_proof + .as_ref() + .is_none_or(crate::services::notification_sys::cross_pool_fence_fleet_proof_matches) + } + + pub(crate) async fn mark_commit_started(&mut self) -> Result<()> { + if !self.capability_proof_matches() { + let ledger = self.ledger.as_ref().ok_or(StorageError::PartMissingOrCorrupt)?; + return Err(quota_capability_error(&ledger.bucket, &ledger.ledger_object)); + } + if let Some(ledger) = self.ledger.as_ref() { + mark_commit_started(ledger).await?; + } + self.state = ReservationState::CommitStarted; + Ok(()) + } + + pub(crate) async fn commit(mut self) { + self.state = ReservationState::Committed; + let Some(ledger) = self.ledger.as_ref() else { + return; + }; + crate::store::list_objects::observe_list_objects_mutation(&ledger.store, &ledger.bucket).await; + match settle(ledger, true).await { + Ok(()) => self.ledger = None, + Err(err) => log_deferred_settlement(ledger, "commit_deferred", &err), + } + } + + pub(crate) async fn abort(mut self) { + let Some(ledger) = self.ledger.as_ref() else { + return; + }; + match settle(ledger, false).await { + Ok(()) => self.ledger = None, + Err(err) => log_deferred_settlement(ledger, "abort_deferred", &err), + } + } + + pub(crate) fn defer_after_fence(mut self) { + self.state = ReservationState::FenceReleaseUncertain; + } +} + +fn should_settle_on_drop(state: ReservationState) -> bool { + !matches!(state, ReservationState::CommitStarted | ReservationState::FenceReleaseUncertain) +} + +impl Drop for QuotaReservation { + fn drop(&mut self) { + let Some(ledger) = self.ledger.take() else { + return; + }; + if !should_settle_on_drop(self.state) { + return; + } + let committed = matches!(self.state, ReservationState::Committed); + let operation_guard = self.operation_guard.take(); + let metadata_guard = self.metadata_guard.take(); + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + runtime.spawn(async move { + let _operation_guard = operation_guard; + let _metadata_guard = metadata_guard; + if let Err(err) = settle(&ledger, committed).await { + log_deferred_settlement(&ledger, "background_retry_failed", &err); + } + }); + } +} + +pub(crate) async fn begin( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, + object: &str, + snapshot_admission: Option, + data_movement: bool, + pool_index: usize, + set_index: usize, +) -> Result { + if crate::bucket::utils::is_meta_bucketname(bucket) { + return Ok(QuotaContext { + store: None, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: None, + quota_revision: None, + quota_limit: None, + capability_proof: None, + snapshot_admission: None, + legacy_data_movement: false, + metadata_guard: None, + pool_index: None, + set_index: None, + }); + } + #[cfg(test)] + if let Some(snapshot_admission) = snapshot_admission { + return Ok(QuotaContext { + store: None, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: None, + quota_revision: None, + quota_limit: Some(snapshot_admission.quota_limit()), + capability_proof: None, + snapshot_admission: Some(snapshot_admission), + legacy_data_movement: false, + metadata_guard: None, + pool_index: Some(pool_index), + set_index: Some(set_index), + }); + } + #[cfg(any(test, feature = "test-util"))] + if ctx.bucket_metadata_sys().is_none() { + return Ok(QuotaContext { + store: None, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: None, + quota_revision: None, + quota_limit: None, + capability_proof: None, + snapshot_admission: None, + legacy_data_movement: false, + metadata_guard: None, + pool_index: None, + set_index: None, + }); + } + + let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?; + let (quota, bucket_incarnation, quota_revision) = + metadata_sys::get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await?; + if metadata_guard.is_lock_lost() { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_config", + bucket: bucket.to_string(), + object: ledger_object(bucket), + required: 1, + achieved: 0, + }); + } + if quota + .as_ref() + .is_some_and(|quota| quota.has_unsupported_reservation_protocol()) + { + return Err(StorageError::PartMissingOrCorrupt); + } + let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations()); + let capability_proof = if durable_quota.is_some() { + Some( + crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof() + .ok_or_else(|| quota_capability_error(bucket, &ledger_object(bucket)))?, + ) + } else { + None + }; + let durable_quota_limit = durable_quota.and_then(|quota| quota.quota); + let snapshot_admission = match quota.as_ref().filter(|quota| !quota.uses_durable_reservations()) { + Some(quota) => match (quota.quota, snapshot_admission) { + (Some(limit), Some(admission)) if admission.quota_limit() == limit => Some(admission), + (Some(_), None) if data_movement => None, + (Some(_), _) => return Err(StorageError::PartMissingOrCorrupt), + (None, _) => None, + }, + None => None, + }; + let legacy_data_movement = durable_quota_limit.is_none() + && quota.as_ref().and_then(|quota| quota.quota).is_some() + && snapshot_admission.is_none() + && data_movement; + let quota_limit = durable_quota_limit + .or_else(|| snapshot_admission.map(QuotaAdmission::quota_limit)) + .or_else(|| { + legacy_data_movement + .then(|| quota.as_ref().and_then(|quota| quota.quota)) + .flatten() + }); + let store = if durable_quota_limit.is_some() { + Some(metadata_sys::object_store_in(ctx).await?) + } else { + None + }; + Ok(QuotaContext { + store, + bucket: bucket.to_string(), + object: object.to_string(), + ledger_object: ledger_object(bucket), + bucket_incarnation: Some(bucket_incarnation), + quota_revision: Some(quota_revision), + quota_limit, + capability_proof, + snapshot_admission, + legacy_data_movement, + metadata_guard: Some(metadata_guard), + pool_index: Some(pool_index), + set_index: Some(set_index), + }) +} + +fn quota_capability_error(bucket: &str, object: &str) -> StorageError { + StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_capability", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + } +} + +pub(crate) async fn replaced_logical_size(set_disks: &SetDisks, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { + if opts.versioned && !opts.version_suspended && opts.version_id.is_none() { + return Ok(0); + } + let version_id = opts + .version_id + .clone() + .or_else(|| opts.version_suspended.then(|| Uuid::nil().to_string())); + let lookup_opts = ObjectOptions { + version_id, + no_lock: true, + metadata_cache_safe: false, + versioned: opts.versioned, + version_suspended: opts.version_suspended, + ..Default::default() + }; + match set_disks.get_object_info(bucket, object, &lookup_opts).await { + Ok(info) if info.delete_marker => Ok(0), + Ok(info) => logical_object_size(&info), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(0), + Err(err) => Err(err), + } +} + +fn logical_object_size(info: &ObjectInfo) -> Result { + crate::data_usage::quota_object_size(info) +} + +async fn mark_commit_started(data: &LedgerReservationData) -> Result<()> { + let data = data.clone(); + tokio::spawn(async move { + let ledger_lock = data.store.new_ns_lock(RUSTFS_META_BUCKET, &data.ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?; + let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?; + ledger.mark_commit_started(data.operation_id, &data.reservation)?; + save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await + }) + .await + .map_err(|err| StorageError::other(format!("quota commit marker task failed: {err}")))? +} + +async fn settle(data: &LedgerReservationData, committed: bool) -> Result<()> { + let store = Arc::clone(&data.store); + let ledger_object = data.ledger_object.clone(); + let operation_id = data.operation_id; + let reservation = data.reservation.clone(); + tokio::spawn(async move { + // The commit/abort path releases its own object fence before settlement. + // Do not revoke all tokens here: a deferred retry can run after the + // object lock is released and would otherwise revoke a later write's + // newly acquired fence for the same object. + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_object, None).await?; + let mut ledger = load_ledger_locked(Arc::clone(&store), &ledger_object).await?; + if committed { + ledger.commit(operation_id, &reservation)?; + } else { + ledger.abort(operation_id, &reservation)?; + } + save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await + }) + .await + .map_err(|err| StorageError::other(format!("quota ledger settlement task failed: {err}")))? +} + +async fn load_current_ledger_locked( + store: Arc, + bucket: &str, + ledger_object: &str, + bucket_incarnation: Uuid, + quota_revision: OffsetDateTime, +) -> Result { + match load_ledger_locked(Arc::clone(&store), ledger_object).await { + Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger), + Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => { + let usage = exact_bucket_usage(&store, bucket).await?; + Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage)) + } + Ok(_) => Err(StorageError::PartMissingOrCorrupt), + Err(StorageError::ConfigNotFound) => { + let usage = exact_bucket_usage(&store, bucket).await?; + Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage)) + } + Err(err) => Err(err), + } +} + +async fn reap_stale_reservations(store: Arc, bucket: &str, ledger_object: &str) -> Result<()> { + let now = now_unix(); + let (candidates, reconcile_required, next_cursor) = { + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, ledger_object).await?; + let _ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + match load_ledger_locked(Arc::clone(&store), ledger_object).await { + Ok(ledger) => { + let (candidates, next_cursor) = ledger.reap_candidates(now); + (candidates, ledger.reconcile_required, next_cursor) + } + Err(StorageError::ConfigNotFound) => (Vec::new(), false, None), + Err(err) => return Err(err), + } + }; + if candidates.is_empty() && !reconcile_required { + return Ok(()); + } + + let probe_results = stream::iter(candidates) + .map(|operation_id| { + let store = Arc::clone(&store); + async move { + let lock_object = operation_lock_object(ledger_object, operation_id); + let operation_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &lock_object).await?; + Ok::<_, StorageError>( + operation_lock + .get_write_lock_quiet(Duration::from_millis(50)) + .await + .ok() + .map(|guard| (operation_id, guard)), + ) + } + }) + .buffer_unordered(ORPHAN_PROBE_CONCURRENCY) + .collect::>() + .await; + let mut orphan_guards = Vec::new(); + for result in probe_results { + if let Some(guard) = result? { + orphan_guards.push(guard); + if orphan_guards.len() == MAX_ORPHANS_REAPED_PER_WRITE { + break; + } + } + } + + let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, ledger_object).await?; + let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + fence_namespace_mutations(&store, RUSTFS_META_BUCKET, ledger_object, None).await?; + let mut ledger = load_ledger_locked(Arc::clone(&store), ledger_object).await?; + let cursor_changed = next_cursor.is_some() && ledger.reap_cursor != next_cursor; + if next_cursor.is_some() { + ledger.reap_cursor = next_cursor; + } + let orphan_ids = orphan_guards + .iter() + .map(|(operation_id, _)| *operation_id) + .collect::>(); + let orphan_commit_targets = orphan_ids + .iter() + .filter_map(|operation_id| ledger.reservations.get(operation_id)) + .filter(|reservation| reservation.commit_started) + .map(|reservation| (reservation.object.clone(), reservation.target())) + .collect::>(); + for (object, target) in orphan_commit_targets { + fence_namespace_mutations(&store, bucket, &object, target).await?; + } + let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?; + if ledger.reservations.is_empty() && ledger.reconcile_required { + reconcile_exact(&store, bucket, &mut ledger).await?; + removed = true; + } + if !removed && !cursor_changed { + return Ok(()); + } + save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await +} + +fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result { + let mut removed = false; + for operation_id in operation_ids { + let Some(reservation) = ledger.reservations.get(operation_id).cloned() else { + continue; + }; + if reservation.commit_started { + ledger.reservations.remove(operation_id); + ledger.reconcile_required = true; + } else { + ledger.abort(*operation_id, &reservation)?; + } + removed = true; + } + Ok(removed) +} + +async fn reconcile_exact(store: &Arc, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> { + if !ledger.reservations.is_empty() { + return Err(StorageError::PartMissingOrCorrupt); + } + ledger.accounted_usage = exact_bucket_usage(store, bucket).await?; + ledger.reconcile_required = false; + Ok(()) +} + +async fn exact_bucket_usage(store: &Arc, bucket: &str) -> Result { + crate::store::list_objects::observe_list_objects_mutation(store, bucket).await; + Ok(compute_bucket_usage(Arc::clone(store), bucket).await?.size) +} + +fn ledger_object(bucket: &str) -> String { + format!("{CONFIG_PREFIX}/quota-ledger/{bucket}.json") +} + +fn operation_lock_object(ledger_object: &str, operation_id: Uuid) -> String { + format!("{ledger_object}.operations/{operation_id}") +} + +fn now_unix() -> i64 { + OffsetDateTime::now_utc().unix_timestamp() +} + +async fn fence_namespace_mutations( + store: &Arc, + bucket: &str, + object: &str, + target: Option<(usize, usize)>, +) -> Result<()> { + crate::bucket::utils::check_object_args(bucket, object)?; + let sets = match target { + Some((pool_index, set_index)) => { + let set = store + .pools + .get(pool_index) + .and_then(|pool| pool.disk_set.get(set_index)) + .cloned() + .ok_or(StorageError::PartMissingOrCorrupt)?; + vec![set] + } + None => store.pools.iter().map(|pool| pool.get_disks_by_key(object)).collect(), + }; + for set in sets { + let write_quorum = set.default_write_quorum(); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + let revoke_results = stream::iter(disks) + .map(|disk| { + let fence_path = fence_path.clone(); + async move { + let result = disk + .release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, crate::disk::SnapshotLeaseToken::revoke_all()) + .await; + (disk, result) + } + }) + .buffer_unordered(ORPHAN_PROBE_CONCURRENCY) + .collect::>() + .await; + let revoked_disks = revoke_results + .into_iter() + .filter_map(|(disk, result)| result.is_ok().then_some(disk)) + .collect::>(); + if revoked_disks.len() < write_quorum { + return Err(StorageError::ErasureWriteQuorum); + } + + let drain_results = stream::iter(revoked_disks) + .map(|disk| async move { + match disk.acquire_snapshot_lease(bucket, object).await { + Ok(token) => disk.release_snapshot_lease(bucket, object, token).await, + Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => Ok(()), + Err(err) => Err(err), + } + }) + .buffer_unordered(ORPHAN_PROBE_CONCURRENCY) + .collect::>() + .await; + if drain_results.iter().filter(|result| result.is_ok()).count() < write_quorum { + return Err(StorageError::ErasureWriteQuorum); + } + } + Ok(()) +} + +#[cfg(test)] +pub(crate) async fn fence_namespace_mutations_for_test( + store: &Arc, + bucket: &str, + object: &str, + target: Option<(usize, usize)>, +) -> Result<()> { + fence_namespace_mutations(store, bucket, object, target).await +} + +async fn load_ledger_locked(store: Arc, ledger_object: &str) -> Result { + let data = read_config_no_lock(store, ledger_object).await?; + let ledger: QuotaLedger = serde_json::from_slice(&data)?; + if ledger.version != QUOTA_LEDGER_FORMAT_VERSION { + return Err(StorageError::CorruptedFormat); + } + ledger.admitted_usage()?; + Ok(ledger) +} + +async fn save_ledger_locked( + store: Arc, + ledger_object: &str, + ledger: &QuotaLedger, + ledger_guard: &NamespaceLockGuard, +) -> Result<()> { + if ledger_guard.is_lock_lost() { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_ledger", + bucket: RUSTFS_META_BUCKET.to_string(), + object: ledger_object.to_string(), + required: 1, + achieved: 0, + }); + } + #[cfg(any(test, feature = "test-util"))] + if FAIL_NEXT_LEDGER_SAVE.swap(false, std::sync::atomic::Ordering::SeqCst) { + return Err(StorageError::Unexpected); + } + let mut opts = ObjectOptions { + max_parity: true, + no_lock: true, + ..Default::default() + }; + let _ = opts.set_quota_admission(0, u64::MAX); + opts.add_namespace_lock_guard(ledger_guard); + save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await +} + +#[cfg(any(test, feature = "test-util"))] +pub fn fail_next_quota_ledger_save_for_test() { + FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst); +} + +fn log_deferred_settlement(data: &LedgerReservationData, state: &'static str, err: &StorageError) { + warn!( + event = EVENT_QUOTA_LEDGER_SETTLEMENT, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_QUOTA, + state, + bucket = %data.bucket, + operation_id = %data.operation_id, + error = %err, + "quota ledger settlement deferred" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ledger(accounted_usage: u64) -> QuotaLedger { + QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage) + } + + #[test] + fn ledger_rejects_reserved_growth_overflow() { + let mut ledger = ledger(u64::MAX); + let result = ledger.reserve( + Uuid::new_v4(), + PersistedReservation { + object: "object".to_string(), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ); + + assert!(matches!(result, Err(StorageError::PartMissingOrCorrupt))); + } + + #[test] + fn legacy_reservation_without_topology_uses_conservative_fallback() { + let reservation: PersistedReservation = + serde_json::from_str(r#"{"object":"object","old_size":0,"new_size":1,"created_at":0,"commit_started":true}"#) + .expect("legacy reservation should deserialize"); + + assert_eq!(reservation.target(), None); + } + + #[test] + fn ledger_rejects_persisted_reservations_larger_than_accounted_usage() { + let mut ledger = ledger(0); + ledger.reservations.insert( + Uuid::new_v4(), + PersistedReservation { + object: "object".to_string(), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ); + + assert!(matches!(ledger.admitted_usage(), Err(StorageError::PartMissingOrCorrupt))); + } + + #[test] + fn ledger_accounts_overwrite_delta_and_reserved_growth() { + let mut ledger = ledger(10); + let operation_id = Uuid::new_v4(); + let overwrite = PersistedReservation { + object: "object".to_string(), + old_size: 8, + new_size: 5, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }; + ledger + .reserve(operation_id, overwrite.clone()) + .expect("shrinking overwrite should reserve"); + ledger + .reserve( + Uuid::new_v4(), + PersistedReservation { + object: "new-object".to_string(), + old_size: 0, + new_size: 7, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ) + .expect("new object should reserve positive growth"); + + assert_eq!( + ledger + .admitted_usage() + .expect("ledger usage should count positive growth only"), + 17 + ); + ledger + .commit(operation_id, &overwrite) + .expect("overwrite should settle exactly"); + assert_eq!(ledger.accounted_usage, 17); + assert!(ledger.reconcile_required); + assert_eq!(ledger.admitted_usage().expect("remaining reservation should stay counted"), 17); + } + + #[test] + fn commit_started_reservation_stays_precharged_until_reconciled() { + let mut ledger = ledger(10); + let operation_id = Uuid::new_v4(); + ledger + .reserve( + operation_id, + PersistedReservation { + object: "object".to_string(), + old_size: 0, + new_size: 7, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ) + .expect("new object should reserve positive growth"); + assert_eq!(ledger.accounted_usage, 17); + + let expected = ledger + .reservations + .get(&operation_id) + .expect("reservation should exist") + .clone(); + ledger + .mark_commit_started(operation_id, &expected) + .expect("commit marker should persist"); + + assert_eq!(ledger.accounted_usage, 17); + assert!(!ledger.should_reconcile_after_denial()); + } + + #[test] + fn commit_started_orphans_make_progress_across_bounded_batches() { + let mut ledger = ledger(65); + let operation_ids = (0..65).map(|_| Uuid::new_v4()).collect::>(); + for operation_id in &operation_ids { + ledger.reservations.insert( + *operation_id, + PersistedReservation { + object: format!("object-{operation_id}"), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: true, + }, + ); + } + + assert!(remove_orphan_reservations(&mut ledger, &operation_ids[..64]).expect("first orphan batch should apply")); + assert_eq!(ledger.reservations.len(), 1); + assert!(ledger.reconcile_required); + assert!(remove_orphan_reservations(&mut ledger, &operation_ids[64..]).expect("final orphan batch should apply")); + assert!(ledger.reservations.is_empty()); + } + + #[test] + fn orphan_probe_cursor_rotates_across_the_bounded_window() { + let mut ledger = ledger(129); + let operation_ids = (1..=129).map(Uuid::from_u128).collect::>(); + for operation_id in &operation_ids { + ledger.reservations.insert( + *operation_id, + PersistedReservation { + object: format!("object-{operation_id}"), + old_size: 0, + new_size: 1, + created_at: 0, + pool_index: Some(0), + set_index: Some(0), + commit_started: false, + }, + ); + } + + let (first, cursor) = ledger.reap_candidates(1); + assert_eq!(first.len(), MAX_ORPHAN_PROBES_PER_WRITE); + assert_eq!(first.first(), operation_ids.first()); + ledger.reap_cursor = cursor; + + let (second, _) = ledger.reap_candidates(1); + assert_eq!(second.first(), operation_ids.last()); + } + + #[test] + fn uncertain_fence_release_does_not_schedule_abort_on_drop() { + assert!(!should_settle_on_drop(ReservationState::FenceReleaseUncertain)); + assert!(!should_settle_on_drop(ReservationState::CommitStarted)); + assert!(should_settle_on_drop(ReservationState::Pending)); + assert!(should_settle_on_drop(ReservationState::Committed)); + } +} diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 46f246d6b..1426fc91f 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -248,6 +248,16 @@ fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) Ok(server_epoch) } +fn decode_cross_pool_fence_capability(expected_member: &str, result: &[u8]) -> Result<(u32, Uuid)> { + let version = result + .get(..4) + .and_then(|value| value.try_into().ok()) + .map(u32::from_be_bytes) + .ok_or_else(|| Error::other("peer returned an invalid cross-pool fence capability version"))?; + let epoch = decode_remote_version_state_capability(expected_member, &result[4..])?; + Ok((version, epoch)) +} + #[derive(Clone, Debug)] pub struct PeerLiveEventsBatch { pub events: Vec, @@ -1288,6 +1298,16 @@ impl PeerRestClient { Ok((self.topology_member.clone(), epoch)) } + pub async fn probe_cross_pool_fence(&self, topology_fingerprint: String) -> Result<(String, u32, Uuid)> { + let mut probe = rustfs_protos::CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX.to_vec(); + probe.extend_from_slice(Uuid::new_v4().as_bytes()); + let result = self + .heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe) + .await?; + let (supported_version, epoch) = decode_cross_pool_fence_capability(&self.topology_member, &result)?; + Ok((self.topology_member.clone(), supported_version, epoch)) + } + pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> { self.finalize_result( async { @@ -2738,6 +2758,24 @@ mod tests { assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err()); } + #[test] + fn cross_pool_fence_capability_decoder_fails_closed() { + let epoch = Uuid::new_v4(); + let result = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", epoch.as_bytes()) + .expect("small capability response should encode"); + assert_eq!( + decode_cross_pool_fence_capability("node-a:9000", &result).expect("valid capability should decode"), + (1, epoch) + ); + for malformed in [&[][..], &[0, 0, 0][..], &result[..result.len() - 1]] { + assert!(decode_cross_pool_fence_capability("node-a:9000", malformed).is_err()); + } + assert!(decode_cross_pool_fence_capability("node-b:9000", &result).is_err()); + let nil = rustfs_protos::encode_cross_pool_fence_capability(1, "node-a:9000", Uuid::nil().as_bytes()) + .expect("small capability response should encode"); + assert!(decode_cross_pool_fence_capability("node-a:9000", &nil).is_err()); + } + struct TierMutationResponseFixture<'a> { version: u32, phase: TierMutationRpcPhase, diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 41495f068..3d0c122de 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -1355,7 +1355,7 @@ impl BucketUsageAccumulator { return Ok(()); } - let object_size = object.size.max(0) as u64; + let object_size = quota_object_size(object)?; self.current_live_versions = self.current_live_versions.saturating_add(1); self.size_histogram.add(object_size); self.total_size = self.total_size.saturating_add(object_size); @@ -1385,6 +1385,31 @@ impl BucketUsageAccumulator { } } +pub fn quota_object_size(object: &ObjectInfo) -> Result { + let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?; + let persisted_part_size = if object.parts.is_empty() { + u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)? + } else { + object.parts.iter().try_fold(0_u64, |total, part| { + // Compressed streaming objects persist -1 when the transformed + // part size is unknown. The physical part size remains a valid + // quota floor; reject only non-negative values that overflow. + let actual_size = if part.actual_size < 0 { + if object.is_compressed() { + 0 + } else { + return Err(Error::PartMissingOrCorrupt); + } + } else { + u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)? + }; + let part_size = actual_size.max(u64::try_from(part.size).map_err(|_| Error::PartMissingOrCorrupt)?); + total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt) + })? + }; + Ok(logical_size.max(persisted_part_size)) +} + type UsageVersionPage = StorageListObjectVersionsInfo; pub async fn compute_bucket_usage(store: Arc, bucket_name: &str) -> Result { @@ -3124,6 +3149,102 @@ mod tests { assert_eq!(usage.object_versions_histogram.get("BETWEEN_1000_AND_10000"), Some(&1)); } + #[test] + fn bucket_usage_uses_the_larger_of_logical_and_physical_size() { + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "4096".to_string()); + let object = ObjectInfo { + name: "compressed".to_string(), + size: 128, + user_defined: Arc::new(metadata), + ..Default::default() + }; + let mut usage = BucketUsageAccumulator::default(); + usage + .record("bucket", &object) + .expect("valid compressed metadata should be counted"); + assert_eq!(usage.finish().size, 4096); + + let mut framed_metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut framed_metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + rustfs_utils::http::insert_str(&mut framed_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let framed = ObjectInfo { + name: "framed".to_string(), + size: 17, + user_defined: Arc::new(framed_metadata), + ..Default::default() + }; + assert_eq!(quota_object_size(&framed).expect("physical framing must remain quota-accounted"), 17); + + let legacy_compressed_part = ObjectInfo { + name: "legacy-compressed-part".to_string(), + size: 1, + user_defined: Arc::new((*framed.user_defined).clone()), + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 1, + actual_size: -1, + ..Default::default() + }]), + ..Default::default() + }; + assert_eq!( + quota_object_size(&legacy_compressed_part).expect("unknown compressed part size is a valid sentinel"), + 1 + ); + + let uncompressed_negative_part = ObjectInfo { + name: "uncompressed-negative-part".to_string(), + size: 1, + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 1, + actual_size: -1, + ..Default::default() + }]), + ..Default::default() + }; + assert!(matches!(quota_object_size(&uncompressed_negative_part), Err(Error::PartMissingOrCorrupt))); + + let mut corrupt_metadata = (*object.user_defined).clone(); + rustfs_utils::http::insert_str(&mut corrupt_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "-1".to_string()); + let corrupt = ObjectInfo { + user_defined: Arc::new(corrupt_metadata), + ..object + }; + assert!(matches!(quota_object_size(&corrupt), Err(Error::PartMissingOrCorrupt))); + + let mut poisoned_metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut poisoned_metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + rustfs_utils::http::insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let poisoned = ObjectInfo { + name: "legacy-swift-metadata".to_string(), + size: 4096, + user_defined: Arc::new(poisoned_metadata), + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 4096, + actual_size: 4096, + ..Default::default() + }]), + ..Default::default() + }; + assert_eq!( + quota_object_size(&poisoned).expect("persisted part accounting must bound legacy user metadata"), + 4096 + ); + } + #[tokio::test] #[serial] async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() { diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 64e945d1d..c6ff03604 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -27,17 +27,18 @@ use crate::disk::{ BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, - PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, - RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, - conv_part_err_to_int, + PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, + ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, + SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int, endpoint::Endpoint, error::{DiskError, Error, FileAccessDeniedWithContext, Result}, error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}, format::FormatV3, fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename}, - os, + is_quota_mutation_fence_path, os, os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source}, + quota_mutation_fence_path, }; use crate::erasure::coding::{self, bitrot_verify}; use crate::runtime::sources as runtime_sources; @@ -60,9 +61,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Debug; use std::io::{Error as IoError, SeekFrom}; -#[cfg(target_os = "linux")] -use std::sync::atomic::AtomicBool; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; use std::{ @@ -4756,6 +4755,25 @@ struct SnapshotLeaseEntry { tokens: HashSet, pending_delete: Option, deleting: bool, + mutation_fence: Option>, +} + +#[derive(Default)] +struct QuotaMutationFenceState { + revoked: AtomicBool, + running: AtomicUsize, + notify: Notify, +} + +struct QuotaMutationFenceClaim { + state: Arc, +} + +impl Drop for QuotaMutationFenceClaim { + fn drop(&mut self) { + self.state.running.fetch_sub(1, Ordering::AcqRel); + self.state.notify.notify_waiters(); + } } #[derive(Default)] @@ -7320,6 +7338,34 @@ fn normalize_path_components(path: impl AsRef) -> PathBuf { } impl LocalDisk { + async fn claim_quota_mutation_fence( + &self, + volume: &str, + path: &str, + token: SnapshotLeaseToken, + ) -> Result> { + let key = SnapshotLeaseKey { + volume: RUSTFS_META_BUCKET.to_string(), + path: quota_mutation_fence_path(volume, path), + }; + let state = { + let registry = self.snapshot_leases.lock().await; + let entry = registry.entries.get(&key).ok_or(DiskError::FileNotFound)?; + let state = entry.mutation_fence.as_ref().ok_or(DiskError::FileNotFound)?; + if !entry.tokens.contains(&token) || state.revoked.load(Ordering::Acquire) { + return Err(DiskError::FileNotFound); + } + state.running.fetch_add(1, Ordering::AcqRel); + Arc::clone(state) + }; + if state.revoked.load(Ordering::Acquire) { + state.running.fetch_sub(1, Ordering::AcqRel); + state.notify.notify_waiters(); + return Err(DiskError::FileNotFound); + } + Ok(Arc::new(QuotaMutationFenceClaim { state })) + } + async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result { let path = format!("{object}/{data_dir}"); let data_path = self.io_get_object_path(volume, &path)?; @@ -8653,7 +8699,30 @@ impl DiskAPI for LocalDisk { // optimistic; this lease establishes the local commit/delete order and // remains owned by any blocking syscall that outlives async cancellation. let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; + let quota_fence_token = + match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { + Some(value) => { + let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; + Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) + } + None if rustfs_utils::http::metadata_compat::contains_key_str( + &fi.metadata, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + ) => + { + return Err(DiskError::FileCorrupt); + } + None => None, + }; + rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); + let quota_fence_claim = match quota_fence_token { + Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), + None => None, + }; let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; + if let Some(claim) = quota_fence_claim { + mutation_lease.attach_external_guard(claim); + } if fi.is_legacy_indexed_delete_marker() { fi.erasure.index = 0; } @@ -9643,11 +9712,26 @@ impl DiskAPI for LocalDisk { } async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result { - let file_path = self.io_get_object_path(volume, path)?; let key = SnapshotLeaseKey { volume: volume.to_string(), path: path.to_string(), }; + if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) { + let mut registry = self.snapshot_leases.lock().await; + let entry = registry.entries.entry(key).or_default(); + let state = entry + .mutation_fence + .get_or_insert_with(|| Arc::new(QuotaMutationFenceState::default())); + if state.revoked.load(Ordering::Acquire) { + return Err(DiskError::FileNotFound); + } + let token = SnapshotLeaseToken::new(); + entry.tokens.insert(token); + return Ok(token); + } + + let file_path = self.io_get_object_path(volume, path)?; + let _mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, volume, &file_path).await; let token = { let mut registry = self.snapshot_leases.lock().await; if registry.entries.get(&key).is_some_and(|entry| entry.deleting) { @@ -9675,6 +9759,48 @@ impl DiskAPI for LocalDisk { volume: volume.to_string(), path: path.to_string(), }; + if volume == RUSTFS_META_BUCKET && is_quota_mutation_fence_path(path) { + if !token.is_revoke_all() { + let mut registry = self.snapshot_leases.lock().await; + let Some(entry) = registry.entries.get_mut(&key) else { + return Ok(()); + }; + entry.tokens.remove(&token); + let removable = entry.tokens.is_empty() + && entry + .mutation_fence + .as_ref() + .is_none_or(|state| state.running.load(Ordering::Acquire) == 0); + if removable { + registry.entries.remove(&key); + } + return Ok(()); + } + let state = { + let mut registry = self.snapshot_leases.lock().await; + let Some(entry) = registry.entries.get_mut(&key) else { + return Ok(()); + }; + let Some(state) = entry.mutation_fence.as_ref().cloned() else { + registry.entries.remove(&key); + return Ok(()); + }; + state.revoked.store(true, Ordering::Release); + entry.tokens.clear(); + state + }; + loop { + let notified = state.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if state.running.load(Ordering::Acquire) == 0 { + break; + } + notified.await; + } + self.snapshot_leases.lock().await.entries.remove(&key); + return Ok(()); + } let opts = { let mut registry = self.snapshot_leases.lock().await; let Some(entry) = registry.entries.get_mut(&key) else { @@ -18889,6 +19015,48 @@ mod test { assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound))); } + #[tokio::test] + async fn quota_mutation_fence_revoke_waits_for_active_claim_and_rejects_late_claims() { + use tempfile::tempdir; + + let root_dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let bucket = "quota-fence-volume"; + let object = "object"; + let fence_path = quota_mutation_fence_path(bucket, object); + let token = disk + .acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path) + .await + .expect("quota mutation token should be prepared"); + let claim = disk + .claim_quota_mutation_fence(bucket, object, token) + .await + .expect("prepared token should be claimable"); + + let release_disk = Arc::clone(&disk); + let mut release = tokio::spawn(async move { + release_disk + .release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, SnapshotLeaseToken::revoke_all()) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut release).await.is_err(), + "revoke must wait until an already claimed mutation has finished" + ); + + drop(claim); + tokio::time::timeout(Duration::from_secs(1), release) + .await + .expect("revoke should wake after the final claim drops") + .expect("revoke task should not panic") + .expect("revoke should succeed"); + assert!(matches!( + disk.claim_quota_mutation_fence(bucket, object, token).await, + Err(DiskError::FileNotFound) + )); + } + #[tokio::test] async fn delete_version_keeps_later_part_until_snapshot_release() { use tempfile::tempdir; diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 26047fdfc..43259b8c8 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -72,6 +72,28 @@ use time::OffsetDateTime; use tokio::io::{AsyncRead, AsyncWrite}; use uuid::Uuid; +const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/"; +pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token"; + +pub(crate) fn quota_mutation_fence_path(bucket: &str, object: &str) -> String { + use sha2::{Digest, Sha256}; + + let mut input = Vec::with_capacity(bucket.len() + object.len() + 1); + input.extend_from_slice(bucket.as_bytes()); + input.push(0); + input.extend_from_slice(object.as_bytes()); + let digest = Sha256::digest(input); + format!( + "{QUOTA_MUTATION_FENCE_PREFIX}{}", + hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower) + ) +} + +pub(crate) fn is_quota_mutation_fence_path(path: &str) -> bool { + path.strip_prefix(QUOTA_MUTATION_FENCE_PREFIX) + .is_some_and(|digest| digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit())) +} + pub type DiskStore = Arc; pub type FileReader = Box; @@ -96,6 +118,20 @@ impl SnapshotLeaseToken { pub fn as_bytes(&self) -> &[u8; 16] { self.0.as_bytes() } + + pub(crate) fn as_uuid(self) -> Uuid { + self.0 + } + + #[doc(hidden)] + pub fn revoke_all() -> Self { + Self(Uuid::nil()) + } + + #[doc(hidden)] + pub fn is_revoke_all(self) -> bool { + self.0.is_nil() + } } impl Default for SnapshotLeaseToken { diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index bd3ec77a5..fc9c8027c 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -306,12 +306,20 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc { pub(crate) struct NamespaceMutationLease { _namespace_guard: OwnedMutexGuard<()>, _volume_guard: Option>, + external_guard: Mutex>>, +} + +impl NamespaceMutationLease { + pub(crate) fn attach_external_guard(&self, guard: Arc) { + *self.external_guard.lock() = Some(guard); + } } async fn acquire_namespace_mutation_lease(path: &Path) -> Arc { Arc::new(NamespaceMutationLease { _namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await, _volume_guard: None, + external_guard: Mutex::new(None), }) } @@ -327,6 +335,7 @@ pub(crate) async fn acquire_rename_data_mutation_lease( Arc::new(NamespaceMutationLease { _namespace_guard: namespace_guard, _volume_guard: Some(volume_guard), + external_guard: Mutex::new(None), }) } diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index c63dbc401..41fc15970 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -44,12 +44,14 @@ const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification"; const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation"; +const EVENT_NOTIFICATION_CAPABILITY_PROBE: &str = "notification_capability_probe"; const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100); const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10); const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30); +const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1; /// Cached result from the last successful admin call to a peer. struct PeerAdminCache { @@ -95,15 +97,15 @@ lazy_static! { } #[derive(Clone)] -struct RemoteVersionStateFleetProof { +struct FleetCapabilityProof { topology_fingerprint: String, peer_epochs: Arc>, expires_at: Instant, } -impl RemoteVersionStateFleetProof { - fn token(&self) -> RemoteVersionStateFleetProofToken { - RemoteVersionStateFleetProofToken { +impl FleetCapabilityProof { + fn token(&self) -> FleetCapabilityProofToken { + FleetCapabilityProofToken { topology_fingerprint: self.topology_fingerprint.clone(), peer_epochs: self.peer_epochs.clone(), } @@ -111,37 +113,41 @@ impl RemoteVersionStateFleetProof { } #[derive(Clone, PartialEq, Eq)] -pub(crate) struct RemoteVersionStateFleetProofToken { +struct FleetCapabilityProofToken { topology_fingerprint: String, peer_epochs: Arc>, } #[derive(Default)] -struct RemoteVersionStateFleetProofState { - proof: Option, +struct FleetCapabilityProofState { + proof: Option, topology_conflict: bool, } -static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock> = OnceLock::new(); +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken); + +#[derive(Clone, PartialEq, Eq)] +pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken); + +static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock> = OnceLock::new(); +static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock> = OnceLock::new(); static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock = OnceLock::new(); -fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock { - REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default())) +fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock { + CROSS_POOL_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) } -fn replace_remote_version_state_fleet_proof(proof: Option) { - replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof); +fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock { + REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) } -fn replace_remote_version_state_fleet_proof_in( - slot: &std::sync::RwLock, - proof: Option, -) { +fn replace_fleet_capability_proof(slot: &std::sync::RwLock, proof: Option) { slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof; } -fn publish_remote_version_state_probe_result( - slot: &std::sync::RwLock, +fn publish_fleet_capability_probe_result( + slot: &std::sync::RwLock, topology_fingerprint: &str, result: Result>, observed_at: Instant, @@ -155,7 +161,7 @@ fn publish_remote_version_state_probe_result( .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs) .map(|proof| Arc::clone(&proof.peer_epochs)) .unwrap_or_else(|| Arc::new(peer_epochs)); - state.proof = Some(RemoteVersionStateFleetProof { + state.proof = Some(FleetCapabilityProof { topology_fingerprint: topology_fingerprint.to_string(), peer_epochs, expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL, @@ -163,7 +169,7 @@ fn publish_remote_version_state_probe_result( None } Err(err) => { - replace_remote_version_state_fleet_proof_in(slot, None); + replace_fleet_capability_proof(slot, None); Some(err) } } @@ -174,27 +180,60 @@ pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option Option { - if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) { +) -> Option { + if state.topology_conflict || !fleet_capability_proof_valid_at(state.proof.as_ref(), expected_topology, now) { return None; } - state.proof.as_ref().map(RemoteVersionStateFleetProof::token) + state.proof.as_ref().map(FleetCapabilityProof::token) } pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool { + fleet_capability_proof_matches(remote_version_state_fleet_proof_slot(), &proof.0) +} + +pub fn acquire_cross_pool_fence_fleet_proof() -> Option { + let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?; + let state = cross_pool_fence_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(CrossPoolFenceFleetProofToken) +} + +pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToken) -> bool { + fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0) +} + +#[cfg(any(test, feature = "test-util"))] +pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool { + let mut state = cross_pool_fence_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(current) = state.proof.as_ref() else { + return false; + }; + state.proof = Some(FleetCapabilityProof { + topology_fingerprint: current.topology_fingerprint.clone(), + peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()), + expires_at: current.expires_at, + }); + true +} + +fn fleet_capability_proof_matches( + slot: &std::sync::RwLock, + proof: &FleetCapabilityProofToken, +) -> bool { let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else { return false; }; - let state = remote_version_state_fleet_proof_slot() - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner); + let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner); if state.topology_conflict { return false; } @@ -206,13 +245,17 @@ pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStat }) } +fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool { + proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at) +} + #[cfg(test)] pub(crate) struct RemoteVersionStateFleetProofGuard; #[cfg(test)] impl Drop for RemoteVersionStateFleetProofGuard { fn drop(&mut self) { - replace_remote_version_state_fleet_proof(None); + replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None); } } @@ -227,7 +270,7 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp Err(_) => panic!("remote version state test topology is already bound to another fingerprint"), } let peer_epochs = BTreeMap::new(); - if let Some(err) = publish_remote_version_state_probe_result( + if let Some(err) = publish_fleet_capability_probe_result( remote_version_state_fleet_proof_slot(), topology_fingerprint, Ok(peer_epochs), @@ -238,14 +281,6 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp RemoteVersionStateFleetProofGuard } -fn remote_version_state_fleet_proof_valid_at( - proof: Option<&RemoteVersionStateFleetProof>, - expected_topology: &str, - now: Instant, -) -> bool { - proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at) -} - fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap, peer: String, epoch: Uuid) -> Result<()> { if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() { return Err(Error::other("remote version state capability peer identity is invalid")); @@ -256,11 +291,11 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap, pe pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) { - let mut state = remote_version_state_fleet_proof_slot() - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner); - state.topology_conflict = true; - state.proof = None; + for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] { + let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner); + state.topology_conflict = true; + state.proof = None; + } } return; } @@ -281,13 +316,23 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { } None => Err(Error::other("remote version state fleet capability notification system is unavailable")), }; + let fence_result = match get_global_notification_sys() { + Some(notification_sys) => timeout( + REMOTE_VERSION_STATE_PROBE_TIMEOUT, + notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint), + ) + .await + .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))), + None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")), + }; let topology_conflict = remote_version_state_fleet_proof_slot() .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .topology_conflict; if topology_conflict { - replace_remote_version_state_fleet_proof(None); - } else if let Some(err) = publish_remote_version_state_probe_result( + replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None); + replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None); + } else if let Some(err) = publish_fleet_capability_probe_result( remote_version_state_fleet_proof_slot(), &topology_fingerprint, result, @@ -295,6 +340,24 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { ) { debug!(error = %err, "remote version state fleet capability probe failed closed"); } + if !topology_conflict + && let Some(err) = publish_fleet_capability_probe_result( + cross_pool_fence_fleet_proof_slot(), + &topology_fingerprint, + fence_result, + Instant::now(), + ) + { + debug!( + event = EVENT_NOTIFICATION_CAPABILITY_PROBE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + capability = "cross_pool_fence_v1", + state = "failed_closed", + error = %err, + "notification capability probe" + ); + } sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await; } }); @@ -362,6 +425,27 @@ impl NotificationSys { } Ok(peer_epochs) } + + async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result> { + if self.peer_clients.len() != self.peer_topology_hosts.len() { + return Err(Error::other("cross-pool fence capability fleet membership is incomplete")); + } + let probes = self.peer_clients.iter().map(|client| async { + let client = client + .as_ref() + .ok_or_else(|| Error::other("cross-pool fence capability peer is unreachable"))?; + client.probe_cross_pool_fence(topology_fingerprint.to_string()).await + }); + let mut peer_epochs = BTreeMap::new(); + for result in join_all(probes).await { + let (peer, version, epoch) = result?; + if version < CROSS_POOL_FENCE_SUPPORTED_VERSION { + return Err(Error::other("cross-pool fence capability version is unsupported")); + } + insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?; + } + Ok(peer_epochs) + } } pub struct NotificationPeerErr { @@ -2177,16 +2261,16 @@ mod tests { let now = Instant::now(); let mut peer_epochs = BTreeMap::new(); peer_epochs.insert("peer-a".to_string(), Uuid::new_v4()); - let proof = RemoteVersionStateFleetProof { + let proof = FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(peer_epochs), expires_at: now + Duration::from_secs(1), }; - assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now)); - assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now)); - assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at)); - assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now)); + assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now)); + assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now)); + assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-a", proof.expires_at)); + assert!(!fleet_capability_proof_valid_at(None, "topology-a", now)); } #[test] @@ -2200,25 +2284,25 @@ mod tests { #[test] fn remote_version_state_fleet_proof_accepts_single_node_membership() { let now = Instant::now(); - let proof = RemoteVersionStateFleetProof { + let proof = FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(BTreeMap::new()), expires_at: now + Duration::from_secs(1), }; - assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now)); + assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now)); } #[test] fn remote_version_state_fleet_proof_token_changes_with_process_epoch() { let now = Instant::now(); - let proof = RemoteVersionStateFleetProof { + let proof = FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), expires_at: now + Duration::from_secs(1), }; let captured = proof.token(); - let restarted = RemoteVersionStateFleetProof { + let restarted = FleetCapabilityProof { topology_fingerprint: proof.topology_fingerprint.clone(), peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])), expires_at: proof.expires_at, @@ -2229,11 +2313,11 @@ mod tests { #[test] fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() { - let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()); + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); let now = Instant::now(); let epoch = Uuid::new_v4(); let peers = BTreeMap::from([("peer-a".to_string(), epoch)]); - assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none()); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none()); let original = slot .read() .expect("proof slot should not poison") @@ -2242,9 +2326,7 @@ mod tests { .expect("successful probe should publish proof") .token(); - assert!( - publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none() - ); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()); let renewed = slot .read() .expect("proof slot should not poison") @@ -2256,8 +2338,7 @@ mod tests { let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]); assert!( - publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2)) - .is_none() + publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2)).is_none() ); let replaced = slot .read() @@ -2272,18 +2353,18 @@ mod tests { #[test] fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() { let now = Instant::now(); - let mut state = RemoteVersionStateFleetProofState { - proof: Some(RemoteVersionStateFleetProof { + let mut state = FleetCapabilityProofState { + proof: Some(FleetCapabilityProof { topology_fingerprint: "topology-a".to_string(), peer_epochs: Arc::new(BTreeMap::new()), expires_at: now + Duration::from_secs(1), }), topology_conflict: false, }; - assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some()); + assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some()); state.topology_conflict = true; - assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none()); + assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_none()); } #[test] @@ -2299,19 +2380,19 @@ mod tests { #[test] fn remote_version_state_fleet_probe_failure_revokes_previous_proof() { - let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()); + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); let now = Instant::now(); let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]); - assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); assert!(slot.read().expect("proof slot should not poison").proof.is_some()); assert!( - publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some() + publish_fleet_capability_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some() ); assert!(slot.read().expect("proof slot should not poison").proof.is_none()); let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]); - assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none()); assert!(slot.read().expect("proof slot should not poison").proof.is_some()); } diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 1fd1b02d5..91b6c99fa 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -590,7 +590,7 @@ impl MetadataQuorumAccumulator { }) } - pub(in crate::set_disk) fn default_write_quorum(&self) -> usize { + pub(crate) fn default_write_quorum(&self) -> usize { if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks { return self.total_disks; } @@ -3016,7 +3016,7 @@ impl SetDisks { self.set_drive_count - self.default_parity_count } - pub(in crate::set_disk) fn default_write_quorum(&self) -> usize { + pub(crate) fn default_write_quorum(&self) -> usize { let mut data_count = self.set_drive_count - self.default_parity_count; if data_count == self.default_parity_count { data_count += 1 @@ -3025,6 +3025,89 @@ impl SetDisks { data_count } + pub(in crate::set_disk) async fn prepare_quota_mutation_fences( + disks: &[Option], + bucket: &str, + object: &str, + write_quorum: usize, + ) -> crate::error::Result<(Vec>, Vec>)> { + let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + let results = join_all(disks.iter().map(|disk| { + let disk = disk.clone(); + let fence_path = fence_path.clone(); + async move { + let disk = disk?; + match disk.acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path).await { + Ok(token) => Some((disk, token)), + Err(_) => None, + } + } + })) + .await; + if results.iter().flatten().count() < write_quorum { + for (disk, token) in results.iter().flatten() { + let _ = disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, *token).await; + } + return Err(StorageError::ErasureWriteQuorum); + } + let mut fenced_disks = Vec::with_capacity(results.len()); + let mut tokens = Vec::with_capacity(results.len()); + for result in results { + match result { + Some((disk, token)) => { + fenced_disks.push(Some(disk)); + tokens.push(Some(token)); + } + None => { + fenced_disks.push(None); + tokens.push(None); + } + } + } + Ok((fenced_disks, tokens)) + } + + pub(in crate::set_disk) async fn release_quota_mutation_fences( + disks: &[Option], + tokens: &[Option], + bucket: &str, + object: &str, + write_quorum: usize, + ) -> crate::error::Result<()> { + let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| { + let disk = disk.as_ref()?.clone(); + let token = (*token)?; + let fence_path = fence_path.clone(); + Some(async move { disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, token).await }) + })) + .await; + if results.iter().filter(|result| result.is_ok()).count() < write_quorum { + return Err(StorageError::ErasureWriteQuorum); + } + Ok(()) + } + + pub(in crate::set_disk) async fn abort_quota_reservation_after_fence( + reservation: crate::bucket::quota::reservation::QuotaReservation, + disks: &[Option], + tokens: &[Option], + bucket: &str, + object: &str, + write_quorum: usize, + fenced: bool, + ) { + let safe_to_abort = !fenced + || Self::release_quota_mutation_fences(disks, tokens, bucket, object, write_quorum) + .await + .is_ok(); + if safe_to_abort { + reservation.abort().await; + } else { + reservation.defer_after_fence(); + } + } + #[tracing::instrument(level = "debug", skip(disks, file_infos))] #[allow(clippy::type_complexity)] pub(in crate::set_disk) async fn rename_data( diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 27e382a3e..87f4624ef 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -719,8 +719,8 @@ pub(crate) use core::io_primitives::disk_call_counters; mod ctx; mod metadata; mod ops; -#[cfg(test)] -pub(crate) use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; +#[cfg(any(test, feature = "test-util"))] +pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; #[cfg(feature = "test-util")] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; pub(crate) use ops::object::body_cache_plaintext_len; diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 236c8af3b..a11c0030a 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -27,11 +27,12 @@ use super::object::{ object_transaction_fencing_requested, old_data_cleanup_receipt_path, read_object_transaction_epoch_fence, verify_object_transaction_epoch_fence, }; +use crate::bucket::quota::reservation; use crate::crash_inject::{self, CrashPoint}; use crate::multipart_listing::paginate_multipart_listing; use futures::{StreamExt, stream}; use std::future::Future; -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use tokio::task::JoinSet; @@ -61,20 +62,21 @@ impl StaleMultipartCleanupGuard { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] #[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum MultipartCommitPause { +pub enum MultipartCommitPause { PutPartBeforeLockAcquire, PutPartBeforeLockLost, PutPartAfterRename, BeforeLockLost, + BeforeQuotaRename, BeforeTransactionEpochVerify, BeforeObjectPublication, AfterObjectPublication, AfterRename, } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] struct MultipartCommitBarrierState { bucket: String, object: String, @@ -85,27 +87,22 @@ struct MultipartCommitBarrierState { release: tokio::sync::Semaphore, } -#[cfg(test)] -pub(crate) struct MultipartCommitBarrier { +#[cfg(any(test, feature = "test-util"))] +pub struct MultipartCommitBarrier { state: Arc, } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] impl MultipartCommitBarrier { - pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self { + pub fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self { Self::install_for_arrivals(bucket, object, pause, 1) } - pub(crate) fn install_for_arrivals( - bucket: &str, - object: &str, - pause: MultipartCommitPause, - expected_arrivals: usize, - ) -> Self { + pub fn install_for_arrivals(bucket: &str, object: &str, pause: MultipartCommitPause, expected_arrivals: usize) -> Self { assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival"); let state = Arc::new(MultipartCommitBarrierState { bucket: bucket.to_string(), @@ -126,7 +123,7 @@ impl MultipartCommitBarrier { Self { state } } - pub(crate) async fn wait_until_paused(&self) { + pub async fn wait_until_paused(&self) { tokio::time::timeout(Duration::from_secs(30), async { loop { let arrived = self.state.arrived.notified(); @@ -140,12 +137,12 @@ impl MultipartCommitBarrier { .expect("multipart completion should reach the deterministic commit barrier"); } - pub(crate) fn release(&self) { + pub fn release(&self) { self.state.release.add_permits(self.state.expected_arrivals); } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] impl Drop for MultipartCommitBarrier { fn drop(&mut self) { self.release(); @@ -159,7 +156,7 @@ impl Drop for MultipartCommitBarrier { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) { let barrier = { let mut slot = MULTIPART_COMMIT_BARRIER @@ -1892,6 +1889,24 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { fi.parts = Vec::with_capacity(uploaded_parts.len()); + let quota_context = reservation::begin( + &self.ctx, + bucket, + object, + opts.quota_admission, + opts.data_movement, + self.pool_index, + self.set_index, + ) + .await?; + let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some(); + let preserve_replication_ciphertext = opts.replication_request + && contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT); + if quota_context.is_enforced() && preserve_replication_ciphertext { + return Err(Error::PartMissingOrCorrupt); + } + let transformed_object = fi.is_compressed() || should_persist_encryption_original_size(&fi.metadata); + let mut object_size: usize = 0; let mut object_actual_size: i64 = 0; @@ -2018,15 +2033,23 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { checksum_combined.extend_from_slice(cs.raw.as_slice()); } - object_size += ext_part.size; - if opts.quota_admission.is_some() && ext_part.actual_size < 0 { + object_size = object_size.checked_add(ext_part.size).ok_or(Error::PartMissingOrCorrupt)?; + if ext_part.actual_size < 0 && (!opts.replication_request || quota_context.is_enforced()) { return Err(Error::PartMissingOrCorrupt); } + let normalized_actual_size = if ext_part.actual_size >= 0 && !transformed_object { + ext_part + .actual_size + .max(i64::try_from(ext_part.size).map_err(|_| Error::PartMissingOrCorrupt)?) + } else { + ext_part.actual_size + }; object_actual_size = object_actual_size - .checked_add(ext_part.actual_size) + .checked_add(normalized_actual_size) .ok_or(Error::PartMissingOrCorrupt)?; - - fi.parts.push(completed_multipart_object_part(p.part_num, ext_part)); + let mut completed_part = completed_multipart_object_part(p.part_num, ext_part); + completed_part.actual_size = normalized_actual_size; + fi.parts.push(completed_part); } if let Some(wtcs) = opts.want_checksum.as_ref() { @@ -2053,15 +2076,34 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } } - if let Some(admission) = opts.quota_admission { - let quota_operation_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; - if quota_operation_size > admission.remaining() { - return Err(Error::QuotaExceeded { - current: admission.current_usage(), - limit: admission.quota_limit(), - }); - } - } + let declared_replication_actual_size = opts + .replication_request + .then(|| get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP)) + .flatten(); + let replication_actual_size = if opts.replication_request && quota_context.is_enforced() { + let observed_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let declared_cap = declared_replication_actual_size + .as_deref() + .map(|value| value.parse::().map_err(|_| Error::PartMissingOrCorrupt)) + .transpose()? + .unwrap_or(0); + let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(&fi.metadata) + .map_err(Error::other)? + .map(u64::try_from) + .transpose() + .map_err(|_| Error::PartMissingOrCorrupt)? + .unwrap_or(0); + Some(observed_size.max(declared_cap).max(declared_encryption_size)) + } else { + None + }; + let quota_new_size = match replication_actual_size { + Some(size) => size.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?), + None if quota_context.is_enforced() => u64::try_from(object_actual_size) + .map_err(|_| Error::PartMissingOrCorrupt)? + .max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?), + None => 0, + }; if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) { if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) { fi.checksum = Some(Bytes::from(rc_crc_bytes)); @@ -2134,7 +2176,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string()); } } else if opts.replication_request { - if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) { + if let Some(actual_size) = replication_actual_size { + insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); + if persist_encryption_original_size { + fi.metadata + .insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string()); + } + } else if let Some(actual_size) = declared_replication_actual_size { insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone()); if persist_encryption_original_size { fi.metadata @@ -2324,8 +2372,41 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } else { None }; + + let quota_old_size = if quota_context.is_enforced() { + if opts.data_movement { + quota_new_size + } else { + reservation::replaced_logical_size(&self, bucket, object, opts).await? + } + } else { + 0 + }; + let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?; + let (commit_disks, quota_fence_tokens) = if quota_mutation_fence { + match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await { + Ok((disks, tokens)) => { + for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) { + if let Some(token) = token { + insert_str( + &mut metadata.metadata, + crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + token.as_uuid().to_string(), + ); + } + } + (disks, tokens) + } + Err(err) => { + quota_reservation.abort().await; + return Err(err); + } + } + } else { + (shuffle_disks.clone(), vec![None; shuffle_disks.len()]) + }; let transaction_epoch = - transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas)); + transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas)); let commit_set = self.clone(); let commit_bucket = bucket.to_owned(); @@ -2333,41 +2414,121 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let commit_upload_id = upload_id.to_owned(); let commit_upload_id_path = upload_id_path.clone(); let commit_version_suspended = opts.version_suspended; + let commit_versioned = opts.versioned; + let commit_version_id = opts.version_id.clone(); + let commit_namespace_lock_fence = opts.namespace_lock_fence.clone(); + let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone(); let commit_is_versioned = opts.versioned || opts.version_suspended; let commit_capacity_scope_token = opts.capacity_scope_token; let commit_object_lock_guard = object_lock_guard.take(); - let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some(); + let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some() || quota_mutation_fence; let commit = async move { let _object_lock_guard = commit_object_lock_guard; let _upload_guard = upload_guard; + let mut quota_reservation = quota_reservation; let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); - // Crash-consistency injection: hard power loss after the upload is fully - // staged and locked but before the authoritative rename_data commit. No - // disk has moved the staged data, so a crash here must leave any prior - // committed version byte-for-byte intact (rustfs/backlog#864) and the - // upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`. - if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) { - return Err(StorageError::Unexpected); + let pre_rename_result: Result<()> = async { + // Crash-consistency injection: hard power loss after the upload is fully + // staged and locked but before the authoritative rename_data commit. No + // disk has moved the staged data, so a crash here must leave any prior + // committed version byte-for-byte intact (rustfs/backlog#864) and the + // upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`. + if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) { + return Err(StorageError::Unexpected); + } + quota_reservation.mark_commit_started().await?; + #[cfg(any(test, feature = "test-util"))] + pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeQuotaRename).await; + if quota_reservation.is_lock_lost() + || !quota_reservation.capability_proof_matches() + || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || commit_namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || commit_bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: commit_bucket.clone(), + object: commit_object.clone(), + required: 1, + achieved: 0, + }); + } + let restore_opts = ObjectOptions { + version_id: commit_version_id.clone(), + versioned: commit_versioned, + version_suspended: commit_version_suspended, + no_lock: true, + ..Default::default() + }; + commit_set + .require_current_restore_operation_id( + &commit_bucket, + &commit_object, + &restore_opts, + expected_restore_operation_id, + "complete_multipart_upload_quota_reservation", + ) + .await?; + if let Some(proof) = transaction_fencing_proof.as_ref() + && !object_transaction_fencing_fleet_proof_matches(proof) + { + return Err(Error::other( + "object transaction fencing fleet capability changed during complete_multipart_upload", + )); + } + if let Some(expected) = transaction_epoch_fence { + #[cfg(test)] + pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify) + .await; + verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?; + } + if quota_reservation.is_lock_lost() + || !quota_reservation.capability_proof_matches() + || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || commit_namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || commit_bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: commit_bucket.clone(), + object: commit_object.clone(), + required: 1, + achieved: 0, + }); + } + Ok(()) + } + .await; + if let Err(err) = pre_rename_result { + SetDisks::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + &commit_bucket, + &commit_object, + write_quorum, + quota_mutation_fence, + ) + .await; + return Err(err); } // The trailing `_` drops the rename_data old-size backfill // (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit // `get_object_info` lookup, so the backfill has no consumer here yet. - if let Some(proof) = transaction_fencing_proof.as_ref() - && !object_transaction_fencing_fleet_proof_matches(proof) - { - return Err(Error::other( - "object transaction fencing fleet capability changed during complete_multipart_upload", - )); - } - if let Some(expected) = transaction_epoch_fence { - #[cfg(test)] - pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify).await; - verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?; - } - let (online_disks, convergence, op_old_dir, cleanup_disks, _) = SetDisks::rename_data( - &shuffle_disks, + let rename_result = SetDisks::rename_data( + &commit_disks, RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, &parts_metadatas, @@ -2375,7 +2536,24 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { &commit_object, write_quorum, ) - .await?; + .await; + if quota_mutation_fence { + let _ = SetDisks::release_quota_mutation_fences( + &commit_disks, + "a_fence_tokens, + &commit_bucket, + &commit_object, + write_quorum, + ) + .await; + } + if rename_result.is_ok() { + quota_reservation.commit().await; + } + let (online_disks, convergence, op_old_dir, cleanup_disks, _) = match rename_result { + Ok(result) => result, + Err(err) => return Err(err.into()), + }; // Detach admission before any post-commit await: client cancellation // must not couple durable convergence repair to cleanup work. @@ -3162,7 +3340,7 @@ mod tests { let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await; let mut denied_opts = ObjectOptions::default(); - assert!(denied_opts.set_quota_admission(100, 4195)); + assert!(denied_opts.set_quota_admission(100, 4180)); let err = set_disks .clone() @@ -3173,7 +3351,7 @@ mod tests { err, StorageError::QuotaExceeded { current: 100, - limit: 4195 + limit: 4180 } )); @@ -3191,7 +3369,7 @@ mod tests { ); let mut allowed_opts = ObjectOptions::default(); - assert!(allowed_opts.set_quota_admission(100, 4196)); + assert!(allowed_opts.set_quota_admission(100, 4181)); let completed = set_disks .clone() .complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts) @@ -3232,6 +3410,120 @@ mod tests { ); } + #[tokio::test] + async fn replication_quota_uses_server_observed_part_size_as_lower_bound() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-replication-quota-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let mut create_opts = ObjectOptions::default(); + insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string()); + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("replication multipart upload should be created"); + let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x72; 4096], 1).await; + let mut complete_opts = ObjectOptions { + replication_request: true, + ..Default::default() + }; + assert!(complete_opts.set_quota_admission(0, 4095)); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part.clone()], &complete_opts) + .await + .expect_err("a forged tiny replication logical size must not reduce quota admission"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + assert!( + set_disks + .check_upload_id_exists(bucket, object, &upload.upload_id, false) + .await + .is_ok(), + "quota rejection must leave replicated multipart parts retryable" + ); + + assert!(complete_opts.set_quota_admission(0, 4096)); + let completed = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect("the physical safety boundary should admit the transformed replica"); + assert_eq!(completed.get_actual_size().expect("replica logical size should parse"), 1); + } + + #[tokio::test] + async fn direct_multipart_quota_uses_server_observed_part_size_as_lower_bound() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-direct-quota-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let upload = set_disks + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("multipart upload should be created"); + let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x73; 4096], 1).await; + let mut complete_opts = ObjectOptions::default(); + assert!(complete_opts.set_quota_admission(0, 4095)); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect_err("a forged tiny direct logical size must not reduce quota admission"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + assert!( + set_disks + .check_upload_id_exists(bucket, object, &upload.upload_id, false) + .await + .is_ok(), + "quota rejection must leave direct multipart parts retryable" + ); + } + + #[tokio::test] + async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-compressed-ciphertext-quota-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let mut create_opts = ObjectOptions::default(); + insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string()); + insert_str( + &mut create_opts.user_defined, + rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, + "true".to_string(), + ); + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("ciphertext multipart upload should be created"); + let payload = vec![0x74; 4096]; + let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &payload, 1).await; + let mut complete_opts = ObjectOptions { + replication_request: true, + ..Default::default() + }; + assert!(complete_opts.set_quota_admission(0, u64::MAX)); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect_err("ciphertext replication has no server-observed logical quota size"); + assert!(matches!(err, StorageError::PartMissingOrCorrupt)); + assert!( + set_disks + .check_upload_id_exists(bucket, object, &upload.upload_id, false) + .await + .is_ok(), + "rejection must leave ciphertext multipart parts retryable" + ); + } + #[tokio::test] async fn complete_multipart_quota_rejects_invalid_logical_sizes() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index cd450dac3..65c3de2b5 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -40,6 +40,7 @@ use crate::bucket::lifecycle::{ save_transition_transaction_record, }, }; +use crate::bucket::quota::reservation; use crate::bucket::replication::{ DeleteReplicationConfigSnapshot, VersionPurgeStatusType, replication_state_to_filemeta, version_purge_status_to_filemeta, }; @@ -57,9 +58,40 @@ use http::HeaderValue; use rustfs_utils::path::decode_dir_object; use std::future::Future; use std::sync::OnceLock; +use tokio_util::sync::CancellationToken; const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json"; +struct PutObjectCommitCancellation { + token: CancellationToken, + armed: bool, +} + +impl PutObjectCommitCancellation { + fn new() -> Self { + Self { + token: CancellationToken::new(), + armed: true, + } + } + + fn child_token(&self) -> CancellationToken { + self.token.clone() + } + + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for PutObjectCommitCancellation { + fn drop(&mut self) { + if self.armed { + self.token.cancel(); + } + } +} + #[inline] fn duration_millis_f64(duration: std::time::Duration) -> f64 { duration.as_secs_f64() * 1000.0 @@ -1341,6 +1373,26 @@ impl SetDisks { object: &str, data: &mut PutObjReader, opts: &ObjectOptions, + ) -> Result<(ObjectInfo, Option)> { + self.put_object_with_old_current_size_boxed(bucket, object, data, opts).await + } + + fn put_object_with_old_current_size_boxed<'a>( + &'a self, + bucket: &'a str, + object: &'a str, + data: &'a mut PutObjReader, + opts: &'a ObjectOptions, + ) -> impl Future)>> + Send + 'a { + Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts)) + } + + async fn put_object_with_old_current_size_inner( + &self, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, ) -> Result<(ObjectInfo, Option)> { crate::hp_guard!("SetDisks::put_object"); let storage_class_config = self.storage_class_config_snapshot(); @@ -1929,8 +1981,125 @@ impl SetDisks { } else { None }; + + let quota_context = reservation::begin( + &self.ctx, + bucket, + object, + opts.quota_admission, + opts.data_movement, + self.pool_index, + self.set_index, + ) + .await?; + let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some(); + let mut replication_quota_size = None; + + if opts.replication_request { + if quota_context.is_enforced() && opts.preserve_ciphertext { + return Err(Error::PartMissingOrCorrupt); + } + if quota_context.is_enforced() { + let persisted_metadata = &parts_metadatas[response_metadata_slot].metadata; + let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let physical_size = u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let transformed = contains_key_str(persisted_metadata, SUFFIX_COMPRESSION) + || should_persist_encryption_original_size(persisted_metadata); + let declared_size = get_str(persisted_metadata, SUFFIX_ACTUAL_SIZE) + .map(|value| value.parse::().map_err(|_| Error::PartMissingOrCorrupt)) + .transpose()? + .unwrap_or(0); + let declared_encryption_size = rustfs_utils::http::get_object_encryption_original_size(persisted_metadata) + .map_err(Error::other)? + .map(u64::try_from) + .transpose() + .map_err(|_| Error::PartMissingOrCorrupt)? + .unwrap_or(0); + let logical_size = observed_size.max(declared_size).max(declared_encryption_size); + let persisted_size = if transformed { + logical_size + } else { + logical_size.max(physical_size) + }; + replication_quota_size = Some(logical_size.max(physical_size)); + actual_size = i64::try_from(persisted_size).map_err(|_| Error::PartMissingOrCorrupt)?; + for metadata in &mut parts_metadatas { + insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, persisted_size.to_string()); + if should_persist_encryption_original_size(&metadata.metadata) { + metadata + .metadata + .insert("x-rustfs-encryption-original-size".to_string(), persisted_size.to_string()); + } + if let Some(part) = metadata.parts.first_mut() { + part.actual_size = actual_size; + } + } + } + } else if actual_size >= 0 { + let observed_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?; + let persisted_metadata = &parts_metadatas[response_metadata_slot].metadata; + let transformed = contains_key_str(persisted_metadata, SUFFIX_COMPRESSION) + || should_persist_encryption_original_size(persisted_metadata); + let server_observed_size = if transformed { + observed_size + } else { + observed_size.max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?) + }; + actual_size = i64::try_from(server_observed_size).map_err(|_| Error::PartMissingOrCorrupt)?; + for metadata in &mut parts_metadatas { + insert_str(&mut metadata.metadata, SUFFIX_ACTUAL_SIZE, server_observed_size.to_string()); + if should_persist_encryption_original_size(&metadata.metadata) { + metadata + .metadata + .insert("x-rustfs-encryption-original-size".to_string(), server_observed_size.to_string()); + } + if let Some(part) = metadata.parts.first_mut() { + part.actual_size = actual_size; + } + } + } + + let (quota_old_size, quota_new_size) = if quota_context.is_enforced() { + let new_size = match replication_quota_size { + Some(size) => size, + None => u64::try_from(actual_size) + .map_err(|_| Error::PartMissingOrCorrupt)? + .max(u64::try_from(w_size).map_err(|_| Error::PartMissingOrCorrupt)?), + }; + let old_size = if opts.data_movement { + new_size + } else { + reservation::replaced_logical_size(self, bucket, object, opts).await? + }; + (old_size, new_size) + } else { + (0, 0) + }; + let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?; + let (commit_disks, quota_fence_tokens) = if quota_mutation_fence { + match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await { + Ok((disks, tokens)) => { + for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) { + if let Some(token) = token { + insert_str( + &mut metadata.metadata, + crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + token.as_uuid().to_string(), + ); + } + } + (disks, tokens) + } + Err(err) => { + quota_reservation.abort().await; + return Err(err); + } + } + } else { + (shuffle_disks.clone(), vec![None; shuffle_disks.len()]) + }; let transaction_epoch = - transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas)); + transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas)); let commit_set = self.clone(); let commit_bucket = bucket.to_owned(); @@ -1938,18 +2107,65 @@ impl SetDisks { let commit_tmp_dir = tmp_dir.clone(); let commit_object_lock_guard = object_lock_guard.take(); let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take(); - let detach_commit_owner = commit_object_lock_guard.is_some() || commit_bucket_lifecycle_guard.is_some(); + let detach_commit_owner = + commit_object_lock_guard.is_some() || commit_bucket_lifecycle_guard.is_some() || quota_mutation_fence; let commit_write_path_label = write_path.metric_label(); let commit_is_versioned = opts.versioned || opts.version_suspended; + let commit_versioned = opts.versioned; + let commit_version_suspended = opts.version_suspended; + let commit_version_id = opts.version_id.clone(); + let commit_namespace_lock_fence = opts.namespace_lock_fence.clone(); + let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone(); let commit_capacity_scope_token = opts.capacity_scope_token; let commit_replication_state = replication_state_to_filemeta(&opts.put_replication_state()); tmp_cleanup_owned = true; - let commit = async move { + let commit = move |cancellation: Option| async move { let _object_lock_guard = commit_object_lock_guard; let _bucket_lifecycle_guard = commit_bucket_lifecycle_guard; + let mut quota_reservation = quota_reservation; let rename_stage_start = Instant::now(); - let pre_rename_result: Result<()> = async { + let pre_rename = async { + #[cfg(any(test, feature = "test-util"))] + pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterQuotaReservation).await; + quota_reservation.mark_commit_started().await?; + #[cfg(any(test, feature = "test-util"))] + pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::BeforeQuotaRename).await; + if quota_reservation.is_lock_lost() + || !quota_reservation.capability_proof_matches() + || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || commit_namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || commit_bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: commit_bucket.clone(), + object: commit_object.clone(), + required: 1, + achieved: 0, + }); + } + let restore_opts = ObjectOptions { + version_id: commit_version_id.clone(), + versioned: commit_versioned, + version_suspended: commit_version_suspended, + no_lock: true, + ..Default::default() + }; + commit_set + .require_current_restore_operation_id( + &commit_bucket, + &commit_object, + &restore_opts, + expected_restore_operation_id, + "put_object_quota_reservation", + ) + .await?; if let Some(proof) = transaction_fencing_proof.as_ref() && !object_transaction_fencing_fleet_proof_matches(proof) { @@ -1965,10 +2181,47 @@ impl SetDisks { .await; verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?; } + if quota_reservation.is_lock_lost() + || !quota_reservation.capability_proof_matches() + || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || commit_namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || commit_bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + bucket: commit_bucket.clone(), + object: commit_object.clone(), + required: 1, + achieved: 0, + }); + } Ok(()) - } - .await; + }; + let pre_rename_result = if let Some(cancellation) = cancellation { + tokio::select! { + biased; + _ = cancellation.cancelled() => Err(StorageError::OperationCanceled), + result = pre_rename => result, + } + } else { + pre_rename.await + }; if let Err(err) = pre_rename_result { + SetDisks::abort_quota_reservation_after_fence( + quota_reservation, + &commit_disks, + "a_fence_tokens, + &commit_bucket, + &commit_object, + write_quorum, + quota_mutation_fence, + ) + .await; if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { @@ -1982,8 +2235,9 @@ impl SetDisks { } return Err(err); } + let rename_result = SetDisks::rename_data( - &shuffle_disks, + &commit_disks, RUSTFS_META_TMP_BUCKET, commit_tmp_dir.as_str(), &parts_metadatas, @@ -1992,6 +2246,19 @@ impl SetDisks { write_quorum, ) .await; + if quota_mutation_fence { + let _ = SetDisks::release_quota_mutation_fences( + &commit_disks, + "a_fence_tokens, + &commit_bucket, + &commit_object, + write_quorum, + ) + .await; + } + if rename_result.is_ok() { + quota_reservation.commit().await; + } let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = match rename_result { Ok(commit) => commit, Err(err) => { @@ -2202,11 +2469,15 @@ impl SetDisks { }; if detach_commit_owner { - tokio::spawn(commit) + let mut cancellation = PutObjectCommitCancellation::new(); + let child_token = cancellation.child_token(); + let result = tokio::spawn(async move { Box::pin(commit(Some(child_token))).await }) .await - .map_err(|err| Error::other(format!("put_object commit task failed: {err}")))? + .map_err(|err| Error::other(format!("put_object commit task failed: {err}")))?; + cancellation.disarm(); + result } else { - commit.await + Box::pin(commit(None)).await } } .await; @@ -3225,6 +3496,8 @@ fn transaction_fencing_gate_requested_for(requested: bool, fleet_confirmed: bool pub enum PutObjectCommitPause { BeforeNamespace, AfterNamespace, + AfterQuotaReservation, + BeforeQuotaRename, BeforeMetadata, BeforeTransactionEpochVerify, } @@ -6295,6 +6568,139 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support { } } +#[cfg(test)] +mod replication_quota_safety_tests { + use super::hermetic_set_disks_support::hermetic_set_disks; + use super::*; + use std::io::Cursor; + + #[tokio::test] + async fn quota_put_future_keeps_commit_state_off_the_caller_stack() { + let (_temp_dirs, _disks, set_disks) = hermetic_set_disks(4).await; + let mut reader = PutObjReader::from_vec(Vec::new()); + let opts = ObjectOptions::default(); + + let future = set_disks.put_object_with_old_current_size("bucket", "object", &mut reader, &opts); + let future_size = std::mem::size_of_val(&future); + + assert!( + future_size <= 1024, + "put_object_with_old_current_size future must stay stack-bounded, got {future_size} bytes" + ); + } + + #[tokio::test] + async fn replication_put_quota_uses_physical_bytes_as_a_safety_floor() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "replication-put-quota-safety"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut user_defined = HashMap::new(); + insert_str( + &mut user_defined, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string()); + let payload = vec![0x61; 4096]; + + let mut denied_opts = ObjectOptions { + replication_request: true, + user_defined: user_defined.clone(), + ..Default::default() + }; + assert!(denied_opts.set_quota_admission(0, 4095)); + let mut denied_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false) + .expect("construct forged replication reader"), + ); + let err = set_disks + .put_object(bucket, "object", &mut denied_reader, &denied_opts) + .await + .expect_err("server-observed bytes must prevent a tiny replication quota claim"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + + let mut allowed_opts = ObjectOptions { + replication_request: true, + user_defined, + ..Default::default() + }; + assert!(allowed_opts.set_quota_admission(0, 4096)); + let mut allowed_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false) + .expect("construct exact-boundary replication reader"), + ); + let stored = set_disks + .put_object(bucket, "object", &mut allowed_reader, &allowed_opts) + .await + .expect("server-observed exact quota boundary should succeed"); + assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1); + } + + #[tokio::test] + async fn direct_put_cannot_persist_a_tiny_logical_size() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "direct-put-quota-safety"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let payload = vec![0x62; 4096]; + let mut denied_opts = ObjectOptions::default(); + assert!(denied_opts.set_quota_admission(0, 4095)); + let mut denied_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload.clone()), 4096, 1, None, None, false) + .expect("construct forged direct reader"), + ); + let err = set_disks + .put_object(bucket, "object", &mut denied_reader, &denied_opts) + .await + .expect_err("server-observed bytes must prevent a tiny direct quota claim"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4095 })); + + let mut allowed_opts = ObjectOptions::default(); + assert!(allowed_opts.set_quota_admission(0, 4096)); + let mut allowed_reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload), 4096, 1, None, None, false) + .expect("construct exact-boundary direct reader"), + ); + let stored = set_disks + .put_object(bucket, "object", &mut allowed_reader, &allowed_opts) + .await + .expect("server-observed exact quota boundary should succeed"); + assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 4096); + } + + #[tokio::test] + async fn quota_rejects_ciphertext_replication_without_a_server_observed_logical_size() { + let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await; + let bucket = "ciphertext-replication-quota-safety"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut user_defined = HashMap::new(); + user_defined.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()); + insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1".to_string()); + let mut opts = ObjectOptions { + replication_request: true, + preserve_ciphertext: true, + user_defined, + ..Default::default() + }; + assert!(opts.set_quota_admission(0, u64::MAX)); + let payload = vec![0x63; 4096]; + let mut reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(payload), 4096, 4096, None, None, false) + .expect("construct ciphertext replication reader"), + ); + let err = set_disks + .put_object(bucket, "object", &mut reader, &opts) + .await + .expect_err("ciphertext replication without a server-observed logical size must fail closed"); + assert!(matches!(err, StorageError::PartMissingOrCorrupt)); + } +} + #[cfg(test)] mod inline_put_commit_path_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 077caf714..1e4b706d2 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -1174,6 +1174,32 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn quota_object_fence_ignores_an_unrelated_offline_pool() { + let temp_dir = tempfile::tempdir().expect("create quota fence store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "quota-object-fence", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = format!("quota-object-fence-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create quota fence bucket"); + store.pools[1].disk_set[0].disks.write().await.fill(None); + + crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, Some((0, 0))) + .await + .expect("the selected pool fence should ignore an unrelated offline pool"); + let err = crate::bucket::quota::reservation::fence_namespace_mutations_for_test(&store, &bucket, object, None) + .await + .expect_err("legacy reservations must conservatively fence every pool"); + assert!(matches!(err, StorageError::ErasureWriteQuorum)); + + shutdown.cancel(); + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn tag_updates_skip_active_rebalance_source_pool() { diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 2e8ee9968..10fc766a2 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -641,7 +641,7 @@ pub(crate) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) { .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta))); } -pub(super) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 { +pub(crate) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 { observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default() } diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index ac6b520f0..0ba89a7ec 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -4741,9 +4741,10 @@ mod tests { #[tokio::test] #[serial_test::serial(body_cache_hook)] async fn select_snapshot_rejects_latest_versioned_delete_marker_during_prepare() { - let (_first_dirs, first_set) = make_local_set_disks(4, 2).await; - let (_second_dirs, second_set) = make_local_set_disks(4, 2).await; - let store = new_prepared_reader_test_store(&[Arc::clone(&first_set), Arc::clone(&second_set)]).await; + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await; let bucket = "select-snapshot-latest-delete-marker"; let object = "versioned-object.bin"; let versioned_opts = ObjectOptions { diff --git a/crates/protocols/src/swift/handler.rs b/crates/protocols/src/swift/handler.rs index 0d9cdc455..b8018ab5a 100644 --- a/crates/protocols/src/swift/handler.rs +++ b/crates/protocols/src/swift/handler.rs @@ -748,9 +748,10 @@ async fn handle_authenticated_request( } for (key, value) in info.user_defined.iter() { - if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } @@ -817,9 +818,10 @@ async fn handle_authenticated_request( // Add custom metadata headers (X-Object-Meta-*) for (key, value) in info.user_defined.iter() { - if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } @@ -856,9 +858,10 @@ async fn handle_authenticated_request( // Add custom metadata headers (X-Object-Meta-*) for (key, value) in info.user_defined.iter() { - if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } @@ -1168,9 +1171,10 @@ async fn handle_object_get( } for (key, value) in info.user_defined.iter() { - if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } @@ -1237,9 +1241,10 @@ async fn handle_object_get( if key == "x-delete-at" { // Add X-Delete-At header directly (not as X-Object-Meta-*) response = response.header("x-delete-at", value.as_str()); - } else if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + } else if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } @@ -1293,9 +1298,10 @@ async fn handle_object_head( if key == "x-delete-at" { // Add X-Delete-At header directly (not as X-Object-Meta-*) response = response.header("x-delete-at", value.as_str()); - } else if key != "content-type" { - let header_name = format!("x-object-meta-{}", key); - response = response.header(header_name, value.as_str()); + } else if key != "content-type" + && let Some(key) = object::swift_response_user_metadata_key(key) + { + response = response.header(format!("x-object-meta-{key}"), value.as_str()); } } diff --git a/crates/protocols/src/swift/object.rs b/crates/protocols/src/swift/object.rs index 85581a323..7169b152a 100644 --- a/crates/protocols/src/swift/object.rs +++ b/crates/protocols/src/swift/object.rs @@ -67,10 +67,57 @@ const LOG_COMPONENT_PROTOCOLS: &str = "protocols"; const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object"; const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state"; const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at"; +const USER_METADATA_PREFIX: &str = "x-amz-meta-"; /// Maximum object size in bytes (5GB - Swift default) const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024; +fn stored_swift_user_metadata_key(key: &str) -> String { + if rustfs_utils::http::is_internal_key(key) + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-amz-") + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-") + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-") + { + format!("{USER_METADATA_PREFIX}{key}") + } else { + key.to_string() + } +} + +pub(super) fn swift_response_user_metadata_key(key: &str) -> Option<&str> { + if rustfs_utils::http::is_internal_key(key) + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-") + || rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-") + { + return None; + } + if let Some(unescaped) = key.strip_prefix(USER_METADATA_PREFIX) + && (rustfs_utils::http::is_internal_key(unescaped) + || rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-amz-") + || rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-rustfs-encryption-") + || rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-minio-encryption-")) + { + return Some(unescaped); + } + Some(key) +} + +fn swift_user_metadata(headers: &HeaderMap) -> Option> { + let mut metadata = HashMap::new(); + let mut present = false; + for (header_name, header_value) in headers.iter() { + let header_name = header_name.as_str().to_lowercase(); + let Some(key) = header_name.strip_prefix("x-object-meta-") else { + continue; + }; + present = true; + if let Ok(value) = header_value.to_str() { + metadata.insert(stored_swift_user_metadata_key(key), value.to_string()); + } + } + present.then_some(metadata) +} + /// Object key translator for Swift object names /// /// Handles URL encoding/decoding and path normalization for Swift object keys. @@ -303,15 +350,7 @@ where let bucket = mapper.swift_to_s3_bucket(container, &project_id); // 5. Extract Swift metadata from X-Object-Meta-* headers - let mut user_metadata = HashMap::new(); - for (header_name, header_value) in headers.iter() { - let header_str = header_name.as_str().to_lowercase(); - if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") - && let Ok(value_str) = header_value.to_str() - { - user_metadata.insert(meta_key.to_string(), value_str.to_string()); - } - } + let mut user_metadata = swift_user_metadata(headers).unwrap_or_default(); // 6. Extract Content-Type if provided if let Some(content_type) = headers.get("content-type") @@ -739,15 +778,7 @@ pub async fn update_object_metadata( } // 8. Extract new metadata from X-Object-Meta-* headers - let mut new_metadata = HashMap::new(); - for (header_name, header_value) in headers.iter() { - let header_str = header_name.as_str().to_lowercase(); - if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") - && let Ok(value_str) = header_value.to_str() - { - new_metadata.insert(meta_key.to_string(), value_str.to_string()); - } - } + let mut new_metadata = swift_user_metadata(headers).unwrap_or_default(); // 9. Also update Content-Type if provided if let Some(content_type) = headers.get("content-type") @@ -889,19 +920,8 @@ pub async fn copy_object( let mut new_metadata = (*src_info.user_defined).clone(); // 11. If custom metadata headers provided, use those instead (Swift behavior) - let mut has_custom_meta = false; - for (header_name, header_value) in headers.iter() { - let header_str = header_name.as_str().to_lowercase(); - if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") { - if !has_custom_meta { - // First custom meta header - clear source metadata - new_metadata.clear(); - has_custom_meta = true; - } - if let Ok(value_str) = header_value.to_str() { - new_metadata.insert(meta_key.to_string(), value_str.to_string()); - } - } + if let Some(custom_metadata) = swift_user_metadata(headers) { + new_metadata = custom_metadata; } // 12. Also check for Content-Type override @@ -1123,6 +1143,36 @@ mod tests { assert!(ObjectKeyMapper::validate_object_name("unicode-文件.txt").is_ok()); } + #[test] + fn swift_user_metadata_cannot_materialize_internal_storage_keys() { + let mut headers = HeaderMap::new(); + headers.insert("x-object-meta-x-rustfs-internal-actual-size", "1".parse().expect("valid metadata value")); + headers.insert("x-object-meta-description", "safe".parse().expect("valid metadata value")); + let metadata = swift_user_metadata(&headers).expect("custom metadata should be detected"); + + assert_eq!(metadata.get("x-amz-meta-x-rustfs-internal-actual-size").map(String::as_str), Some("1")); + assert_eq!(metadata.get("description").map(String::as_str), Some("safe")); + assert!(!metadata.contains_key("x-rustfs-internal-actual-size")); + assert_eq!( + stored_swift_user_metadata_key("x-minio-encryption-original-size"), + "x-amz-meta-x-minio-encryption-original-size" + ); + assert_eq!(stored_swift_user_metadata_key("description"), "description"); + } + + #[test] + fn swift_user_metadata_response_mapping_is_reversible_and_filters_internal_keys() { + assert_eq!( + swift_response_user_metadata_key("x-amz-meta-x-rustfs-internal-actual-size"), + Some("x-rustfs-internal-actual-size") + ); + assert_eq!(swift_response_user_metadata_key("x-amz-meta-x-amz-checksum"), Some("x-amz-checksum")); + assert_eq!(swift_response_user_metadata_key("x-amz-meta-description"), Some("x-amz-meta-description")); + assert_eq!(swift_response_user_metadata_key("description"), Some("description")); + assert_eq!(swift_response_user_metadata_key("x-rustfs-internal-actual-size"), None); + assert_eq!(swift_response_user_metadata_key("x-minio-internal-actual-size"), None); + } + #[test] fn test_validate_object_name_empty() { let result = ObjectKeyMapper::validate_object_name(""); diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 93f958ef2..9f721f17b 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -16,7 +16,6 @@ for later deletion. - `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2. - `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration. - `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract. -- `cross-pool-fence-v1` authenticated unsupported advertisement: predeployment servers recognize the versioned cross-pool fence capability probe but report support version 0, allowing a later all-peer probe to distinguish predeployment nodes without activating a second lock domain. Replace the unsupported advertisement only when composite lock acquisition, a cluster-wide activation fence, complete fleet proof, commit-time proof revalidation, and fail-closed revocation ship together. - `replacement-recovery-status-v1` replacement recovery status capability: new peers treat an unimplemented ReplacementRecoveryStatus RPC as an explicitly non-definitive rolling-upgrade response, so Admin v4 cannot claim distributed replacement completion from old peers. Remove the fallback after the minimum supported RustFS peer version implements ReplacementRecoveryStatus. - `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration. - `rustfs-5509` FileInfo positional MessagePack decoding: beta.11 serialized 28 fields, while beta.12 inserted transition-version fields in the middle and serialized an incompatible 30-field array. New releases write named maps and retain readers for both shipped array layouts so direct and rolling upgrades can read either release. Remove the positional-array readers after every supported direct-upgrade release writes named maps and no retained RPC payload can contain a pre-map FileInfo array. diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index 1a3c2b383..c73d4c487 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -465,6 +465,16 @@ impl Operation for ImportBucketMetadata { file_contents.push((file_path, content)); } + let durable_quota_import = imported_quota_requires_fleet_proof(&file_contents)?; + let quota_fleet_proof = + if durable_quota_import { + Some(crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| { + s3_error!(ServiceUnavailable, "durable quota capability is not confirmed across the cluster") + })?) + } else { + None + }; + // Extract bucket names let mut bucket_names = Vec::new(); for (file_path, _) in &file_contents { @@ -707,21 +717,6 @@ impl Operation for ImportBucketMetadata { } BUCKET_QUOTA_CONFIG_FILE => { - if let Err(e) = serde_json::from_slice::(&content) { - warn!( - event = EVENT_ADMIN_BUCKET_META_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_BUCKET_META, - action = "import_bucket_metadata", - result = "config_deserialize_failed", - bucket = %bucket_name, - config_name = %conf_name, - error = %e, - "admin bucket meta state" - ); - continue; - } - let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, @@ -830,10 +825,6 @@ impl Operation for ImportBucketMetadata { } } - // Persist the assembled metadata to disk. Prior to this, the import only mutated the - // in-memory `bucket_metadatas` map and returned 200, silently dropping every imported - // config. `metadata_sys::update` loads the on-disk metadata, overwrites the given config - // field and saves it, preserving any configs not present in the import archive. for (bucket_name, metadata) in &bucket_metadatas { for (config_file, data) in imported_configs_to_persist(metadata) { let site_replication_item = imported_config_to_site_replication_item(bucket_name, metadata, config_file, &data)?; @@ -842,7 +833,21 @@ impl Operation for ImportBucketMetadata { } else { None }; - if let Err(e) = metadata_sys::update(bucket_name, config_file, data).await { + let persist_result = if config_file == BUCKET_QUOTA_CONFIG_FILE { + let quota: BucketQuota = + serde_json::from_slice(&data).map_err(|e| s3_error!(InvalidRequest, "invalid bucket quota: {e}"))?; + if quota.uses_durable_reservations() { + let proof = quota_fleet_proof.as_ref().ok_or_else(|| { + s3_error!(ServiceUnavailable, "durable quota capability is not confirmed across the cluster") + })?; + metadata_sys::update_quota_if_incarnation(bucket_name, data, metadata.bucket_incarnation_id, proof).await + } else { + metadata_sys::update_if_incarnation(bucket_name, config_file, data, metadata.bucket_incarnation_id).await + } + } else { + metadata_sys::update_if_incarnation(bucket_name, config_file, data, metadata.bucket_incarnation_id).await + }; + if let Err(e) = persist_result { warn!( event = EVENT_ADMIN_BUCKET_META_STATE, component = LOG_COMPONENT_ADMIN, @@ -885,6 +890,26 @@ impl Operation for ImportBucketMetadata { } } +fn imported_quota_requires_fleet_proof(file_contents: &[(String, Vec)]) -> S3Result { + let mut durable = false; + for (file_path, content) in file_contents { + let mut parts = file_path.split(SLASH_SEPARATOR); + let Some(_bucket) = parts.next() else { + continue; + }; + if parts.next() != Some(BUCKET_QUOTA_CONFIG_FILE) { + continue; + } + let quota: BucketQuota = + serde_json::from_slice(content).map_err(|e| s3_error!(InvalidRequest, "invalid bucket quota: {e}"))?; + if quota.has_unsupported_reservation_protocol() { + return Err(s3_error!(InvalidRequest, "unsupported bucket quota reservation protocol")); + } + durable |= quota.uses_durable_reservations(); + } + Ok(durable) +} + /// The `(config_file, data)` pairs to persist for an imported bucket's metadata: every non-empty /// config field keyed by its on-disk config-file name, as owned data ready for /// `metadata_sys::update`. Empty fields are skipped so an import never overwrites an existing @@ -1076,4 +1101,31 @@ mod import_persist_tests { assert!(!has_site_replication_item); } + + #[test] + fn quota_import_preflight_rejects_invalid_and_unknown_protocols() { + let missing_limit = vec![( + format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"), + br#"{"quota":0,"reservation_protocol":1}"#.to_vec(), + )]; + assert!(imported_quota_requires_fleet_proof(&missing_limit).is_err()); + + let unknown_protocol = vec![( + format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"), + br#"{"quota":0,"reservation_protocol":2,"reservation_quota":1024}"#.to_vec(), + )]; + assert!(imported_quota_requires_fleet_proof(&unknown_protocol).is_err()); + } + + #[test] + fn quota_import_preflight_requires_proof_only_for_durable_quota() { + let legacy = vec![(format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"), br#"{"quota":1024}"#.to_vec())]; + assert!(!imported_quota_requires_fleet_proof(&legacy).expect("legacy quota should remain compatible")); + + let durable = vec![( + format!("bucket/{BUCKET_QUOTA_CONFIG_FILE}"), + serde_json::to_vec(&BucketQuota::new(Some(1024))).expect("durable quota should encode"), + )]; + assert!(imported_quota_requires_fleet_proof(&durable).expect("durable quota should pass preflight")); + } } diff --git a/rustfs/src/admin/handlers/quota.rs b/rustfs/src/admin/handlers/quota.rs index 35124b038..7dc2621be 100644 --- a/rustfs/src/admin/handlers/quota.rs +++ b/rustfs/src/admin/handlers/quota.rs @@ -22,6 +22,7 @@ use crate::admin::storage_api::bucket::metadata_sys::{self, BucketMetadataSys}; use crate::admin::storage_api::bucket::quota::checker::QuotaChecker; use crate::admin::storage_api::bucket::quota::{BucketQuota, QuotaError, QuotaOperation}; use crate::auth::{check_key_valid, get_session_token}; +use crate::error::ApiError; use crate::server::ADMIN_PREFIX; use hyper::{Method, StatusCode}; use matchit::Params; @@ -293,16 +294,36 @@ impl Operation for SetBucketQuotaHandler { return Err(s3_error!(InvalidArgument, "{}", rustfs_config::QUOTA_INVALID_TYPE_ERROR_MSG)); } + let fleet_proof = if request.quota.is_some() { + Some(crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| { + S3Error::with_message( + s3s::S3ErrorCode::ServiceUnavailable, + "durable quota capability is not confirmed across the cluster".to_string(), + ) + })?) + } else { + None + }; + let quota = BucketQuota::new(request.quota); let metadata_sys_lock = bucket_metadata_from_context() .ok_or_else(|| s3_error!(InternalError, "{}", rustfs_config::QUOTA_METADATA_SYSTEM_ERROR_MSG))?; let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); - let updated_at = quota_checker - .set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id) - .await - .map_err(|e| s3_error!(InternalError, "failed to set quota: {}", e))?; + let updated_at = match fleet_proof.as_ref() { + Some(fleet_proof) => { + quota_checker + .set_durable_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id, fleet_proof) + .await + } + None => { + quota_checker + .set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id) + .await + } + } + .map_err(ApiError::from)?; if let Err(err) = site_replication_bucket_meta_hook(SRBucketMeta { bucket: bucket.clone(), diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 85d529836..bf946c0c4 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -29,6 +29,7 @@ use crate::admin::storage_api::bucket::metadata::{ BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG, }; use crate::admin::storage_api::bucket::metadata_sys; +use crate::admin::storage_api::bucket::quota::BucketQuota; use crate::admin::storage_api::bucket::replication; use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; use crate::admin::storage_api::bucket::target_sys::BucketTargetSys; @@ -7895,9 +7896,35 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { if !skip_config_write { if let Some(data) = data { - metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) - .await - .map_err(ApiError::from)?; + if item.r#type == "quota-config" { + let quota: BucketQuota = serde_json::from_slice(&data) + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid bucket quota: {e}")))?; + if quota.has_unsupported_reservation_protocol() { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "unsupported bucket quota reservation protocol".to_string(), + )); + } + if quota.uses_durable_reservations() { + let proof = crate::admin::storage_api::acquire_cross_pool_fence_fleet_proof().ok_or_else(|| { + S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "durable quota capability is not confirmed across the cluster".to_string(), + ) + })?; + metadata_sys::update_quota_if_incarnation(&item.bucket, data, expected_incarnation_id, &proof) + .await + .map_err(ApiError::from)?; + } else { + metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + } + } else { + metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + } } else { metadata_sys::delete_if_incarnation(&item.bucket, config_file, expected_incarnation_id) .await diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 13a45fd35..1f3ac8856 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -2901,7 +2901,7 @@ async fn handle_misc_extension_request(req: &mut S3Request, route: &MiscEx MiscExtRoute::ObjectLambda { bucket, object } => { let get_req = build_object_lambda_get_request(req, bucket, object)?; let usecase = default_object_usecase(); - let get_resp = Box::pin(usecase.execute_get_object(get_req)).await?; + let get_resp = usecase.execute_get_object(get_req).await?; invoke_object_lambda_target(req, bucket, object, get_resp).await } MiscExtRoute::ListenNotification { bucket } => { diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index fd01de999..17841d1de 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -64,7 +64,9 @@ mod ecstore_metrics { } mod ecstore_notification { - pub(crate) use crate::storage::storage_api::ecstore_notification::NotificationSys; + pub(crate) use crate::storage::storage_api::ecstore_notification::{ + CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof, + }; } #[allow(unused_imports)] @@ -112,6 +114,10 @@ pub(crate) type TierCreds = ecstore_tier::tier_admin::TierCreds; pub(crate) type TierType = ecstore_tier::tier_config::TierType; pub(crate) type TierConfigUpdateError = crate::storage::storage_api::TierConfigUpdateError; +pub(crate) fn acquire_cross_pool_fence_fleet_proof() -> Option { + ecstore_notification::acquire_cross_pool_fence_fleet_proof() +} + pub(crate) mod runtime_sources { pub(crate) type DailyAllTierStats = super::DailyAllTierStats; pub(crate) type ECStore = super::ECStore; @@ -296,6 +302,15 @@ pub(crate) mod metadata_sys { super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await } + pub(crate) async fn update_quota_if_incarnation( + bucket: &str, + data: Vec, + expected_incarnation_id: uuid::Uuid, + proof: &super::ecstore_notification::CrossPoolFenceFleetProofToken, + ) -> Result { + super::ecstore_bucket::metadata_sys::update_quota_if_incarnation(bucket, data, expected_incarnation_id, proof).await + } + pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result { super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await } diff --git a/rustfs/src/app/gating_test_env.rs b/rustfs/src/app/gating_test_env.rs index e25a4471e..f8d5bfd9b 100644 --- a/rustfs/src/app/gating_test_env.rs +++ b/rustfs/src/app/gating_test_env.rs @@ -23,6 +23,9 @@ //! `ECStore` and one metadata-sys initialization exist per test binary. use super::storage_api::test::bucket::metadata_sys; +use super::storage_api::test::bucket::quota::BucketQuota; +use super::storage_api::test::bucket::quota::checker::QuotaChecker; +use super::storage_api::test::contract::bucket::MakeBucketOptions; use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions}; use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints}; use super::{context::AppContext, object_traffic_health::ObjectTrafficHealth}; @@ -87,6 +90,17 @@ pub(crate) async fn shared_gating_ecstore() -> Arc { crate::storage::storage_api::new_global_notification_sys(endpoint_pools.clone()) .await .expect("initialize notification system for gating test env"); + let topology_fingerprint = + crate::storage::storage_api::heal_control_startup_consumer::heal_topology_fingerprint(&endpoint_pools) + .expect("single-node gating topology should hash"); + crate::storage::storage_api::start_remote_version_state_fleet_probe(topology_fingerprint); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while crate::storage::storage_api::ecstore_notification::acquire_cross_pool_fence_fleet_proof().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .expect("single-node cross-pool fence capability proof should publish"); let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) @@ -107,6 +121,24 @@ pub(crate) async fn shared_gating_ecstore() -> Arc { ecstore } +pub(crate) async fn durable_quota_test_bucket(prefix: &str, limit: u64) -> (Arc, String) { + let store = shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let bucket = format!("{prefix:.30}-{}", uuid::Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create durable quota test bucket"); + super::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 0).await; + let metadata_sys = + crate::app::storage_api::test::get_global_bucket_metadata_sys().expect("test app context should expose bucket metadata"); + QuotaChecker::new(metadata_sys) + .set_quota_config(&bucket, BucketQuota::new(Some(limit))) + .await + .expect("configure durable quota test bucket"); + (store, bucket) +} + pub(crate) async fn shared_gating_ambient() -> Arc { let store = shared_gating_ecstore().await; if let Some(ambient) = crate::runtime_sources::current_app_context() { diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 8dd561da1..66ffa26cc 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -33,7 +33,7 @@ use super::storage_api::multipart_usecase::contract::multipart::{CompletePart, M use super::storage_api::multipart_usecase::contract::object::{ObjectIO as _, ObjectOperations as _}; use super::storage_api::multipart_usecase::contract::range::HTTPRangeSpec; use super::storage_api::multipart_usecase::data_usage::{ - record_bucket_object_version_write_memory, record_bucket_object_write_memory, + quota_object_size, record_bucket_object_version_write_memory, record_bucket_object_write_memory, }; use super::storage_api::multipart_usecase::error::{StorageError, is_err_object_not_found, is_err_version_not_found}; use super::storage_api::multipart_usecase::helper::OperationHelper; @@ -64,10 +64,10 @@ use super::storage_api::multipart_usecase::{ }; use crate::app::object_data_cache::{ ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success, - invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation, + invalidate_object_data_cache_before_mutation, }; use crate::app::object_usecase::{ - acquire_copy_bucket_lifecycle_locks, build_put_like_object_lock_metadata, map_quota_check_outcome, + acquire_copy_bucket_lifecycle_locks, apply_quota_admission, build_put_like_object_lock_metadata, map_quota_check_outcome, validate_existing_object_lock_for_write, }; use crate::app::runtime_sources::{ @@ -83,6 +83,8 @@ use rustfs_io_metrics::record_s3_op; use rustfs_s3_ops::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; +#[cfg(test)] +use rustfs_utils::http::insert_header; use rustfs_utils::http::{ SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST, contains_key_str, get_header, get_source_scheme, @@ -226,12 +228,8 @@ fn internal_object_info_lookup_opts(mut opts: ObjectOptions) -> ObjectOptions { opts } -fn logical_object_size(info: &ObjectInfo) -> Result { - u64::try_from(info.get_actual_size()?).map_err(|_| StorageError::PartMissingOrCorrupt) -} - fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result { - match logical_object_size(info) { + match quota_object_size(info) { Ok(size) => Ok(size), Err(err) if fail_closed => Err(ApiError::from(err).into()), Err(_) => Ok(info.size.max(0) as u64), @@ -507,11 +505,7 @@ impl DefaultMultipartUsecase { Ok(existing_obj_info) => { validate_existing_object_lock_for_write(&existing_obj_info, ¤t_opts)?; let physical_size = existing_obj_info.size.max(0) as u64; - let logical_size = if opts.replication_request { - Ok(physical_size) - } else { - logical_object_size(&existing_obj_info) - }; + let logical_size = quota_object_size(&existing_obj_info); Some((physical_size, logical_size)) } Err(err) => { @@ -560,32 +554,20 @@ impl DefaultMultipartUsecase { }; let quota_metadata_sys = self.bucket_metadata_sys(); + let quota_tracking = quota_metadata_sys.is_some(); + let mut quota_enabled = false; if let Some(metadata_sys) = quota_metadata_sys.as_ref() { let quota_checker = QuotaChecker::new(metadata_sys.clone()); let check_result = map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?; - // Ciphertext-passthrough replication parts use a different size basis and retain - // the existing post-commit accounting path until they carry a trusted logical-size proof. - if !opts.replication_request - && let Some(quota_limit) = check_result.quota_limit - { - let installed = check_result - .current_usage - .is_some_and(|current_usage| opts.set_quota_admission(current_usage, quota_limit)); - if !installed { - return Err(S3Error::with_message( - S3ErrorCode::ServiceUnavailable, - "Bucket quota check temporarily unavailable, please retry".to_string(), - )); - } - } + quota_enabled = check_result.quota_limit.is_some(); + apply_quota_admission(&mut opts, &check_result)?; } let previous_current_size = match previous_current_sizes { - Some((physical_size, _)) if opts.replication_request => Some(physical_size), - Some((_, Ok(logical_size))) => Some(logical_size), - Some((_, Err(err))) if opts.quota_admission.is_some() => return Err(ApiError::from(err).into()), - Some((physical_size, Err(_))) => Some(physical_size), + Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size), + Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err).into()), + Some((physical_size, _)) => Some(physical_size), None => None, }; @@ -595,7 +577,6 @@ impl DefaultMultipartUsecase { let key = key.clone(); let upload_id = upload_id.clone(); let opts = opts.clone(); - let quota_metadata_sys = quota_metadata_sys.clone(); async move { let obj_info = store .clone() @@ -605,37 +586,9 @@ impl DefaultMultipartUsecase { let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await; record_capacity_write(Some(capacity_scope_token)).await; - if let Some(metadata_sys) = quota_metadata_sys.as_ref() { - if opts.replication_request { - let quota_checker = QuotaChecker::new(metadata_sys.clone()); - match quota_checker - .check_quota(&bucket, QuotaOperation::PutObject, obj_info.size.max(0) as u64) - .await - { - Ok(check_result) if !check_result.allowed => { - let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await; - let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await; - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!( - "Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes", - check_result.current_usage.unwrap_or(0), - check_result.quota_limit.unwrap_or(0) - ), - )); - } - Err(err) => { - warn!("Quota check failed for bucket {} after multipart completion: {}", bucket, err); - } - Ok(_) => {} - } - } + if quota_tracking { + let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?; - let committed_size = if opts.replication_request { - obj_info.size.max(0) as u64 - } else { - quota_accounting_object_size(&obj_info, opts.quota_admission.is_some())? - }; if versioned { record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { @@ -845,6 +798,20 @@ impl DefaultMultipartUsecase { let ciphertext_passthrough = replication_authorized && get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") && rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some(); + if ciphertext_passthrough && let Some(metadata_sys) = self.bucket_metadata_sys() { + let check_result = map_quota_check_outcome( + &bucket, + QuotaChecker::new(metadata_sys) + .check_quota(&bucket, QuotaOperation::PutObject, 0) + .await, + )?; + if check_result.quota_limit.is_some() { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(), + )); + } + } if ciphertext_passthrough { insert_str(&mut metadata, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, "true".to_string()); } @@ -1678,6 +1645,24 @@ mod tests { assert_eq!(quota_accounting_object_size(&info, true).expect("logical size should resolve"), 8192); assert_eq!(quota_accounting_object_size(&info, false).expect("logical size should resolve"), 8192); + + let mut poisoned_metadata = HashMap::new(); + insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string()); + insert_str(&mut poisoned_metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let poisoned = ObjectInfo { + size: 17, + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 4096, + actual_size: 4096, + ..Default::default() + }]), + user_defined: Arc::new(poisoned_metadata), + ..Default::default() + }; + assert_eq!( + quota_accounting_object_size(&poisoned, true).expect("persisted part size must be charged"), + 4096 + ); } #[test] @@ -2086,30 +2071,14 @@ mod tests { #[tokio::test] #[serial_test::serial] async fn compressed_complete_records_logical_quota_usage_and_overwrite_delta() { - use crate::app::storage_api::multipart_usecase::bucket::quota::BucketQuota; - use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; - use crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test; - - let store = crate::app::gating_test_env::shared_gating_ecstore().await; - crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; - - let bucket = format!("compressed-complete-quota-{}", Uuid::new_v4()); + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("compressed-complete-quota", 16_384).await; let object = "object"; - store - .make_bucket(&bucket, &MakeBucketOptions::default()) - .await - .expect("create compressed quota bucket"); - seed_bucket_usage_memory_for_test(&bucket, 0).await; let usecase = DefaultMultipartUsecase::from_global(); let metadata_sys = usecase .bucket_metadata_sys() .expect("test app context should expose bucket metadata"); - let mut quota_checker = QuotaChecker::new(metadata_sys); - quota_checker - .set_quota_config(&bucket, BucketQuota::new(Some(16_384))) - .await - .expect("configure bucket quota"); + let quota_checker = QuotaChecker::new(metadata_sys); for (actual_size, payload_byte) in [(8192_i64, 0x61), (4096_i64, 0x62)] { let mut create_opts = ObjectOptions::default(); @@ -2154,6 +2123,173 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial] + async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() { + let (_store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await; + let usecase = DefaultMultipartUsecase::from_global(); + let input = CreateMultipartUploadInput::builder() + .bucket(bucket) + .key("object".to_string()) + .build() + .expect("create multipart request should build"); + let mut request = build_request(input, Method::POST); + insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + request + .headers + .insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256")); + request.extensions.insert(crate::storage::access::ReqInfo { + replication_request_authorized: true, + ..Default::default() + }); + + let err = usecase + .execute_create_multipart_upload(request) + .await + .expect_err("quota-enabled ciphertext multipart replication should fail before upload creation"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[tokio::test] + #[serial_test::serial] + async fn concurrent_completions_share_durable_bucket_quota_reservations() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-complete-quota", 6000).await; + + let usecase = DefaultMultipartUsecase::from_global(); + + let mut inputs = Vec::new(); + for object in ["first", "second"] { + let upload = store + .new_multipart_upload(&bucket, object, &ObjectOptions::default()) + .await + .expect("create concurrent multipart upload"); + let mut reader = PutObjReader::from_vec(vec![0x71; 4096]); + let part = store + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .expect("stage concurrent multipart part"); + inputs.push( + CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .upload_id(upload.upload_id) + .multipart_upload(Some(CompletedMultipartUpload { + parts: Some(vec![CompletedPart { + part_number: Some(1), + e_tag: part.etag.map(|etag| to_s3s_etag(&etag)), + ..Default::default() + }]), + })) + .build() + .expect("build concurrent completion input"), + ); + } + + let first_usecase = usecase.clone(); + let first = first_usecase.execute_complete_multipart_upload(build_request(inputs.remove(0), Method::POST)); + let second = usecase.execute_complete_multipart_upload(build_request(inputs.remove(0), Method::POST)); + let (first, second) = tokio::join!(first, second); + + assert_eq!(usize::from(first.is_ok()) + usize::from(second.is_ok()), 1); + let denied = first.err().or_else(|| second.err()).expect("one completion must be denied"); + assert_eq!(denied.code(), &S3ErrorCode::InvalidRequest); + } + + #[tokio::test] + #[serial_test::serial] + async fn multipart_completion_rejects_rotated_quota_capability_before_rename() { + use crate::app::storage_api::test::set_disk::{MultipartCommitBarrier, MultipartCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("rotated-proof-mpu-quota", 4096).await; + let object = "object"; + let upload = store + .new_multipart_upload(&bucket, object, &ObjectOptions::default()) + .await + .expect("create multipart upload"); + let mut reader = PutObjReader::from_vec(vec![0x78; 4096]); + let part = store + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .expect("stage multipart part"); + let barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::BeforeQuotaRename); + let complete_store = Arc::clone(&store); + let complete_bucket = bucket.clone(); + let upload_id = upload.upload_id.clone(); + let complete = tokio::spawn(async move { + complete_store + .complete_multipart_upload( + &complete_bucket, + object, + &upload_id, + vec![CompletePart { + part_num: 1, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + }); + barrier.wait_until_paused().await; + assert!( + crate::storage::storage_api::ecstore_notification::rotate_cross_pool_fence_fleet_proof_for_test(), + "the gating environment must have a current fleet proof" + ); + barrier.release(); + + let err = complete + .await + .expect("completion task should not panic") + .expect_err("a replaced fleet proof must fence multipart rename"); + assert!(matches!( + err, + StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + .. + } + )); + store + .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default()) + .await + .expect("proof rotation must preserve the multipart upload for retry"); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_multipart_completion_has_zero_quota_growth() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("data-movement-mpu-quota", 0).await; + let object = "object"; + let mut movement_opts = ObjectOptions { + data_movement: true, + ..Default::default() + }; + let upload = store + .new_multipart_upload(&bucket, object, &movement_opts) + .await + .expect("create data-movement multipart upload"); + let mut reader = PutObjReader::from_vec(vec![0x7a; 4096]); + let part = store + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &movement_opts) + .await + .expect("stage data-movement multipart part"); + movement_opts.preserve_etag = Some("movement-etag".to_string()); + let completed = store + .complete_multipart_upload( + &bucket, + object, + &upload.upload_id, + vec![CompletePart { + part_num: 1, + etag: part.etag, + ..Default::default() + }], + &movement_opts, + ) + .await + .expect("moving an already-accounted multipart object between pools must have zero quota growth"); + assert_eq!(completed.size, 4096); + } + #[tokio::test] #[serial_test::serial] async fn rejected_empty_parts_preserve_existing_object_and_staging() { diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index b48430d28..3470b48d8 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -24,6 +24,8 @@ use super::storage_api::object_usecase::access::{ has_bypass_governance_header, load_bucket_generation_from_store, recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref, }; +#[cfg(test)] +use super::storage_api::object_usecase::bucket::quota::BucketQuota; use super::storage_api::object_usecase::bucket::quota::checker::QuotaChecker; #[cfg(test)] use super::storage_api::object_usecase::bucket::replication::{ReplicationState, replication_statuses_map}; @@ -63,8 +65,9 @@ use super::storage_api::object_usecase::contract::namespace::NamespaceLocking; use super::storage_api::object_usecase::contract::object::{ObjectIO as _, ObjectOperations as _}; use super::storage_api::object_usecase::contract::range::HTTPRangeSpec; use super::storage_api::object_usecase::data_usage::{ - record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, - record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, + quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, + record_bucket_object_version_write_memory, record_bucket_object_write_memory, + record_bucket_object_write_unknown_previous_memory, }; use super::storage_api::object_usecase::deadlock_detector; use super::storage_api::object_usecase::ecfs::FS; @@ -136,6 +139,8 @@ use rustfs_targets::{EventName, get_request_host, get_request_port, get_request_ use rustfs_utils::CompressionAlgorithm; #[cfg(test)] use rustfs_utils::http::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; +#[cfg(test)] +use rustfs_utils::http::insert_header; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, @@ -538,6 +543,69 @@ pub(super) fn map_quota_check_outcome(bucket: &str, outcome: Result S3Result<()> { + if result.uses_durable_reservations { + return Ok(()); + } + let Some(quota_limit) = result.quota_limit else { + return Ok(()); + }; + let Some(current_usage) = result.current_usage else { + return Err(S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "Bucket quota check temporarily unavailable, please retry".to_string(), + )); + }; + if current_usage > quota_limit { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"), + )); + } + let _ = opts.set_quota_admission(current_usage, quota_limit); + Ok(()) +} + +fn ensure_object_size_within_quota(result: &QuotaCheckResult, new_size: u64) -> S3Result<()> { + let (Some(current_usage), Some(quota_limit)) = (result.current_usage, result.quota_limit) else { + return Ok(()); + }; + if new_size > quota_limit { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"), + )); + } + Ok(()) +} + +fn ensure_legacy_archive_size_within_quota(result: &QuotaCheckResult, total_unpacked_size: u64) -> S3Result<()> { + if result.uses_durable_reservations { + return Ok(()); + } + let (Some(current_usage), Some(quota_limit)) = (result.current_usage, result.quota_limit) else { + return Ok(()); + }; + let expected_usage = current_usage + .checked_add(total_unpacked_size) + .ok_or_else(|| s3_error!(InvalidArgument, "Archive total size overflowed quota accounting"))?; + if expected_usage > quota_limit { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"), + )); + } + Ok(()) +} + +fn quota_accounting_object_size(info: &ObjectInfo, fail_closed: bool) -> S3Result { + match quota_object_size(info) { + Ok(size) => Ok(size), + Err(err) if fail_closed => Err(ApiError::from(err).into()), + Err(_) => Ok(info.size.max(0) as u64), + } +} + fn request_uses_aws_chunked(headers: &HeaderMap) -> bool { let has_aws_chunked = |header_name: &str| { headers @@ -3838,13 +3906,6 @@ fn put_object_extract_limits() -> ArchiveLimits { ArchiveLimits::default() } -fn put_object_extract_quota_exceeded(current_usage: u64, quota_limit: u64) -> S3Error { - S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!("Bucket quota exceeded. Current usage: {current_usage} bytes, limit: {quota_limit} bytes"), - ) -} - fn validate_put_object_extract_entry_count(count: usize, limits: ArchiveLimits) -> S3Result<()> { if count > limits.max_entries { return Err(s3_error!( @@ -4067,12 +4128,12 @@ impl DefaultObjectUsecase { .unwrap_or_else(|| RustFSBufferConfig::default().base_config.default_unknown) } - async fn check_bucket_quota(&self, bucket: &str, op: QuotaOperation, size: u64) -> S3Result<()> { + async fn check_bucket_quota(&self, bucket: &str, op: QuotaOperation, size: u64) -> S3Result> { let Some(metadata_sys) = self.bucket_metadata_sys() else { - return Ok(()); + return Ok(None); }; let quota_checker = QuotaChecker::new(metadata_sys); - map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(|_| ()) + map_quota_check_outcome(bucket, quota_checker.check_quota(bucket, op, size).await).map(Some) } fn build_memory_bytes_blob( @@ -5468,9 +5529,24 @@ impl DefaultObjectUsecase { } } - #[instrument(level = "info", skip(self, _fs, req))] - #[hotpath::measure(impl_type = "DefaultObjectUsecase")] + #[instrument(name = "execute_put_object", level = "info", skip(self, _fs, req))] pub async fn execute_put_object(&self, _fs: &FS, req: S3Request) -> S3Result> { + self.execute_put_object_boxed(_fs, req).await + } + + fn execute_put_object_boxed<'a>( + &'a self, + _fs: &'a FS, + req: S3Request, + ) -> impl std::future::Future>> + Send + 'a { + Box::pin(self.execute_put_object_inner(_fs, req)) + } + + #[hotpath::measure( + label = "rustfs::app::object_usecase::DefaultObjectUsecase::execute_put_object", + impl_type = "DefaultObjectUsecase" + )] + async fn execute_put_object_inner(&self, _fs: &FS, req: S3Request) -> S3Result> { let start_time = std::time::Instant::now(); let mut req = req; @@ -5572,8 +5648,22 @@ impl DefaultObjectUsecase { // Resolve the authoritative decoded/plain object length (rejecting negative/unknown) before anything else consumes it. let mut size = resolve_put_object_authoritative_size(&req.headers, content_length)?; - // Bucket-quota admission runs exactly once, and only now that the authoritative object length is known. `size` is the same basis the settle phase records via ObjectInfo.size (actual, pre-compression/pre-encryption logical size), NOT the aws-chunked wire Content-Length. When no quota is configured this stays a zero-extra-I/O fast path; once a hard quota is set, checker/config/usage faults fail closed with a retryable error. - self.check_bucket_quota(&bucket, quota_operation, size as u64).await?; + // The app check preserves the existing S3 error contract; the storage + // commit path reserves the exact net logical growth under its locks. + let quota_check = self + .check_bucket_quota( + &bucket, + quota_operation, + u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + ) + .await?; + let quota_enabled = quota_check.as_ref().is_some_and(|result| result.quota_limit.is_some()); + if quota_enabled && ciphertext_passthrough { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(), + )); + } let put_stage_metrics_enabled = rustfs_io_metrics::put_stage_metrics_enabled(); let ingress_stage_start = put_stage_metrics_enabled.then(Instant::now); @@ -5747,6 +5837,9 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; + if let Some(quota_check) = quota_check.as_ref() { + apply_quota_admission(&mut opts, quota_check)?; + } rustfs_io_metrics::record_put_object_stage_duration_from("app_put_opts_build", put_opts_stage_start); apply_bucket_generation_guard(&req, &bucket, &mut opts)?; apply_put_request_object_lock_opts( @@ -5784,7 +5877,11 @@ impl DefaultObjectUsecase { Some(match previous_current_info { Ok(existing_obj_info) => { validate_existing_object_lock_for_write(&existing_obj_info, &opts)?; - Some(existing_obj_info.size.max(0) as u64) + Some(if quota_enabled { + quota_object_size(&existing_obj_info).map_err(ApiError::from)? + } else { + existing_obj_info.size.max(0) as u64 + }) } Err(err) => { if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { @@ -5799,6 +5896,12 @@ impl DefaultObjectUsecase { rustfs_io_metrics::record_put_object_stage_duration_from("app_prelookup", prelookup_stage_start); let actual_size = size; + if !ciphertext_passthrough && let Some(quota_check) = quota_check.as_ref() { + ensure_object_size_within_quota( + quota_check, + u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + )?; + } let mut md5hex = if let Some(base64_md5) = content_md5 { let md5 = base64_simd::STANDARD @@ -6084,18 +6187,14 @@ impl DefaultObjectUsecase { // backfill reproduces the lookup's observation bit for bit (latest // version's ObjectInfo.size — 0 for a delete-marker latest — or // not-found → None). + let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?; match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size)) { Some(previous_current_size) => { if put_versioned { - record_bucket_object_version_write_memory( - &bucket, - previous_current_size, - obj_info.size.max(0) as u64, - ) - .await; + record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { - record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; + record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; } } None => { @@ -6111,8 +6210,7 @@ impl DefaultObjectUsecase { put_versioned, "put_object old-size backfill unknown; recording degraded usage delta" ); - record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, put_versioned) - .await; + record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, put_versioned).await; } } @@ -6489,9 +6587,23 @@ impl DefaultObjectUsecase { }) } - #[instrument(level = "trace", skip(self, req))] - #[hotpath::measure(impl_type = "DefaultObjectUsecase")] + #[instrument(name = "execute_get_object", level = "trace", skip(self, req))] pub async fn execute_get_object(&self, req: S3Request) -> S3Result> { + self.execute_get_object_boxed(req).await + } + + fn execute_get_object_boxed( + &self, + req: S3Request, + ) -> impl std::future::Future>> + Send + '_ { + Box::pin(self.execute_get_object_inner(req)) + } + + #[hotpath::measure( + label = "rustfs::app::object_usecase::DefaultObjectUsecase::execute_get_object", + impl_type = "DefaultObjectUsecase" + )] + async fn execute_get_object_inner(&self, req: S3Request) -> S3Result> { if let Some(context) = &self.context { let _ = context.object_store(); } @@ -6945,8 +7057,15 @@ impl DefaultObjectUsecase { result } - #[instrument(level = "debug", skip(self, req))] - pub async fn execute_copy_object(&self, req: S3Request) -> S3Result> { + pub fn execute_copy_object( + &self, + req: S3Request, + ) -> impl std::future::Future>> + Send + '_ { + Box::pin(self.execute_copy_object_inner(req)) + } + + #[instrument(name = "execute_copy_object", level = "debug", skip(self, req))] + async fn execute_copy_object_inner(&self, req: S3Request) -> S3Result> { if let Some(context) = &self.context { let _ = context.object_store(); } @@ -7205,7 +7324,7 @@ impl DefaultObjectUsecase { if _self_copy_lock_guard.is_some() { current_opts.no_lock = true; } - let previous_current_size = match store.get_object_info(&bucket, &key, ¤t_opts).await { + let previous_current_sizes = match store.get_object_info(&bucket, &key, ¤t_opts).await { Ok(existing_obj_info) => { validate_existing_object_lock_for_write(&existing_obj_info, &dst_opts)?; if let Some(expected) = expected_current_version_id.as_deref() @@ -7213,7 +7332,7 @@ impl DefaultObjectUsecase { { return Err(s3_error!(PreconditionFailed)); } - Some(existing_obj_info.size.max(0) as u64) + Some((existing_obj_info.size.max(0) as u64, quota_object_size(&existing_obj_info))) } Err(err) => { if expected_current_version_id.is_some() { @@ -7534,8 +7653,29 @@ impl DefaultObjectUsecase { src_info.user_defined = Arc::new(user_defined); - self.check_bucket_quota(&bucket, QuotaOperation::CopyObject, src_info.size as u64) + let quota_check = self + .check_bucket_quota( + &bucket, + QuotaOperation::CopyObject, + u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + ) .await?; + let quota_enabled = quota_check.as_ref().is_some_and(|result| result.quota_limit.is_some()); + if let Some(quota_check) = quota_check.as_ref() { + apply_quota_admission(&mut dst_opts, quota_check)?; + } + let previous_current_size = match previous_current_sizes { + Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size), + Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err).into()), + Some((physical_size, _)) => Some(physical_size), + None => None, + }; + if let Some(quota_check) = quota_check.as_ref() { + ensure_object_size_within_quota( + quota_check, + u64::try_from(actual_size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + )?; + } let has_bucket_metadata = self.bucket_metadata_sys().is_some(); let cache_adapter = self.object_data_cache(); let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; @@ -7570,10 +7710,11 @@ impl DefaultObjectUsecase { let dest_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; if has_bucket_metadata { + let committed_size = quota_accounting_object_size(&oi, quota_enabled)?; if dest_versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; + record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { - record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; + record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; } } @@ -9053,6 +9194,17 @@ impl DefaultObjectUsecase { #[instrument(level = "debug", skip(self, req))] #[hotpath::measure(impl_type = "DefaultObjectUsecase")] pub async fn execute_put_object_extract(&self, req: S3Request) -> S3Result> { + self.execute_put_object_extract_boxed(req).await + } + + fn execute_put_object_extract_boxed( + &self, + req: S3Request, + ) -> impl std::future::Future>> + Send + '_ { + Box::pin(self.execute_put_object_extract_inner(req)) + } + + async fn execute_put_object_extract_inner(&self, req: S3Request) -> S3Result> { let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, S3Operation::PutObject).suppress_event(); let request_context = helper.request_context_or_from_request(&req); let auth_method = req.method.clone(); @@ -9168,7 +9320,12 @@ impl DefaultObjectUsecase { } validate_object_key(&key, "PUT")?; validate_table_catalog_object_mutation(&bucket, &key).await?; - self.check_bucket_quota(&bucket, QuotaOperation::PutObject, size as u64) + let _ = self + .check_bucket_quota( + &bucket, + QuotaOperation::PutObject, + u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, + ) .await?; // Apply adaptive buffer sizing based on file size for optimal streaming performance. @@ -9225,14 +9382,17 @@ impl DefaultObjectUsecase { let extract_options = resolve_put_object_extract_options(&req.headers)?; let extract_limits = put_object_extract_limits(); - let extract_quota_snapshot = if let Some(metadata_sys) = self.bucket_metadata_sys() { + let extract_quota_check = if let Some(metadata_sys) = self.bucket_metadata_sys() { let quota_checker = QuotaChecker::new(metadata_sys); let check_result = map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)?; - check_result.current_usage.zip(check_result.quota_limit) + Some(check_result) } else { None }; + let extract_quota_enabled = extract_quota_check + .as_ref() + .is_some_and(|result| result.quota_limit.is_some()); let version_id = match event_version_id { Some(v) => v.to_string(), None => String::new(), @@ -9312,10 +9472,8 @@ impl DefaultObjectUsecase { .checked_add(entry_size) .ok_or_else(|| s3_error!(InvalidArgument, "Archive total unpacked size overflowed while processing entries"))?; validate_put_object_extract_total_size(total_unpacked_size, extract_limits)?; - if let Some((current_usage, quota_limit)) = extract_quota_snapshot - && current_usage.saturating_add(total_unpacked_size) > quota_limit - { - return Err(put_object_extract_quota_exceeded(current_usage, quota_limit)); + if let Some(quota_check) = extract_quota_check.as_ref() { + ensure_legacy_archive_size_within_quota(quota_check, total_unpacked_size)?; } let mut size = i64::try_from(entry_size).map_err(|_| s3_error!(InvalidArgument, "Archive entry size does not fit into i64"))?; @@ -9360,6 +9518,9 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; + if let Some(quota_check) = extract_quota_check.as_ref() { + apply_quota_admission(&mut opts, quota_check)?; + } opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id; opts.object_lock_config_snapshot = Some(Arc::clone(&object_lock_config_snapshot)); let pax_authorization = @@ -9503,19 +9664,18 @@ impl DefaultObjectUsecase { return Err(ApiError::from(e).into()); } }; + let committed_size = quota_accounting_object_size(&obj_info, extract_quota_enabled)?; let extract_versioned = BucketVersioningSys::prefix_enabled(&bucket, &fpath).await; match previous_current_size_from_backfill(backfilled_old_current_size) { Some(previous_current_size) => { if extract_versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64) - .await; + record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; } else { - record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; + record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; } } None => { - record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, extract_versioned) - .await; + record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, extract_versioned).await; } } let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await; @@ -15931,13 +16091,6 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); } - #[test] - fn put_object_extract_quota_exceeded_matches_existing_error_shape() { - let err = put_object_extract_quota_exceeded(10, 8); - assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); - assert_eq!(err.message(), Some("Bucket quota exceeded. Current usage: 10 bytes, limit: 8 bytes")); - } - #[tokio::test] async fn execute_put_object_rejects_post_object_sse_kms_from_input() { let input = PutObjectInput::builder() @@ -16422,8 +16575,12 @@ mod tests { .expect("create self-copy test bucket"); let payload = b"object whose key equals its bucket".to_vec(); let mut reader = PutObjReader::from_vec(payload.clone()); + let setup_opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; store - .put_object(&bucket, &bucket, &mut reader, &ObjectOptions::default()) + .put_object(&bucket, &bucket, &mut reader, &setup_opts) .await .expect("put object whose key equals its bucket"); @@ -17660,9 +17817,99 @@ mod tests { quota_limit: Some(2048), operation_size: 512, remaining: Some(512), + uses_durable_reservations: true, } } + #[tokio::test] + #[serial_test::serial] + async fn quota_rejects_ciphertext_replication_before_polling_the_body() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let (_store, bucket) = + crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-replication-early-reject", 4096).await; + let body_polled = Arc::new(AtomicBool::new(false)); + let body_polled_in_stream = Arc::clone(&body_polled); + let body = StreamingBlob::wrap(futures::stream::once(async move { + body_polled_in_stream.store(true, Ordering::Release); + Ok::(Bytes::from_static(b"ciphertext")) + })); + let input = PutObjectInput::builder() + .bucket(bucket) + .key("object".to_string()) + .body(Some(body)) + .content_length(Some(10)) + .build() + .expect("ciphertext replication PUT input should build"); + let mut request = build_request(input, Method::PUT); + insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + request + .headers + .insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256")); + request.extensions.insert(crate::storage::access::ReqInfo { + replication_request_authorized: true, + ..Default::default() + }); + + let err = DefaultObjectUsecase::from_global() + .execute_put_object(&FS::new(), request) + .await + .expect_err("quota-enabled ciphertext replication should fail at ingress"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(!body_polled.load(Ordering::Acquire), "rejected ciphertext body must not be consumed"); + } + + #[tokio::test] + #[serial_test::serial] + async fn legacy_quota_rejects_full_put_before_polling_the_body() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + use std::sync::atomic::{AtomicBool, Ordering}; + + const GI_B: u64 = 1024 * 1024 * 1024; + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + let bucket = format!("legacy-quota-{}", Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create legacy quota test bucket"); + crate::app::storage_api::test::data_usage::seed_bucket_usage_memory_for_test(&bucket, 4 * GI_B).await; + let metadata_sys = DefaultObjectUsecase::from_global() + .bucket_metadata_sys() + .expect("test app context should expose bucket metadata"); + QuotaChecker::new(metadata_sys) + .set_quota_config( + &bucket, + BucketQuota { + quota: Some(5 * GI_B), + ..Default::default() + }, + ) + .await + .expect("configure legacy quota"); + + let body_polled = Arc::new(AtomicBool::new(false)); + let body_polled_in_stream = Arc::clone(&body_polled); + let body = StreamingBlob::wrap(futures::stream::once(async move { + body_polled_in_stream.store(true, Ordering::Release); + Ok::(Bytes::new()) + })); + let input = PutObjectInput::builder() + .bucket(bucket) + .key("object".to_string()) + .body(Some(body)) + .content_length(Some(i64::try_from(2 * GI_B).expect("test size should fit i64"))) + .build() + .expect("legacy quota PUT input should build"); + + let err = DefaultObjectUsecase::from_global() + .execute_put_object(&FS::new(), build_request(input, Method::PUT)) + .await + .expect_err("4 GiB used plus a 2 GiB PUT must exceed a 5 GiB legacy quota"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(!body_polled.load(Ordering::Acquire), "legacy quota rejection must not consume the body"); + } + #[test] fn quota_admission_allows_within_limit() { let result = map_quota_check_outcome("bucket", Ok(quota_result(true))).expect("an allowed result admits the write"); @@ -17673,12 +17920,421 @@ mod tests { assert_eq!(result.remaining, Some(512)); } + #[tokio::test] + #[serial_test::serial] + async fn concurrent_puts_share_durable_bucket_quota_reservations() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-put-quota", 6000).await; + + let first_opts = ObjectOptions::default(); + let second_opts = ObjectOptions::default(); + let first_store = Arc::clone(&store); + let first_bucket = bucket.clone(); + let first = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x73; 4096]); + first_store.put_object(&first_bucket, "first", &mut reader, &first_opts).await + }); + let second = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x74; 4096]); + store.put_object(&bucket, "second", &mut reader, &second_opts).await + }); + let (first, second) = tokio::join!(first, second); + let first = first.expect("first PUT task should not panic"); + let second = second.expect("second PUT task should not panic"); + + assert_eq!(usize::from(first.is_ok()) + usize::from(second.is_ok()), 1); + let denied = first.err().or_else(|| second.err()).expect("one PUT must be denied"); + assert!(matches!( + denied, + StorageError::QuotaExceeded { + current: 4096, + limit: 6000 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn concurrent_within_limit_puts_keep_independent_mutation_fences() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("concurrent-fence-quota", 8192).await; + let first_barrier = PutObjectCommitBarrier::install(&bucket, "first", PutObjectCommitPause::BeforeQuotaRename); + let second_barrier = PutObjectCommitBarrier::install(&bucket, "second", PutObjectCommitPause::BeforeQuotaRename); + + let first_store = Arc::clone(&store); + let first_bucket = bucket.clone(); + let first = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x75; 4096]); + first_store + .put_object(&first_bucket, "first", &mut reader, &ObjectOptions::default()) + .await + }); + let second_store = Arc::clone(&store); + let second_bucket = bucket.clone(); + let second = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x76; 4096]); + second_store + .put_object(&second_bucket, "second", &mut reader, &ObjectOptions::default()) + .await + }); + + first_barrier.wait_until_paused().await; + second_barrier.wait_until_paused().await; + first_barrier.release(); + second_barrier.release(); + + first + .await + .expect("first PUT task should not panic") + .expect("first within-limit PUT should commit"); + second + .await + .expect("second PUT task should not panic") + .expect("second within-limit PUT should commit"); + } + + #[tokio::test] + #[serial_test::serial] + async fn put_rejects_rotated_quota_capability_before_rename() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("rotated-proof-put-quota", 4096).await; + let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename); + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x77; 4096]); + put_store + .put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + assert!( + crate::storage::storage_api::ecstore_notification::rotate_cross_pool_fence_fleet_proof_for_test(), + "the gating environment must have a current fleet proof" + ); + barrier.release(); + + let err = put + .await + .expect("PUT task should not panic") + .expect_err("a replaced fleet proof must fence the authoritative rename"); + assert!(matches!( + err, + StorageError::NamespaceLockQuorumUnavailable { + mode: "quota_reservation", + .. + } + )); + store + .get_object_info(&bucket, "object", &ObjectOptions::default()) + .await + .expect_err("proof rotation before rename must leave no committed object"); + } + + #[tokio::test] + #[serial_test::serial] + async fn durable_quota_reclaims_overwrites_and_deleted_bytes() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("quota-delta-reconcile", 4096).await; + + for byte in [0x41, 0x42] { + let mut reader = PutObjReader::from_vec(vec![byte; 4096]); + store + .put_object(&bucket, "object", &mut reader, &ObjectOptions::default()) + .await + .expect("same-size overwrite must consume no additional quota"); + } + + store + .delete_object(&bucket, "object", ObjectOptions::default()) + .await + .expect("delete quota-tracked object"); + let mut replacement = PutObjReader::from_vec(vec![0x43; 4096]); + store + .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) + .await + .expect("deleted bytes must be reclaimed before rejecting a replacement"); + + let mut excess = PutObjReader::from_vec(vec![0x44]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("one byte beyond the reclaimed exact quota must be denied"); + assert!(matches!( + err, + StorageError::QuotaExceeded { + current: 4096, + limit: 4096 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn data_movement_put_has_zero_quota_growth() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("data-movement-put-quota", 0).await; + let mut reader = PutObjReader::from_vec(vec![0x79; 4096]); + let stored = store + .put_object( + &bucket, + "object", + &mut reader, + &ObjectOptions { + data_movement: true, + ..Default::default() + }, + ) + .await + .expect("moving an already-accounted object between pools must have zero quota growth"); + assert_eq!(stored.size, 4096); + } + + #[tokio::test] + #[serial_test::serial] + async fn cancelled_put_releases_durable_quota_reservation() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("cancelled-put-quota", 4096).await; + + let barrier = PutObjectCommitBarrier::install(&bucket, "cancelled", PutObjectCommitPause::AfterQuotaReservation); + let cancelled_store = Arc::clone(&store); + let cancelled_bucket = bucket.clone(); + let cancelled = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x51; 4096]); + cancelled_store + .put_object(&cancelled_bucket, "cancelled", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + cancelled.abort(); + let cancelled_result = cancelled.await; + assert!(cancelled_result.is_err(), "the paused request must be cancelled"); + drop(barrier); + + let mut replacement = PutObjReader::from_vec(vec![0x52; 4096]); + store + .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) + .await + .expect("cancelling before commit must release the complete reservation"); + } + + #[tokio::test] + #[serial_test::serial] + async fn cancelled_put_after_commit_marker_is_reconciled() { + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("cancelled-spawned-put-quota", 4096).await; + let commit_barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename); + let first_store = Arc::clone(&store); + let first_bucket = bucket.clone(); + let first = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x53; 4096]); + first_store + .put_object(&first_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + commit_barrier.wait_until_paused().await; + first.abort(); + assert!(first.await.is_err(), "the outer request task must be cancelled"); + drop(commit_barrier); + + store + .get_object_info(&bucket, "object", &ObjectOptions::default()) + .await + .expect_err("cancelling before rename must not commit the object"); + let mut replacement = PutObjReader::from_vec(vec![0x54; 4096]); + store + .put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default()) + .await + .expect("the next admission must reap the abandoned commit marker"); + } + + #[tokio::test] + #[serial_test::serial] + async fn committed_put_survives_quota_ledger_settlement_failure() { + use crate::app::storage_api::test::set_disk::{ + PutObjectCommitBarrier, PutObjectCommitPause, fail_next_quota_ledger_save_for_test, + }; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("settlement-failure-quota", 4096).await; + let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename); + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x59; 4096]); + put_store + .put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + fail_next_quota_ledger_save_for_test(); + barrier.release(); + put.await + .expect("PUT task should not panic") + .expect("a post-commit ledger failure must not change the successful write result"); + let stored = store + .get_object_info(&bucket, "object", &ObjectOptions::default()) + .await + .expect("the committed object must remain visible"); + assert_eq!(stored.size, 4096); + } + + #[tokio::test] + #[serial_test::serial] + async fn suspended_null_version_overwrite_uses_exact_quota_delta() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("suspended-version-quota", 6200).await; + let mut versioned_reader = PutObjReader::from_vec(vec![0x61; 4096]); + store + .put_object( + &bucket, + "object", + &mut versioned_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("write UUID version"); + + for (size, byte) in [(1024, 0x62), (2048, 0x63)] { + let mut reader = PutObjReader::from_vec(vec![byte; size]); + store + .put_object( + &bucket, + "object", + &mut reader, + &ObjectOptions { + version_suspended: true, + ..Default::default() + }, + ) + .await + .expect("suspended write should replace only the exact null version"); + } + + let mut excess = PutObjReader::from_vec(vec![0x64; 57]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("UUID plus replacement null version must consume 6144 bytes"); + assert!(matches!( + err, + StorageError::QuotaExceeded { + current: 6144, + limit: 6200 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn durable_quota_reservation_observes_lowered_config_revision() { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("lowered-quota-revision", 8192).await; + let mut initial = PutObjReader::from_vec(vec![0x71; 4096]); + store + .put_object(&bucket, "initial", &mut initial, &ObjectOptions::default()) + .await + .expect("write under original quota"); + + let metadata_sys = DefaultObjectUsecase::from_global() + .bucket_metadata_sys() + .expect("test app context should expose bucket metadata"); + QuotaChecker::new(metadata_sys) + .set_quota_config(&bucket, BucketQuota::new(Some(4096))) + .await + .expect("lower bucket quota"); + let mut excess = PutObjReader::from_vec(vec![0x72]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("reservation must not use the stale larger quota revision"); + assert!(matches!( + err, + StorageError::QuotaExceeded { + current: 4096, + limit: 4096 + } + )); + } + + #[tokio::test] + #[serial_test::serial] + async fn quota_enable_waits_for_unlimited_commit() { + use crate::app::storage_api::test::metadata_sys::ConfigWriteLockProbe; + use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("quota-config-fence", 8192).await; + let metadata_sys = DefaultObjectUsecase::from_global() + .bucket_metadata_sys() + .expect("test app context should expose bucket metadata"); + QuotaChecker::new(Arc::clone(&metadata_sys)) + .set_quota_config(&bucket, BucketQuota::new(None)) + .await + .expect("clear quota before the fenced write"); + let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::AfterQuotaReservation); + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x73; 4096]); + put_store + .put_object(&put_bucket, "object", &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + + let update_probe = ConfigWriteLockProbe::install(&bucket); + let update_bucket = bucket.clone(); + let update = tokio::spawn(async move { + QuotaChecker::new(metadata_sys) + .set_quota_config(&update_bucket, BucketQuota::new(Some(0))) + .await + }); + update_probe.wait_until_attempted().await; + assert!( + !update.is_finished(), + "quota mutation must wait for the reservation's metadata transaction guard" + ); + + barrier.release(); + put.await + .expect("PUT task should not panic") + .expect("the write linearized before the quota update must commit"); + update + .await + .expect("quota update task should not panic") + .expect("quota update should proceed after commit"); + + let mut excess = PutObjReader::from_vec(vec![0x74]); + let err = store + .put_object(&bucket, "excess", &mut excess, &ObjectOptions::default()) + .await + .expect_err("writes after the zero-byte quota update must be denied"); + assert!(matches!(err, StorageError::QuotaExceeded { current: 4096, limit: 0 })); + } + #[test] fn quota_admission_rejects_over_limit() { let err = map_quota_check_outcome("bucket", Ok(quota_result(false))).expect_err("an over-limit result rejects the write"); assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); } + #[test] + fn legacy_quota_admission_rejects_already_over_limit() { + let result = QuotaCheckResult { + allowed: true, + current_usage: Some(6), + quota_limit: Some(5), + operation_size: 0, + remaining: Some(0), + uses_durable_reservations: false, + }; + let mut opts = ObjectOptions::default(); + let err = + apply_quota_admission(&mut opts, &result).expect_err("legacy completion must not bypass an already exceeded quota"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + #[test] fn quota_admission_fails_closed_on_checker_error() { // A configured hard quota must never be bypassed by an internal fault: a checker error becomes a retryable ServiceUnavailable, not a silent allow. @@ -17692,6 +18348,45 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); } + #[test] + fn legacy_archive_quota_rejects_cumulative_size_and_overflow() { + let legacy = QuotaCheckResult { + allowed: true, + current_usage: Some(4), + quota_limit: Some(5), + operation_size: 0, + remaining: Some(1), + uses_durable_reservations: false, + }; + assert!(ensure_legacy_archive_size_within_quota(&legacy, 2).is_err()); + assert!(ensure_legacy_archive_size_within_quota(&legacy, 1).is_ok()); + + let maxed = QuotaCheckResult { + current_usage: Some(u64::MAX), + quota_limit: Some(u64::MAX), + ..legacy + }; + assert!(ensure_legacy_archive_size_within_quota(&maxed, 1).is_err()); + } + + #[test] + fn early_quota_filter_rejects_only_an_individually_impossible_object() { + let stale_full_usage = QuotaCheckResult { + allowed: true, + current_usage: Some(4096), + quota_limit: Some(4096), + operation_size: 0, + remaining: Some(0), + uses_durable_reservations: true, + }; + + ensure_object_size_within_quota(&stale_full_usage, 4096) + .expect("commit-time ledger must decide whether stale usage was reclaimed"); + let err = ensure_object_size_within_quota(&stale_full_usage, 4097) + .expect_err("an object larger than the whole quota can never fit"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + #[test] fn quota_admission_fails_closed_on_unknown_authoritative_usage() { let err = map_quota_check_outcome( diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 39d7d7685..20e937790 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -47,6 +47,8 @@ pub(crate) mod capacity { pub(crate) mod data_usage { use std::sync::Arc; + pub(crate) use crate::storage::storage_api::ecstore_data_usage::quota_object_size; + pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) { crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await; } @@ -1214,4 +1216,14 @@ pub(crate) mod test { pub(crate) use crate::storage::storage_api::{ ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, }; + pub(crate) mod set_disk { + pub(crate) use crate::storage::storage_api::ecstore_set_disk::{ + MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause, + fail_next_quota_ledger_save_for_test, + }; + } + + pub(crate) mod metadata_sys { + pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::ConfigWriteLockProbe; + } } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index e27c6dc29..ce4be49d5 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -301,7 +301,7 @@ impl S3 for FS { #[instrument(level = "debug", skip(self, req))] async fn copy_object(&self, req: S3Request) -> S3Result> { let usecase = s3_api::object_usecase_for(self); - Box::pin(usecase.execute_copy_object(req)).await + usecase.execute_copy_object(req).await } #[instrument( @@ -704,7 +704,7 @@ impl S3 for FS { async fn get_object(&self, req: S3Request) -> S3Result> { crate::hp_guard!("S3::get_object"); let usecase = s3_api::object_usecase_for(self); - Box::pin(usecase.execute_get_object(req)).await + usecase.execute_get_object(req).await } async fn get_object_acl(&self, req: S3Request) -> S3Result> { @@ -1261,7 +1261,7 @@ impl S3 for FS { async fn put_object(&self, req: S3Request) -> S3Result> { crate::hp_guard!("S3::put_object"); let usecase = s3_api::object_usecase_for(self); - Box::pin(usecase.execute_put_object(self, req)).await + usecase.execute_put_object(self, req).await } async fn put_object_acl(&self, req: S3Request) -> S3Result> { diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 131f3ce1f..34bb5f915 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -153,9 +153,7 @@ fn remove_heal_control_replay( static HEAL_CONTROL_REPLAY_CACHE: OnceLock>>> = OnceLock::new(); static NODE_CAPABILITY_SERVER_EPOCH: LazyLock = LazyLock::new(Uuid::new_v4); -// RUSTFS_COMPAT_TODO(cross-pool-fence-v1): advertise unsupported during predeployment. Remove after composite acquisition, -// activation fencing, fleet proof, commit-time proof revalidation, and fail-closed revocation ship together. -const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 0; +const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 1; fn admit_heal_control_replay( replay_cache: &mut HashMap>, @@ -3342,7 +3340,7 @@ mod tests { } #[tokio::test] - async fn cross_pool_fence_probe_authenticates_unsupported_rollout_state() { + async fn cross_pool_fence_probe_authenticates_supported_v1_state() { let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string()); let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true); assert!( @@ -3407,7 +3405,7 @@ mod tests { assert!(response.success); assert_eq!(response.error_info, None); - assert_eq!(&response.result[..4], &0_u32.to_be_bytes()); + assert_eq!(&response.result[..4], &1_u32.to_be_bytes()); let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(&response.result[4..]) .expect("capability identity should decode"); assert_eq!(topology_member, "node-a:9000"); diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index 0e0f2defe..1dc45beba 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -37,19 +37,28 @@ const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512; const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024; const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1; +fn snapshot_lease_response(result: Result) -> Response { + match result { + Ok(token) => Response::new(SnapshotLeaseResponse { + success: true, + token: token.as_bytes().to_vec().into(), + protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION, + error: None, + }), + Err(err) => Response::new(SnapshotLeaseResponse { + success: false, + token: Bytes::new(), + protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION, + error: Some(err.into()), + }), + } +} + struct DecodedRpcPayload { value: T, from_msgpack: bool, } -fn snapshot_lease_disabled_response() -> SnapshotLeaseResponse { - SnapshotLeaseResponse { - success: false, - token: Bytes::new(), - protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION, - error: Some(DiskError::UnsupportedDisk.into()), - } -} fn decode_msgpack_or_json( binary: &[u8], json: &str, @@ -241,7 +250,12 @@ impl NodeService { rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref()), "acquire_snapshot_lease", )?; - Ok(Response::new(snapshot_lease_disabled_response())) + let request = request.into_inner(); + let result = match self.find_disk(&request.disk).await { + Some(disk) => disk.acquire_snapshot_lease(&request.volume, &request.path).await, + None => Err(DiskError::other("cannot find disk")), + }; + Ok(snapshot_lease_response(result)) } pub(super) async fn handle_renew_snapshot_lease( @@ -253,7 +267,14 @@ impl NodeService { rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref()), "renew_snapshot_lease", )?; - Ok(Response::new(snapshot_lease_disabled_response())) + let request = request.into_inner(); + let token = + SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?; + let result = match self.find_disk(&request.disk).await { + Some(disk) => disk.renew_snapshot_lease(&request.volume, &request.path, token).await, + None => Err(DiskError::other("cannot find disk")), + }; + Ok(snapshot_lease_response(result)) } pub(super) async fn handle_release_snapshot_lease( @@ -266,8 +287,11 @@ impl NodeService { "release_snapshot_lease", )?; let request = request.into_inner(); - let token = - SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?; + let token = if request.token.as_ref() == SnapshotLeaseToken::revoke_all().as_bytes() { + SnapshotLeaseToken::revoke_all() + } else { + SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))? + }; let Some(disk) = self.find_disk(&request.disk).await else { return Ok(Response::new(SnapshotLeaseMutationResponse { success: false, @@ -1494,11 +1518,11 @@ mod tests { use super::{ compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, encode_batch_read_version_response_payloads, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, - encode_read_multiple_response_payloads, encode_rename_data_response_payloads, snapshot_lease_disabled_response, + encode_read_multiple_response_payloads, encode_rename_data_response_payloads, }; use crate::storage::storage_api::ReadMultipleResp; + use crate::storage::storage_api::RenameDataResp; use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp; - use crate::storage::storage_api::{DiskError, RenameDataResp}; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::global_internode_metrics; use serde::{Deserialize, Serialize}; @@ -1509,17 +1533,6 @@ mod tests { count: u32, } - #[test] - fn snapshot_lease_acquire_and_renew_fail_closed() { - let response = snapshot_lease_disabled_response(); - let expected_error = DiskError::UnsupportedDisk.into(); - - assert!(!response.success); - assert!(response.token.is_empty()); - assert_eq!(response.protocol_version, 1); - assert_eq!(response.error, Some(expected_error)); - } - #[test] fn decode_msgpack_or_json_prefers_binary_payload() { let payload = SamplePayload { diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index b12169c71..d192e38a2 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -430,7 +430,7 @@ pub(crate) mod ecstore_config { pub(crate) mod ecstore_data_usage { pub(crate) use rustfs_ecstore::api::data_usage::{ apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached, - load_data_usage_from_backend, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, + load_data_usage_from_backend, quota_object_size, record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend, }; @@ -487,8 +487,12 @@ pub(crate) mod ecstore_metrics { #[allow(unused_imports)] pub(crate) mod ecstore_notification { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::notification::rotate_cross_pool_fence_fleet_proof_for_test; pub(crate) use rustfs_ecstore::api::notification::{ - NotificationSys, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe, + CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof, + cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys, + start_remote_version_state_fleet_probe, }; } @@ -546,6 +550,11 @@ pub(crate) mod ecstore_test_support { } pub(crate) mod ecstore_set_disk { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::set_disk::test_util::{ + MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause, + fail_next_quota_ledger_save_for_test, + }; pub(crate) use rustfs_ecstore::api::set_disk::{ DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class, }; @@ -1130,7 +1139,9 @@ pub(crate) trait StorageDiskRpcExt { ) -> DiskResult<()>; async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult; async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>; + async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult; async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()>; + async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult; async fn stat_volume(&self, volume: &str) -> DiskResult; async fn list_volumes(&self) -> DiskResult>; async fn make_volume(&self, volume: &str) -> DiskResult<()>; @@ -1258,10 +1269,18 @@ where ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await } + async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult { + ecstore_disk::DiskAPI::acquire_snapshot_lease(self, volume, path).await + } + async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()> { ecstore_disk::DiskAPI::release_snapshot_lease(self, volume, path, token).await } + async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult { + ecstore_disk::DiskAPI::renew_snapshot_lease(self, volume, path, token).await + } + async fn stat_volume(&self, volume: &str) -> DiskResult { ecstore_disk::DiskAPI::stat_volume(self, volume).await } diff --git a/scripts/check_s3s_footprint.sh b/scripts/check_s3s_footprint.sh index 30cdcb3f0..56fb60d35 100755 --- a/scripts/check_s3s_footprint.sh +++ b/scripts/check_s3s_footprint.sh @@ -26,7 +26,7 @@ cd "$(dirname "$0")/.." # Excludes crates/e2e_test/ — test infrastructure legitimately uses s3s # to verify S3 behavior and does not widen the production s3s surface. S3S_IMPORT_FILES_BASELINE=213 -S3_ERROR_LINES_BASELINE=1621 +S3_ERROR_LINES_BASELINE=1620 S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::' E2E_TEST_GLOB='--glob=!crates/e2e_test/**' From e2be34cade9c79ec2429e41597796228b198bcb7 Mon Sep 17 00:00:00 2001 From: houseme Date: Fri, 14 Aug 2026 16:59:21 +0800 Subject: [PATCH 02/71] test(rpc): align snapshot lease missing-disk expectation (#6108) Co-authored-by: heihutu --- rustfs/src/storage/rpc/node_service.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 34bb5f915..d06e0892f 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -2943,7 +2943,7 @@ mod tests { } #[tokio::test] - async fn snapshot_lease_acquire_and_renew_handlers_fail_closed() { + async fn snapshot_lease_acquire_and_renew_handlers_fail_closed_for_missing_disk() { let service = make_server(); let disk = "http://node-a:9000/data/rustfs0".to_string(); @@ -2960,7 +2960,7 @@ mod tests { let acquire = service .acquire_snapshot_lease(acquire) .await - .expect("disabled acquire should return a protocol response") + .expect("missing-disk acquire should return a protocol response") .into_inner(); let mut renew = Request::new(SnapshotLeaseRenewRequest { @@ -2977,14 +2977,14 @@ mod tests { let renew = service .renew_snapshot_lease(renew) .await - .expect("disabled renew should return a protocol response") + .expect("missing-disk renew should return a protocol response") .into_inner(); for response in [acquire, renew] { assert!(!response.success); assert!(response.token.is_empty()); assert_eq!(response.protocol_version, 1); - assert_eq!(response.error, Some(DiskError::UnsupportedDisk.into())); + assert_eq!(response.error, Some(DiskError::other("cannot find disk").into())); } } From 5a4c063d16a27a08456a55a2c42b81afcfacf325 Mon Sep 17 00:00:00 2001 From: houseme Date: Fri, 14 Aug 2026 17:25:10 +0800 Subject: [PATCH 03/71] perf(get): avoid materialized body clone (#6109) Stream materialized GET bodies by moving the buffered Bytes once instead of wrapping the stream in an extra bytes_stream layer. Add an operations runbook for object I/O tuning A/B sweeps. Co-authored-by: heihutu --- docs/operations/object-io-tuning-ab-matrix.md | 218 ++++++++++++++++++ rustfs/src/app/object_usecase.rs | 29 ++- 2 files changed, 231 insertions(+), 16 deletions(-) create mode 100644 docs/operations/object-io-tuning-ab-matrix.md diff --git a/docs/operations/object-io-tuning-ab-matrix.md b/docs/operations/object-io-tuning-ab-matrix.md new file mode 100644 index 000000000..baf0b906b --- /dev/null +++ b/docs/operations/object-io-tuning-ab-matrix.md @@ -0,0 +1,218 @@ +# Object I/O (GET/PUT) tuning A/B matrix runbook + +> Scope: **parameter tuning** — measuring the effect of changing one +> `RUSTFS_*` runtime knob at a time, against a fixed binary. This is +> deliberately different from the code-change A/B gate in +> [`hotpath-warp-ab-runbook.md`](hotpath-warp-ab-runbook.md) and the formal +> ABBA validation in +> [`hotpath-warp-abba-runbook.md`](hotpath-warp-abba-runbook.md), which compare +> a baseline binary against a candidate binary. +> +> When a knob change turns out to need a code change, use those two runbooks +> for the code-level validation and come back here for the knob-level sweep. + +## 1. What this runbook answers + +For each tuning knob it answers three questions: + +1. Which stage is actually slow — `set_disk_encode`, `set_disk_rename`, + `metadata_fanout`, `bitrot_verify`, etc.? +2. Is the knob the real bottleneck, or is the stage slow for another reason? +3. Does widening/loosening the knob buy throughput without an unacceptable + memory (RSS) or tail-latency regression? + +The core discipline is **one variable per A/B cell**. Never change two knobs in +the same cell, or the result is unexplainable. + +## 2. Prerequisites + +- Linux bench host (or an ansible-managed cluster); a laptop smoke run is too + noisy to decide anything. +- `warp` on `PATH` (or pass `--warp-bin` to the driver). +- The observability metrics runtime **enabled**. The stage histograms below are + not emitted when `RUSTFS_OBS_METRICS_EXPORT_ENABLED=false` or the runtime is + otherwise off — see + [`hotpath-warp-ab-runbook.md`](hotpath-warp-ab-runbook.md) for the no-log / + no-monitor baseline env. +- A warm, disposable data set. Recreate the bucket per run; do not bench against + production data. + +Load driver and gate are reused, not reimplemented: + +- `scripts/run_object_batch_bench_enhanced.sh` — warp driver with rounds, + median aggregation, `baseline_compare.csv`, and Prometheus service-metric + capture. +- `scripts/hotpath_warp_ab_gate.sh` — relative budget gate over the deltas. +- `scripts/run_hotpath_warp_ab.sh` — optional orchestrator when a knob needs + the full baseline-vs-candidate treatment (e.g. two different defaults). + +## 3. Fixed test conditions (lock before you start) + +Record these per run; a result without them is not reproducible: + +```text +nodes, disks_per_node, total_disks, cpu_per_node, mem_per_node, +network, erasure_set_drive_count, endpoint_mode (direct|lb), +rustfs_commit_sha, warp --version, durability mode +``` + +Workload matrix (the same shapes the hotpath gate uses, expanded for the +stage-breakdown object sizes): + +| Workload | mode | sizes | +| --- | --- | --- | +| small-fixed | put / get | 4KiB, 100KiB | +| ec-boundary | put / get | 1MiB, 4MiB | +| large-stream | put / get | 10MiB, 16MiB, 32MiB | +| mixed | mixed | 256KiB | + +Concurrency ladder: `8, 16, 32, 64` (add `96, 128` on a bigger rig). Duration +`120s`, `--rounds >= 3`, cooldown `>= 30s`. + +Isolate background noise before the sweep: scanner deep-verify, heal, +replication, lifecycle transition, periodic capacity refresh — record whether +each is on rather than silently assuming it is off. + +## 4. Measurement stack + +The code already instruments every stage below. Drive each A/B cell with these +histograms (names verified against `crates/io-metrics/src/lib.rs`): + +- PUT stages: `rustfs_s3_put_object_stage_duration_ms{stage=...}` — compute + P50/P95/P99 per stage. Stages: `app_bucket_validate`, `app_sse_config_lookup`, + `app_object_lock_config_lookup`, `app_put_opts_build`, `app_prelookup`, + `ingress_prepare`, `app_encryption_prepare`, `app_replication_decision`, + `app_store_put`, `app_post_store_bookkeeping`, `app_capacity_update`, + `set_disk_writer_setup`, `set_disk_encode`, `set_disk_rename`, + `set_disk_old_data_cleanup`. +- GET stages: `rustfs_io_get_object_stage_duration_seconds{path=..., stage=...}` — + the `path` label separates the read paths: `legacy_duplex`, `codec_streaming`, + `direct_memory`, `body_cache`, `inline_direct`, `internal_meta`, + `remote_transition`, `set_disk`, `empty`. Stages: `metadata`, + `metadata_cache_lookup`, `metadata_fanout`, `metadata_resolve`, `object_info`, + `path_decision`, `quorum_reached`, `range`, `reader_setup`, + `stripe_read`, `stripe_read_first_shard`, `stripe_read_quorum`, `decode`, + `reconstruct`, `emit`, `fill`, `output_poll`, `output_lock_wait`, + `bitrot_verify`, `first_byte`, `full_body`, `response_handoff`, + `lock_acquire`. +- EC memory pressure: `rustfs_ec_encode_inflight_bytes_current` and the + allocator reclaim gauge; plus node RSS and CPU. + +Host telemetry (collect alongside every cell): + +```bash +pidstat -durh 5 > telemetry/pidstat.txt & +mpstat 5 > telemetry/mpstat.txt & +iostat -xz 5 > telemetry/iostat.txt & +``` + +## 5. Tuning knob catalog + +Defaults are verified against `crates/config/src/constants/object.rs` and +`crates/ecstore/src/erasure/coding/encode.rs`. + +### 5.1 PUT + +| Knob | Default | Controls | Validating stage | Risk if widened | +| --- | --- | --- | --- | --- | +| `RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES` | 32MiB | EC encode producer/consumer memory budget (blocks queued between encode and shard write) | `set_disk_encode` P95 + `rustfs_ec_encode_inflight_bytes_current` | RSS growth under high concurrency | +| `RUSTFS_OBJECT_IO_BUFFER_SIZE` | 128KiB | Streaming read-in / write-out block size | `ingress_prepare`, `set_disk_encode` | Larger buffers = fewer polls, more resident memory | +| `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` | 4MiB | duplex pipe capacity (shared, but PUT path uses it less than GET) | `set_disk_encode` feed smoothness | Memory per in-flight request | +| `RUSTFS_DURABILITY_MODE` / `RUSTFS_DRIVE_SYNC_ENABLE` | mode-dependent | per-shard fsync/sync discipline on commit | `set_disk_rename` P99 | Weakening it changes the durability contract — treat as a deliberate tradeoff, not a free win | +| `RUSTFS_RUNTIME_WORKER_THREADS` / `RUSTFS_RUNTIME_MAX_BLOCKING_THREADS` | Tokio defaults | async workers + `spawn_blocking` pool feeding per-block encode | `set_disk_encode` P95 + mpstat | Oversubscription | + +### 5.2 GET + +| Knob | Default | Controls | Validating stage | Risk if enabled | +| --- | --- | --- | --- | --- | +| `RUSTFS_GET_CODEC_STREAMING_ROLLOUT` | `off` | switches the read path from `legacy_duplex` to the pull-based `ErasureDecodeReader` (`codec_streaming`) | compare `path="legacy_duplex"` vs `path="codec_streaming"` for `decode`/`emit`/`output_lock_wait`/`stripe_read` | behavioral change to the read path; rollout is `off` by default for a reason | +| `RUSTFS_GET_CODEC_STREAMING_ENGINE` | `legacy` | `legacy` vs `rustfs` decode engine under the streaming reader | `reconstruct`/`decode` per `path` | engine swap on a correctness-critical path | +| `RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE` | `false` | multipart objects on the streaming reader | same, multipart cells | wider format coverage | +| `RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE` (+ `_MAX_SIZE`, `_FIRST_READER_SETUP`) | `false` / 512KiB | prefer data-shard readers before parity | `stripe_read_first_shard`/`stripe_read_quorum` | shard-selection order change | +| `RUSTFS_OBJECT_GET_SKIP_BITROT_VERIFY` | `false` | skip per-shard HighwayHash verify | `bitrot_verify` | **do not default on** — measures the theoretical ceiling only | +| `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` | 4MiB | legacy GET in-process pipe capacity | `output_lock_wait`/`output_poll` | memory per in-flight GET | +| `RUSTFS_GET_SEEK_BUFFER_ENABLE` | `false` | in-memory seek buffer for small GET | `first_byte` | experimental, startup-latched — see [`get-path-experimental-switches.md`](get-path-experimental-switches.md) | +| `RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE` | `false` | adds `response_handoff` attribution (metrics only) | `response_handoff` | small per-request bookkeeping cost | + +## 6. A/B matrix + +Run each row as an independent cell. Baseline is the shipped default; candidate +is one knob moved. Everything else (topology, sizes, concurrency, rounds, +durability) stays fixed. + +### 6.1 PUT + +| # | Knob | Baseline | Candidates | Signal metric | Judgement | +| --- | --- | --- | --- | --- | --- | +| P1 | encode in-flight | 32MiB | 48MiB, 64MiB, 96MiB | `set_disk_encode` P95 + throughput + RSS | throughput up AND `set_disk_encode` down AND RSS tolerable → budget was the binding constraint | +| P2 | object I/O buffer | 128KiB | 256KiB, 512KiB, 1MiB | `ingress_prepare` + `set_disk_encode` | encode smooths with bounded RSS → upstream feed was too small | +| P3 | duplex buffer | 4MiB | 8MiB, 16MiB | `set_disk_encode` feed variance | lower priority than P1/P2 | +| P4 | blocking threads | default | 512, 768, 1024 | `set_disk_encode` P95 + mpstat | per-block `spawn_blocking` is scheduler-bound if P95 falls | +| P5 | durability | current mode | weaker/stronger mode | `set_disk_rename` P99 | only as a **deliberate durability tradeoff**, never a silent default change | +| P6 | set drive count | current | other valid widths | all PUT stages | highest cost — only after P1–P5, and only on a rebuildable topology | + +### 6.2 GET + +| # | Knob | Baseline | Candidates | Signal metric | Judgement | +| --- | --- | --- | --- | --- | --- | +| G1 | codec streaming rollout | `off` | `on` (pct ramp 10/50/100) | `path="legacy_duplex"` vs `path="codec_streaming"` for `decode`/`emit`/`output_lock_wait`/`stripe_read` + throughput | streaming beats duplex on `decode`+`output_lock_wait` and byte-for-byte output matches → candidate for default | +| G2 | codec streaming engine | `legacy` | `rustfs` | `reconstruct`/`decode` per `path` | engine swap is neutral-or-better on CPU with identical bytes | +| G3 | multipart streaming | `false` | `true` | multipart GET cells | only after G1 is stable on single-part | +| G4 | data-blocks-first | `false` | `true` | `stripe_read_first_shard`/`stripe_read_quorum` | fewer shard reads without a correctness regression | +| G5 | duplex buffer | 4MiB | 8MiB, 16MiB | `output_lock_wait`/`output_poll` (legacy path) | only if still on `legacy_duplex` | +| G6 | skip bitrot verify | `false` | `true` | `bitrot_verify` | **ceiling measurement only**; do not carry into production | + +## 7. Execution sequence + +1. Freeze the conditions in §3 and record the provenance block. +2. Run the baseline cell (all defaults) and capture stage histograms + host + telemetry. +3. Pick the **one** most-likely knob from the analysis. For large-object PUT + that is almost always `set_disk_encode` → P1; for GET it is G1 (the + `legacy_duplex` → `codec_streaming` switch). +4. Sweep that knob's candidate column one value at a time, same workload. +5. Read the decision table in §8; if the stage did not move, the knob is not + the bottleneck — stop widening it and pick the next stage. + +Driver invocation for one PUT cell: + +```bash +scripts/run_object_batch_bench_enhanced.sh \ + --tool warp --endpoint http://127.0.0.1:9000 \ + --access-key "$RUSTFS_ACCESS_KEY" --secret-key "$RUSTFS_SECRET_KEY" \ + --bucket rustfs-put-tuning --warp-mode put \ + --sizes 16MiB,32MiB --concurrency 32 --duration 120s --rounds 3 \ + --out-dir target/bench/put-tuning-p1-64mib +``` + +## 8. Interpretation / decision table + +| Stage high | Most likely cause | Next action | +| --- | --- | --- | +| `set_disk_encode` | per-block EC encode scheduling + in-flight budget | P1 → P4 → P2, in that order | +| `set_disk_rename` | commit tail (rename fan-out / RPC / fsync) | P5 (durability) and cluster tail analysis, not encode | +| `set_disk_writer_setup` | per-disk `BitrotWriter` + temp-file create | disk/filesystem metadata; per-disk fan-out cost | +| `set_disk_old_data_cleanup` | overwrite / versioned-object directory delete | confirm overwrite-vs-new-write; defer cleanup further | +| `metadata_fanout` / `metadata_resolve` | cross-disk `xl.meta` read + quorum | metadata cache hit rate; small-object fixed cost | +| `bitrot_verify` | HighwayHash verify on the read path | G6 ceiling only; do not default on | +| `output_lock_wait` / `output_poll` | legacy duplex backpressure | G1 (move off duplex) or G5 | +| `stripe_read*` | shard concurrency / selection | G4 shard-selection, disk/network tail | + +## 9. Guardrails + +- **Never weaken correctness for throughput**: read/write quorum, bitrot verify, + `xl.meta` validation, and durability (`RUSTFS_DURABILITY_MODE`) are integrity + contracts, not knobs. P5 and G6 are ceiling measurements and must be labelled + as such; do not carry their values into production without an explicit + durability/correctness decision. +- **One variable per cell.** A cell that changes two knobs is thrown away. +- **Memory is part of the result.** A throughput win with unbounded RSS growth + is a regression; record RSS and the EC in-flight gauge for every PUT cell. +- **Startup-latched knobs** (`RUSTFS_GET_SEEK_BUFFER_ENABLE`, + `RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE`, and the codec-streaming + switches) require a process restart to change — see + [`get-path-experimental-switches.md`](get-path-experimental-switches.md). +- **Archive the raw data.** Keep the `baseline_compare.csv`, `median_summary.csv`, + stage histograms, and host telemetry per cell; the conclusion must trace back + to them. Do not commit benchmark result snapshots to the repo — record them in + the issue tracker. diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 3470b48d8..d404add26 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -134,7 +134,6 @@ use rustfs_notify::EventArgsBuilder; use rustfs_object_capacity::capacity_manager::get_capacity_manager; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object}; -use rustfs_s3select_api::object_store::bytes_stream; use rustfs_targets::{EventName, get_request_host, get_request_port, get_request_user_agent}; use rustfs_utils::CompressionAlgorithm; #[cfg(test)] @@ -918,7 +917,7 @@ pin_project! { } struct MemoryTrackedBytesStream { - bytes: Bytes, + bytes: Option, emitted: bool, completed: bool, expected: usize, @@ -1032,7 +1031,7 @@ impl MemoryTrackedBytesStream { ) -> Self { let length_mismatch = bytes.len() != expected; Self { - bytes, + bytes: Some(bytes), emitted: false, completed: !length_mismatch && expected == 0, expected, @@ -1100,35 +1099,36 @@ impl futures::Stream for MemoryTrackedBytesStream { // differently sized body. This is a defense-in-depth backstop; the // buffered/cache callers reject the mismatch before headers are sent. if this.length_mismatch { + let actual = this.bytes.as_ref().map_or(0, Bytes::len); this.emitted = true; this.finish_err(); return Poll::Ready(Some(Err(std::io::Error::new( std::io::ErrorKind::InvalidData, - format!( - "materialized GET body length mismatch: expected {}, got {}", - this.expected, - this.bytes.len() - ), + format!("materialized GET body length mismatch: expected {}, got {}", this.expected, actual), )))); } - let first_byte_elapsed = (!this.bytes.is_empty()).then(|| this.started.elapsed()); + let Some(bytes) = this.bytes.take() else { + return Poll::Ready(None); + }; + let bytes_len = bytes.len(); + let first_byte_elapsed = (!bytes.is_empty()).then(|| this.started.elapsed()); this.emitted = true; if let Some(elapsed) = first_byte_elapsed { rustfs_io_metrics::record_get_object_first_byte_latency(GET_OBJECT_STAGE_PATH_S3_HANDLER, elapsed.as_secs_f64()); } - if this.bytes.len() >= this.expected { + if bytes_len >= this.expected { this.finish_ok(); } if let Some(poll_start) = poll_start { rustfs_io_metrics::record_get_object_memory_body_stream_poll( this.source, GET_READER_STREAM_POLL_READY_DATA, - this.bytes.len(), + bytes_len, poll_start.elapsed().as_secs_f64(), ); } - Poll::Ready(Some(Ok(this.bytes.clone()))) + Poll::Ready(Some(Ok(bytes))) } } @@ -4148,10 +4148,7 @@ impl DefaultObjectUsecase { let bytes_len = bytes.len(); let guard = rustfs_io_metrics::track_get_object_buffered_bytes(bytes_len); let remaining = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX); - let blob = StreamingBlob::wrap(bytes_stream( - MemoryTrackedBytesStream::new(bytes, remaining, source, guard, lifecycle), - remaining, - )); + let blob = StreamingBlob::wrap(MemoryTrackedBytesStream::new(bytes, remaining, source, guard, lifecycle)); if let Some(handoff_start) = handoff_start { rustfs_io_metrics::record_get_object_response_handoff( "single_chunk", From 6f29431a65a179cb0ef03f2f3846b84d4457fd6c Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 14 Aug 2026 19:07:34 +0800 Subject: [PATCH 04/71] test(ecstore): isolate rename publication hooks (#6106) --- crates/ecstore/src/disk/local.rs | 32 +++++++++++--------- crates/ecstore/src/set_disk/ops/multipart.rs | 2 +- crates/scanner/src/scanner_io.rs | 12 +++++--- crates/scanner/src/storage_api.rs | 2 +- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index c6ff03604..e8059319f 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -2028,14 +2028,17 @@ static RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT: std::sync::Mutex; #[cfg(test)] +type RenameDataPublicationHookKey = (PathBuf, String, String); +#[cfg(test)] static INLINE_PREPARATION_BEFORE_BACKUP: std::sync::LazyLock>> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); #[cfg(test)] static INLINE_BEFORE_FILE_SYNC_ADMISSION: std::sync::LazyLock>> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); #[cfg(test)] -static RENAME_DATA_AFTER_FIRST_PUBLICATION: std::sync::LazyLock>> = - std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); +static RENAME_DATA_AFTER_FIRST_PUBLICATION: std::sync::LazyLock< + std::sync::Mutex>, +> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); #[cfg(test)] static OWNED_FILE_WRITE_BEFORE_OPEN: std::sync::LazyLock>> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new())); @@ -2107,11 +2110,11 @@ fn set_inline_before_file_sync_admission(dst_path: &str, hook: impl FnOnce() + S } #[cfg(test)] -fn set_rename_data_after_first_publication(dst_path: &str, hook: impl FnOnce() + Send + 'static) { +fn set_rename_data_after_first_publication(root: &Path, dst_volume: &str, dst_path: &str, hook: impl FnOnce() + Send + 'static) { RENAME_DATA_AFTER_FIRST_PUBLICATION .lock() .expect("test publication hook lock should not be poisoned") - .insert(dst_path.to_string(), Box::new(hook)); + .insert((root.to_path_buf(), dst_volume.to_string(), dst_path.to_string()), Box::new(hook)); } #[cfg(test)] @@ -2263,11 +2266,11 @@ fn run_inline_before_file_sync_admission(dst_path: &str) { } #[cfg(test)] -fn run_rename_data_after_first_publication(dst_path: &str) { +fn run_rename_data_after_first_publication(root: &Path, dst_volume: &str, dst_path: &str) { let hook = RENAME_DATA_AFTER_FIRST_PUBLICATION .lock() .expect("test publication hook lock should not be poisoned") - .remove(dst_path); + .remove(&(root.to_path_buf(), dst_volume.to_string(), dst_path.to_string())); if let Some(hook) = hook { hook(); } @@ -2365,9 +2368,6 @@ async fn remove_dst_base_before_commit( #[cfg(not(test))] fn run_inline_preparation_before_backup(_dst_path: &str) {} -#[cfg(not(test))] -fn run_rename_data_after_first_publication(_dst_path: &str) {} - #[cfg(not(test))] fn should_fail_after_delete_data_staged(_path: &str) -> bool { false @@ -9015,8 +9015,9 @@ impl DiskAPI for LocalDisk { .await?; return Err(err); } + #[cfg(test)] if has_data_dir_path.is_some() { - run_rename_data_after_first_publication(dst_path); + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); } // Crash-consistency injection: hard power loss after the data dir @@ -9449,7 +9450,8 @@ impl DiskAPI for LocalDisk { let _ = remove_file_if_exists(staged_backup); return Err(err); } - run_rename_data_after_first_publication(dst_path); + #[cfg(test)] + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); if sync { file_sync_admission = Some( os::acquire_file_sync_admission(self.file_sync_permits.clone()) @@ -13133,7 +13135,7 @@ mod test { let replacement_staging_parent_for_hook = replacement_staging_parent.clone(); let staged_metadata_for_hook = staged_metadata.clone(); let replacement_staged_metadata_for_hook = replacement_staged_metadata.clone(); - set_rename_data_after_first_publication(object, move || { + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { std::fs::rename(&object_dir_for_hook, &replacement_dir_for_hook) .expect_err("the destination object identity must remain pinned until xl.meta commits"); std::fs::rename(&staging_parent_for_hook, &replacement_staging_parent_for_hook) @@ -13400,7 +13402,7 @@ mod test { let replacement_dir_for_hook = replacement_dir.clone(); let staged_metadata_for_hook = staged_metadata.clone(); let replacement_staged_metadata_for_hook = replacement_staged_metadata.clone(); - set_rename_data_after_first_publication(object, move || { + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { std::fs::rename(&object_dir_for_hook, &replacement_dir_for_hook) .expect_err("the destination object identity must remain pinned after publishing its rollback backup"); std::fs::rename(&staged_metadata_for_hook, &replacement_staged_metadata_for_hook) @@ -13766,7 +13768,7 @@ mod test { let (entered_tx, entered_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel(); - set_rename_data_after_first_publication(object, move || { + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { entered_tx.send(()).expect("signal first publication"); release_rx.recv().expect("wait while delete_volume is blocked"); }); @@ -14360,7 +14362,7 @@ mod test { let (published_tx, published_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel(); - set_rename_data_after_first_publication(object, move || { + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { published_tx.send(()).expect("signal backup publication"); release_rx.recv().expect("wait for lock-order assertion"); }); diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index a11c0030a..008542b8e 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -2034,7 +2034,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } object_size = object_size.checked_add(ext_part.size).ok_or(Error::PartMissingOrCorrupt)?; - if ext_part.actual_size < 0 && (!opts.replication_request || quota_context.is_enforced()) { + if ext_part.actual_size < 0 && (quota_context.is_enforced() || (!opts.replication_request && !opts.data_movement)) { return Err(Error::PartMissingOrCorrupt); } let normalized_actual_size = if ext_part.actual_size >= 0 && !transformed_object { diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 019a1fe64..6b0586ecf 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -4023,7 +4023,7 @@ mod tests { use crate::scanner_budget::ScannerCycleBudgetConfig; use crate::scanner_folder::ScannerItem; use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats}; - use crate::storage_api::scan::{BucketOperations as _, MakeBucketOptions, ObjectIO as _}; + use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _}; use crate::{ DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions, ScannerPutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, @@ -4248,16 +4248,20 @@ mod tests { async fn multi_pool_scanner_cycle_zero_fills_bucket_absent_from_first_pool() { let (_temp_dir, store) = setup_two_pool_scanner_store().await; let bucket = format!("scanner-second-pool-{}", Uuid::new_v4().simple()); - store.pools[1].disk_set[0] + store .make_bucket(&bucket, &MakeBucketOptions::default()) .await - .expect("bucket should be created only in the second pool"); + .expect("bucket and its authoritative metadata should be created"); let body = b"second-only"; let mut reader = ScannerPutObjReader::from_vec(body.to_vec()); - store.pools[1].disk_set[0] + store.pools[1] .put_object(&bucket, "pool-b", &mut reader, &ScannerObjectOptions::default()) .await .expect("object should be written only to the second pool"); + store.pools[0] + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("bucket should be removed from the first pool only"); let ctx = CancellationToken::new(); let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()); diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index 634f20e45..e77033aa2 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -278,7 +278,7 @@ pub(crate) mod scan { SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, }; #[cfg(test)] - pub(crate) use super::storage_contracts::{MakeBucketOptions, ObjectIO}; + pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO}; } pub(crate) mod scanner_io { From 0ff3d4cbf400ab2e2b09a28c35b90747d0c16bbc Mon Sep 17 00:00:00 2001 From: houseme Date: Fri, 14 Aug 2026 19:56:36 +0800 Subject: [PATCH 05/71] perf(ecstore): borrow rename metadata during commit fanout (#6104) * perf(ecstore): borrow rename metadata during commit fanout Co-Authored-By: heihutu * fix(ecstore): preserve rename_data API compatibility Co-Authored-By: heihutu --------- Co-authored-by: heihutu Co-authored-by: Zhengchao An --- crates/ecstore/src/cluster/rpc/remote_disk.rs | 119 ++++++++++-------- crates/ecstore/src/disk/disk_store.rs | 43 +++++-- crates/ecstore/src/disk/local.rs | 16 ++- crates/ecstore/src/disk/mod.rs | 31 ++++- .../src/set_disk/core/io_primitives.rs | 101 ++++++++++++--- crates/ecstore/src/set_disk/ops/heal.rs | 5 +- crates/ecstore/src/set_disk/ops/multipart.rs | 16 ++- crates/ecstore/src/set_disk/ops/object.rs | 52 ++------ rustfs/src/storage/rpc/node_service/disk.rs | 2 +- rustfs/src/storage/storage_api.rs | 6 +- 10 files changed, 253 insertions(+), 138 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 9b78fc846..13e45fbd3 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -1359,6 +1359,71 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> { file_info.validate_for_metadata_read().map_err(Into::into) } +impl RemoteDisk { + #[tracing::instrument(level = "trace", skip_all)] + pub(crate) async fn rename_data_borrowed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result { + trace!( + event = EVENT_REMOTE_DISK_RPC, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REMOTE_DISK, + endpoint = %self.endpoint, + src_volume, + src_path, + dst_volume, + dst_path, + op = "rename_data", + state = "started", + "Remote disk RPC started" + ); + + self.execute_with_timeout_for_op( + "rename_data", + || async { + let file_info = compat_json(fi)?; + let file_info_bin = encode_file_info_msgpack(fi)?; + let mut client = self + .get_client() + .await + .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; + let mut request = Request::new(RenameDataRequest { + disk: self.endpoint.to_string(), + src_volume: src_volume.to_string(), + src_path: src_path.to_string(), + file_info, + dst_volume: dst_volume.to_string(), + dst_path: dst_path.to_string(), + file_info_bin: file_info_bin.into(), + }); + let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref()); + attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?; + + let response = client.rename_data(request).await?.into_inner(); + + if !response.success { + return Err(response.error.unwrap_or_default().into()); + } + + let rename_data_resp = decode_msgpack_or_json::( + &response.rename_data_resp_bin, + &response.rename_data_resp, + "RenameDataResp", + )?; + + Ok(rename_data_resp) + }, + get_max_timeout_duration(), + ) + .await + } +} + #[async_trait::async_trait] impl DiskAPI for RemoteDisk { #[tracing::instrument(level = "trace", skip_all)] @@ -2284,58 +2349,8 @@ impl DiskAPI for RemoteDisk { dst_volume: &str, dst_path: &str, ) -> Result { - trace!( - event = EVENT_REMOTE_DISK_RPC, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REMOTE_DISK, - endpoint = %self.endpoint, - src_volume, - src_path, - dst_volume, - dst_path, - op = "rename_data", - state = "started", - "Remote disk RPC started" - ); - - self.execute_with_timeout_for_op( - "rename_data", - || async { - let file_info = compat_json(&fi)?; - let file_info_bin = encode_file_info_msgpack(&fi)?; - let mut client = self - .get_client() - .await - .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; - let mut request = Request::new(RenameDataRequest { - disk: self.endpoint.to_string(), - src_volume: src_volume.to_string(), - src_path: src_path.to_string(), - file_info, - dst_volume: dst_volume.to_string(), - dst_path: dst_path.to_string(), - file_info_bin: file_info_bin.into(), - }); - let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref()); - attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?; - - let response = client.rename_data(request).await?.into_inner(); - - if !response.success { - return Err(response.error.unwrap_or_default().into()); - } - - let rename_data_resp = decode_msgpack_or_json::( - &response.rename_data_resp_bin, - &response.rename_data_resp, - "RenameDataResp", - )?; - - Ok(rename_data_resp) - }, - get_max_timeout_duration(), - ) - .await + self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path) + .await } #[tracing::instrument(level = "trace", skip_all)] diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index deb40a5cb..4ad8b22dc 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -241,6 +241,40 @@ pub fn get_drive_list_dir_timeout() -> Duration { ) } +pub(crate) trait DiskStoreRenameDataExt { + async fn rename_data_borrowed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result; +} + +impl DiskStoreRenameDataExt for LocalDiskWrapper { + async fn rename_data_borrowed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result { + self.track_disk_health_mutation( + "rename_data", + DiskMetricMutation::Write, + || async { + self.disk + .rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path) + .await + }, + get_max_timeout_duration(), + ) + .await + } +} + pub fn get_drive_walkdir_timeout() -> Duration { get_drive_timeout_duration( rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, @@ -1985,13 +2019,8 @@ impl DiskAPI for LocalDiskWrapper { dst_volume: &str, dst_path: &str, ) -> Result { - self.track_disk_health_mutation( - "rename_data", - DiskMetricMutation::Write, - || async { self.disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await }, - get_max_timeout_duration(), - ) - .await + self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path) + .await } async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result> { diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index e8059319f..1dc3a79c4 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -8689,11 +8689,12 @@ impl DiskAPI for LocalDisk { &self, src_volume: &str, src_path: &str, - mut fi: FileInfo, + fi: FileInfo, dst_volume: &str, dst_path: &str, ) -> Result { crate::hp_guard!("LocalDisk::rename_data"); + let mut fi = fi; // A non-force DeleteBucket must not remove a directory while a local // object commit is publishing into it. The peer's empty scan remains // optimistic; this lease establishes the local commit/delete order and @@ -10581,6 +10582,19 @@ impl DiskAPI for LocalDisk { } } +impl LocalDisk { + pub(crate) async fn rename_data_borrowed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result { + ::rename_data(self, src_volume, src_path, fi.clone(), dst_volume, dst_path).await + } +} + async fn wait_for_startup_cleanup_signal( startup_cleanup_ready: &AtomicU32, startup_cleanup_notify: &Notify, diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 43259b8c8..428bd61a9 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -55,6 +55,7 @@ pub fn part_transaction_path(part_path: &str) -> String { use crate::cluster::rpc::RemoteDisk; use crate::cluster::rpc::build_internode_data_transport_from_env; +use crate::disk::disk_store::DiskStoreRenameDataExt; use crate::disk::disk_store::LocalDiskWrapper; use crate::disk::health_state::RuntimeDriveHealthState; use crate::disk::local::ScanGuard; @@ -434,10 +435,8 @@ impl DiskAPI for Disk { dst_volume: &str, dst_path: &str, ) -> Result { - match self { - Disk::Local(local_disk) => local_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await, - Disk::Remote(remote_disk) => remote_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await, - } + self.rename_data_borrowed(src_volume, src_path, &fi, dst_volume, dst_path) + .await } #[tracing::instrument(level = "trace", skip_all)] @@ -667,6 +666,30 @@ impl DiskAPI for Disk { } } +impl Disk { + pub(crate) async fn rename_data_borrowed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result { + match self { + Disk::Local(local_disk) => { + local_disk + .rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path) + .await + } + Disk::Remote(remote_disk) => { + remote_disk + .rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path) + .await + } + } + } +} + impl Disk { pub async fn ns_scanner_server_epoch(&self) -> Result> { match self { diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 91b6c99fa..521cc1a33 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -46,6 +46,7 @@ use crate::diagnostics::get::{ GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, }; +use crate::disk::disk_store::DiskStoreRenameDataExt; use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX; use crate::disk::{ DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, @@ -3011,6 +3012,35 @@ impl RenameConvergence { } } +pub(in crate::set_disk) struct RenameDataCommit { + pub(in crate::set_disk) online_disks: Vec>, + pub(in crate::set_disk) convergence: RenameConvergence, + pub(in crate::set_disk) data_dir: Option, + pub(in crate::set_disk) cleanup_disks: Vec>, + pub(in crate::set_disk) old_current_size: Option, + pub(in crate::set_disk) committed_file_info: FileInfo, +} + +type RenameDataLegacyTuple = ( + Vec>, + RenameConvergence, + Option, + Vec>, + Option, +); + +impl RenameDataCommit { + fn into_legacy_tuple(self) -> RenameDataLegacyTuple { + ( + self.online_disks, + self.convergence, + self.data_dir, + self.cleanup_disks, + self.old_current_size, + ) + } +} + impl SetDisks { pub(in crate::set_disk) fn default_read_quorum(&self) -> usize { self.set_drive_count - self.default_parity_count @@ -3088,6 +3118,14 @@ impl SetDisks { Ok(()) } + pub(in crate::set_disk) fn assign_rename_data_indexes(file_infos: &mut [FileInfo]) { + for (index, file_info) in file_infos.iter_mut().enumerate() { + if file_info.erasure.index == 0 { + file_info.erasure.index = index + 1; + } + } + } + pub(in crate::set_disk) async fn abort_quota_reservation_after_fence( reservation: crate::bucket::quota::reservation::QuotaReservation, disks: &[Option], @@ -3118,13 +3156,22 @@ impl SetDisks { dst_bucket: &str, dst_object: &str, write_quorum: usize, - ) -> disk::error::Result<( - Vec>, - RenameConvergence, - Option, - Vec>, - Option, - )> { + ) -> disk::error::Result { + Self::rename_data_owned(disks, src_bucket, src_object, file_infos.to_vec(), dst_bucket, dst_object, write_quorum) + .await + .map(RenameDataCommit::into_legacy_tuple) + } + + #[tracing::instrument(level = "debug", skip(disks, file_infos))] + pub(in crate::set_disk) async fn rename_data_owned( + disks: &[Option], + src_bucket: &str, + src_object: &str, + file_infos: Vec, + dst_bucket: &str, + dst_object: &str, + write_quorum: usize, + ) -> disk::error::Result { if let Some(file_info) = disks .iter() .zip(file_infos.iter()) @@ -3149,7 +3196,7 @@ impl SetDisks { let disk_count = disks.len(); let fanout_disks = disks.to_vec(); - let fanout_file_infos = file_infos.to_vec(); + let fanout_file_infos = file_infos; let fanout_src_bucket = src_bucket.clone(); let fanout_src_object = src_object.clone(); let fanout_dst_bucket = dst_bucket.clone(); @@ -3161,9 +3208,9 @@ impl SetDisks { let fanout = tokio::spawn(async move { let futures = fanout_disks .into_iter() - .zip(fanout_file_infos) + .zip(fanout_file_infos.iter()) .enumerate() - .map(|(i, (disk, mut file_info))| { + .map(|(i, (disk, file_info))| { let src_bucket = fanout_src_bucket.clone(); let src_object = fanout_src_object.clone(); let dst_object = fanout_dst_object.clone(); @@ -3180,11 +3227,15 @@ impl SetDisks { }; let is_delete_marker = file_info.is_canonical_delete_marker(); - if file_info.erasure.index == 0 { - file_info.erasure.index = i + 1; - } - - if !is_delete_marker && !file_info.has_valid_erasure_geometry() { + let mut local_file_info; + let file_info = if file_info.erasure.index == 0 { + local_file_info = file_info.clone(); + local_file_info.erasure.index = i + 1; + &local_file_info + } else { + file_info + }; + if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) { return Err(DiskError::FileCorrupt); } @@ -3192,12 +3243,13 @@ impl SetDisks { // A no-op immediately-ready future in production. Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await; - disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) + disk.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) .await }) .catch_unwind() }); - join_all(futures).await + let results = join_all(futures).await; + (results, fanout_file_infos) }); let mut disk_versions = vec![None; disk_count]; @@ -3205,7 +3257,7 @@ impl SetDisks { let mut cleanup_data_dirs = vec![None; disk_count]; let mut old_current_sizes = vec![None; disk_count]; - let results = fanout.await.map_err(|_| DiskError::Unexpected)?; + let (results, mut file_infos) = fanout.await.map_err(|_| DiskError::Unexpected)?; for (idx, result) in results.iter().enumerate() { match result { @@ -3261,7 +3313,7 @@ impl SetDisks { } if let Some(disk) = disks[i].as_ref() { - let fi = file_infos[i].clone(); + let fi = std::mem::take(&mut file_infos[i]); let old_data_dir = data_dirs[i]; let disk = disk.clone(); let dst_bucket = dst_bucket.clone(); @@ -3384,6 +3436,8 @@ impl SetDisks { let convergence = Self::classify_rename_convergence(&disk_versions, &errs); let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum); let online_disks = Self::eval_disks(disks, &errs); + let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?; + let committed_file_info = std::mem::take(&mut file_infos[committed_slot]); let cleanup_disks = if let Some(data_dir) = data_dir { disks .iter() @@ -3401,7 +3455,14 @@ impl SetDisks { vec![None; disks.len()] }; - Ok((online_disks, convergence, data_dir, cleanup_disks, old_current_size)) + Ok(RenameDataCommit { + online_disks, + convergence, + data_dir, + cleanup_disks, + old_current_size, + committed_file_info, + }) } /// rustfs/backlog#1009: reduce the per-disk observations of the diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 089515ec6..20e1f0fd6 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::super::*; +use crate::disk::disk_store::DiskStoreRenameDataExt; use crate::io_support::bitrot::object_mmap_read_enabled; use crate::storage_api_contracts::namespace::NamespaceLocking as _; use tracing::trace; @@ -1164,10 +1165,10 @@ impl SetDisks { let rename_result = if should_fail_heal_rename(bucket, object, index) { Err(DiskError::Unexpected) } else { - disk.rename_data( + disk.rename_data_borrowed( RUSTFS_META_TMP_BUCKET, &tmp_id, - parts_metadata[index].clone(), + &parts_metadata[index], bucket, object, ) diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 008542b8e..48c6d3076 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -2527,11 +2527,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { // The trailing `_` drops the rename_data old-size backfill // (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit // `get_object_info` lookup, so the backfill has no consumer here yet. - let rename_result = SetDisks::rename_data( + Self::assign_rename_data_indexes(&mut parts_metadatas); + let rename_result = SetDisks::rename_data_owned( &commit_disks, RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, - &parts_metadatas, + parts_metadatas, &commit_bucket, &commit_object, write_quorum, @@ -2550,10 +2551,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { if rename_result.is_ok() { quota_reservation.commit().await; } - let (online_disks, convergence, op_old_dir, cleanup_disks, _) = match rename_result { + let rename_commit = match rename_result { Ok(result) => result, Err(err) => return Err(err.into()), }; + let online_disks = rename_commit.online_disks; + let convergence = rename_commit.convergence; + let op_old_dir = rename_commit.data_dir; + let cleanup_disks = rename_commit.cleanup_disks; + let committed_file_info = rename_commit.committed_file_info; // Detach admission before any post-commit await: client cancellation // must not couple durable convergence repair to cleanup work. @@ -2598,9 +2604,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { return Err(StorageError::Unexpected); } - if let Some(committed_slot) = online_disks.iter().position(Option::is_some) { - fi = parts_metadatas[committed_slot].clone(); - } + fi = committed_file_info; let committed_dir = fi.data_dir.unwrap_or_default().to_string(); commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &online_disks); diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 65c3de2b5..95fe3438a 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -97,10 +97,6 @@ fn duration_millis_f64(duration: std::time::Duration) -> f64 { duration.as_secs_f64() * 1000.0 } -fn committed_response_metadata_slot(committed_disks: &[Option], fallback_slot: usize) -> usize { - committed_disks.iter().position(Option::is_some).unwrap_or(fallback_slot) -} - pub(in crate::set_disk::ops) fn assign_object_transaction_epoch( shuffle_disks: &[Option], parts_metadatas: &mut [FileInfo], @@ -239,38 +235,6 @@ mod duration_metrics_tests { } } -#[cfg(test)] -mod put_metadata_tests { - use super::*; - - #[test] - fn committed_file_info_follows_exact_quorum_success_slot() { - let mut first_success = FileInfo::new("bucket/object", 2, 2); - first_success.name = "first-success".to_string(); - let mut second_success = first_success.clone(); - second_success.name = "second-success".to_string(); - let mut parts_metadata = [FileInfo::default(), first_success, second_success, FileInfo::default()]; - let committed_disks = [None, Some(()), Some(()), None]; - - assert_eq!( - committed_disks.iter().filter(|disk| disk.is_some()).count(), - 2, - "fixture must meet exact quorum" - ); - let selected_slot = committed_response_metadata_slot(&committed_disks, 3); - let selected = std::mem::take(&mut parts_metadata[selected_slot]); - - assert_eq!(selected.name, "first-success"); - assert_eq!(parts_metadata[1], FileInfo::default(), "selected metadata should move without cloning"); - assert_eq!(parts_metadata[2].name, "second-success", "other committed metadata must remain available"); - assert_eq!( - committed_response_metadata_slot::<()>(&[None, None, None, None], 3), - 3, - "a violated post-commit success-mask invariant must not turn a durable PUT into an error" - ); - } -} - fn is_restore_control_metadata(key: &str) -> bool { key.eq_ignore_ascii_case(X_AMZ_RESTORE.as_str()) || key.eq_ignore_ascii_case(rustfs_utils::http::headers::AMZ_RESTORE_EXPIRY_DAYS) @@ -2236,11 +2200,12 @@ impl SetDisks { return Err(err); } - let rename_result = SetDisks::rename_data( + Self::assign_rename_data_indexes(&mut parts_metadatas); + let rename_result = SetDisks::rename_data_owned( &commit_disks, RUSTFS_META_TMP_BUCKET, commit_tmp_dir.as_str(), - &parts_metadatas, + parts_metadatas, &commit_bucket, &commit_object, write_quorum, @@ -2259,7 +2224,7 @@ impl SetDisks { if rename_result.is_ok() { quota_reservation.commit().await; } - let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = match rename_result { + let rename_commit = match rename_result { Ok(commit) => commit, Err(err) => { if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { @@ -2276,6 +2241,12 @@ impl SetDisks { return Err(err.into()); } }; + let online_disks = rename_commit.online_disks; + let convergence = rename_commit.convergence; + let op_old_dir = rename_commit.data_dir; + let cleanup_disks = rename_commit.cleanup_disks; + let old_current_size = rename_commit.old_current_size; + let mut fi = rename_commit.committed_file_info; // Do this before any post-commit await so request cancellation cannot // bypass best-effort admission. A process crash before admission // remains subject to the existing scanner reconciliation path. @@ -2384,9 +2355,6 @@ impl SetDisks { } } - let committed_metadata_slot = committed_response_metadata_slot(&online_disks, response_metadata_slot); - let mut fi = std::mem::take(&mut parts_metadatas[committed_metadata_slot]); - if is_compressed { record_compression_total_memory(actual_size as u64, w_size as u64).await; } diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index 1dc45beba..ce0ba42e5 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -1049,7 +1049,7 @@ impl NodeService { .rename_data( &request.src_volume, &request.src_path, - decoded_file_info.value, + &decoded_file_info.value, &request.dst_volume, &request.dst_path, ) diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index d192e38a2..aacd91f8a 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -1150,7 +1150,7 @@ pub(crate) trait StorageDiskRpcExt { &self, src_volume: &str, src_path: &str, - file_info: rustfs_filemeta::FileInfo, + file_info: &rustfs_filemeta::FileInfo, dst_volume: &str, dst_path: &str, ) -> DiskResult; @@ -1301,11 +1301,11 @@ where &self, src_volume: &str, src_path: &str, - file_info: rustfs_filemeta::FileInfo, + file_info: &rustfs_filemeta::FileInfo, dst_volume: &str, dst_path: &str, ) -> DiskResult { - ecstore_disk::DiskAPI::rename_data(self, src_volume, src_path, file_info, dst_volume, dst_path).await + ecstore_disk::DiskAPI::rename_data(self, src_volume, src_path, file_info.clone(), dst_volume, dst_path).await } async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult> { From 67a19021b55f879aaf38c616c8662eba19b00e76 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 14 Aug 2026 21:00:01 +0800 Subject: [PATCH 06/71] fix(ecstore): allow migrated unknown part sizes (#6112) --- crates/ecstore/src/set_disk/ops/multipart.rs | 238 +++++++++++++++---- 1 file changed, 187 insertions(+), 51 deletions(-) diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 48c6d3076..9684cc1f6 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -1906,6 +1906,23 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { return Err(Error::PartMissingOrCorrupt); } let transformed_object = fi.is_compressed() || should_persist_encryption_original_size(&fi.metadata); + let data_movement_actual_size = if opts.data_movement { + rustfs_utils::http::get_consistent_str(&opts.user_defined, SUFFIX_ACTUAL_SIZE) + .map(|value| { + value + .parse::() + .ok() + .filter(|value| *value >= 0) + .ok_or(Error::PartMissingOrCorrupt) + }) + .transpose()? + } else { + None + }; + let data_movement_actual_size_u64 = data_movement_actual_size + .map(u64::try_from) + .transpose() + .map_err(|_| Error::PartMissingOrCorrupt)?; let mut object_size: usize = 0; let mut object_actual_size: i64 = 0; @@ -2034,7 +2051,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } object_size = object_size.checked_add(ext_part.size).ok_or(Error::PartMissingOrCorrupt)?; - if ext_part.actual_size < 0 && (quota_context.is_enforced() || (!opts.replication_request && !opts.data_movement)) { + let unknown_actual_size_allowed = opts.data_movement && transformed_object && data_movement_actual_size.is_some() + || opts.replication_request && !quota_context.is_enforced(); + if ext_part.actual_size < 0 && !unknown_actual_size_allowed { return Err(Error::PartMissingOrCorrupt); } let normalized_actual_size = if ext_part.actual_size >= 0 && !transformed_object { @@ -2051,7 +2070,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { completed_part.actual_size = normalized_actual_size; fi.parts.push(completed_part); } - + if !transformed_object && data_movement_actual_size.is_some_and(|actual_size| actual_size < object_actual_size) { + return Err(Error::PartMissingOrCorrupt); + } if let Some(wtcs) = opts.want_checksum.as_ref() { if checksum_type.full_object_requested() { if wtcs.encoded != checksum.encoded { @@ -2097,7 +2118,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } else { None }; - let quota_new_size = match replication_actual_size { + let data_movement_quota_size = data_movement_actual_size_u64.filter(|_| quota_context.is_enforced()); + let quota_new_size = match data_movement_quota_size.or(replication_actual_size) { Some(size) => size.max(u64::try_from(object_size).map_err(|_| Error::PartMissingOrCorrupt)?), None if quota_context.is_enforced() => u64::try_from(object_actual_size) .map_err(|_| Error::PartMissingOrCorrupt)? @@ -2155,20 +2177,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { ); } - let data_movement_actual_size = if opts.data_movement { - rustfs_utils::http::get_consistent_str(&opts.user_defined, SUFFIX_ACTUAL_SIZE) - .map(|value| { - value - .parse::() - .ok() - .filter(|value| *value >= 0) - .ok_or_else(|| Error::other("data movement actual size metadata is invalid")) - }) - .transpose()? - } else { - None - }; - if let Some(actual_size) = data_movement_actual_size { insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); if persist_encryption_original_size { @@ -3109,12 +3117,34 @@ mod tests { content: &[u8], actual_size: i64, ) -> CompletePart { + put_test_part_with_opts( + set_disks, + bucket, + object, + upload_id, + part_number, + (content, actual_size), + &ObjectOptions::default(), + ) + .await + } + + async fn put_test_part_with_opts( + set_disks: &Arc, + bucket: &str, + object: &str, + upload_id: &str, + part_number: usize, + part: (&[u8], i64), + opts: &ObjectOptions, + ) -> CompletePart { + let (content, actual_size) = part; let mut reader = PutObjReader::new( HashReader::from_stream(Cursor::new(content.to_vec()), content.len() as i64, actual_size, None, None, false) .expect("hash reader should be constructed"), ); let part = set_disks - .put_object_part(bucket, object, upload_id, part_number, &mut reader, &ObjectOptions::default()) + .put_object_part(bucket, object, upload_id, part_number, &mut reader, opts) .await .expect("uploading the part should succeed"); CompletePart { @@ -3850,7 +3880,6 @@ mod tests { async fn data_movement_complete_accepts_unknown_compressed_part_actual_size() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; let bucket = "data-movement-unknown-actual-size-bucket"; - let object = "object"; make_bucket_on_all(&disk_stores, bucket).await; let mut metadata = HashMap::new(); @@ -3864,6 +3893,63 @@ mod tests { rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, "source-generation".to_string(), ); + + for (object, quota_limit) in [("without-quota", None), ("with-quota", Some(u64::MAX))] { + let create_opts = ObjectOptions { + data_movement: true, + user_defined: metadata.clone(), + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("data movement upload should be created"); + + let mut completed_parts = Vec::new(); + for (number, actual_size) in [(1, -1), (2, 1)] { + let mut reader = PutObjReader::new( + HashReader::from_stream(Cursor::new(vec![number as u8]), 1, actual_size, None, None, false) + .expect("part reader should be constructed"), + ); + let part = set_disks + .put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &create_opts) + .await + .expect("data movement part should be written"); + completed_parts.push(CompletePart { + part_num: number, + etag: part.etag, + ..Default::default() + }); + } + + let missing_size_err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts.clone(), &create_opts) + .await + .expect_err("data movement completion must require an authoritative total size"); + assert!(matches!(missing_size_err, StorageError::PartMissingOrCorrupt)); + + let mut complete_opts = create_opts.clone(); + rustfs_utils::http::insert_str( + &mut complete_opts.user_defined, + rustfs_utils::http::SUFFIX_ACTUAL_SIZE, + "2".to_string(), + ); + if let Some(quota_limit) = quota_limit { + assert!(complete_opts.set_quota_admission(0, quota_limit)); + } + let completion = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &complete_opts) + .await; + + let completed = completion.expect("data movement completion should accept the persisted unknown-size sentinel"); + + assert_eq!(completed.parts[0].actual_size, -1); + assert_eq!(completed.get_actual_size().expect("completed object actual size"), 2); + } + + let object = "all-unknown-with-quota"; let create_opts = ObjectOptions { data_movement: true, user_defined: metadata.clone(), @@ -3873,43 +3959,93 @@ mod tests { .new_multipart_upload(bucket, object, &create_opts) .await .expect("data movement upload should be created"); - - let mut completed_parts = Vec::new(); - for (number, actual_size) in [(1, -1), (2, 1)] { - let mut reader = PutObjReader::new( - HashReader::from_stream(Cursor::new(vec![number as u8]), 1, actual_size, None, None, false) - .expect("part reader should be constructed"), - ); - let part = set_disks - .put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &create_opts) - .await - .expect("data movement part should be written"); - completed_parts.push(CompletePart { - part_num: number, - etag: part.etag, - ..Default::default() - }); - } - - rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "2".to_string()); + let part = put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x40], -1), &create_opts).await; + let mut complete_opts = create_opts; + rustfs_utils::http::insert_str(&mut complete_opts.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "2".to_string()); + assert!(complete_opts.set_quota_admission(0, u64::MAX)); let completed = set_disks .clone() - .complete_multipart_upload( - bucket, - object, - &upload.upload_id, - completed_parts, - &ObjectOptions { - data_movement: true, - user_defined: metadata, - ..Default::default() - }, - ) + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) .await - .expect("data movement completion should accept the persisted unknown-size sentinel"); - + .expect("quota must accept an all-unknown data movement upload with an authoritative total"); assert_eq!(completed.parts[0].actual_size, -1); assert_eq!(completed.get_actual_size().expect("completed object actual size"), 2); + + let object = "legacy-zero-fallback"; + let create_opts = ObjectOptions { + data_movement: true, + user_defined: metadata.clone(), + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("data movement upload should be created"); + let part = + put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x41; 128], 128), &create_opts).await; + let mut complete_opts = create_opts; + rustfs_utils::http::insert_str( + &mut complete_opts.user_defined, + rustfs_utils::http::SUFFIX_ACTUAL_SIZE, + "100".to_string(), + ); + assert!(complete_opts.set_quota_admission(0, u64::MAX)); + let completed = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect("legacy zero-to-physical part fallback must remain migratable"); + assert_eq!(completed.parts[0].actual_size, 128); + assert_eq!(completed.get_actual_size().expect("completed object actual size"), 100); + + let object = "inconsistent-untransformed-total"; + let mut untransformed_metadata = metadata.clone(); + rustfs_utils::http::remove_str(&mut untransformed_metadata, rustfs_utils::http::SUFFIX_COMPRESSION); + let create_opts = ObjectOptions { + data_movement: true, + user_defined: untransformed_metadata, + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("data movement upload should be created"); + let part = put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x42; 2], 2), &create_opts).await; + let mut complete_opts = create_opts; + rustfs_utils::http::insert_str(&mut complete_opts.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1".to_string()); + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect_err("authoritative total below known logical part sizes must fail closed"); + assert!(matches!(err, StorageError::PartMissingOrCorrupt)); + + for (object, declared_size) in [("invalid-total", "invalid"), ("negative-total", "-1")] { + let create_opts = ObjectOptions { + data_movement: true, + user_defined: metadata.clone(), + ..Default::default() + }; + let upload = set_disks + .new_multipart_upload(bucket, object, &create_opts) + .await + .expect("data movement upload should be created"); + let part = + put_test_part_with_opts(&set_disks, bucket, object, &upload.upload_id, 1, (&[0x41], 1), &create_opts).await; + let mut complete_opts = create_opts.clone(); + rustfs_utils::http::insert_str( + &mut complete_opts.user_defined, + rustfs_utils::http::SUFFIX_ACTUAL_SIZE, + declared_size.to_string(), + ); + + let err = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts) + .await + .expect_err("invalid authoritative total size must fail closed"); + assert!(matches!(err, StorageError::PartMissingOrCorrupt)); + } } async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) { From 69719c257e648dedf2195b06bc41ff97e6baa121 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 14 Aug 2026 21:19:06 +0800 Subject: [PATCH 07/71] chore(ecstore): remove the pool-level ListObjects pagination copy (#6078) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(ecstore): remove the pool-level ListObjects pagination copy The ListObjects pagination pipeline existed in three near-copies in one file; production listing never reaches the Sets copy, which ECStore bypasses by expanding straight to per-set disks. This removes it: impl ListOperations for Sets (61 lines of pure forwarding in core/sets.rs) and the impl Sets pagination block (826 lines of inner_list_objects_v2 / list_objects_generic / inner_list_object_versions / list_path / list_merged / walk_internal in store/list_objects.rs). Two preconditions verified before deleting rather than taken on faith: the architecture guard pins only set_disks_implements_storage_list_operations_contract, so nothing requires the Sets trait impl; and the four Sets pagination methods had no cross-file caller besides that trait impl. The single test consumer moves to the surviving pipeline instead of being deleted: writes still go through the pool, and the listing assertion now targets the set-level implementation. It is renamed accordingly so the name still describes what it covers. The logging guardrail's TRACE-only requirement for Sets::list_objects_v2 retires in the same diff — the wrapper it pinned no longer exists. The ECStore and SetDisks entries are untouched. The SetDisks copy stays for now: its trait impl is guard-pinned, so replacing the duplicate pipeline behind it needs the generic helper the issue schedules for post-1.0. Verification: cargo nextest run -p rustfs-ecstore 4020 passed; check_architecture_migration_rules.sh and check_logging_guardrails.sh pass; clippy --lib --tests -D warnings clean; make pre-commit green. Ref rustfs/backlog#1821 (PR1). * chore(ecstore): fold the ListObjects forwarders into the ECStore impl store/list.rs held two thin forwarders, handle_list_objects_v2 and handle_list_object_versions, that only re-entered the inner_* implementations. The ListOperations impl now calls those directly and the file goes away. The logging guardrail's trace_hot_spans list pinned handle_list_objects_v2 as TRACE-only; that entry is retired in the same diff, adjacent to the sets.rs entry retired by the preceding commit. Ref rustfs/backlog#1821. * chore(ecstore): drop the type aliases orphaned by the pagination removal core/sets.rs declared four local type aliases — ListObjectsV2Info, ListObjectVersionsInfo, ObjectInfoOrErr and WalkOptions — used only by the pool-level pagination pipeline removed earlier in this branch. store/list_objects.rs keeps its own live copies of the same aliases. They only surface now that #6087 removed the core module's dead_code blanket: on that older base each PR was warning-free on its own, and the combination is what exposes them. Their storage_api_contracts imports go with them. Ref rustfs/backlog#1823, rustfs/backlog#1821. * fix(ecstore): preserve Sets listing compatibility --- crates/ecstore/src/store/list.rs | 81 ------------------- crates/ecstore/src/store/list_objects.rs | 2 +- crates/ecstore/src/store/mod.rs | 7 +- .../tests/ecstore_contract_compat_test.rs | 12 +++ scripts/check_logging_guardrails.sh | 4 +- 5 files changed, 19 insertions(+), 87 deletions(-) delete mode 100644 crates/ecstore/src/store/list.rs diff --git a/crates/ecstore/src/store/list.rs b/crates/ecstore/src/store/list.rs deleted file mode 100644 index 5f7f480b5..000000000 --- a/crates/ecstore/src/store/list.rs +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::*; - -impl ECStore { - #[instrument(level = "trace", skip(self))] - #[allow(clippy::too_many_arguments)] - pub(super) async fn handle_list_objects_v2( - self: Arc, - bucket: &str, - prefix: &str, - continuation_token: Option, - delimiter: Option, - max_keys: i32, - fetch_owner: bool, - start_after: Option, - incl_deleted: bool, - ) -> Result { - self.inner_list_objects_v2( - bucket, - prefix, - continuation_token, - delimiter, - max_keys, - fetch_owner, - start_after, - incl_deleted, - ) - .await - } - - #[instrument(skip(self))] - pub(super) async fn handle_list_object_versions( - self: Arc, - bucket: &str, - prefix: &str, - marker: Option, - version_marker: Option, - delimiter: Option, - max_keys: i32, - ) -> Result { - self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys) - .await - } - - pub(crate) async fn list_object_versions_for_lifecycle( - self: Arc, - bucket: &str, - prefix: &str, - marker: Option, - version_marker: Option, - delimiter: Option, - max_keys: i32, - ) -> Result { - self.inner_list_object_versions_for_lifecycle(bucket, prefix, marker, version_marker, delimiter, max_keys) - .await - } - - pub(super) async fn handle_walk( - self: Arc, - rx: CancellationToken, - bucket: &str, - prefix: &str, - result: tokio::sync::mpsc::Sender, - opts: WalkOptions, - ) -> Result<()> { - self.walk_internal(rx, bucket, prefix, result, opts).await - } -} diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 10fc766a2..558bb7c94 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -3845,7 +3845,7 @@ impl ECStore { .await } - pub(crate) async fn inner_list_object_versions_for_lifecycle( + pub(crate) async fn list_object_versions_for_lifecycle( self: Arc, bucket: &str, prefix: &str, diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 30a76bb69..f51fa6df5 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -148,7 +148,6 @@ mod heal_walk; pub use heal_walk::HealWalkVersion; mod init; pub(crate) mod init_format; -mod list; pub(crate) mod list_objects; mod multipart; mod object; @@ -601,7 +600,7 @@ impl crate::storage_api_contracts::list::ListOperations for ECStore { start_after: Option, incl_deleted: bool, ) -> Result { - self.handle_list_objects_v2( + self.inner_list_objects_v2( bucket, prefix, continuation_token, @@ -624,7 +623,7 @@ impl crate::storage_api_contracts::list::ListOperations for ECStore { delimiter: Option, max_keys: i32, ) -> Result { - self.handle_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys) + self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys) .await } @@ -636,7 +635,7 @@ impl crate::storage_api_contracts::list::ListOperations for ECStore { result: tokio::sync::mpsc::Sender, opts: WalkOptions, ) -> Result<()> { - self.handle_walk(rx, bucket, prefix, result, opts).await + self.walk_internal(rx, bucket, prefix, result, opts).await } } diff --git a/crates/ecstore/tests/ecstore_contract_compat_test.rs b/crates/ecstore/tests/ecstore_contract_compat_test.rs index 496eab291..d1d303f7a 100644 --- a/crates/ecstore/tests/ecstore_contract_compat_test.rs +++ b/crates/ecstore/tests/ecstore_contract_compat_test.rs @@ -157,6 +157,18 @@ fn ecstore_implements_storage_list_operations_contract() { assert!(storage_list_operations_type_name::().ends_with("::ECStore")); } +#[test] +fn ecstore_pools_expose_storage_list_operations_contract() { + fn assert_contract(store: &ECStore) { + let future = store.pools[0] + .clone() + .list_objects_v2("bucket", "", None, None, 1, false, None, false); + drop(future); + } + + let _ = assert_contract; +} + #[test] fn ecstore_implements_storage_multipart_operations_contract() { assert!(storage_multipart_operations_type_name::().ends_with("::ECStore")); diff --git a/scripts/check_logging_guardrails.sh b/scripts/check_logging_guardrails.sh index a5095d8b3..eb0640635 100755 --- a/scripts/check_logging_guardrails.sh +++ b/scripts/check_logging_guardrails.sh @@ -984,7 +984,9 @@ trace_hot_spans=( "crates/ecstore/src/store/object.rs:handle_get_object_info" "crates/ecstore/src/set_disk/ops/object.rs:get_object_info" "crates/ecstore/src/store/mod.rs:list_objects_v2" - "crates/ecstore/src/store/list.rs:handle_list_objects_v2" + # The ECStore handle_list_objects_v2 forwarder was folded into the trait impl + # above, so store/mod.rs now carries this hot path's TRACE requirement + # directly (backlog#1821). "crates/ecstore/src/core/sets.rs:list_objects_v2" "crates/ecstore/src/set_disk/ops/list.rs:list_objects_v2" "rustfs/src/app/bucket_usecase.rs:execute_list_objects_v2" From d91086d09466b5101af16173f92467ee3de6d213 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 14 Aug 2026 21:37:51 +0800 Subject: [PATCH 08/71] test(scanner): refresh metadata after fixture mutation (#6113) --- crates/scanner/src/scanner_io.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 6b0586ecf..2b234ac59 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -4262,6 +4262,7 @@ mod tests { .delete_bucket(&bucket, &DeleteBucketOptions::default()) .await .expect("bucket should be removed from the first pool only"); + init_bucket_metadata_sys_for_scanner_tests(store.clone()).await; let ctx = CancellationToken::new(); let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()); From d6c62b96014b568c35dce25c7956e2bc3947b44d Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 14 Aug 2026 21:56:11 +0800 Subject: [PATCH 09/71] chore(ecstore): drop the services dead_code blanket (#6103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing the blanket exposes twenty-five items across tier, notification and rebalance. Only eight are deleted — the lowest ratio of this burn-down so far, and the reason is that these subsystems carry heavy test coverage, so the blanket was mostly hiding test-only seams rather than dead weight. Deleted: - crates/ecstore/src/services/tier/warm_backend_s3sdk.rs entirely (200 lines). Its WarmBackendS3 is never constructed; the type of the same name in warm_backend_s3.rs is the live one, wrapped by the Azure backend. Two implementations of one S3 warm backend, one of them never wired. - TierConfigMgr::begin_publish_transition and publish_candidate_inner, thin wrappers whose _with_allowed_mutation_blocks siblings carry every real caller, plus retire_driver. - The GCS backend's MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PARTS_COUNT and MIN_PART_SIZE, and its write-only storage_class field. - mark_started_rebalance_pools_stopped and the RStats alias. Two deletions were withdrawn after a per-name grep, both because of an inference rather than a check: AsyncBatchProcessor::new was deleted on the strength of grepping only BATCH_PROCESSOR_OPERATION_CUSTOM, whose two hits are its definition and its use inside new. That looked like a self-contained dead pair; new in fact has seven test callers. The warning listed both items, and only one of them was actually checked. Deleting the two dead publish wrappers then revealed a second layer — publish_candidate_owned, remove_and_save_with, clear_and_save_with, save_tiering_config_if_current. These are not dead: publish_candidate, their caller, is #[cfg(test)], so a callee that lives in the main body has no caller in the lib build and a live one in the test build. rustc reports the roots of a dead subgraph, and the next layer down can have a different character, so each layer needs its own grep. Kept with allows: the tier mutation-intent record helpers (asserted by store::init tests), affected_targets, tier_object_blocks_target_rebind, the rebalance snapshot and retry-wait helpers, notification_sys's tier_config_reload_worker_active and call_peer_with_timeout, and active_operation_lease_count, whose only caller sits behind #[cfg(feature = "test-util")]. Also kept, with a module note rather than removal: the ecstore-side EventNotifier. All four of its methods are unreachable and init_bucket_targets logs that it is a no-op in this build; the working stack is rustfs-notify, whose own EventNotifier drives bucket configuration. Removing it means also retiring the InstanceContext slot that holds it (backlog#939 Phase 5), which belongs in its own PR. Worth a separate issue: MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PARTS_COUNT and MIN_PART_SIZE are declared independently in eight warm-backend files plus client/constants.rs. Only the GCS copies were dead; the other seven backends each use their own. Verification, four lanes warning-free: default, --tests, --features rio-v2 --tests, --features test-util --tests. cargo nextest run -p rustfs-ecstore 4041 passed; clippy --lib --tests -D warnings clean; make pre-commit exit 0. Ref rustfs/backlog#1823 (step 2). --- crates/ecstore/src/event/targetlist.rs | 4 + .../ecstore/src/services/batch_processor.rs | 5 + .../src/services/event_notification.rs | 15 ++ crates/ecstore/src/services/mod.rs | 1 - .../ecstore/src/services/notification_sys.rs | 2 + crates/ecstore/src/services/rebalance/meta.rs | 5 + .../src/services/rebalance/migration.rs | 1 + .../ecstore/src/services/rebalance/types.rs | 3 - crates/ecstore/src/services/tier/mod.rs | 1 - crates/ecstore/src/services/tier/tier.rs | 30 +-- .../src/services/tier/tier_mutation_intent.rs | 16 ++ .../src/services/tier/warm_backend_gcs.rs | 5 - .../src/services/tier/warm_backend_s3sdk.rs | 200 ------------------ 13 files changed, 58 insertions(+), 230 deletions(-) delete mode 100644 crates/ecstore/src/services/tier/warm_backend_s3sdk.rs diff --git a/crates/ecstore/src/event/targetlist.rs b/crates/ecstore/src/event/targetlist.rs index b63b786ef..96f4ea91f 100644 --- a/crates/ecstore/src/event/targetlist.rs +++ b/crates/ecstore/src/event/targetlist.rs @@ -20,6 +20,10 @@ use std::sync::atomic::AtomicI64; /// this type never grew past its counter. `total_events` is read by the /// notifier's log line but nothing increments it, so that field reports zero. #[derive(Default)] +#[allow( + dead_code, + reason = "held only by the dead ecstore EventNotifier; see services/event_notification.rs (backlog#1823)" +)] pub struct TargetList { pub total_events: AtomicI64, } diff --git a/crates/ecstore/src/services/batch_processor.rs b/crates/ecstore/src/services/batch_processor.rs index 61212ec00..006fd1f78 100644 --- a/crates/ecstore/src/services/batch_processor.rs +++ b/crates/ecstore/src/services/batch_processor.rs @@ -23,6 +23,10 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tokio::task::JoinSet; +#[allow( + dead_code, + reason = "default operation label for the test-only AsyncBatchProcessor::new (backlog#1823)" +)] const BATCH_PROCESSOR_OPERATION_CUSTOM: &str = "custom"; const BATCH_PROCESSOR_OPERATION_READ: &str = "read"; const BATCH_PROCESSOR_OPERATION_WRITE: &str = "write"; @@ -211,6 +215,7 @@ pub struct AsyncBatchProcessor { } impl AsyncBatchProcessor { + #[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")] pub fn new(max_concurrent: usize) -> Self { Self::new_with_operation(max_concurrent, BATCH_PROCESSOR_OPERATION_CUSTOM) } diff --git a/crates/ecstore/src/services/event_notification.rs b/crates/ecstore/src/services/event_notification.rs index a5a5c7acd..d9f5267be 100644 --- a/crates/ecstore/src/services/event_notification.rs +++ b/crates/ecstore/src/services/event_notification.rs @@ -26,11 +26,26 @@ use std::sync::atomic::Ordering; use tokio::sync::RwLock; use tracing::warn; +/// Dead ecstore-side notification skeleton. +/// +/// The working notification stack is `rustfs-notify`, whose own `EventNotifier` +/// is the one bucket configuration actually drives. Nothing calls the methods +/// below; `init_bucket_targets` even logs that it is a no-op in this build. +/// Removing it means also retiring the `InstanceContext` slot that holds it +/// (backlog#939 Phase 5), so it is left explicit here rather than half-removed. +#[allow( + dead_code, + reason = "ecstore-side notification skeleton superseded by rustfs-notify; see module note (backlog#1823)" +)] pub struct EventNotifier { target_list: TargetList, //bucket_rules_map: HashMap>, } +#[allow( + dead_code, + reason = "ecstore-side notification skeleton superseded by rustfs-notify; see module note (backlog#1823)" +)] impl EventNotifier { pub fn new() -> Arc> { Arc::new(RwLock::new(Self { diff --git a/crates/ecstore/src/services/mod.rs b/crates/ecstore/src/services/mod.rs index 303ceb94b..6335c441f 100644 --- a/crates/ecstore/src/services/mod.rs +++ b/crates/ecstore/src/services/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: background service owners still contain staged notification/rebalance/tier paths. -#![allow(dead_code)] pub(crate) mod batch_processor; pub(crate) mod event_notification; diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index 41fc15970..f3dd0a416 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -1623,6 +1623,7 @@ impl NotificationSys { workers.peers.remove(host); } + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn tier_config_reload_worker_active(&self, host: &str) -> bool { self.tier_config_reload_workers .lock() @@ -1796,6 +1797,7 @@ where .map_err(|_| Error::other(format!("scanner activity peer {host} timed out after {timeout_duration:?}")))? } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] async fn call_peer_with_timeout( timeout_dur: Duration, host_label: &str, diff --git a/crates/ecstore/src/services/rebalance/meta.rs b/crates/ecstore/src/services/rebalance/meta.rs index 39f2a2da1..2a1ea259b 100644 --- a/crates/ecstore/src/services/rebalance/meta.rs +++ b/crates/ecstore/src/services/rebalance/meta.rs @@ -864,6 +864,10 @@ pub(super) fn merge_rebalance_meta(remote: &mut RebalanceMeta, local: &Rebalance RebalanceMetaMergeOutcome::Merged } +#[allow( + dead_code, + reason = "stop-transition helper retained beside stop_rebalance_meta_snapshot; no caller yet (backlog#1823)" +)] pub(super) fn mark_started_rebalance_pools_stopped(meta: &mut RebalanceMeta, stop_time: OffsetDateTime) { for pool_stat in meta.pool_stats.iter_mut() { if pool_stat.info.status == RebalStatus::Started { @@ -964,6 +968,7 @@ pub(super) fn rollback_rebalance_start_meta_snapshot_for_id( }) } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(super) fn stop_rebalance_meta_snapshot(meta: Option<&mut RebalanceMeta>, now: OffsetDateTime) -> Option { let meta = meta?; stop_rebalance_state(meta, now); diff --git a/crates/ecstore/src/services/rebalance/migration.rs b/crates/ecstore/src/services/rebalance/migration.rs index 7e23c9ed1..c3e1e5e69 100644 --- a/crates/ecstore/src/services/rebalance/migration.rs +++ b/crates/ecstore/src/services/rebalance/migration.rs @@ -171,6 +171,7 @@ where } #[allow(clippy::too_many_arguments)] +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(super) async fn migrate_entry_version_with_retry_wait( set: &Backend, bucket: String, diff --git a/crates/ecstore/src/services/rebalance/types.rs b/crates/ecstore/src/services/rebalance/types.rs index c8c8378ea..b43e075b8 100644 --- a/crates/ecstore/src/services/rebalance/types.rs +++ b/crates/ecstore/src/services/rebalance/types.rs @@ -1,5 +1,4 @@ use serde::{Deserialize, Serialize}; -use std::sync::Arc; use time::OffsetDateTime; use tokio_util::sync::CancellationToken; @@ -32,8 +31,6 @@ pub struct RebalanceStats { pub cleanup_warnings: RebalanceCleanupWarnings, } -pub type RStats = Vec>; - #[derive(Debug, Default)] pub(super) struct RebalanceBucketConfigs { pub(super) bucket_incarnation_id: Option, diff --git a/crates/ecstore/src/services/tier/mod.rs b/crates/ecstore/src/services/tier/mod.rs index acc0bd93b..801c1e3c9 100644 --- a/crates/ecstore/src/services/tier/mod.rs +++ b/crates/ecstore/src/services/tier/mod.rs @@ -30,6 +30,5 @@ pub mod warm_backend_minio; pub mod warm_backend_r2; pub mod warm_backend_rustfs; pub mod warm_backend_s3; -pub mod warm_backend_s3sdk; pub mod warm_backend_tencent; pub mod warm_backend_wasabi; diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index f6c9114f2..581f1a5a4 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -488,6 +488,7 @@ impl TierCandidateMutation { targets } + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn affected_targets( &self, manager: &TierConfigMgr, @@ -802,6 +803,7 @@ fn tier_persisted_reference_blocks_any_target( .any(|target| tier_persisted_reference_blocks_target(tier_name, backend_identity, target)) } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn tier_object_blocks_target_rebind(object: &ObjectInfo, target: &TierMutationIntentTarget) -> io::Result { tier_object_blocks_any_target_rebind(object, std::slice::from_ref(target)) } @@ -2726,14 +2728,6 @@ impl TierConfigMgr { Self::publish_candidate_owned(handle, candidate, driver_tier.map(str::to_string), update).await } - fn begin_publish_transition( - handle: &Arc>, - manager: &mut Self, - candidate: &Self, - ) -> std::result::Result { - Self::begin_publish_transition_with_allowed_mutation_blocks(handle, manager, candidate, None) - } - fn begin_publish_transition_with_allowed_mutation_blocks( handle: &Arc>, manager: &mut Self, @@ -2819,14 +2813,6 @@ impl TierConfigMgr { }) } - async fn publish_candidate_inner( - handle: &Arc>, - candidate: Self, - driver_tier: Option<&str>, - ) -> std::result::Result<(), AdminError> { - Self::publish_candidate_inner_with_allowed_mutation_blocks(handle, candidate, driver_tier, None).await - } - async fn publish_candidate_inner_with_allowed_mutation_blocks( handle: &Arc>, candidate: Self, @@ -2939,6 +2925,7 @@ impl TierConfigMgr { admin_err } + #[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")] async fn publish_candidate_owned( handle: &Arc>, candidate: Self, @@ -3541,6 +3528,7 @@ impl TierConfigMgr { Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Remove(tier_name.to_string(), force)).await } + #[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")] async fn remove_and_save_with( handle: &Arc>, api: Arc, @@ -3574,6 +3562,7 @@ impl TierConfigMgr { Self::update_candidate_with_config_lock(handle, api, TierCandidateMutation::Clear(force)).await } + #[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")] async fn clear_and_save_with( handle: &Arc>, api: Arc, @@ -3612,6 +3601,10 @@ impl TierConfigMgr { } #[cfg(test)] + #[allow( + dead_code, + reason = "lease accounting asserted by a bucket_lifecycle_ops test behind `--features test-util` (backlog#1823)" + )] pub(crate) async fn active_operation_lease_count(handle: &Arc>, tier_name: &str) -> usize { let manager = handle.read().await; let Some(runtime) = registered_tier_driver_runtime(&manager) else { @@ -3717,10 +3710,6 @@ impl TierConfigMgr { Ok(()) } - fn retire_driver(&mut self, tier_name: &str) { - self.revoke_driver(tier_name); - } - fn revoke_all_drivers(&mut self) { if let Some(runtime) = registered_tier_driver_runtime(self) { let mut runtime = lock_unpoisoned(&runtime); @@ -3884,6 +3873,7 @@ impl TierConfigMgr { self.save_config(api, &config_file, data).await } + #[allow(dead_code, reason = "reached only through #[cfg(test)] helpers in this file (backlog#1823)")] async fn save_tiering_config_if_current( &self, api: Arc, diff --git a/crates/ecstore/src/services/tier/tier_mutation_intent.rs b/crates/ecstore/src/services/tier/tier_mutation_intent.rs index 28a303e32..94c940653 100644 --- a/crates/ecstore/src/services/tier/tier_mutation_intent.rs +++ b/crates/ecstore/src/services/tier/tier_mutation_intent.rs @@ -305,6 +305,10 @@ impl TierMutationIntent { } } +#[allow( + dead_code, + reason = "intent-record persistence asserted by store::init tests (backlog#1823)" +)] pub(crate) fn tier_mutation_intent_record_object_name(mutation_id: Uuid) -> Result { tier_mutation_intent_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id) } @@ -317,6 +321,10 @@ fn tier_mutation_intent_record_object_name_with_prefix(prefix: &str, mutation_id Ok(format!("{}/{}/{}/{}.json", prefix, &mutation_key[..2], &mutation_key[2..4], mutation_key)) } +#[allow( + dead_code, + reason = "intent-record persistence asserted by store::init tests (backlog#1823)" +)] pub(crate) fn tier_mutation_intent_id_from_record_object_name(object: &str) -> Result { tier_mutation_intent_id_from_record_object_name_with_prefix(TIER_MUTATION_INTENT_RECORD_PREFIX, object) } @@ -355,6 +363,10 @@ fn tier_mutation_intent_id_from_record_object_name_with_prefix(prefix: &str, obj Uuid::parse_str(mutation_key).map_err(|_| TierMutationIntentError::Corrupt("intent record path has invalid uuid")) } +#[allow( + dead_code, + reason = "intent-record persistence asserted by store::init tests (backlog#1823)" +)] pub(crate) async fn save_tier_mutation_intent_record(api: Arc, intent: &TierMutationIntent) -> EcstoreResult<()> where S: EcstoreObjectIO, @@ -446,6 +458,10 @@ where Ok((intent, etag)) } +#[allow( + dead_code, + reason = "intent-record persistence asserted by store::init tests (backlog#1823)" +)] pub(crate) async fn save_tier_mutation_intent_record_if_current( api: Arc, intent: &TierMutationIntent, diff --git a/crates/ecstore/src/services/tier/warm_backend_gcs.rs b/crates/ecstore/src/services/tier/warm_backend_gcs.rs index ef537002f..35aecfb6d 100644 --- a/crates/ecstore/src/services/tier/warm_backend_gcs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_gcs.rs @@ -41,10 +41,7 @@ use crate::services::tier::{ }; use tracing::warn; -const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; -const MAX_PARTS_COUNT: i64 = 10000; const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; -const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; fn parse_generation(remote_version: &str) -> Result, Error> { if remote_version.is_empty() { @@ -64,7 +61,6 @@ pub struct WarmBackendGCS { pub control: Arc, pub bucket: String, pub prefix: String, - pub storage_class: String, } impl WarmBackendGCS { @@ -104,7 +100,6 @@ impl WarmBackendGCS { control, bucket: conf.bucket.clone(), prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(), - storage_class: "".to_string(), }) } diff --git a/crates/ecstore/src/services/tier/warm_backend_s3sdk.rs b/crates/ecstore/src/services/tier/warm_backend_s3sdk.rs deleted file mode 100644 index fc1790e0a..000000000 --- a/crates/ecstore/src/services/tier/warm_backend_s3sdk.rs +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#![allow(unused_imports)] -#![allow(unused_variables)] -#![allow(unused_mut)] -#![allow(unused_assignments)] -#![allow(unused_must_use)] -#![allow(clippy::all)] - -use std::collections::HashMap; -use std::sync::Arc; -use url::Url; - -use aws_config::meta::region::RegionProviderChain; -use aws_sdk_s3::Client; -use aws_sdk_s3::config::{Credentials, Region}; -use aws_sdk_s3::primitives::ByteStream; - -use crate::client::{ - api_get_options::GetObjectOptions, - api_put_object::PutObjectOptions, - api_remove::RemoveObjectOptions, - transition_api::{ReadCloser, ReaderImpl}, -}; -use crate::error::ErrorResponse; -use crate::error::error_resp_to_object_err; -use crate::services::tier::{ - tier_config::TierS3, - warm_backend::{WarmBackend, WarmBackendGetOpts}, -}; - -pub struct WarmBackendS3 { - pub client: Arc, - pub bucket: String, - pub prefix: String, - pub storage_class: String, -} - -impl WarmBackendS3 { - pub async fn new(conf: &TierS3, tier: &str) -> Result { - let u = match Url::parse(&conf.endpoint) { - Ok(u) => u, - Err(err) => { - return Err(std::io::Error::other(err.to_string())); - } - }; - - if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != "" - || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == "" - { - return Err(std::io::Error::other("both the token file and the role ARN are required")); - } else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" { - return Err(std::io::Error::other("both the access and secret keys are required")); - } else if conf.aws_role - && (conf.aws_role_web_identity_token_file != "" - || conf.aws_role_arn != "" - || conf.access_key != "" - || conf.secret_key != "") - { - return Err(std::io::Error::other( - "AWS Role cannot be activated with static credentials or the web identity token file", - )); - } else if conf.bucket == "" { - return Err(std::io::Error::other("no bucket name was provided")); - } - - let creds; - if conf.access_key != "" && conf.secret_key != "" { - creds = Credentials::new( - conf.access_key.clone(), // access_key_id - conf.secret_key.clone(), // secret_access_key - None, // session_token (optional) - None, - "Static", - ); - } else { - return Err(std::io::Error::other("insufficient parameters for S3 backend authentication")); - } - let region_provider = RegionProviderChain::default_provider().or_else(Region::new(conf.region.clone())); - #[allow(deprecated)] - let config = aws_config::from_env() - .endpoint_url(conf.endpoint.clone()) - .region(region_provider) - .credentials_provider(creds) - .load() - .await; - let client = Client::new(&config); - let client = Arc::new(client); - Ok(Self { - client, - bucket: conf.bucket.clone(), - prefix: conf.prefix.clone().trim_matches('/').to_string(), - storage_class: conf.storage_class.clone(), - }) - } - - pub fn get_dest(&self, object: &str) -> String { - let mut dest_obj = object.to_string(); - if self.prefix != "" { - dest_obj = format!("{}/{}", &self.prefix, object); - } - return dest_obj; - } -} - -#[async_trait::async_trait] -impl WarmBackend for WarmBackendS3 { - async fn put_with_meta( - &self, - object: &str, - r: ReaderImpl, - length: i64, - meta: HashMap, - ) -> Result { - let client = self.client.clone(); - let Ok(res) = client - .put_object() - .bucket(&self.bucket) - .key(&self.get_dest(object)) - .body(match r { - ReaderImpl::Body(content_body) => ByteStream::from(content_body.to_vec()), - ReaderImpl::ObjectBody(mut content_body) => ByteStream::from(content_body.read_all().await?), - }) - .send() - .await - else { - return Err(std::io::Error::other("put_object error")); - }; - - Ok(res.version_id().unwrap_or("").to_string()) - } - - async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result { - self.put_with_meta(object, r, length, HashMap::new()).await - } - - async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { - let client = self.client.clone(); - let mut req = client.get_object().bucket(&self.bucket).key(&self.get_dest(object)); - - if !rv.is_empty() { - req = req.version_id(rv); - } - - if opts.start_offset >= 0 && opts.length > 0 { - let end = opts - .start_offset - .checked_add(opts.length) - .and_then(|v| v.checked_sub(1)) - .ok_or_else(|| std::io::Error::other("invalid range: overflow"))?; - req = req.range(format!("bytes={}-{}", opts.start_offset, end)); - } - - let res = req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?; - - Ok(ReadCloser::new(std::io::Cursor::new( - res.body.collect().await.map(|data| data.into_bytes().to_vec())?, - ))) - } - - async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> { - let client = self.client.clone(); - let mut req = client.delete_object().bucket(&self.bucket).key(&self.get_dest(object)); - - if !rv.is_empty() { - req = req.version_id(rv); - } - - req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?; - - Ok(()) - } - - async fn in_use(&self) -> Result { - let client = self.client.clone(); - let Ok(res) = client - .list_objects_v2() - .bucket(&self.bucket) - //.max_keys(10) - //.into_paginator() - .send() - .await - else { - return Err(std::io::Error::other("list_objects_v2 error")); - }; - - Ok(res.common_prefixes.unwrap_or_default().len() > 0 || res.contents.unwrap_or_default().len() > 0) - } -} From 4421d4829fcdfe767d9267130b6518bce2f066a6 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 14 Aug 2026 21:56:20 +0800 Subject: [PATCH 10/71] test(table-catalog): share the store test doubles and fold three commit-rejection cases (#6076) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes PR3 of the issue. Store doubles: NoopTableCatalogStore (193 lines, a pure stub answering "nothing here") and TestTableCatalogStore (416 lines, a stateful fake with commit pauses and failure injection) move into test_support.rs, along with TestCatalogPublishPause which the latter needs. Per the issue's ruling both shapes are kept — they are different tools, not duplicates of each other. Being honest about the benefit: this does not reduce the number of TableCatalogStore implementations, it puts both in one file so a trait change is one file to edit instead of two. row_level_conflict fold: rejects_stale_new_manifest_sequence, rejects_stale_added_entry_sequence, and rejects_historical_change_in_new_manifest were identical apart from four literals (manifest-list sequence, data-file name, manifest-entry snapshot id, failure message). They become one table-driven test with three rows, each row keeping its original values, and every assertion carries the case name. Verification: cargo test -p rustfs --lib table_catalog 479 passed and --lib admin::handlers::table_catalog 165 passed (both down exactly 2 from the 3->1 fold; the store filter is a substring match that also covers the admin tests); clippy --lib --tests -D warnings clean; make pre-commit green. Ref rustfs/backlog#1837 (PR3). --- .../src/admin/handlers/table_catalog/tests.rs | 689 +++--------------- rustfs/src/table_catalog/test_support.rs | 639 +++++++++++++++- rustfs/src/table_catalog/tests.rs | 198 +---- 3 files changed, 724 insertions(+), 802 deletions(-) diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index f1d26f1ae..454d12bad 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -12,8 +12,8 @@ use datafusion::{ use std::sync::Arc; use crate::table_catalog::test_support::{ - TestCatalogObjectBackend as TestTableCatalogObjectBackend, TestCatalogObjectRecord, - manifest_avro_bytes as test_manifest_avro_bytes, + TestCatalogObjectBackend as TestTableCatalogObjectBackend, TestCatalogObjectRecord, TestCatalogPublishPause, + TestTableCatalogStore, manifest_avro_bytes as test_manifest_avro_bytes, manifest_avro_bytes_with_nullable_sequences as test_manifest_avro_bytes_with_nullable_sequences, manifest_list_avro_bytes as test_manifest_list_avro_bytes, manifest_list_avro_entries as test_manifest_list_avro_entries, table_metadata_json as test_table_metadata_json, @@ -6328,181 +6328,96 @@ async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { assert_eq!(unchanged.generation, current.generation); } +/// Table-driven fold of the three commit-rejection cases whose bodies were +/// identical apart from four literals (backlog#1837 PR3). Each row keeps its +/// original manifest-list sequence, data-file name, manifest-entry snapshot +/// id, and failure message, so no poison combination is lost. #[tokio::test] -async fn row_level_conflict_rejects_stale_new_manifest_sequence() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let manifest = format!("{table_location}/metadata/manifest-11.avro"); - let data_file = format!("{table_location}/data/part-11.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 1, 11).await; - seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 11, 1)]).await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": manifest_list, - "summary": { - "operation": "append" +async fn row_level_conflict_rejects_stale_or_historical_manifest_sequences() { + // (case, manifest-list sequence, data-file suffix, manifest-entry snapshot id, expected failure) + let cases: &[(&str, i64, &str, i64, &str)] = &[ + ( + "stale-new-manifest-sequence", + 1, + "11", + 11, + "new manifest sequence must match the committed snapshot", + ), + ( + "stale-added-entry-sequence", + 2, + "11", + 11, + "added file sequence must match the new manifest", + ), + ( + "historical-change-in-new-manifest", + 2, + "10", + 10, + "new manifest must not claim a historical changed entry", + ), + ]; + + for (case, manifest_list_sequence, data_file_suffix, entry_snapshot_id, failure) in cases { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let manifest_list = format!("{table_location}/metadata/snap-11.avro"); + let manifest = format!("{table_location}/metadata/manifest-11.avro"); + let data_file = format!("{table_location}/data/part-{data_file_suffix}.parquet"); + seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], *manifest_list_sequence, 11).await; + seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, *entry_snapshot_id, 1)]).await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": manifest_list, + "summary": { + "operation": "append" + } } } - } - ] - })) - .expect("append request should parse"); + ] + })) + .expect("append request should parse"); - let error = commit_table_response( - &store, - &trusted_table_commit_backend(&metadata_backend), - "warehouse", - &namespace, - "events", - append_request, - ) - .await - .expect_err("new manifest sequence must match the committed snapshot"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - let unchanged = store - .load_table("warehouse", "analytics", "events") + let Err(error) = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, current.metadata_location); - assert_eq!(unchanged.version_token, current.version_token); - assert_eq!(unchanged.generation, current.generation); -} + else { + panic!("[{case}] {failure}"); + }; -#[tokio::test] -async fn row_level_conflict_rejects_stale_added_entry_sequence() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let manifest = format!("{table_location}/metadata/manifest-11.avro"); - let data_file = format!("{table_location}/data/part-11.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 2, 11).await; - seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 11, 1)]).await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - } - ] - })) - .expect("append request should parse"); - - let error = commit_table_response( - &store, - &trusted_table_commit_backend(&metadata_backend), - "warehouse", - &namespace, - "events", - append_request, - ) - .await - .expect_err("added file sequence must match the new manifest"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, current.metadata_location); - assert_eq!(unchanged.version_token, current.version_token); - assert_eq!(unchanged.generation, current.generation); -} - -#[tokio::test] -async fn row_level_conflict_rejects_historical_change_in_new_manifest() { - let store = TestTableCatalogStore::default(); - let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); - let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); - let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; - let table_location = created.metadata["location"] - .as_str() - .expect("created metadata should have table location"); - let current = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should exist"); - let manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let manifest = format!("{table_location}/metadata/manifest-11.avro"); - let data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 2, 11).await; - seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, 1)]).await; - let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": manifest_list, - "summary": { - "operation": "append" - } - } - } - ] - })) - .expect("append request should parse"); - - let error = commit_table_response( - &store, - &trusted_table_commit_backend(&metadata_backend), - "warehouse", - &namespace, - "events", - append_request, - ) - .await - .expect_err("new manifest must not claim a historical changed entry"); - - assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); - let unchanged = store - .load_table("warehouse", "analytics", "events") - .await - .expect("table lookup should succeed") - .expect("table should still exist"); - assert_eq!(unchanged.metadata_location, current.metadata_location); - assert_eq!(unchanged.version_token, current.version_token); - assert_eq!(unchanged.generation, current.generation); + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest, "[{case}] {failure}"); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, current.metadata_location, "[{case}] {failure}"); + assert_eq!(unchanged.version_token, current.version_token, "[{case}] {failure}"); + assert_eq!(unchanged.generation, current.generation, "[{case}] {failure}"); + } } #[tokio::test] @@ -7922,34 +7837,6 @@ fn commit_table_request_uses_rest_commit_fields() { assert_eq!(request.writer.as_deref(), Some("pyiceberg")); } -#[derive(Default)] -struct TestTableCatalogStore { - table_buckets: tokio::sync::Mutex>, - namespaces: tokio::sync::Mutex>, - tables: tokio::sync::Mutex>, - views: tokio::sync::Mutex>, - commits: tokio::sync::Mutex>, - fail_put_table_bucket: tokio::sync::Mutex, - register_table_pause: Option, - commit_table_pause: Option, -} - -#[derive(Clone, Default)] -struct TestCatalogPublishPause { - started: Arc, - release: Arc, -} - -impl TestCatalogPublishPause { - async fn wait_started(&self) { - self.started.notified().await; - } - - fn release(&self) { - self.release.notify_one(); - } -} - fn trusted_table_commit_backend( backend: &TestTableCatalogObjectBackend, ) -> TableCommitObjectBackend { @@ -8318,410 +8205,6 @@ async fn seed_object_table_for_metadata_maintenance( .await; } -#[async_trait::async_trait] -impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { - async fn get_table_bucket( - &self, - table_bucket: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .table_buckets - .lock() - .await - .iter() - .find(|entry| entry.table_bucket == table_bucket) - .cloned()) - } - - async fn put_table_bucket( - &self, - entry: crate::table_catalog::TableBucketEntry, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - let mut fail_put_table_bucket = self.fail_put_table_bucket.lock().await; - if *fail_put_table_bucket { - *fail_put_table_bucket = false; - return Err(crate::table_catalog::TableCatalogStoreError::Internal( - "injected table bucket write failure".to_string(), - )); - } - drop(fail_put_table_bucket); - - let mut table_buckets = self.table_buckets.lock().await; - table_buckets.retain(|existing| existing.table_bucket != entry.table_bucket); - table_buckets.push(entry); - Ok(()) - } - - async fn create_namespace( - &self, - entry: crate::table_catalog::NamespaceEntry, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - if self.get_table_bucket(&entry.table_bucket).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "table bucket {}", - entry.table_bucket - ))); - } - self.namespaces.lock().await.push(entry); - Ok(()) - } - - async fn list_namespaces( - &self, - table_bucket: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .namespaces - .lock() - .await - .iter() - .filter(|entry| entry.table_bucket == table_bucket) - .cloned() - .collect()) - } - - async fn get_namespace( - &self, - table_bucket: &str, - namespace: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .namespaces - .lock() - .await - .iter() - .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) - .cloned()) - } - - async fn update_namespace_properties( - &self, - table_bucket: &str, - namespace: &str, - update: crate::table_catalog::NamespacePropertiesUpdate, - ) -> crate::table_catalog::TableCatalogStoreResult { - let mut namespaces = self.namespaces.lock().await; - let entry = namespaces - .iter_mut() - .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) - .ok_or_else(|| { - crate::table_catalog::TableCatalogStoreError::NotFound(format!("namespace {table_bucket}/{namespace}")) - })?; - Ok(update.apply_to(entry)) - } - - async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { - self.namespaces - .lock() - .await - .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace)); - Ok(()) - } - - async fn create_table(&self, entry: crate::table_catalog::TableEntry) -> crate::table_catalog::TableCatalogStoreResult<()> { - if self.get_table_bucket(&entry.table_bucket).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "table bucket {}", - entry.table_bucket - ))); - } - if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - self.tables.lock().await.push(entry); - Ok(()) - } - - async fn register_table(&self, entry: crate::table_catalog::TableEntry) -> crate::table_catalog::TableCatalogStoreResult<()> { - if self.get_table_bucket(&entry.table_bucket).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "table bucket {}", - entry.table_bucket - ))); - } - if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - if let Some(pause) = &self.register_table_pause { - pause.started.notify_one(); - pause.release.notified().await; - } - self.tables.lock().await.push(entry); - Ok(()) - } - - async fn register_table_with_publication( - &self, - entry: crate::table_catalog::TableEntry, - publication: &(dyn crate::table_catalog::TableCommitPublication + Sync), - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - publication.begin_table_bucket(&entry.table_bucket).await?; - if !publication.holds_table_bucket(&entry.table_bucket) { - return Err(crate::table_catalog::TableCatalogStoreError::Internal( - "table registration requires a table-bucket publication fence".to_string(), - )); - } - let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication); - publication - .prepare(&entry.table_bucket, &entry.namespace, &entry.table) - .await?; - if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) { - return Err(crate::table_catalog::TableCatalogStoreError::Internal( - "table registration requires a table publication fence".to_string(), - )); - } - self.register_table(entry).await - } - - async fn list_tables( - &self, - table_bucket: &str, - namespace: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .tables - .lock() - .await - .iter() - .filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) - .cloned() - .collect()) - } - - async fn list_all_tables( - &self, - table_bucket: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .tables - .lock() - .await - .iter() - .filter(|entry| entry.table_bucket == table_bucket) - .cloned() - .collect()) - } - - async fn load_table( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .tables - .lock() - .await - .iter() - .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table) - .cloned()) - } - - async fn commit_table( - &self, - request: crate::table_catalog::TableCommitRequest, - ) -> crate::table_catalog::TableCatalogStoreResult { - let mut tables = self.tables.lock().await; - let Some(index) = tables.iter().position(|entry| { - entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.table == request.table - }) else { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "table {}/{}/{}", - request.table_bucket, request.namespace, request.table - ))); - }; - - let current = tables[index].clone(); - if current.version_token != request.expected_version_token { - return Err(crate::table_catalog::TableCatalogStoreError::Conflict( - "current table version token does not match expected token".to_string(), - )); - } - if current.metadata_location != request.expected_metadata_location { - return Err(crate::table_catalog::TableCatalogStoreError::Conflict( - "current table metadata location does not match expected location".to_string(), - )); - } - if let Some(pause) = &self.commit_table_pause { - pause.started.notify_one(); - pause.release.notified().await; - } - - let mut next = current.clone(); - next.metadata_location = request.new_metadata_location.clone(); - next.version_token = "token-committed".to_string(); - next.generation = next.generation.saturating_add(1); - tables[index] = next.clone(); - drop(tables); - - let commit_log = crate::table_catalog::CommitLogEntry { - version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, - commit_id: request.commit_id, - idempotency_key: request.idempotency_key, - table_id: current.table_id, - operation: request.operation, - expected_version_token: request.expected_version_token, - new_version_token: next.version_token.clone(), - previous_metadata_location: request.expected_metadata_location, - new_metadata_location: request.new_metadata_location, - requirements: request.requirements, - status: crate::table_catalog::CommitLogStatus::Committed, - writer: request.writer, - created_at: None, - updated_at: None, - }; - self.commits.lock().await.push(commit_log.clone()); - - Ok(crate::table_catalog::TableCommitResult { table: next, commit_log }) - } - - async fn commit_table_with_publication( - &self, - request: crate::table_catalog::TableCommitRequest, - publication: &(dyn crate::table_catalog::TableCommitPublication + Sync), - ) -> crate::table_catalog::TableCatalogStoreResult { - publication - .prepare(&request.table_bucket, &request.namespace, &request.table) - .await?; - if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) { - return Err(crate::table_catalog::TableCatalogStoreError::Internal( - "table commit requires a table publication fence".to_string(), - )); - } - let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication); - self.commit_table(request).await - } - - async fn drop_table( - &self, - table_bucket: &str, - namespace: &str, - table: &str, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - self.tables - .lock() - .await - .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table)); - Ok(()) - } - - async fn create_view(&self, entry: crate::table_catalog::ViewEntry) -> crate::table_catalog::TableCatalogStoreResult<()> { - if self.get_table_bucket(&entry.table_bucket).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "table bucket {}", - entry.table_bucket - ))); - } - if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "namespace {}/{}", - entry.table_bucket, entry.namespace - ))); - } - self.views.lock().await.push(entry); - Ok(()) - } - - async fn list_views( - &self, - table_bucket: &str, - namespace: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .views - .lock() - .await - .iter() - .filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) - .cloned() - .collect()) - } - - async fn load_view( - &self, - table_bucket: &str, - namespace: &str, - view: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(self - .views - .lock() - .await - .iter() - .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view) - .cloned()) - } - - async fn replace_view( - &self, - request: crate::table_catalog::ViewCommitRequest, - ) -> crate::table_catalog::TableCatalogStoreResult { - let mut views = self.views.lock().await; - let Some(index) = views.iter().position(|entry| { - entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.view == request.view - }) else { - return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( - "view {}/{}/{}", - request.table_bucket, request.namespace, request.view - ))); - }; - let current = views[index].clone(); - if current.version_token != request.expected_version_token { - return Err(crate::table_catalog::TableCatalogStoreError::Conflict( - "current view version token does not match expected token".to_string(), - )); - } - if current.metadata_location != request.expected_metadata_location { - return Err(crate::table_catalog::TableCatalogStoreError::Conflict( - "current view metadata location does not match expected location".to_string(), - )); - } - let mut next = current; - next.metadata_location = request.new_metadata_location; - next.version_token = "token-view-committed".to_string(); - next.generation = next.generation.saturating_add(1); - views[index] = next.clone(); - Ok(crate::table_catalog::ViewCommitResult { view: next }) - } - - async fn drop_view( - &self, - table_bucket: &str, - namespace: &str, - view: &str, - ) -> crate::table_catalog::TableCatalogStoreResult<()> { - self.views - .lock() - .await - .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view)); - Ok(()) - } - - async fn get_commit_by_id( - &self, - _table_bucket: &str, - _table_id: &str, - _commit_id: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(None) - } - - async fn get_commit_by_idempotency_key( - &self, - _table_bucket: &str, - _table_id: &str, - _idempotency_key: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(None) - } -} - #[tokio::test] async fn ensure_table_bucket_entry_seeds_enabled_bucket_before_namespace_create() { let store = TestTableCatalogStore::default(); diff --git a/rustfs/src/table_catalog/test_support.rs b/rustfs/src/table_catalog/test_support.rs index 517eb8923..3dac79309 100644 --- a/rustfs/src/table_catalog/test_support.rs +++ b/rustfs/src/table_catalog/test_support.rs @@ -26,10 +26,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use time::OffsetDateTime; -use super::{ - StrongTableCatalogRuntime, TableCatalogObject, TableCatalogObjectBackend, TableCatalogObjectMetadata, - TableCatalogPutPrecondition, TableCatalogStoreError, TableCatalogStoreResult, TableCommitPublication, -}; +use super::*; pub(crate) fn table_metadata_json(table_uuid: &str, location: &str) -> serde_json::Value { serde_json::json!({ @@ -939,3 +936,637 @@ impl TestCatalogObjectBackend { .expect("lock acquisition attempts should be observable"); } } + +#[derive(Clone, Default)] +pub(crate) struct TestCatalogPublishPause { + started: Arc, + release: Arc, +} + +impl TestCatalogPublishPause { + pub(crate) async fn wait_started(&self) { + self.started.notified().await; + } + + pub(crate) fn release(&self) { + self.release.notify_one(); + } +} + +// --- TableCatalogStore test doubles (backlog#1837 PR3) --- +// +// Two deliberately different shapes, per the issue's ruling: NoopTableCatalogStore +// is a pure stub whose methods answer "nothing here", used where a store must +// exist but never matter; TestTableCatalogStore is a stateful fake with commit +// pauses and failure injection. Both live here so a TableCatalogStore trait +// change is one file to update instead of two. + +pub(crate) struct NoopTableCatalogStore; + +#[async_trait::async_trait] +impl TableCatalogStore for NoopTableCatalogStore { + async fn get_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult> { + Ok(None) + } + + async fn put_table_bucket(&self, _entry: TableBucketEntry) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn create_namespace(&self, _entry: NamespaceEntry) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn list_namespaces(&self, _table_bucket: &str) -> TableCatalogStoreResult> { + Ok(Vec::new()) + } + + async fn get_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { + Ok(None) + } + + async fn drop_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn create_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn register_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn register_table_with_publication( + &self, + entry: TableEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "table registration requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.table) + .await?; + if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) { + return Err(TableCatalogStoreError::Internal( + "table registration requires a table publication fence".to_string(), + )); + } + self.register_table(entry).await + } + + async fn list_tables(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { + Ok(Vec::new()) + } + + async fn list_all_tables(&self, _table_bucket: &str) -> TableCatalogStoreResult> { + Ok(Vec::new()) + } + + async fn load_table( + &self, + _table_bucket: &str, + _namespace: &str, + _table: &str, + ) -> TableCatalogStoreResult> { + Ok(None) + } + + async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { + let table = TableEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: request.table_bucket, + namespace: request.namespace, + table: request.table, + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://analytics/tables/table-id".to_string(), + metadata_location: request.new_metadata_location.clone(), + version_token: "token-v2".to_string(), + generation: 2, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + let commit_log = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: request.commit_id, + idempotency_key: request.idempotency_key, + table_id: table.table_id.clone(), + operation: request.operation, + expected_version_token: request.expected_version_token, + new_version_token: table.version_token.clone(), + previous_metadata_location: request.expected_metadata_location, + new_metadata_location: table.metadata_location.clone(), + requirements: request.requirements, + status: CommitLogStatus::Committed, + writer: request.writer, + created_at: None, + updated_at: None, + }; + + Ok(TableCommitResult { table, commit_log }) + } + + async fn commit_table_with_publication( + &self, + request: TableCommitRequest, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { + publication + .prepare(&request.table_bucket, &request.namespace, &request.table) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) { + return Err(TableCatalogStoreError::Internal( + "table commit requires a table publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + self.commit_table(request).await + } + + async fn drop_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn create_view(&self, _entry: ViewEntry) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn list_views(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { + Ok(Vec::new()) + } + + async fn load_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult> { + Ok(None) + } + + async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { + Ok(ViewCommitResult { + view: ViewEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: request.table_bucket, + namespace: request.namespace, + view: request.view, + view_id: "view-id".to_string(), + view_uuid: "view-uuid".to_string(), + format: "ICEBERG_VIEW".to_string(), + format_version: 1, + warehouse_location: "s3://analytics/views/view-id".to_string(), + metadata_location: request.new_metadata_location, + version_token: "token-v2".to_string(), + generation: 2, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }, + }) + } + + async fn drop_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn get_commit_by_id( + &self, + _table_bucket: &str, + _table_id: &str, + _commit_id: &str, + ) -> TableCatalogStoreResult> { + Ok(None) + } + + async fn get_commit_by_idempotency_key( + &self, + _table_bucket: &str, + _table_id: &str, + _idempotency_key: &str, + ) -> TableCatalogStoreResult> { + Ok(None) + } +} + +#[derive(Default)] +pub(crate) struct TestTableCatalogStore { + pub(crate) table_buckets: tokio::sync::Mutex>, + pub(crate) namespaces: tokio::sync::Mutex>, + pub(crate) tables: tokio::sync::Mutex>, + pub(crate) views: tokio::sync::Mutex>, + pub(crate) commits: tokio::sync::Mutex>, + pub(crate) fail_put_table_bucket: tokio::sync::Mutex, + pub(crate) register_table_pause: Option, + pub(crate) commit_table_pause: Option, +} + +#[async_trait::async_trait] +impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { + async fn get_table_bucket( + &self, + table_bucket: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .table_buckets + .lock() + .await + .iter() + .find(|entry| entry.table_bucket == table_bucket) + .cloned()) + } + + async fn put_table_bucket( + &self, + entry: crate::table_catalog::TableBucketEntry, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + let mut fail_put_table_bucket = self.fail_put_table_bucket.lock().await; + if *fail_put_table_bucket { + *fail_put_table_bucket = false; + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "injected table bucket write failure".to_string(), + )); + } + drop(fail_put_table_bucket); + + let mut table_buckets = self.table_buckets.lock().await; + table_buckets.retain(|existing| existing.table_bucket != entry.table_bucket); + table_buckets.push(entry); + Ok(()) + } + + async fn create_namespace( + &self, + entry: crate::table_catalog::NamespaceEntry, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + if self.get_table_bucket(&entry.table_bucket).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "table bucket {}", + entry.table_bucket + ))); + } + self.namespaces.lock().await.push(entry); + Ok(()) + } + + async fn list_namespaces( + &self, + table_bucket: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .namespaces + .lock() + .await + .iter() + .filter(|entry| entry.table_bucket == table_bucket) + .cloned() + .collect()) + } + + async fn get_namespace( + &self, + table_bucket: &str, + namespace: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .namespaces + .lock() + .await + .iter() + .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) + .cloned()) + } + + async fn update_namespace_properties( + &self, + table_bucket: &str, + namespace: &str, + update: crate::table_catalog::NamespacePropertiesUpdate, + ) -> crate::table_catalog::TableCatalogStoreResult { + let mut namespaces = self.namespaces.lock().await; + let entry = namespaces + .iter_mut() + .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) + .ok_or_else(|| { + crate::table_catalog::TableCatalogStoreError::NotFound(format!("namespace {table_bucket}/{namespace}")) + })?; + Ok(update.apply_to(entry)) + } + + async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.namespaces + .lock() + .await + .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace)); + Ok(()) + } + + async fn create_table(&self, entry: crate::table_catalog::TableEntry) -> crate::table_catalog::TableCatalogStoreResult<()> { + if self.get_table_bucket(&entry.table_bucket).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "table bucket {}", + entry.table_bucket + ))); + } + if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + self.tables.lock().await.push(entry); + Ok(()) + } + + async fn register_table(&self, entry: crate::table_catalog::TableEntry) -> crate::table_catalog::TableCatalogStoreResult<()> { + if self.get_table_bucket(&entry.table_bucket).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "table bucket {}", + entry.table_bucket + ))); + } + if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + if let Some(pause) = &self.register_table_pause { + pause.started.notify_one(); + pause.release.notified().await; + } + self.tables.lock().await.push(entry); + Ok(()) + } + + async fn register_table_with_publication( + &self, + entry: crate::table_catalog::TableEntry, + publication: &(dyn crate::table_catalog::TableCommitPublication + Sync), + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table registration requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication); + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.table) + .await?; + if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table registration requires a table publication fence".to_string(), + )); + } + self.register_table(entry).await + } + + async fn list_tables( + &self, + table_bucket: &str, + namespace: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .tables + .lock() + .await + .iter() + .filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) + .cloned() + .collect()) + } + + async fn list_all_tables( + &self, + table_bucket: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .tables + .lock() + .await + .iter() + .filter(|entry| entry.table_bucket == table_bucket) + .cloned() + .collect()) + } + + async fn load_table( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .tables + .lock() + .await + .iter() + .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table) + .cloned()) + } + + async fn commit_table( + &self, + request: crate::table_catalog::TableCommitRequest, + ) -> crate::table_catalog::TableCatalogStoreResult { + let mut tables = self.tables.lock().await; + let Some(index) = tables.iter().position(|entry| { + entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.table == request.table + }) else { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + request.table_bucket, request.namespace, request.table + ))); + }; + + let current = tables[index].clone(); + if current.version_token != request.expected_version_token { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict( + "current table version token does not match expected token".to_string(), + )); + } + if current.metadata_location != request.expected_metadata_location { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict( + "current table metadata location does not match expected location".to_string(), + )); + } + if let Some(pause) = &self.commit_table_pause { + pause.started.notify_one(); + pause.release.notified().await; + } + + let mut next = current.clone(); + next.metadata_location = request.new_metadata_location.clone(); + next.version_token = "token-committed".to_string(); + next.generation = next.generation.saturating_add(1); + tables[index] = next.clone(); + drop(tables); + + let commit_log = crate::table_catalog::CommitLogEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + commit_id: request.commit_id, + idempotency_key: request.idempotency_key, + table_id: current.table_id, + operation: request.operation, + expected_version_token: request.expected_version_token, + new_version_token: next.version_token.clone(), + previous_metadata_location: request.expected_metadata_location, + new_metadata_location: request.new_metadata_location, + requirements: request.requirements, + status: crate::table_catalog::CommitLogStatus::Committed, + writer: request.writer, + created_at: None, + updated_at: None, + }; + self.commits.lock().await.push(commit_log.clone()); + + Ok(crate::table_catalog::TableCommitResult { table: next, commit_log }) + } + + async fn commit_table_with_publication( + &self, + request: crate::table_catalog::TableCommitRequest, + publication: &(dyn crate::table_catalog::TableCommitPublication + Sync), + ) -> crate::table_catalog::TableCatalogStoreResult { + publication + .prepare(&request.table_bucket, &request.namespace, &request.table) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table commit requires a table publication fence".to_string(), + )); + } + let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication); + self.commit_table(request).await + } + + async fn drop_table( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.tables + .lock() + .await + .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.table == table)); + Ok(()) + } + + async fn create_view(&self, entry: crate::table_catalog::ViewEntry) -> crate::table_catalog::TableCatalogStoreResult<()> { + if self.get_table_bucket(&entry.table_bucket).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "table bucket {}", + entry.table_bucket + ))); + } + if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "namespace {}/{}", + entry.table_bucket, entry.namespace + ))); + } + self.views.lock().await.push(entry); + Ok(()) + } + + async fn list_views( + &self, + table_bucket: &str, + namespace: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .views + .lock() + .await + .iter() + .filter(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace) + .cloned() + .collect()) + } + + async fn load_view( + &self, + table_bucket: &str, + namespace: &str, + view: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(self + .views + .lock() + .await + .iter() + .find(|entry| entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view) + .cloned()) + } + + async fn replace_view( + &self, + request: crate::table_catalog::ViewCommitRequest, + ) -> crate::table_catalog::TableCatalogStoreResult { + let mut views = self.views.lock().await; + let Some(index) = views.iter().position(|entry| { + entry.table_bucket == request.table_bucket && entry.namespace == request.namespace && entry.view == request.view + }) else { + return Err(crate::table_catalog::TableCatalogStoreError::NotFound(format!( + "view {}/{}/{}", + request.table_bucket, request.namespace, request.view + ))); + }; + let current = views[index].clone(); + if current.version_token != request.expected_version_token { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict( + "current view version token does not match expected token".to_string(), + )); + } + if current.metadata_location != request.expected_metadata_location { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict( + "current view metadata location does not match expected location".to_string(), + )); + } + let mut next = current; + next.metadata_location = request.new_metadata_location; + next.version_token = "token-view-committed".to_string(); + next.generation = next.generation.saturating_add(1); + views[index] = next.clone(); + Ok(crate::table_catalog::ViewCommitResult { view: next }) + } + + async fn drop_view( + &self, + table_bucket: &str, + namespace: &str, + view: &str, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.views + .lock() + .await + .retain(|entry| !(entry.table_bucket == table_bucket && entry.namespace == namespace && entry.view == view)); + Ok(()) + } + + async fn get_commit_by_id( + &self, + _table_bucket: &str, + _table_id: &str, + _commit_id: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(None) + } + + async fn get_commit_by_idempotency_key( + &self, + _table_bucket: &str, + _table_id: &str, + _idempotency_key: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + Ok(None) + } +} diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs index f8a0346c2..1572c3b16 100644 --- a/rustfs/src/table_catalog/tests.rs +++ b/rustfs/src/table_catalog/tests.rs @@ -3,7 +3,9 @@ use super::identifier::{ default_table_lifecycle_path, default_table_marker_path, default_table_root_prefix, is_valid_table_metadata_file_name, namespace_name_from_marker_path, table_name_from_marker_path, validate_object_mutation, }; -use super::test_support::{BlockingObjectPublication, TestCatalogObjectBackend, UnserializedTestPublication}; +use super::test_support::{ + BlockingObjectPublication, NoopTableCatalogStore, TestCatalogObjectBackend, UnserializedTestPublication, +}; use super::*; use datafusion::{ arrow::{ @@ -222,200 +224,6 @@ fn catalog_object_listing_rejects_missing_or_stalled_continuation_tokens() { ); } -struct NoopTableCatalogStore; - -#[async_trait::async_trait] -impl TableCatalogStore for NoopTableCatalogStore { - async fn get_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult> { - Ok(None) - } - - async fn put_table_bucket(&self, _entry: TableBucketEntry) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn create_namespace(&self, _entry: NamespaceEntry) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn list_namespaces(&self, _table_bucket: &str) -> TableCatalogStoreResult> { - Ok(Vec::new()) - } - - async fn get_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { - Ok(None) - } - - async fn drop_namespace(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn create_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn register_table(&self, _entry: TableEntry) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn register_table_with_publication( - &self, - entry: TableEntry, - publication: &(dyn TableCommitPublication + Sync), - ) -> TableCatalogStoreResult<()> { - publication.begin_table_bucket(&entry.table_bucket).await?; - if !publication.holds_table_bucket(&entry.table_bucket) { - return Err(TableCatalogStoreError::Internal( - "table registration requires a table-bucket publication fence".to_string(), - )); - } - let _publication_completion = TableCommitPublicationCompletion::new(publication); - publication - .prepare(&entry.table_bucket, &entry.namespace, &entry.table) - .await?; - if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) { - return Err(TableCatalogStoreError::Internal( - "table registration requires a table publication fence".to_string(), - )); - } - self.register_table(entry).await - } - - async fn list_tables(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { - Ok(Vec::new()) - } - - async fn list_all_tables(&self, _table_bucket: &str) -> TableCatalogStoreResult> { - Ok(Vec::new()) - } - - async fn load_table( - &self, - _table_bucket: &str, - _namespace: &str, - _table: &str, - ) -> TableCatalogStoreResult> { - Ok(None) - } - - async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { - let table = TableEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: request.table_bucket, - namespace: request.namespace, - table: request.table, - table_id: "table-id".to_string(), - table_uuid: "table-uuid".to_string(), - format: "ICEBERG".to_string(), - format_version: 2, - warehouse_location: "s3://analytics/tables/table-id".to_string(), - metadata_location: request.new_metadata_location.clone(), - version_token: "token-v2".to_string(), - generation: 2, - state: TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }; - let commit_log = CommitLogEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - commit_id: request.commit_id, - idempotency_key: request.idempotency_key, - table_id: table.table_id.clone(), - operation: request.operation, - expected_version_token: request.expected_version_token, - new_version_token: table.version_token.clone(), - previous_metadata_location: request.expected_metadata_location, - new_metadata_location: table.metadata_location.clone(), - requirements: request.requirements, - status: CommitLogStatus::Committed, - writer: request.writer, - created_at: None, - updated_at: None, - }; - - Ok(TableCommitResult { table, commit_log }) - } - - async fn commit_table_with_publication( - &self, - request: TableCommitRequest, - publication: &(dyn TableCommitPublication + Sync), - ) -> TableCatalogStoreResult { - publication - .prepare(&request.table_bucket, &request.namespace, &request.table) - .await?; - if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) { - return Err(TableCatalogStoreError::Internal( - "table commit requires a table publication fence".to_string(), - )); - } - let _publication_completion = TableCommitPublicationCompletion::new(publication); - self.commit_table(request).await - } - - async fn drop_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn create_view(&self, _entry: ViewEntry) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn list_views(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { - Ok(Vec::new()) - } - - async fn load_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult> { - Ok(None) - } - - async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { - Ok(ViewCommitResult { - view: ViewEntry { - version: TABLE_CATALOG_ENTRY_VERSION, - table_bucket: request.table_bucket, - namespace: request.namespace, - view: request.view, - view_id: "view-id".to_string(), - view_uuid: "view-uuid".to_string(), - format: "ICEBERG_VIEW".to_string(), - format_version: 1, - warehouse_location: "s3://analytics/views/view-id".to_string(), - metadata_location: request.new_metadata_location, - version_token: "token-v2".to_string(), - generation: 2, - state: TableCatalogEntryState::Active, - properties: BTreeMap::new(), - created_at: None, - updated_at: None, - }, - }) - } - - async fn drop_view(&self, _table_bucket: &str, _namespace: &str, _view: &str) -> TableCatalogStoreResult<()> { - Ok(()) - } - - async fn get_commit_by_id( - &self, - _table_bucket: &str, - _table_id: &str, - _commit_id: &str, - ) -> TableCatalogStoreResult> { - Ok(None) - } - - async fn get_commit_by_idempotency_key( - &self, - _table_bucket: &str, - _table_id: &str, - _idempotency_key: &str, - ) -> TableCatalogStoreResult> { - Ok(None) - } -} - #[tokio::test] async fn table_catalog_store_trait_covers_entry_read_write_shapes() { let store: &dyn TableCatalogStore = &NoopTableCatalogStore; From ebd0531124568e660629e90a135623d22e3de991 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 14 Aug 2026 21:57:29 +0800 Subject: [PATCH 11/71] chore(ecstore): drop the data_usage dead_code blanket (#6089) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing the blanket exposes eighteen items. Six are deleted; the rest belong to one feature that was never wired up. crates/ecstore/src/data_usage/local_snapshot.rs and its aggregation entry point, aggregate_local_snapshots, form a complete per-disk usage-snapshot feature: read/write of snapshot files under the metadata bucket, cross-disk aggregation, and tests. Nothing calls it. It landed in #5307 on 2026-07-27 and git log -S over the whole history shows the entry point has never had a caller; the live data-usage path is load_data_usage_from_backend / store_data_usage_in_backend. Rather than delete a recent, tested feature on cleanup grounds, the module gets a header explaining the situation and its items carry individual allows, so the gap stays greppable without reintroducing a blanket. One commit flips this to deletion if that is preferred. Deleted: - DATA_USAGE_BLOOM_NAME together with DATA_USAGE_BLOOM_NAME_PATH. Both are a dead duplicate of the pair in crates/scanner/src/data_usage_define.rs, which has roughly fifty consumers across scanner.rs and remote_scanner.rs; the ecstore copies have none. - increment_bucket_usage_memory and decrement_bucket_usage_memory, thin wrappers over the live record_bucket_object_write_memory and record_bucket_object_delete_memory. - sync_memory_cache_with_backend, whose doc comment says it is "called by scanner" — nothing calls it. - create_cache_entry_from_summary and cache_to_data_usage_info, neither with a consumer in any lane. resolve_loaded_snapshot is kept with an allow: nine tests in the same file assert its primary/backup fallback. Verification, four lanes warning-free: default, --tests, --features rio-v2 --tests, --features test-util --tests. rustfs-scanner still compiles clean. cargo nextest run -p rustfs-ecstore 4041 passed; clippy --lib --tests -D warnings clean; make pre-commit exit 0. Ref rustfs/backlog#1823 (step 2). --- .../ecstore/src/data_usage/local_snapshot.rs | 19 +++ crates/ecstore/src/data_usage/mod.rs | 118 +++--------------- 2 files changed, 33 insertions(+), 104 deletions(-) diff --git a/crates/ecstore/src/data_usage/local_snapshot.rs b/crates/ecstore/src/data_usage/local_snapshot.rs index 0ed2b6e1b..8262bee95 100644 --- a/crates/ecstore/src/data_usage/local_snapshot.rs +++ b/crates/ecstore/src/data_usage/local_snapshot.rs @@ -12,6 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Per-disk usage snapshots persisted under the metadata bucket. +//! +//! **Nothing calls into this module.** It landed complete with tests in #5307 +//! (2026-07-27) and its aggregation entry point, +//! [`crate::data_usage::aggregate_local_snapshots`], has never had a caller in +//! the tree's history. The live data-usage path is +//! `load_data_usage_from_backend` / `store_data_usage_in_backend`. The items +//! below therefore carry individual `dead_code` allows rather than a module +//! blanket, so the gap stays greppable until it is either wired up or removed. + use crate::data_usage::BucketUsageInfo; use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result}; @@ -26,10 +36,12 @@ pub const DATA_USAGE_DIR: &str = "datausage"; /// Directory used to store incremental scan state files under the metadata bucket. pub const DATA_USAGE_STATE_DIR: &str = "datausage/state"; /// Snapshot file format version, allows forward compatibility if the structure evolves. +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub const LOCAL_USAGE_SNAPSHOT_VERSION: u32 = 1; /// Additional metadata describing which disk produced the snapshot. #[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub struct LocalUsageSnapshotMeta { /// Disk UUID stored as a string for simpler serialization. pub disk_id: String, @@ -43,6 +55,7 @@ pub struct LocalUsageSnapshotMeta { /// Usage snapshot produced by a single disk. #[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub struct LocalUsageSnapshot { /// Format version recorded in the snapshot. pub format_version: u32, @@ -64,6 +77,7 @@ pub struct LocalUsageSnapshot { pub objects_total_size: u64, } +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] impl LocalUsageSnapshot { /// Create an empty snapshot with the default format version filled in. pub fn new(meta: LocalUsageSnapshotMeta) -> Self { @@ -99,11 +113,13 @@ impl LocalUsageSnapshot { } /// Build the snapshot file name `.json`. +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub fn snapshot_file_name(disk_id: &str) -> String { format!("{disk_id}.json") } /// Build the object path relative to `RUSTFS_META_BUCKET`, e.g. `datausage/.json`. +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub fn snapshot_object_path(disk_id: &str) -> String { format!("{}/{}", DATA_USAGE_DIR, snapshot_file_name(disk_id)) } @@ -119,11 +135,13 @@ pub fn data_usage_state_dir(root: &Path) -> PathBuf { } /// Build the absolute path to the snapshot file for the provided disk ID. +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub fn snapshot_path(root: &Path, disk_id: &str) -> PathBuf { data_usage_dir(root).join(snapshot_file_name(disk_id)) } /// Read a snapshot from disk if it exists. +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub async fn read_snapshot(root: &Path, disk_id: &str) -> Result> { let path = snapshot_path(root, disk_id); match fs::read(&path).await { @@ -138,6 +156,7 @@ pub async fn read_snapshot(root: &Path, disk_id: &str) -> Result Result<()> { let dir = data_usage_dir(root); fs::create_dir_all(&dir).await.map_err(Error::other)?; diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 3d0c122de..f034b3a3e 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: scanner/data-usage state is partially migrated and still owns staged cache helpers. -#![allow(dead_code)] pub mod local_snapshot; @@ -34,8 +33,8 @@ use crate::{ pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path}; use rustfs_data_usage::{ BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, - DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary, - VersionsHistogram, observed_data_usage_is_newer, + DataUsageCache, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, VersionsHistogram, + observed_data_usage_is_newer, }; use rustfs_io_metrics::record_system_path_failure; use rustfs_utils::path::SLASH_SEPARATOR; @@ -55,7 +54,6 @@ use tracing::{debug, error, info, instrument}; // Data usage storage constants pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json"; -const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; const DATA_USAGE_CACHE_TTL_SECS: u64 = 30; const LIVE_BUCKET_USAGE_MAX_ENTRIES: u64 = 1024; @@ -313,11 +311,6 @@ lazy_static::lazy_static! { LEGACY_DATA_USAGE_OBJECT_NAME ); static ref LEGACY_DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()); - pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}", - crate::disk::BUCKET_META_PREFIX, - SLASH_SEPARATOR, - DATA_USAGE_BLOOM_NAME - ); pub static ref DATA_COMPRESSION_TOTAL_NAME_PATH: String = format!("{}{}{}", crate::disk::BUCKET_META_PREFIX, SLASH_SEPARATOR, @@ -858,6 +851,10 @@ async fn resolve_loaded_snapshot_pair_with_source( } } +#[allow( + dead_code, + reason = "primary/backup snapshot fallback asserted by this file's tests (backlog#1823)" +)] async fn resolve_loaded_snapshot( primary: Result, Error>, backup: impl Future, Error>>, @@ -1187,6 +1184,10 @@ pub async fn invalidate_admin_data_usage_snapshot_cache() { } /// Aggregate usage information from local disk snapshots. +#[allow( + dead_code, + reason = "reached only through aggregate_local_snapshots, which has no caller (backlog#1823)" +)] fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapshot, latest_update: &mut Option) { if let Some(update) = snapshot.last_update && latest_update.is_none_or(|current| update > current) @@ -1220,6 +1221,10 @@ fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapsh } } +#[allow( + dead_code, + reason = "entry point of the local usage-snapshot feature, which has had no caller since it landed in #5307 (backlog#1823)" +)] pub async fn aggregate_local_snapshots(store: Arc) -> Result<(Vec, DataUsageInfo), Error> { let mut aggregated = DataUsageInfo::default(); let mut latest_update: Option = None; @@ -1767,11 +1772,6 @@ pub async fn record_bucket_object_write_unknown_previous_memory(bucket: &str, ne entry.pending_scanner_position = None; } -/// Fast in-memory increment for immediate quota consistency. -pub async fn increment_bucket_usage_memory(bucket: &str, size_increment: u64) { - record_bucket_object_write_memory(bucket, None, size_increment).await; -} - /// Fast in-memory update for successful object deletes. pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) { ensure_bucket_usage_cached(bucket).await; @@ -1814,11 +1814,6 @@ pub async fn record_bucket_delete_marker_memory(bucket: &str) { entry.pending_scanner_position = None; } -/// Fast in-memory decrement for immediate quota consistency -pub async fn decrement_bucket_usage_memory(bucket: &str, size_decrement: u64) { - record_bucket_object_delete_memory(bucket, size_decrement, size_decrement > 0).await; -} - /// Get bucket usage from the authoritative cache for this topology. async fn get_persisted_bucket_usage(bucket: &str) -> Option { let store = runtime_sources::object_store_handle()?; @@ -2013,91 +2008,6 @@ pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageIn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info, authoritative).await; } -/// Sync memory cache with backend data (called by scanner) -pub async fn sync_memory_cache_with_backend() -> Result<(), Error> { - if let Some(store) = runtime_sources::object_store_handle() { - match load_data_usage_from_backend(store.clone()).await { - Ok(data_usage_info) => { - replace_bucket_usage_memory_from_info(&data_usage_info).await; - } - Err(e) => { - debug!("Failed to sync memory cache with backend: {}", e); - } - } - } - Ok(()) -} - -/// Create a data usage cache entry from size summary -pub fn create_cache_entry_from_summary(summary: &SizeSummary) -> DataUsageEntry { - let mut entry = DataUsageEntry::default(); - entry.add_sizes(summary); - entry -} - -/// Convert data usage cache to DataUsageInfo -pub fn cache_to_data_usage_info( - cache: &DataUsageCache, - path: &str, - buckets: &[crate::storage_api_contracts::bucket::BucketInfo], -) -> DataUsageInfo { - let e = match cache.find(path) { - Some(e) => e, - None => return DataUsageInfo::default(), - }; - let flat = cache.flatten(&e); - - let mut buckets_usage = HashMap::new(); - for bucket in buckets.iter() { - let e = match cache.find(&bucket.name) { - Some(e) => e, - None => continue, - }; - let flat = cache.flatten(&e); - let mut bui = BucketUsageInfo { - size: flat.size as u64, - versions_count: flat.versions as u64, - objects_count: flat.objects as u64, - delete_markers_count: flat.delete_markers as u64, - object_size_histogram: flat.obj_sizes.to_map(), - object_versions_histogram: flat.obj_versions.to_map(), - ..Default::default() - }; - - if let Some(rs) = &flat.replication_stats { - bui.replica_size = rs.replica_size; - bui.replica_count = rs.replica_count; - - for (arn, stat) in rs.targets.iter() { - bui.replication_info.insert( - arn.clone(), - BucketTargetUsageInfo { - replication_pending_size: stat.pending_size, - replicated_size: stat.replicated_size, - replication_failed_size: stat.failed_size, - replication_pending_count: stat.pending_count, - replication_failed_count: stat.failed_count, - replicated_count: stat.replicated_count, - ..Default::default() - }, - ); - } - } - buckets_usage.insert(bucket.name.clone(), bui); - } - - DataUsageInfo { - last_update: cache.info.last_update, - objects_total_count: flat.objects as u64, - versions_total_count: flat.versions as u64, - delete_markers_total_count: flat.delete_markers as u64, - objects_total_size: flat.size as u64, - buckets_count: e.children.len() as u64, - buckets_usage, - ..Default::default() - } -} - // Helper functions for DataUsageCache operations pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result { use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; From ebbcfa3ac2bdfaaa5b392d0272db3d3219e28d6a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 14 Aug 2026 21:59:38 +0800 Subject: [PATCH 12/71] fix(tier): decrypt transitioned objects instead of serving their ciphertext (#6107) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(tier): decrypt transitioned objects instead of serving their ciphertext A GET on a managed-SSE object that lifecycle had transitioned to a remote tier returned the ciphertext with the plaintext's Content-Length and no error: silent corruption on read-through, and worse than a failed request because nothing signals it. Restore of the same object failed server-side with IncompleteBody while POST ?restore still answered 200, so the object simply never came back and HEAD never showed an x-amz-restore marker. Both symptoms are one cause. The transitioned read path built its fetch through new_getobjectreader, which decides nothing about encryption: it derived the range from the parts table — whose sizes are PLAINTEXT sizes — then used that range to fetch the object's STORED bytes from the tier, and handed the stream to the caller without any decrypt transform. The GET therefore served the first plaintext-length bytes of ciphertext; the restore copy-back, which validates against the stored size, came up short by exactly the encryption overhead. The path now builds the same ReadPlan the local read path uses, so a single place decides how stored bytes map to requested bytes. ReadPlan gains a two-phase API — build_for_request to learn the storage coordinates before issuing the tier fetch, into_object_reader to wrap the returned stream — because the tier fetch has to be positioned before a stream exists. The encryption resolver reaches the path from InstanceContext, the same source the local read uses. A restore read additionally stops synthesizing a range from the part number. A restore serves the stored representation (restore_request_active already forces the Plain branch), so a plaintext-coordinate range would be reinterpreted as a storage range and truncate the payload by its encoding overhead. An explicit caller range is already in storage coordinates on that path and is still honored, which two existing tests pin. crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs drops its #[ignore]: the transition test now runs and asserts the plaintext round-trips byte-identically through transition, read-through and restore. The same file had its enforcement switch stuck at false from a control experiment; it is back to true, so the test again exercises what its name and module docs claim. Fixes #6025. Refs rustfs/backlog#1582, rustfs/backlog#1637. * test(tier): pass resolver to transitioned reader tests --- .../e2e_test/src/kms/kms_ilm_sse_kms_test.rs | 3 +- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 39 +++++++++---- crates/ecstore/src/object_api/readers.rs | 56 ++++++++++++++++++- crates/ecstore/src/set_disk/ops/object.rs | 3 + 4 files changed, 87 insertions(+), 14 deletions(-) diff --git a/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs b/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs index e1fad39cd..173e87767 100644 --- a/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs +++ b/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs @@ -97,7 +97,7 @@ async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestRe let envs = [ ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"), - ("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "false"), + ("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"), ("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_ILM_PROCESS_TIME", "1"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "2"), @@ -486,7 +486,6 @@ async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult { /// depend on scanner scheduling; the 1s scanner cycle stays on as a backstop. #[tokio::test] #[serial] -#[ignore = "pins rustfs/rustfs#6025: GET on a transitioned managed-SSE object silently returns corrupt bytes (fails with enforcement on AND off, so it is not an authorization regression); un-ignore with the fix"] async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> TestResult { init_logging(); diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 72d9cca86..e0849a637 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -46,15 +46,13 @@ use crate::bucket::lifecycle::transition_transaction::run_transition_transaction use crate::bucket::object_lock::ObjectLockApi; use crate::bucket::versioning::VersioningApi as _; use crate::bucket::versioning_sys::BucketVersioningSys; -use crate::client::object_api_utils::new_getobjectreader; use crate::disk::error::DiskError; use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE}; use crate::error::Error; use crate::error::StorageError; -use crate::error::{ - error_resp_to_object_err, is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down, -}; +use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions}; +use crate::object_api::{ObjectEncryptionResolver, ReadPlan}; use crate::services::tier::{ tier::{TierConfigMgr, TierOperationLease, tier_destination_id_from_metadata}, warm_backend::WarmBackendGetOpts, @@ -4400,9 +4398,10 @@ pub async fn get_transitioned_object_reader( h: &HeaderMap, oi: &ObjectInfo, opts: &ObjectOptions, + resolver: Option<&dyn ObjectEncryptionResolver>, ) -> Result { let tier_config_mgr = runtime_sources::tier_config_mgr_handle(); - get_transitioned_object_reader_with_tier_manager(bucket, object, rs, h, oi, opts, &tier_config_mgr).await + get_transitioned_object_reader_with_tier_manager(bucket, object, rs, h, oi, opts, &tier_config_mgr, resolver).await } fn validate_transition_remote_version(oi: &ObjectInfo) -> Result { @@ -4422,6 +4421,10 @@ fn validate_transition_remote_version(oi: &ObjectInfo) -> Result>, + resolver: Option<&dyn ObjectEncryptionResolver>, ) -> Result { validate_transition_remote_version(oi)?; let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?; @@ -4447,11 +4451,16 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager( tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?; - let ret = new_getobjectreader(rs, oi, opts, h); - if let Err(err) = ret { - return Err(error_resp_to_object_err(err, vec![bucket, object])); - } - let (get_fn, off, length) = ret.expect("get_transitioned_object_reader should succeed after error check"); + // The same read plan the local path uses, so the tier fetch is positioned in + // the object's *stored* coordinate system and the stream is handed the same + // decrypt/decompress transforms. Reading an encrypted object's ciphertext + // through a plaintext-coordinate range and skipping the transform is how a + // transitioned SSE object used to come back as silently corrupt bytes of the + // right length (rustfs/rustfs#6025). + let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver) + .await + .map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?; + let (off, length) = (plan.storage_offset() as i64, plan.storage_length()); let mut gopts = WarmBackendGetOpts::default(); if off >= 0 && length >= 0 { @@ -4488,7 +4497,10 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager( ); e })?; - Ok(attach_tier_operation_lease(get_fn(reader, h.clone()), tgt_client)) + let object_reader = plan + .into_object_reader(Box::new(reader), oi) + .map_err(|err| std::io::Error::other(format!("wrapping the tier stream for {bucket}/{object} failed: {err}")))?; + Ok(attach_tier_operation_lease(object_reader, tgt_client)) } struct TierOperationLeaseReader { @@ -5776,6 +5788,7 @@ mod tests { &object_info, &ObjectOptions::default(), &manager, + None, ) .await .expect("transitioned reader should open"); @@ -5840,6 +5853,7 @@ mod tests { &object_info, &ObjectOptions::default(), &manager, + None, ) .await { @@ -5880,6 +5894,7 @@ mod tests { &object_info, &ObjectOptions::default(), &manager, + None, ) .await { @@ -6117,6 +6132,7 @@ mod tests { &oi, &ObjectOptions::default(), &manager, + None, ) .await { @@ -6140,6 +6156,7 @@ mod tests { &oi, &ObjectOptions::default(), &manager, + None, ) .await { diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index cc3a1986a..6e5985c10 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -479,7 +479,15 @@ enum ReadTransform { }, } -struct ReadPlan { +/// How an object's stored bytes must be fetched and transformed to serve a +/// request. +/// +/// Public so callers that fetch the stored bytes from somewhere other than the +/// local erasure set — the remote-tier read path — can position their own fetch +/// with [`ReadPlan::storage_offset`] / [`ReadPlan::storage_length`] and then +/// hand the resulting stream to [`ReadPlan::into_object_reader`], instead of +/// reimplementing the transform decisions (rustfs/rustfs#6025). +pub struct ReadPlan { storage_offset: usize, storage_length: i64, object_size: i64, @@ -487,6 +495,43 @@ struct ReadPlan { } impl ReadPlan { + /// Byte offset into the object's **stored** bytes where the fetch must + /// start. Encrypted and compressed objects address their storage in a + /// different coordinate system than the plaintext range the caller asked + /// for, which is exactly the distinction this plan resolves. + pub fn storage_offset(&self) -> usize { + self.storage_offset + } + + /// Number of **stored** bytes the fetch must deliver, in the same + /// coordinate system as [`Self::storage_offset`]. + pub fn storage_length(&self) -> i64 { + self.storage_length + } + + /// Build the plan for a request without consuming a stream, so a caller + /// that has to issue its own positioned fetch can read the offsets first. + pub async fn build_for_request( + rs: Option, + oi: &ObjectInfo, + opts: &ObjectOptions, + h: &HeaderMap, + resolver: Option<&dyn ObjectEncryptionResolver>, + ) -> Result { + Self::build_with_resolver(rs, oi, opts, h, resolver).await + } + + /// Wrap `reader` — the stored bytes this plan asked for, already positioned + /// at [`Self::storage_offset`] — in the transforms that turn them into the + /// bytes the caller requested. + pub fn into_object_reader( + self, + reader: Box, + oi: &ObjectInfo, + ) -> Result { + self.into_reader(reader, oi).map(|(reader, _, _)| reader) + } + #[cfg(test)] async fn build(rs: Option, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap) -> Result { Self::build_with_resolver(rs, oi, opts, h, Some(&tests::TEST_RESOLVER)).await @@ -500,8 +545,17 @@ impl ReadPlan { resolver: Option<&dyn ObjectEncryptionResolver>, ) -> Result { let mut rs = rs; + // A part number addresses the object's PLAINTEXT bytes. A restore read + // serves the stored representation instead (see + // [`restore_request_active`]), where that synthesized range would be + // reinterpreted as a storage range and truncate an encrypted or + // compressed payload by exactly its encoding overhead — the copy-back + // then fails its length check partway through + // (rustfs/rustfs#6025). An explicit caller range is already in storage + // coordinates on that path and is still honored. if let Some(part_number) = opts.part_number && rs.is_none() + && !restore_request_active(opts) { rs = http_range_spec_from_object_info(oi, part_number); } diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 95fe3438a..db77a1f8b 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -899,6 +899,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { &object_info, &opts, &self.ctx.tier_config_mgr(), + self.ctx.object_encryption_resolver(), ) .await?; return Ok(finish_set_disk_read_lock(gr, read_lock_guard.take(), bucket, object)); @@ -6065,6 +6066,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &oi, &opts, &self_.ctx.tier_config_mgr(), + self_.ctx.object_encryption_resolver(), ) .await; if let Err(err) = gr { @@ -6134,6 +6136,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &oi, &part_opts, &self_.ctx.tier_config_mgr(), + self_.ctx.object_encryption_resolver(), ) .await .map_err(StorageError::Io)?; From 85be26b3c1fcdc7fb5adf42aa95a7d42e5080345 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 14 Aug 2026 22:07:44 +0800 Subject: [PATCH 13/71] test(ecstore): cover cancelled PUT tmp cleanup (#6105) --- crates/ecstore/src/set_disk/ops/object.rs | 59 +++++++++++++++++------ 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index db77a1f8b..9c7088030 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -11183,7 +11183,7 @@ mod put_object_tmp_cleanup_tests { use tokio::io::AsyncReadExt; /// Large enough that the erasure shards are written as real tmp files - /// (never inlined into xl.meta), so both tests exercise actual cleanup. + /// (never inlined into xl.meta), so the cleanup tests exercise actual cleanup. const TEST_OBJECT_SIZE: usize = 1 << 20; /// Entries under `.rustfs.sys/tmp` on every disk, excluding the `.trash` @@ -11207,6 +11207,18 @@ mod put_object_tmp_cleanup_tests { leftovers } + async fn wait_for_tmp_workspace_to_drain(temp_dirs: &[TempDir], failure_context: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + let leftovers = non_trash_tmp_entries(temp_dirs).await; + if leftovers.is_empty() { + break; + } + assert!(tokio::time::Instant::now() < deadline, "{failure_context}, leftovers: {leftovers:?}"); + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + #[tokio::test] async fn put_object_success_eventually_cleans_tmp_workspace() { let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; @@ -11222,22 +11234,39 @@ mod put_object_tmp_cleanup_tests { .await .expect("put_object should succeed"); - // The speculative cleanup runs on a spawned task off the PUT response - // path, so poll for the tmp workspace to drain instead of asserting - // immediately. - let deadline = tokio::time::Instant::now() + Duration::from_secs(10); - loop { - let leftovers = non_trash_tmp_entries(&temp_dirs).await; - if leftovers.is_empty() { - break; - } - assert!( - tokio::time::Instant::now() < deadline, - "tmp workspace should drain after a successful PUT, leftovers: {leftovers:?}" - ); - tokio::time::sleep(Duration::from_millis(25)).await; + wait_for_tmp_workspace_to_drain(&temp_dirs, "tmp workspace should drain after a successful PUT").await; + + drop(temp_dirs); + } + + #[tokio::test] + async fn cancelled_put_before_rename_cleans_tmp_workspace() { + let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + + let bucket = "tmp-clean-cancelled-bucket"; + let object = "cancelled-object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); } + let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterQuotaReservation); + let cancelled_set = set_disks.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![8u8; TEST_OBJECT_SIZE]); + cancelled_set + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + put.abort(); + let join_error = put.await.expect_err("the paused PUT task must be cancelled"); + assert!(join_error.is_cancelled(), "the paused PUT task must not panic"); + + // Keep the barrier armed so a detached child cannot proceed and hide + // missing cancellation cleanup. + wait_for_tmp_workspace_to_drain(&temp_dirs, "cancelling before rename should drain the tmp workspace").await; + + drop(barrier); drop(temp_dirs); } From eca6bc1600012b6a94faa0e51edf9fdbf6c17c9f Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 14 Aug 2026 22:12:42 +0800 Subject: [PATCH 14/71] fix(ecstore): preserve CopyObject producer errors (#6090) * fix(ecstore): preserve CopyObject producer errors * fix(app): resume preserved relocation I/O errors * fix(copy): preserve transformed source errors --- crates/ecstore/src/object_api/mod.rs | 2 +- crates/ecstore/src/object_api/readers.rs | 6 +- crates/ecstore/src/set_disk/ops/object.rs | 461 +++++++++++++++++++++- crates/rio/src/hardlimit_reader.rs | 109 +++-- crates/rio/src/http_reader.rs | 60 ++- rustfs/src/app/object_usecase.rs | 10 +- rustfs/src/error.rs | 53 ++- rustfs/src/storage/sse.rs | 21 +- 8 files changed, 666 insertions(+), 56 deletions(-) diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index 41c4ff403..30a605962 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -22,7 +22,7 @@ use crate::bucket::replication::{ use crate::bucket::versioning::VersioningApi as _; use crate::config::storageclass; use crate::error::{Error, Result}; -use crate::io_support::rio::{HashReader, LimitReader}; +use crate::io_support::rio::{HardLimitReader, HashReader}; use crate::storage_api_contracts::{ lifecycle::{ExpirationOptions, TransitionedObject}, range::HTTPRangeSpec, diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index 6e5985c10..b1731b466 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -808,7 +808,7 @@ impl ReadPlan { } } } else { - Box::new(LimitReader::new(dec_reader, total_plaintext_size)) + Box::new(HardLimitReader::new(dec_reader, decompressed_length)) }; let mut object_info = oi.clone(); @@ -900,7 +900,7 @@ impl ReadPlan { )?; Box::new(ranged_reader) } else { - Box::new(LimitReader::new(decompressed_reader, total_plaintext_size)) + Box::new(HardLimitReader::new(decompressed_reader, total_plaintext_size_i64)) } } else if plaintext_offset > 0 || plaintext_length != total_plaintext_size_i64 { Box::new(RangedDecompressReader::new( @@ -910,7 +910,7 @@ impl ReadPlan { total_plaintext_size, )?) } else { - Box::new(LimitReader::new(decrypted_reader, total_plaintext_size)) + Box::new(HardLimitReader::new(decrypted_reader, total_plaintext_size_i64)) }; let mut object_info = oi.clone(); diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 9c7088030..1c64de30d 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -315,6 +315,41 @@ async fn get_object_reader_with_context( GetObjectReader::new_with_resolver(reader, range, object_info, opts, headers, ctx.object_encryption_resolver()).await } +async fn get_legacy_object_reader_with_context( + ctx: &InstanceContext, + reader: R, + terminal: tokio::sync::oneshot::Receiver>, + range: Option, + object_info: &ObjectInfo, + opts: &ObjectOptions, + headers: &HeaderMap, +) -> Result<(GetObjectReader, usize, i64)> +where + R: AsyncRead + Unpin + Send + Sync + 'static, +{ + // ReadPlan validates this size below; failure here only keeps the terminal + // guard inside the transform until that validation returns its typed error. + let full_plaintext_size = object_info.get_actual_size().ok(); + let whole_object = opts.part_number.is_none() + && match (&range, full_plaintext_size) { + (None, _) => true, + (Some(range), Some(size)) => range + .get_offset_length(size) + .is_ok_and(|(offset, length)| offset == 0 && length == size), + (Some(_), None) => false, + }; + let (source, terminal): (Box, _) = if whole_object { + (Box::new(reader), Some(terminal)) + } else { + (Box::new(LegacyDuplexProducerReader::new(reader, terminal)), None) + }; + let (mut reader, offset, length) = get_object_reader_with_context(ctx, source, range, object_info, opts, headers).await?; + if let Some(terminal) = terminal { + reader.stream = Box::new(LegacyDuplexProducerReader::new(reader.stream, terminal)); + } + Ok((reader, offset, length)) +} + fn data_read_metadata_early_stop_request_shape_allowed(range: &Option, opts: &ObjectOptions) -> bool { range.is_none() && opts.part_number.is_none() @@ -1090,8 +1125,9 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let (rd, wd) = tokio::io::duplex(duplex_buffer_size); debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer"); + let (producer_terminal_tx, producer_terminal_rx) = tokio::sync::oneshot::channel(); let (mut reader, offset, length) = - get_object_reader_with_context(&self.ctx, Box::new(rd), range, &object_info, opts, &h).await?; + get_legacy_object_reader_with_context(&self.ctx, rd, producer_terminal_rx, range, &object_info, opts, &h).await?; // Carry the hook probe result so the app layer skips its now-redundant // lookup on the streaming miss path (ODC-16). reader.body_source = body_source; @@ -1111,7 +1147,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { // `get_object_with_fileinfo` also waits on `writer`, so an outer timeout // would incorrectly treat downstream backpressure as disk-read latency. // Disk read timeouts must be enforced at the actual disk I/O operations. - if let Err(e) = Self::get_object_with_fileinfo( + let producer_result = Self::get_object_with_fileinfo( &bucket, &object, erasure_cache, @@ -1129,9 +1165,9 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { object_class.as_str(), size_bucket, ) - .await - { - let reason = classify_storage_error(&e); + .await; + if let Err(e) = &producer_result { + let reason = classify_storage_error(e); if reason == GetObjectFailureReason::DownstreamClosed { debug!( event = EVENT_SET_DISK_WRITE, @@ -1170,6 +1206,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { ); } }; + let _ = producer_terminal_tx.send(producer_result.map(|_| ())); }); Ok(reader) @@ -2558,6 +2595,420 @@ impl AsyncRead for TransitionUploadReader { } } +struct LegacyDuplexProducerReader { + inner: Option, + terminal: Option>>, + inner_eof: bool, +} + +impl LegacyDuplexProducerReader { + fn new(inner: R, terminal: tokio::sync::oneshot::Receiver>) -> Self { + Self { + inner: Some(inner), + terminal: Some(terminal), + inner_eof: false, + } + } +} + +impl AsyncRead for LegacyDuplexProducerReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + if !self.inner_eof { + let before = buf.filled().len(); + if let Some(inner) = self.inner.as_mut() { + match Pin::new(inner).poll_read(cx, buf) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Ready(Ok(())) if buf.filled().len() > before => return Poll::Ready(Ok(())), + Poll::Ready(Ok(())) => { + self.inner_eof = true; + self.inner = None; + } + } + } else { + self.inner_eof = true; + } + } + + let Some(terminal) = self.terminal.as_mut() else { + return Poll::Ready(Ok(())); + }; + match Pin::new(terminal).poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(Ok(Ok(()))) => { + self.terminal = None; + Poll::Ready(Ok(())) + } + Poll::Ready(Ok(Err(err))) => { + self.terminal = None; + Poll::Ready(Err(std::io::Error::other(err))) + } + Poll::Ready(Err(_)) => { + self.terminal = None; + Poll::Ready(Err(std::io::Error::other(StorageError::Unexpected))) + } + } + } +} + +#[cfg(test)] +mod legacy_duplex_producer_reader_tests { + use super::*; + use crate::object_api::{EncryptionResolutionError, ObjectEncryptionResolver, ReadEncryptionMaterial, ReadEncryptionMode}; + use rustfs_utils::CompressionAlgorithm; + use std::collections::HashMap; + use std::sync::Arc; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + const TEST_DUPLEX_CAPACITY: usize = 64 * 1024; + + fn storage_error_source(error: &std::io::Error) -> &StorageError { + error + .get_ref() + .and_then(|source| source.downcast_ref::()) + .expect("legacy duplex terminal error should retain StorageError source") + } + + async fn compressed_fixture(plaintext: Vec, recorded_size: usize) -> (Vec, ObjectInfo) { + let mut compressor = rustfs_rio::CompressReader::new(std::io::Cursor::new(plaintext), CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + compressor + .read_to_end(&mut compressed) + .await + .expect("compress test plaintext"); + + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + CompressionAlgorithm::default().to_string(), + ); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, recorded_size.to_string()); + let object_info = ObjectInfo { + size: i64::try_from(compressed.len()).expect("compressed fixture length should fit in i64"), + user_defined: Arc::new(metadata), + ..Default::default() + }; + (compressed, object_info) + } + + #[tokio::test] + async fn legacy_duplex_reader_allows_clean_completion() { + let (mut writer, reader) = tokio::io::duplex(64); + let (terminal_tx, terminal_rx) = tokio::sync::oneshot::channel(); + writer + .write_all(b"complete") + .await + .expect("duplex write should fit in buffer"); + drop(writer); + terminal_tx.send(Ok(())).expect("terminal receiver should remain installed"); + + let mut reader = LegacyDuplexProducerReader::new(reader, terminal_rx); + let mut out = Vec::new(); + reader + .read_to_end(&mut out) + .await + .expect("clean producer completion should surface clean EOF"); + + assert_eq!(out, b"complete"); + } + + #[tokio::test] + async fn legacy_duplex_reader_ignores_zero_capacity_read_buf() { + let (mut writer, reader) = tokio::io::duplex(64); + let (terminal_tx, terminal_rx) = tokio::sync::oneshot::channel(); + writer.write_all(b"body").await.expect("duplex write should fit in buffer"); + drop(writer); + terminal_tx + .send(Err(StorageError::FileCorrupt)) + .expect("terminal receiver should remain installed"); + + let mut reader = LegacyDuplexProducerReader::new(reader, terminal_rx); + let mut empty = []; + std::future::poll_fn(|cx| { + let mut read_buf = ReadBuf::new(&mut empty); + Pin::new(&mut reader).poll_read(cx, &mut read_buf) + }) + .await + .expect("zero-capacity reads should complete without observing EOF or terminal state"); + assert!(!reader.inner_eof); + assert!(reader.terminal.is_some()); + + let mut out = Vec::new(); + let err = reader + .read_to_end(&mut out) + .await + .expect_err("subsequent reads must still receive data and the terminal error"); + assert_eq!(out, b"body"); + assert!(matches!(storage_error_source(&err), StorageError::FileCorrupt)); + } + + #[tokio::test] + async fn legacy_duplex_reader_surfaces_terminal_error_after_partial_data() { + let (mut writer, reader) = tokio::io::duplex(64); + let (terminal_tx, terminal_rx) = tokio::sync::oneshot::channel(); + writer.write_all(b"partial").await.expect("duplex write should fit in buffer"); + drop(writer); + terminal_tx + .send(Err(StorageError::FileCorrupt)) + .expect("terminal receiver should remain installed"); + + let mut reader = LegacyDuplexProducerReader::new(reader, terminal_rx); + let mut out = Vec::new(); + let err = reader + .read_to_end(&mut out) + .await + .expect_err("terminal producer error must not become clean EOF"); + + assert_eq!(out, b"partial"); + assert!(matches!(storage_error_source(&err), StorageError::FileCorrupt)); + } + + #[tokio::test] + async fn legacy_duplex_reader_surfaces_terminal_error_after_declared_length() { + let (mut writer, reader) = tokio::io::duplex(64); + let (terminal_tx, terminal_rx) = tokio::sync::oneshot::channel(); + writer.write_all(b"exact").await.expect("duplex write should fit in buffer"); + drop(writer); + terminal_tx + .send(Err(StorageError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "remote body reset after final byte", + )))) + .expect("terminal receiver should remain installed"); + + let reader = LegacyDuplexProducerReader::new(reader, terminal_rx); + let mut reader = + HashReader::from_stream(reader, 5, 5, None, None, false).expect("hash reader should accept exact declared length"); + let mut out = Vec::new(); + let err = reader + .read_to_end(&mut out) + .await + .expect_err("producer terminal error after the declared length must still fail"); + + assert_eq!(out, b"exact"); + assert!( + matches!(storage_error_source(&err), StorageError::Io(io_error) if io_error.kind() == std::io::ErrorKind::ConnectionReset) + ); + } + + #[tokio::test] + async fn legacy_compressed_reader_surfaces_terminal_error_after_complete_plaintext() { + let plaintext = b"compressed terminal result must survive the plaintext limit".repeat(16); + let (compressed, object_info) = compressed_fixture(plaintext.clone(), plaintext.len()).await; + let full_range = HTTPRangeSpec { + is_suffix_length: false, + start: 0, + end: i64::try_from(plaintext.len()).expect("plaintext fixture length should fit in i64") - 1, + }; + for range in [None, Some(full_range)] { + let (mut writer, reader) = tokio::io::duplex(compressed.len().max(1)); + writer + .write_all(&compressed) + .await + .expect("compressed body should fit in duplex buffer"); + drop(writer); + let (terminal_tx, terminal_rx) = tokio::sync::oneshot::channel(); + terminal_tx + .send(Err(StorageError::FileCorrupt)) + .expect("terminal receiver should remain installed"); + + let (mut reader, _, _) = get_legacy_object_reader_with_context( + &InstanceContext::new(), + reader, + terminal_rx, + range, + &object_info, + &ObjectOptions::default(), + &HeaderMap::new(), + ) + .await + .expect("compressed read plan should build"); + let mut out = Vec::new(); + let err = reader + .read_to_end(&mut out) + .await + .expect_err("terminal error after complete decompression must not become clean EOF"); + + assert_eq!(out, plaintext); + assert!(matches!(storage_error_source(&err), StorageError::FileCorrupt)); + } + } + + #[tokio::test] + async fn legacy_exact_reader_rejects_extra_data_without_backpressure_deadlock() { + let payload = vec![0x5a; TEST_DUPLEX_CAPACITY * 2]; + let (mut writer, reader) = tokio::io::duplex(TEST_DUPLEX_CAPACITY); + let (terminal_tx, terminal_rx) = tokio::sync::oneshot::channel(); + let producer = tokio::spawn(async move { + let result = writer.write_all(&payload).await; + drop(writer); + let terminal_result = result + .as_ref() + .map(|_| ()) + .map_err(|err| StorageError::Io(std::io::Error::new(err.kind(), err.to_string()))); + let _ = terminal_tx.send(terminal_result); + result + }); + let reader = crate::io_support::rio::HardLimitReader::new(reader, 1); + let mut reader = LegacyDuplexProducerReader::new(reader, terminal_rx); + + let mut out = Vec::new(); + tokio::time::timeout(std::time::Duration::from_secs(1), reader.read_to_end(&mut out)) + .await + .expect("extra data beyond the declared size must not deadlock") + .expect_err("extra data beyond the declared size must fail closed"); + assert_eq!(out, [0x5a]); + drop(reader); + let _ = tokio::time::timeout(std::time::Duration::from_secs(1), producer) + .await + .expect("producer must unblock after the read fails") + .expect("producer task should not panic"); + } + + #[tokio::test] + async fn legacy_terminal_reader_releases_unconsumed_source_before_waiting() { + let payload = vec![0x5a; TEST_DUPLEX_CAPACITY * 2]; + let (mut writer, reader) = tokio::io::duplex(TEST_DUPLEX_CAPACITY); + let (terminal_tx, terminal_rx) = tokio::sync::oneshot::channel(); + let producer = tokio::spawn(async move { + let result = writer.write_all(&payload).await; + drop(writer); + let terminal_result = result + .as_ref() + .map(|_| ()) + .map_err(|err| StorageError::Io(std::io::Error::new(err.kind(), err.to_string()))); + let _ = terminal_tx.send(terminal_result); + result + }); + let reader = rustfs_rio::LimitReader::new(reader, 1); + let mut reader = LegacyDuplexProducerReader::new(reader, terminal_rx); + + let mut out = Vec::new(); + let err = tokio::time::timeout(std::time::Duration::from_secs(1), reader.read_to_end(&mut out)) + .await + .expect("terminal wait must not deadlock behind unconsumed source data") + .expect_err("unconsumed source data must fail the producer terminal result"); + assert_eq!(out, [0x5a]); + assert!( + matches!(storage_error_source(&err), StorageError::Io(io_error) if io_error.kind() == std::io::ErrorKind::BrokenPipe) + ); + producer + .await + .expect("producer task should not panic") + .expect_err("source should close early"); + } + + struct FixedEncryptionResolver { + key_bytes: [u8; 32], + base_nonce: [u8; 12], + } + + #[async_trait::async_trait] + impl ObjectEncryptionResolver for FixedEncryptionResolver { + async fn resolve_read_material( + &self, + _request: crate::object_api::ReadEncryptionRequest<'_>, + ) -> std::result::Result, EncryptionResolutionError> { + Ok(Some(ReadEncryptionMaterial { + key_bytes: self.key_bytes, + mode: ReadEncryptionMode::Direct { + base_nonce: self.base_nonce, + }, + })) + } + } + + #[tokio::test] + async fn legacy_encrypted_reader_surfaces_terminal_error_after_complete_plaintext() { + let plaintext = b"encrypted terminal result must survive the plaintext limit".repeat(16); + let key_bytes = [0x31; 32]; + let base_nonce = [0x42; 12]; + let mut encryptor = rustfs_rio::EncryptReader::new(std::io::Cursor::new(plaintext.clone()), key_bytes, base_nonce); + let mut encrypted = Vec::new(); + encryptor.read_to_end(&mut encrypted).await.expect("encrypt test plaintext"); + + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "encrypted-object".to_string(), + size: i64::try_from(encrypted.len()).expect("encrypted fixture length should fit in i64"), + user_defined: Arc::new(HashMap::from([ + ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), + ( + "x-amz-server-side-encryption-customer-original-size".to_string(), + plaintext.len().to_string(), + ), + ])), + ..Default::default() + }; + let ctx = InstanceContext::new(); + assert!( + ctx.set_object_encryption_resolver(Arc::new(FixedEncryptionResolver { key_bytes, base_nonce })) + .is_ok(), + "fresh context should accept resolver" + ); + let full_range = HTTPRangeSpec { + is_suffix_length: false, + start: 0, + end: i64::try_from(plaintext.len()).expect("plaintext fixture length should fit in i64") - 1, + }; + for range in [None, Some(full_range)] { + let (mut writer, reader) = tokio::io::duplex(encrypted.len().max(1)); + writer + .write_all(&encrypted) + .await + .expect("encrypted body should fit in duplex buffer"); + drop(writer); + let (terminal_tx, terminal_rx) = tokio::sync::oneshot::channel(); + terminal_tx + .send(Err(StorageError::FileCorrupt)) + .expect("terminal receiver should remain installed"); + + let (mut reader, _, _) = get_legacy_object_reader_with_context( + &ctx, + reader, + terminal_rx, + range, + &object_info, + &ObjectOptions::default(), + &HeaderMap::new(), + ) + .await + .expect("encrypted read plan should build"); + let mut out = Vec::new(); + let err = reader + .read_to_end(&mut out) + .await + .expect_err("terminal error after complete decryption must not become clean EOF"); + + assert_eq!(out, plaintext); + assert!(matches!(storage_error_source(&err), StorageError::FileCorrupt)); + } + } + + #[tokio::test] + async fn legacy_duplex_reader_fails_closed_when_terminal_channel_closes() { + let (mut writer, reader) = tokio::io::duplex(64); + let (terminal_tx, terminal_rx) = tokio::sync::oneshot::channel::>(); + writer.write_all(b"body").await.expect("duplex write should fit in buffer"); + drop(writer); + drop(terminal_tx); + + let mut reader = LegacyDuplexProducerReader::new(reader, terminal_rx); + let mut out = Vec::new(); + let err = reader + .read_to_end(&mut out) + .await + .expect_err("producer disappearance must fail closed"); + + assert_eq!(out, b"body"); + assert!(matches!(storage_error_source(&err), StorageError::Unexpected)); + } +} + struct TransitionUploadWriter { inner: W, produced: u64, diff --git a/crates/rio/src/hardlimit_reader.rs b/crates/rio/src/hardlimit_reader.rs index 8149e9fe9..5cc5bf4ab 100644 --- a/crates/rio/src/hardlimit_reader.rs +++ b/crates/rio/src/hardlimit_reader.rs @@ -24,12 +24,17 @@ pin_project! { #[pin] pub inner: R, remaining: i64, + scratch: Vec, } } impl HardLimitReader { pub fn new(inner: R, limit: i64) -> Self { - HardLimitReader { inner, remaining: limit } + HardLimitReader { + inner, + remaining: limit, + scratch: Vec::new(), + } } } @@ -37,19 +42,21 @@ impl AsyncRead for HardLimitReader where R: AsyncRead, { - fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - if self.remaining < 0 { + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let mut this = self.project(); + if *this.remaining < 0 { return Poll::Ready(Err(Error::other("input provided more bytes than specified"))); } - let original_filled = buf.filled().len(); - if self.remaining == 0 { + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + if *this.remaining == 0 { let mut discard = [0u8; 8192]; let mut discard_buf = ReadBuf::new(&mut discard); - return match self.as_mut().project().inner.poll_read(cx, &mut discard_buf) { + return match this.inner.as_mut().poll_read(cx, &mut discard_buf) { Poll::Pending => Poll::Pending, Poll::Ready(Ok(())) => { if discard_buf.filled().is_empty() { - debug_assert_eq!(buf.filled().len(), original_filled); Poll::Ready(Ok(())) } else { Poll::Ready(Err(Error::other("input provided more bytes than specified"))) @@ -58,30 +65,46 @@ where Poll::Ready(Err(err)) => Poll::Ready(Err(err)), }; } - // Save the initial length - let before = original_filled; - // Poll the inner reader - let this = self.as_mut().project(); - let poll = this.inner.poll_read(cx, buf); - - if let Poll::Ready(Ok(())) = &poll { - let after = buf.filled().len(); - let read = (after - before) as i64; - if read == 0 && *this.remaining > 0 { - return Poll::Ready(Err(Error::new( - std::io::ErrorKind::UnexpectedEof, - IncompleteBody { - remaining: *this.remaining, - }, - ))); + let remaining = match usize::try_from(*this.remaining) { + Ok(remaining) => remaining, + Err(_) => usize::MAX, + }; + let allowed = remaining.min(buf.remaining()); + let read = if allowed == buf.remaining() { + let before = buf.filled().len(); + match this.inner.as_mut().poll_read(cx, buf) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Ready(Ok(())) => buf.filled().len() - before, } - *this.remaining -= read; - if *this.remaining < 0 { - return Poll::Ready(Err(Error::other("input provided more bytes than specified"))); + } else { + this.scratch.resize(allowed, 0); + let mut scratch_buf = ReadBuf::new(&mut this.scratch[..allowed]); + match this.inner.as_mut().poll_read(cx, &mut scratch_buf) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Ready(Ok(())) => { + let read = scratch_buf.filled().len(); + buf.put_slice(scratch_buf.filled()); + read + } } + }; + if read == 0 { + return Poll::Ready(Err(Error::new( + std::io::ErrorKind::UnexpectedEof, + IncompleteBody { + remaining: *this.remaining, + }, + ))); } - poll + let read = match i64::try_from(read) { + Ok(read) => read, + Err(_) => return Poll::Ready(Err(Error::other("read count exceeds i64::MAX"))), + }; + *this.remaining -= read; + Poll::Ready(Ok(())) } } @@ -140,7 +163,12 @@ mod tests { assert!(err.is_some()); let err = err.unwrap(); - assert_eq!(err.kind(), std::io::ErrorKind::Other); + assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof); + assert!( + err.get_ref() + .and_then(|source| source.downcast_ref::()) + .is_some_and(|source| source.to_string().contains("more bytes than specified")) + ); } #[tokio::test] @@ -155,6 +183,17 @@ mod tests { assert_eq!(&buf, data); } + #[tokio::test] + async fn test_hardlimit_reader_zero_capacity_read_does_not_consume_input() { + let mut reader = HardLimitReader::new(BufReader::new(&b"abc"[..]), 3); + let mut empty = []; + + assert_eq!(reader.read(&mut empty).await.expect("zero-capacity read should succeed"), 0); + let mut out = Vec::new(); + reader.read_to_end(&mut out).await.expect("input should remain readable"); + assert_eq!(out, b"abc"); + } + #[tokio::test] async fn test_hardlimit_reader_short_input_returns_unexpected_eof() { let data = b"abc"; @@ -195,4 +234,18 @@ mod tests { assert_eq!(err.kind(), std::io::ErrorKind::Other); assert!(err.to_string().contains("more bytes than specified")); } + + #[tokio::test] + async fn test_hardlimit_reader_caps_each_read_before_reporting_extra_bytes() { + let mut reader = HardLimitReader::new(BufReader::new(&b"abcdef"[..]), 3); + let mut out = Vec::new(); + + let err = reader + .read_to_end(&mut out) + .await + .expect_err("bytes beyond the declared limit must be rejected"); + + assert_eq!(out, b"abc"); + assert!(err.to_string().contains("more bytes than specified")); + } } diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index 8be68cef4..96fd0971f 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -138,6 +138,12 @@ impl std::fmt::Display for InternodeHttpErrorKind { } } +#[derive(thiserror::Error, Debug, Clone, Copy, Eq, PartialEq)] +#[error("internode body stalled for {timeout:?}")] +pub struct BodyStalled { + pub timeout: Duration, +} + #[derive(Debug, Clone, Eq, PartialEq)] pub struct InternodeHttpRequestContext { method: String, @@ -271,6 +277,10 @@ pub fn internode_http_timeout_error(method: &Method, url: &str) -> io::Error { internode_kind_error(method, url, internode_rpc_operation(url), InternodeHttpErrorKind::ConnectTimeout) } +fn body_stalled_error(stall_timeout: Duration) -> io::Error { + Error::new(io::ErrorKind::TimedOut, BodyStalled { timeout: stall_timeout }) +} + /// Clone an internode HTTP I/O error while retaining its structured classification. /// /// The underlying transport source is intentionally omitted because it is not @@ -1085,10 +1095,7 @@ impl AsyncRead for HttpReader { ); record_internode_stall_timeout(*this.track_internode_metrics, *this.internode_operation); record_internode_error(*this.track_internode_metrics, *this.internode_operation); - Poll::Ready(Err(Error::new( - io::ErrorKind::TimedOut, - "HttpReader stall timeout: no data received before deadline", - ))) + Poll::Ready(Err(body_stalled_error(stall_timeout))) } else { Poll::Pending } @@ -1217,10 +1224,7 @@ impl ChunkReader for HttpChunkReader { ); record_internode_stall_timeout(*this.track_internode_metrics, *this.internode_operation); record_internode_error(*this.track_internode_metrics, *this.internode_operation); - return Poll::Ready(Err(Error::new( - io::ErrorKind::TimedOut, - "HttpReader stall timeout: no data received before deadline", - ))); + return Poll::Ready(Err(body_stalled_error(stall_timeout))); } return Poll::Pending; } @@ -2379,6 +2383,46 @@ mod tests { Err(err) => err, }; assert_eq!(err.kind(), io::ErrorKind::TimedOut); + let stalled = err + .get_ref() + .and_then(|source| source.downcast_ref::()) + .expect("stall timeout should retain typed body-stalled source"); + assert_eq!(stalled.timeout, Duration::from_millis(20)); + + handle.abort(); + } + + #[tokio::test] + async fn http_chunk_reader_stall_timeout_retains_typed_source() { + let state = TestState::default(); + let Some((base_url, handle)) = start_test_server(state).await else { + return; + }; + let url = base_url.replace("/stream", "/stall"); + let mut reader = + HttpChunkReader::new_with_stall_timeout(url, Method::GET, HeaderMap::new(), None, Some(Duration::from_millis(20))) + .await + .expect("chunk reader should open"); + + let first = std::future::poll_fn(|cx| Pin::new(&mut reader).poll_read_chunk(cx, 64)) + .await + .expect("initial body chunk should arrive") + .expect("initial body chunk should not be EOF"); + assert_eq!(first, b"hello"[..]); + + let err = tokio::time::timeout( + Duration::from_secs(1), + std::future::poll_fn(|cx| Pin::new(&mut reader).poll_read_chunk(cx, 64)), + ) + .await + .expect("stall timeout should wake chunk reader") + .expect_err("chunk reader should return a timeout error"); + assert_eq!(err.kind(), io::ErrorKind::TimedOut); + let stalled = err + .get_ref() + .and_then(|source| source.downcast_ref::()) + .expect("chunk stall timeout should retain typed body-stalled source"); + assert_eq!(stalled.timeout, Duration::from_millis(20)); handle.abort(); } diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index d404add26..3bf35bc1b 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -2241,10 +2241,11 @@ fn get_object_resume_control(ctx: GetObjectResumeContext) -> GetObjectResumeCont /// disks" failures keep the existing fail-loud behavior. fn is_object_relocation_error(err: &std::io::Error) -> bool { let Some(inner) = err.get_ref() else { return false }; - matches!( - inner.downcast_ref::(), - Some(StorageError::FileNotFound | StorageError::ObjectNotFound(..) | StorageError::InsufficientReadQuorum(..)) - ) + match inner.downcast_ref::() { + Some(StorageError::FileNotFound | StorageError::ObjectNotFound(..) | StorageError::InsufficientReadQuorum(..)) => true, + Some(StorageError::Io(source)) => source.kind() == std::io::ErrorKind::NotFound, + _ => false, + } } /// Resolve the S3 request-body inter-chunk read timeout from the environment. @@ -13163,6 +13164,7 @@ mod tests { StorageError::FileNotFound, StorageError::ObjectNotFound("test-bucket".to_string(), "relocated-object".to_string()), StorageError::InsufficientReadQuorum("test-bucket".to_string(), "relocated-object".to_string()), + StorageError::Io(std::io::Error::new(std::io::ErrorKind::NotFound, "relocated shard disappeared")), ] { let reopen_count = Arc::new(AtomicUsize::new(0)); let control = counting_resume_control(Arc::clone(&reopen_count), |emitted| { diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 49ac36319..5819b8172 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -326,7 +326,11 @@ impl From for ApiError { _ => S3ErrorCode::InternalError, }; - let message = if matches!(&err, StorageError::QuotaExceeded { .. }) || code == S3ErrorCode::InternalError { + let message = if matches!(&err, StorageError::QuotaExceeded { .. }) { + err.to_string() + } else if code == S3ErrorCode::InternalError && matches!(&err, StorageError::Io(_)) { + ApiError::error_code_to_message(&code) + } else if code == S3ErrorCode::InternalError { err.to_string() } else if let StorageError::InvalidArgument(_, _, reason) = &err && !reason.is_empty() @@ -525,6 +529,25 @@ mod tests { assert!(api_error.source.is_some()); } + #[test] + fn storage_io_internal_error_redacts_public_message_and_retains_source() { + let sensitive_path = "/sensitive/storage/path"; + let api_error = ApiError::from(StorageError::Io(IoError::new( + ErrorKind::PermissionDenied, + format!("permission denied: {sensitive_path}"), + ))); + + assert_eq!(api_error.code, S3ErrorCode::InternalError); + assert_eq!(api_error.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError)); + assert!(!api_error.message.contains(sensitive_path)); + let source = api_error + .source + .as_deref() + .and_then(|source| source.downcast_ref::()) + .expect("API error should retain the storage error source"); + assert!(matches!(source, StorageError::Io(io_error) if io_error.to_string().contains(sensitive_path))); + } + #[test] fn test_kms_service_unavailable_maps_to_retryable_error() { let api_error = ApiError::from(StorageError::other(KmsUnavailableError)); @@ -669,14 +692,36 @@ mod tests { assert!(api_error.source.is_some()); } + #[test] + fn test_api_error_from_storage_io_copy_object_terminal_error_stays_internal() { + let io_error = IoError::other(StorageError::FileCorrupt); + let storage_error: StorageError = io_error.into(); + assert!(matches!(storage_error, StorageError::FileCorrupt)); + + let api_error: ApiError = storage_error.into(); + + assert_eq!(api_error.code, S3ErrorCode::InternalError); + let source = api_error + .source + .as_deref() + .and_then(|source| source.downcast_ref::()) + .expect("API error should retain the storage error source"); + assert!(matches!(source, StorageError::FileCorrupt)); + } + #[test] fn test_api_error_from_iam_error() { let iam_error = rustfs_iam::error::Error::other("IAM test error"); let api_error: ApiError = iam_error.into(); - // IAM error is first converted to StorageError, then to ApiError - assert!(api_error.source.is_some()); - assert!(api_error.message.contains("test error")); + assert_eq!(api_error.code, S3ErrorCode::InternalError); + assert_eq!(api_error.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError)); + let source = api_error + .source + .as_deref() + .and_then(|source| source.downcast_ref::()) + .expect("API error should retain the storage error source"); + assert!(matches!(source, StorageError::Io(io_error) if io_error.to_string().contains("IAM test error"))); } #[test] diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 920d6b6ca..87f5af497 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -5336,7 +5336,16 @@ mod tests { let error = TestSseDekProvider::decrypt_dek(&envelope, [0x55u8; 32]) .expect_err("unknown JSON envelope versions must fail closed"); - assert!(error.message.contains("Unsupported encrypted DEK format version")); + assert_eq!(error.code, S3ErrorCode::InternalError); + assert_eq!(error.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError)); + let source = error + .source + .as_deref() + .and_then(|source| source.downcast_ref::()) + .expect("API error should retain the storage error source"); + assert!(matches!(source, StorageError::Io(io_error) if io_error + .to_string() + .contains("Unsupported encrypted DEK format version"))); } #[tokio::test] @@ -5894,10 +5903,16 @@ mod tests { } #[test] - fn test_map_get_object_reader_error_leaves_non_ssec_errors_unchanged() { + fn test_map_get_object_reader_error_redacts_non_ssec_internal_errors() { let err = map_get_object_reader_error(StorageError::other("plain io failure")); assert_eq!(err.code, S3ErrorCode::InternalError); - assert_eq!(err.message, "Io error: plain io failure"); + assert_eq!(err.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError)); + let source = err + .source + .as_deref() + .and_then(|source| source.downcast_ref::()) + .expect("API error should retain the storage error source"); + assert!(matches!(source, StorageError::Io(io_error) if io_error.to_string().contains("plain io failure"))); } #[test] From e11ce2f132b5234f836cf7a275a68a1d723a3f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Fri, 14 Aug 2026 22:13:37 +0800 Subject: [PATCH 15/71] fix(site-replication): route every state RMW through the locked transaction (#6097) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(site-replication): route every state RMW through the locked transaction P1-15 PR2 (rustfs/backlog#1796, batch B2 of rustfs/backlog#1675), the follow-up promised by rustfs/rustfs#5882. PR1 left ~26 read-modify-write call sites on config/site-replication/state.json in the pre-transaction shape: a process-local mutex around load / mutate / save, each IO taking its own object lock. Nothing held a distributed lock across the whole sequence, so two nodes of one site still lost each other's updates, and the transitional mutex kept the old shape available to copy. Every remaining RMW now runs inside update_site_replication_state; read-only sites use load_site_replication_state, whose object read comes with the object-level read lock. SITE_REPLICATION_STATE_LOCK and its owner helper are gone, together with their architecture-guard allowlist entry and inventory row. The multi-stage flows (add / edit / peer join / peer edit / remove / rotate) keep their updated_at and pending-id CAS, but the CAS now runs inside the transaction that writes, against the state that transaction loaded. Peer probes, IAM work and fan-outs run between transactions and hold no lock at all — the add no longer blocks every writer of the site across its peer join round trips, and it re-checks the precondition right after the capability probes so the common race is rejected before any IAM write or remote join. When the add's commit CAS still fails, the error says the peers may already be joined and that re-running the add reconverges. The add adopts only the fields it computed (exhaustive destructure — adding a state field is a compile error until classified); fields owned by writers that do not bump updated_at keep their freshly loaded values. Ordering of peer-edit deliveries now rests on the generation fence landed in PR1 rather than on a guard that could never order two nodes: the add's finalize fan-out carries the generation allocated in its commit. An accepted peer join PRESERVES the applied-generation high-water marks — join fan-outs are routine (adds and rotations both deliver SRPeerJoin to existing peers), so wiping them would let stalled older edits land after any join; the unilateral-removal rejoin misfence that a wipe would have patched is pre-existing since the fence landed and needs an epoch in the fence instead. The rotation handler now takes the lifecycle guard: the background service-account reconciler runs its repair under a lifecycle try-acquire, and its pending-rotation precheck is only sound if a rotation cannot start mid-repair — an exclusion the removed process mutex used to provide as a side effect. update_site_replication_state_when_changed adds persist-or-skip so ack markers and pending-clearing paths stop rewriting the object on a miss — load-bearing, because the shared persist helper clears the whole object for a ≤1-peer pending-free state — and save_site_replication_state is now cfg(test): the pre-P1-15 shape can no longer be written in production code. No on-disk format change. Verification: cargo nextest run -p rustfs -E 'test(/admin::handlers::site_replication::/)' (181 passed); site-replication dual/three-node e2e (13 passed); cargo clippy -p rustfs --all-targets -D warnings; make pre-commit. Mutation checks: dropping the state-object lock from the boundary reds the separate-node concurrency tests; flipping a persist-or-skip miss to a persist reds test_missed_pending_clear_must_not_rewrite_the_state_object. Reviewed by three independent adversarial passes (correctness/concurrency, security/compatibility, simplicity/test-coverage); their confirmed findings are folded in. * fix(site-replication): serialize peer-join admission around its IAM write Review follow-up (overtrue): two joins accepted by the same node could interleave as "A checks a stale snapshot and pauses reading its body, B applies secret B and commits, A resumes, overwrites IAM with secret A, and A's commit is refused as superseded" — the persisted state advertised B's contract while IAM only accepted A's secret, failing every peer control-plane call. The pre-P1-15 process mutex serialized same-node joins end to end; removing it dropped that exclusion. admit_peer_join now runs the staleness check, the IAM upsert and the state commit under the lifecycle guard, with the authoritative pre-check taken against a load under that guard BEFORE IAM changes anything. The closing transaction still re-checks staleness: the guard is process-local (exactly as far as the old mutex reached) and the state-object lock arbitrates joins accepted by different nodes. The body is fully read before the guard so a stalling sender cannot block add/remove/rotate/reconciler. The IAM step is injected, and the gated-body regression test reproduces the review's ordering: join A is held mid-IAM while a newer join B arrives; B must wait at the guard, and both IAM order and the final persisted state end on B. Mutation-verified: removing the lifecycle guard from admit_peer_join turns the test red. Verification: cargo nextest run -p rustfs -E 'test(/admin::handlers::site_replication::/)' (182 passed); site-replication dual/three-node e2e (13 passed); cargo clippy -p rustfs --all-targets -D warnings; make pre-commit. * fix(site-replication): fence peer-join admission across nodes Review follow-up (overtrue, round 2): the lifecycle guard only serializes joins within one process. Node A could pass the staleness check for an older T1, node B write secret B to IAM and commit a newer T2, and node A then overwrite IAM with secret A while its own state commit is refused as superseded — state advertising T2's contract while IAM only accepts A's secret. The admission (staleness check -> IAM upsert -> state commit) now also runs under a distributed join-admission lock, a namespace-lock key with no backing object, following the repair execution lock's pattern — including its nesting of config-object locks (admission -> state), and delegating crash safety to the lock subsystem's lease expiry instead of a hand-rolled TTL. The staleness check runs against a load taken inside the lock, before IAM changes anything, so a superseded join exits without touching IAM. The closing transaction keeps its re-check for defence in depth and for old-version nodes that do not take the admission lock during a rolling upgrade (that mixed-version window keeps today's behavior and closes when the upgrade completes). admit_peer_join_across_nodes is the admission minus the process-local lifecycle guard — exactly what a second node runs — and the new separate-nodes regression test drives it directly with join A gated mid-IAM: join B must wait at the distributed lock, and both the IAM write order and the final persisted state end on B. Mutation-verified: removing the admission lock turns the test red while the same-node test (which drives the full admit_peer_join) stays green. Verification: cargo nextest run -p rustfs -E 'test(/admin::handlers::site_replication::/)' (183 passed); site-replication dual/three-node e2e (13 passed); cargo clippy -p rustfs --all-targets -D warnings; make pre-commit. --- docs/architecture/global-state-inventory.md | 2 +- rustfs/src/admin/handlers/site_replication.rs | 1823 +++++++++++------ rustfs/src/admin/site_replication_state.rs | 65 +- scripts/check_architecture_migration_rules.sh | 2 +- 4 files changed, 1233 insertions(+), 659 deletions(-) diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index a594bc569..6c06b10c8 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -111,7 +111,7 @@ inventory. Generic function-local names such as `CACHE`, `LOCK`, `INIT`, and | `GET_OBJECT_BUFFER_THRESHOLD_WARNED`, `GET_READER_STREAM_BUFFER_SIZE_OVERRIDE`, function-local `ENABLED`, `OBJECT_SEEK_SUPPORT_THRESHOLD`, `OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS` | `rustfs/src/app/object_usecase.rs` | Cache or constant / owner-local cache | Object GET/seek tuning caches and warning guards stay private to object usecase helpers. | | `SUPPORTED_HEADERS` | `rustfs/src/storage/options.rs` | Cache or constant / owner-local constant | Supported-header lookup state stays private to storage option parsing. | | `AUDIT_TARGET_SPECS`, `NOTIFICATION_TARGET_SPECS` | `rustfs/src/admin/handlers/audit.rs`, `rustfs/src/admin/handlers/event.rs`, `rustfs/src/admin/handlers/plugins_instances.rs` | Cache or constant / owner-local constant | Admin target descriptor tables stay private to their handler owners. | -| `SITE_REPLICATION_PEER_CLIENT`, `SITE_REPLICATION_STATE_LOCK` | `rustfs/src/admin/handlers/site_replication.rs` | Process-global owner-local cache / guard | Site-replication peer client cache and state lock stay private to site-replication handlers. | +| `SITE_REPLICATION_PEER_CLIENT` | `rustfs/src/admin/handlers/site_replication.rs` | Process-global owner-local cache | Site-replication peer client cache stays private to site-replication handlers. The state RMW transaction holds no process-local mutex — see `rustfs/src/admin/site_replication_state.rs`. | | `AUDIT_MODULE_ENABLED`, `NOTIFY_MODULE_ENABLED`, `PERSISTED_NOTIFY_MODULE_ENABLED`, `PERSISTED_AUDIT_MODULE_ENABLED`, `PERSISTED_MODULE_SWITCH_CONFIGURED` | `rustfs/src/server/audit.rs`, `rustfs/src/server/event.rs`, `rustfs/src/server/module_switch.rs` | Process-global owner-local toggles | Audit/notify module snapshots stay private to the server module switch owners. | | `DELETE_TAIL_TOTAL`, `DELETE_CLEANUP_TOTAL`, `DELETE_REPLICATION_TOTAL`, `DELETE_NOTIFY_TOTAL` | `rustfs/src/delete_tail_activity.rs` | Process-global owner-local counters | Delete-tail activity counters stay private behind delete-tail activity helpers. | | `EMBEDDED_SERVER_STARTED` | `rustfs/src/startup_lifecycle.rs` | Process-global owner-local guard | Embedded startup single-start protection stays private to startup lifecycle. | diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index bf946c0c4..be2220d64 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -35,7 +35,9 @@ use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetT use crate::admin::storage_api::bucket::target_sys::BucketTargetSys; use crate::admin::storage_api::bucket::utils::{deserialize, serialize}; use crate::admin::storage_api::bucket::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; -use crate::admin::storage_api::config::{delete_admin_config, read_admin_config, save_admin_config}; +use crate::admin::storage_api::config::read_admin_config; +#[cfg(test)] +use crate::admin::storage_api::config::save_admin_config; use crate::admin::storage_api::contract::bucket::{ BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp, }; @@ -112,13 +114,13 @@ const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_SITE_REPLICATION: &str = "site_replication"; const EVENT_ADMIN_SITE_REPLICATION_STATE: &str = "admin_site_replication_state"; const SERVICE_ACCOUNT_ENVELOPE_VERSION: u64 = 2; -#[cfg(test)] -use crate::admin::site_replication_state::with_site_replication_state_object_lock; -use crate::admin::site_replication_state::{ - SITE_REPLICATION_STATE_PATH, site_replication_state_process_guard, with_site_replication_state_lock, -}; +use crate::admin::site_replication_state::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock}; const SITE_REPLICATION_REPAIR_STATE_PATH: &str = "config/site-replication/repair-state.json"; const SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH: &str = "config/site-replication/repair-execution.lock"; +// Serializes peer-join admission (staleness check -> IAM upsert -> state +// commit) across every node of this site; see admit_peer_join. Never an +// actual object — only a namespace-lock key, like the repair execution lock. +const SITE_REPLICATION_JOIN_ADMISSION_LOCK_PATH: &str = "config/site-replication/join-admission.lock"; const SITE_REPL_ADD_SUCCESS: &str = "Requested sites were configured for replication successfully."; const SITE_REPL_EDIT_SUCCESS: &str = "Requested site was updated successfully."; const SITE_REPL_REMOVE_SUCCESS: &str = "Requested site(s) were removed from cluster replication successfully."; @@ -354,8 +356,11 @@ impl TryFrom<&PeerSite> for PeerConnection { static SITE_REPLICATION_PEER_CLIENT: LazyLock>> = LazyLock::new(|| Mutex::new(None)); // Lock order: lifecycle -> bucket operation -> repair admission -> state -> per-bucket metadata. -// The state mutex lives in crate::admin::site_replication_state and is -// taken through site_replication_state_process_guard (P1-15). +// "state" is the distributed state-object lock in +// crate::admin::site_replication_state, entered through +// update_site_replication_state (P1-15). There is no process-local state +// mutex any more: it could not order two nodes of one site, and the call +// sites that needed ordering carry a generation fence instead. static SITE_REPLICATION_LIFECYCLE_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); static SITE_REPLICATION_BUCKET_OP_LOCK: LazyLock> = LazyLock::new(|| RwLock::new(())); static SITE_REPLICATION_ADD_BOOTSTRAP: LazyLock>> = @@ -469,11 +474,10 @@ struct SiteReplicationState { #[serde(default)] sync_state_initialized: bool, /// Fencing token for peer-edit delivery, allocated inside the state - /// transaction (process mutex + distributed state-object lock). Two nodes - /// of THIS site that accept admin edits concurrently therefore get - /// strictly ordered generations even though the process mutex cannot - /// serialize them, and a delivery that stalls can be recognised as stale - /// by the receiving site. + /// transaction (the distributed state-object lock). Two nodes of THIS + /// site that accept admin edits concurrently therefore get strictly + /// ordered generations, and a delivery that stalls can be recognised as + /// stale by the receiving site. #[serde(default)] edit_generation: u64, /// Per-origin high-water mark of the peer edits already applied here, @@ -1123,44 +1127,49 @@ async fn persist_site_replication_state_no_lock(store: Arc, mut state: } } -/// A second node's view of the same transaction: identical production code -/// path minus the process mutex, which is per process and therefore cannot -/// serialize anything across nodes. Used by the separate-nodes regression -/// test so that removing the distributed lock breaks it. -#[cfg(test)] -async fn update_site_replication_state_as_separate_node(update: F) -> S3Result -where - T: Send + 'static, - F: FnOnce(&mut SiteReplicationState) -> S3Result + Send + 'static, -{ - let store = - current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; - let lock_store = store.clone(); - with_site_replication_state_object_lock(lock_store, move || async move { - let mut state = load_site_replication_state_no_lock(store.clone()).await?; - let result = update(&mut state)?; - persist_site_replication_state_no_lock(store, state).await?; - Ok(result) - }) - .await +/// What a state transaction closure decided to do with the state it was +/// handed. `Unchanged` skips the write entirely: the ack markers and the +/// pending-clearing paths run on every retry and mostly find their pending id +/// already gone, and the retry queue shares this object — rewriting it byte +/// for byte only makes those misses contend with the writers that do have +/// something to say. +enum StateCommit { + Changed(T), + Unchanged(T), } /// The site-replication state RMW transaction: load, mutate, persist — all -/// under the process mutex plus the distributed state-object write lock -/// (see crate::admin::site_replication_state). No peer network calls and no -/// other config locks inside `update`. +/// under the distributed state-object write lock (see +/// crate::admin::site_replication_state). No peer network calls and no other +/// config locks inside `update`; anything that has to talk to a peer belongs +/// between two transactions, with the precondition re-checked inside the +/// second one. async fn update_site_replication_state(update: F) -> S3Result where T: Send + 'static, F: FnOnce(&mut SiteReplicationState) -> S3Result + Send + 'static, +{ + update_site_replication_state_when_changed(move |state| update(state).map(StateCommit::Changed)).await +} + +/// [`update_site_replication_state`] for closures that may find nothing to +/// do — see [`StateCommit`]. +async fn update_site_replication_state_when_changed(update: F) -> S3Result +where + T: Send + 'static, + F: FnOnce(&mut SiteReplicationState) -> S3Result> + Send + 'static, { with_site_replication_state_lock(move || async move { let store = current_object_store_handle() .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; let mut state = load_site_replication_state_no_lock(store.clone()).await?; - let result = update(&mut state)?; - persist_site_replication_state_no_lock(store, state).await?; - Ok(result) + match update(&mut state)? { + StateCommit::Changed(result) => { + persist_site_replication_state_no_lock(store, state).await?; + Ok(result) + } + StateCommit::Unchanged(result) => Ok(result), + } }) .await } @@ -1216,6 +1225,11 @@ where .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock repair state failed: {e}")))? } +/// Test-only seeding of the state object. Every production write goes through +/// [`update_site_replication_state`] — this helper is `cfg(test)` so a new +/// call site cannot reintroduce the pre-P1-15 shape (load through one object +/// lock, save through another, with the mutation in between unprotected). +#[cfg(test)] async fn save_site_replication_state(state: &SiteReplicationState) -> S3Result<()> { let Some(store) = current_object_store_handle() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); @@ -1232,27 +1246,6 @@ async fn save_site_replication_state(state: &SiteReplicationState) -> S3Result<( Ok(()) } -async fn clear_site_replication_state() -> S3Result<()> { - let Some(store) = current_object_store_handle() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - match delete_admin_config(store, SITE_REPLICATION_STATE_PATH).await { - Ok(()) | Err(StorageError::ConfigNotFound) => Ok(()), - Err(err) => Err(S3Error::with_message(S3ErrorCode::InternalError, format!("clear state failed: {err}"))), - } -} - -async fn persist_site_replication_state(state: &SiteReplicationState) -> S3Result<()> { - let mut normalized = state.clone(); - normalized.peers = normalize_peer_map_by_identity(normalized.peers); - if normalized.peers.len() <= 1 && normalized.pending_rotation.is_none() && normalized.pending_remove.is_none() { - clear_site_replication_state().await - } else { - save_site_replication_state(&normalized).await - } -} - fn build_site_replication_peer_client(outbound_tls: &GlobalPublishedOutboundTlsState) -> S3Result { build_site_replication_peer_client_with_resolver(outbound_tls, PeerDnsResolver::new(loopback_replication_targets_allowed())) } @@ -1668,7 +1661,14 @@ fn stored_peer_tls_settings(stored_peer: Option<&PeerInfo>) -> (bool, String) { } fn current_local_peer(req: &S3Request, state: &SiteReplicationState) -> PeerInfo { - let endpoint = site_replication_local_endpoint(&req.uri, &req.headers); + local_peer_at_endpoint(site_replication_local_endpoint(&req.uri, &req.headers), state) +} + +/// The local peer record as the given state describes it. Split out of +/// [`current_local_peer`] so a state transaction can rebuild it against the +/// state it just loaded: the request the endpoint came from cannot cross into +/// the transaction closure, but the endpoint itself can. +fn local_peer_at_endpoint(endpoint: String, state: &SiteReplicationState) -> PeerInfo { let deployment_id = current_deployment_id().unwrap_or_else(|| deployment_id_for_endpoint(&endpoint)); let stored_peer = state.peers.get(&deployment_id); let (skip_tls_verify, ca_cert_pem) = stored_peer_tls_settings(stored_peer); @@ -1695,30 +1695,7 @@ fn current_local_peer(req: &S3Request, state: &SiteReplicationState) -> Pe } fn current_local_runtime_peer(state: &SiteReplicationState) -> PeerInfo { - let endpoint = current_local_runtime_endpoint(); - let deployment_id = current_deployment_id().unwrap_or_else(|| deployment_id_for_endpoint(&endpoint)); - let stored_peer = state.peers.get(&deployment_id); - let (skip_tls_verify, ca_cert_pem) = stored_peer_tls_settings(stored_peer); - - PeerInfo { - endpoint: endpoint.clone(), - name: if state.name.is_empty() { - stored_peer - .map(|peer| peer.name.clone()) - .filter(|name| !name.is_empty()) - .unwrap_or_else(|| infer_site_name(&endpoint)) - } else { - state.name.clone() - }, - deployment_id, - sync_state: stored_peer.map(|peer| peer.sync_state.clone()).unwrap_or(SyncStatus::Unknown), - default_bandwidth: stored_peer.map(|peer| peer.default_bandwidth.clone()).unwrap_or_default(), - replicate_ilm_expiry: stored_peer.is_some_and(|peer| peer.replicate_ilm_expiry), - object_naming_mode: stored_peer.map(|peer| peer.object_naming_mode.clone()).unwrap_or_default(), - skip_tls_verify, - ca_cert_pem, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - } + local_peer_at_endpoint(current_local_runtime_endpoint(), state) } fn normalize_peer_map_by_identity(peers: BTreeMap) -> BTreeMap { @@ -2535,6 +2512,51 @@ fn initialize_join_peer_sync_state(peers: &mut BTreeMap, defer } } +/// Whether an incoming peer join carries a snapshot this site has already +/// moved past — an unstamped join against a configured site, or one whose +/// `updated_at` is not newer. Applying it would roll the local view back to +/// the older topology, so the join is answered as a no-op (MinIO-compatible +/// behaviour, kept verbatim from the pre-transaction handler). +fn join_request_is_superseded(state: &SiteReplicationState, incoming_updated_at: Option) -> bool { + let Some(current_updated_at) = state.updated_at else { + return false; + }; + incoming_updated_at.is_none_or(|incoming_updated_at| incoming_updated_at <= current_updated_at) +} + +/// Adopt an accepted peer join: the sending site's snapshot replaces the local +/// topology wholesale. +/// +/// The peer-edit high-water marks are deliberately KEPT. Wiping them here +/// would reopen the exact window the fence closes: every join fan-out (adds +/// AND service-account rotations deliver `SRPeerJoin` to existing peers) +/// would discard live marks, letting a stalled older edit from a peer that +/// never left roll a record back. The one case a kept mark misfences — a +/// site removed while unreachable rejoining with a restarted generation +/// counter — already misfences its ordinary edits identically (pre-existing +/// since the fence landed) and needs an epoch in the fence to fix, not a +/// blanket reset. Marks of origins that left AND were observed leaving are +/// dropped on load by `parse_site_replication_state`. +fn apply_peer_join( + state: &mut SiteReplicationState, + local_peer: &PeerInfo, + join_req: SRPeerJoinReq, + defer_sync_state_enable: bool, +) { + state.service_account_access_key = join_req.svc_acct_access_key; + state.service_account_parent = join_req.svc_acct_parent; + state.updated_at = join_req.updated_at.or_else(|| Some(OffsetDateTime::now_utc())); + state.peers = normalize_join_peers_for_local(local_peer, join_req.peers); + initialize_join_peer_sync_state(&mut state.peers, defer_sync_state_enable); + state.sync_state_initialized = true; + state.name = state + .peers + .get(&local_peer.deployment_id) + .map(|peer| peer.name.clone()) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| local_peer.name.clone()); +} + fn reconcile_peer_with_actual_identity(mut state: SiteReplicationState, actual_peer: PeerInfo) -> SiteReplicationState { let mut actual_peer = normalize_peer_info(actual_peer); if let Some(requested_peer) = state @@ -2736,7 +2758,8 @@ fn is_missing_service_account_error(err: &rustfs_iam::error::Error) -> bool { /// disables every control-plane push while `replicate info` still reports the site enabled. /// Both used to require deleting and recreating the account by hand. async fn reconcile_site_replicator_service_account() -> S3Result<()> { - let _state_guard = site_replication_state_process_guard().await; + // Read-only against the state: `load_site_replication_state` takes the + // object read lock on its own, and everything after it is IAM work. let state = load_site_replication_state().await?; if !state.enabled() || state.service_account_access_key != SITE_REPLICATOR_SERVICE_ACCOUNT { return Ok(()); @@ -3908,29 +3931,30 @@ async fn persist_site_replication_repair_task( ) -> S3Result<()> { persist_site_replication_repair_operation(operation).await?; - let _state_guard = site_replication_state_process_guard().await; - let mut latest = load_site_replication_state().await?; let family_status = operation .sites .get(&peer.deployment_id) .and_then(|site| site.families.get(family)) .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "repair task status is missing".to_string()))?; - if family_status.failed > 0 { - upsert_site_replication_retry_event( - &mut latest.retry_queue, - peer, - path, - family_status - .errors - .first() - .map(String::as_str) - .unwrap_or("remote-operation-failed"), - None, - ); - } else { - dequeue_site_replication_retry_events(&mut latest.retry_queue, peer, path); - } - persist_site_replication_state(&latest).await + let failure = (family_status.failed > 0).then(|| { + family_status + .errors + .first() + .cloned() + .unwrap_or_else(|| "remote-operation-failed".to_string()) + }); + let peer = peer.clone(); + let path = path.to_string(); + update_site_replication_state(move |state| { + match failure.as_deref() { + Some(error) => upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None), + None => { + dequeue_site_replication_retry_events(&mut state.retry_queue, &peer, &path); + } + } + Ok(()) + }) + .await } fn admit_site_replication_repair_operation( @@ -3986,14 +4010,10 @@ async fn execute_site_replication_repair( async fn execute_site_replication_repair_locked( request: SiteReplicationRepairExecutionRequest, ) -> S3Result> { - let state = { - let _state_guard = site_replication_state_process_guard().await; - let state = load_site_replication_state().await?; - if !state.enabled() || state.service_account_access_key.is_empty() { - return Err(s3_error!(InvalidRequest, "site replication is not configured")); - } - state - }; + let state = load_site_replication_state().await?; + if !state.enabled() || state.service_account_access_key.is_empty() { + return Err(s3_error!(InvalidRequest, "site replication is not configured")); + } let info = build_sr_info(&state, &request.local_peer).await?; let plan = site_replication_bootstrap_plan(&info)?; let plan_token = site_replication_repair_plan_token(&state, &plan)?; @@ -4110,7 +4130,11 @@ async fn execute_site_replication_repair_locked( pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> { let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await; let runtime = { - let _state_guard = site_replication_state_process_guard().await; + // The bucket-op lock is what orders this against add/remove. The + // state is only read here (through the runtime snapshot), and the + // bucket setup below writes bucket metadata, never the state object — + // holding the state transaction across it would put local metadata + // IO inside a distributed lock for nothing. let Some(runtime) = runtime_site_replication_targets().await? else { return Ok(()); }; @@ -5306,6 +5330,29 @@ fn internal_endpoint_refresh_already_committed(state: &SiteReplicationState, inc .is_some_and(|committed| peer_connection_settings_match(committed, incoming)) } +/// An admin add/edit's precondition, re-evaluated inside the transaction that +/// is about to commit: the topology must still be the one the operation was +/// planned against, and the endpoint refresh must still be the same one (or +/// still absent). The planning snapshot is taken before peer probes and +/// fan-outs, none of which may hold the state-object lock, so only the check +/// inside the committing closure binds — the same check between network +/// stages is advisory, fencing the common race off the side-effect path. +/// `stage` names what was in flight for the operator; a rejected commit is +/// safe to re-run. +fn ensure_edit_precondition( + state: &SiteReplicationState, + expected_updated_at: Option, + expected_pending_id: Option<&String>, + stage: &str, +) -> S3Result<()> { + if state.updated_at != expected_updated_at + || pending_endpoint_refresh(state).as_ref().map(|pending| &pending.id) != expected_pending_id + { + return Err(s3_error!(InvalidRequest, "site replication state changed during {stage}")); + } + Ok(()) +} + fn set_pending_endpoint_refresh(state: &mut SiteReplicationState, pending: PendingEndpointRefresh) -> S3Result<()> { state .retry_queue @@ -5924,9 +5971,9 @@ fn peer_edit_fence(queries: &HashMap) -> Option<(String, u64)> { } /// True when a strictly newer edit from the same origin site already landed -/// here. The process mutex on the sending node cannot order deliveries issued -/// by two nodes of that site, so ordering is decided here, on the generation -/// the sender allocated under the distributed lock. Equal generations are NOT +/// here. No lock on the sending side can order deliveries issued by two +/// nodes of that site, so ordering is decided here, on the generation the +/// sender allocated under the distributed lock. Equal generations are NOT /// stale: one edit legitimately fans out several deliveries under a single /// generation (the ILM-expiry edit sends every peer's record), and a replay of /// an applied delivery re-applies the same edit idempotently. @@ -6206,15 +6253,15 @@ async fn record_pending_rotation_secret_candidate(rotation_id: &str, secret: Str return Ok(()); } - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; - if let Some(pending) = state.pending_rotation.as_mut() - && pending.id == rotation_id - { + let rotation_id = rotation_id.to_string(); + update_site_replication_state_when_changed(move |state| { + let Some(pending) = state.pending_rotation.as_mut().filter(|pending| pending.id == rotation_id) else { + return Ok(StateCommit::Unchanged(())); + }; push_unique_secret_candidate(&mut pending.secret_candidates, secret); - save_site_replication_state(&state).await?; - } - Ok(()) + Ok(StateCommit::Changed(())) + }) + .await } async fn record_pending_remove_secret_candidate(remove_id: &str, secret: String) -> S3Result<()> { @@ -6222,27 +6269,26 @@ async fn record_pending_remove_secret_candidate(remove_id: &str, secret: String) return Ok(()); } - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; - if let Some(pending) = state.pending_remove.as_mut() - && pending.id == remove_id - { + let remove_id = remove_id.to_string(); + update_site_replication_state_when_changed(move |state| { + let Some(pending) = state.pending_remove.as_mut().filter(|pending| pending.id == remove_id) else { + return Ok(StateCommit::Unchanged(())); + }; push_unique_secret_candidate(&mut pending.secret_candidates, secret); - save_site_replication_state(&state).await?; - } - Ok(()) + Ok(StateCommit::Changed(())) + }) + .await } async fn mark_pending_rotation_peer_acked(rotation_id: &str, deployment_id: &str) -> S3Result<()> { let rotation_id = rotation_id.to_string(); let deployment_id = deployment_id.to_string(); - update_site_replication_state(move |state| { - if let Some(pending) = state.pending_rotation.as_mut() - && pending.id == rotation_id - { - pending.acked_deployment_ids.insert(deployment_id); - } - Ok(()) + update_site_replication_state_when_changed(move |state| { + let Some(pending) = state.pending_rotation.as_mut().filter(|pending| pending.id == rotation_id) else { + return Ok(StateCommit::Unchanged(())); + }; + pending.acked_deployment_ids.insert(deployment_id); + Ok(StateCommit::Changed(())) }) .await } @@ -6250,37 +6296,37 @@ async fn mark_pending_rotation_peer_acked(rotation_id: &str, deployment_id: &str async fn mark_pending_remove_peer_acked(remove_id: &str, deployment_id: &str) -> S3Result<()> { let remove_id = remove_id.to_string(); let deployment_id = deployment_id.to_string(); - update_site_replication_state(move |state| { - if let Some(pending) = state.pending_remove.as_mut() - && pending.id == remove_id - { - pending.acked_deployment_ids.insert(deployment_id); - } - Ok(()) + update_site_replication_state_when_changed(move |state| { + let Some(pending) = state.pending_remove.as_mut().filter(|pending| pending.id == remove_id) else { + return Ok(StateCommit::Unchanged(())); + }; + pending.acked_deployment_ids.insert(deployment_id); + Ok(StateCommit::Changed(())) }) .await } async fn finalize_pending_rotation_if_complete(rotation_id: &str, local_peer: &PeerInfo) -> S3Result { - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; - let Some(pending) = state.pending_rotation.as_ref() else { - return Ok(true); - }; - if pending.id != rotation_id { - return Ok(false); - } - if !pending_all_remote_peers_acked(&pending.peers, local_peer, &pending.acked_deployment_ids) { - return Ok(false); - } + let rotation_id = rotation_id.to_string(); + let local_peer = local_peer.clone(); + update_site_replication_state_when_changed(move |state| { + let Some(pending) = state.pending_rotation.as_ref() else { + return Ok(StateCommit::Unchanged(true)); + }; + if pending.id != rotation_id { + return Ok(StateCommit::Unchanged(false)); + } + if !pending_all_remote_peers_acked(&pending.peers, &local_peer, &pending.acked_deployment_ids) { + return Ok(StateCommit::Unchanged(false)); + } - state.pending_rotation = None; - persist_site_replication_state(&state).await?; - Ok(true) + state.pending_rotation = None; + Ok(StateCommit::Changed(true)) + }) + .await } async fn pending_remove_ready_to_finalize(remove_id: &str, local_peer: &PeerInfo) -> S3Result> { - let _state_guard = site_replication_state_process_guard().await; let state = load_site_replication_state().await?; let Some(pending) = state.pending_remove.as_ref() else { return Ok(None); @@ -6296,18 +6342,15 @@ async fn pending_remove_ready_to_finalize(remove_id: &str, local_peer: &PeerInfo } async fn clear_pending_remove(remove_id: &str) -> S3Result<()> { - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; - if state - .pending_remove - .as_ref() - .map(|pending| pending.id.as_str() == remove_id) - .unwrap_or(false) - { + let remove_id = remove_id.to_string(); + update_site_replication_state_when_changed(move |state| { + if state.pending_remove.as_ref().is_none_or(|pending| pending.id != remove_id) { + return Ok(StateCommit::Unchanged(())); + } state.pending_remove = None; - persist_site_replication_state(&state).await?; - } - Ok(()) + Ok(StateCommit::Changed(())) + }) + .await } fn removed_deployment_ids_for_pending_remove(pending: &PendingRemove, local_peer: &PeerInfo) -> HashSet { @@ -7375,7 +7418,8 @@ async fn refresh_bucket_targets_after_endpoint_edit(pending_id: &str, service_ac let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(&bucket.name) .await .map_err(ApiError::from)?; - let _state_guard = site_replication_state_process_guard().await; + // Read-only per bucket: the pending refresh is re-read (and re-checked) + // every round, and the writes below are bucket metadata, not state. let state = load_site_replication_state().await?; let Some(pending) = pending_endpoint_refresh(&state).filter(|pending| pending.id == pending_id) else { return Err(s3_error!(InvalidRequest, "endpoint target refresh state changed during update")); @@ -7696,27 +7740,36 @@ async fn refresh_site_resync_status(mut status: SRResyncOpStatus, peer: &PeerInf } async fn persist_site_resync_status(peer_id: &str, status: &SRResyncOpStatus) -> S3Result<()> { - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; - if state - .resync_status - .get(peer_id) - .is_some_and(|current| current.resync_id != status.resync_id || current.generation != status.generation) - { - return Err(s3_error!(InvalidRequest, "site replication resync state changed")); - } - state.resync_status.insert(peer_id.to_string(), status.clone()); - save_site_replication_state(&state).await + let peer_id = peer_id.to_string(); + let status = status.clone(); + update_site_replication_state(move |state| { + // The run identity is checked inside the transaction: a cancel or a + // newer run that committed while this progress snapshot was being + // built must not be overwritten by it. + if state + .resync_status + .get(&peer_id) + .is_some_and(|current| current.resync_id != status.resync_id || current.generation != status.generation) + { + return Err(s3_error!(InvalidRequest, "site replication resync state changed")); + } + state.resync_status.insert(peer_id, status); + Ok(()) + }) + .await } async fn persist_new_site_resync_status(peer_id: &str, status: &SRResyncOpStatus) -> S3Result<()> { - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; - if state.resync_status.get(peer_id).is_some_and(site_resync_is_active) { - return Err(s3_error!(InvalidRequest, "site replication resync is already active")); - } - state.resync_status.insert(peer_id.to_string(), status.clone()); - save_site_replication_state(&state).await + let peer_id = peer_id.to_string(); + let status = status.clone(); + update_site_replication_state(move |state| { + if state.resync_status.get(&peer_id).is_some_and(site_resync_is_active) { + return Err(s3_error!(InvalidRequest, "site replication resync is already active")); + } + state.resync_status.insert(peer_id, status); + Ok(()) + }) + .await } fn apply_state_edit_req(mut state: SiteReplicationState, body: SRStateEditReq) -> SiteReplicationState { @@ -8352,7 +8405,10 @@ impl Operation for SiteReplicationAddHandler { reject_site_replicator_on_public_admin(&cred)?; let replicate_ilm_expiry = sr_add_replicate_ilm_expiry(&req.uri); let lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; - let state_guard = site_replication_state_process_guard().await; + // Everything up to the commit below is preflight: peer probes, IAM + // work and the join fan-out all talk to the network, so none of it may + // run inside the state transaction. The snapshot read here is what the + // `updated_at` CAS in the commit validates. let current_state = load_site_replication_state().await?; if pending_endpoint_refresh(¤t_state).is_some() { return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending")); @@ -8373,13 +8429,14 @@ impl Operation for SiteReplicationAddHandler { } validate_add_preflight_topology(&preflight_infos, &local_peer)?; let expected_updated_at = current_state.updated_at; - drop(state_guard); require_add_peer_tls_capability(&sites, &local_peer).await?; - let _state_guard = site_replication_state_process_guard().await; + // Early exit on a state that moved under the preflight probes, BEFORE + // the IAM write and the join fan-out change anything remote. Advisory + // only — the binding check is the CAS inside the commit — but it fences + // the common race off the side-effect path and refreshes the merge + // base so the CAS window is only the join round trips. let latest_state = load_site_replication_state().await?; - if latest_state.updated_at != expected_updated_at || pending_endpoint_refresh(&latest_state).is_some() { - return Err(s3_error!(InvalidRequest, "site replication state changed during capability probe")); - } + ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?; let current_state = latest_state; let (service_account_access_key, service_account_secret_key) = ensure_site_replicator_service_account(&cred.access_key, false).await?; @@ -8454,13 +8511,63 @@ impl Operation for SiteReplicationAddHandler { } mark_unknown_peer_sync_enabled(&mut state.peers); - persist_site_replication_state(&state).await?; - // The finalize fan-out below delivers peer-edit payloads, so it stays - // under the state guard: ordering against a concurrent edit matters - // and the peer edit handler has no generation fence. It uses the - // plain transport (no retry-event bookkeeping), so nothing re-enters - // the state transaction while the guard is held. + // Commit. The CAS runs inside the transaction, against the state the + // transaction itself loaded — the peer round trips above took however + // long they took, and only this check can tell whether the topology + // this add was planned against is still the current one. The error + // says so: by this point the remote sites already accepted their + // joins, and re-running the add is what reconverges the local side. + let next_state = state; + let (state, edit_generation) = update_site_replication_state(move |state| { + if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() { + return Err(s3_error!( + InvalidRequest, + "site replication state changed during peer join; the peers may already be joined — re-run replicate add" + )); + } + // Adopt only the fields this add computed. Everything else is + // owned by writers that commit without touching `updated_at` + // (retry events, peer-edit generations, resync progress, the + // acks/clears of an already pending rotation or removal), so the + // CAS above cannot vouch for them — they keep the freshly loaded + // value. The exhaustive destructure makes adding a state field a + // compile error here until it is classified. + let SiteReplicationState { + name, + service_account_access_key, + service_account_secret_key: _, + service_account_parent, + peers, + updated_at, + resync_status: _, + pending_rotation: _, + pending_remove: _, + pending_endpoint_refresh: _, + retry_queue: _, + sync_state_initialized, + edit_generation: _, + applied_edit_generations: _, + } = next_state; + state.name = name; + state.service_account_access_key = service_account_access_key; + state.service_account_parent = service_account_parent; + state.peers = peers; + state.updated_at = updated_at; + state.sync_state_initialized = sync_state_initialized; + let edit_generation = next_peer_edit_generation(state); + Ok((state.clone(), edit_generation)) + }) + .await?; + + // The finalize fan-out delivers peer-edit payloads, so it carries the + // generation allocated in the commit above: the receiving site orders + // it against any edit that follows instead of applying whichever + // delivery happens to arrive last. It runs outside the transaction — + // holding the state-object lock across peer traffic would block every + // node of this site, including this add's own retry bookkeeping. + let local_deployment_id = current_deployment_id(); + let finalize_edit_path = peer_edit_path_with_fence(local_deployment_id.as_deref(), edit_generation); for target in state.peers.values() { if target.deployment_id == local_peer.deployment_id || same_identity_endpoint(&target.endpoint, &local_peer.endpoint) { @@ -8477,7 +8584,7 @@ impl Operation for SiteReplicationAddHandler { if let Err(err) = send_peer_admin_request_with_client( &transport.client, &transport.connection, - SITE_REPLICATION_PEER_EDIT_PATH, + &finalize_edit_path, &state.service_account_access_key, &service_account_secret_key, peer, @@ -8490,12 +8597,6 @@ impl Operation for SiteReplicationAddHandler { } } - // Bootstrap and back-fill send bucket-ops, not peer edits, so their - // ordering is not state-sensitive — and their transports do record - // retry events, which re-enter the state transaction. Release the - // guard before them. - drop(_state_guard); - initial_sync_errors.extend(bootstrap_existing_metadata_after_add(&state, &local_peer, &service_account_secret_key).await); // Fix 1: back-fill pre-existing buckets so objects created before `replicate add` @@ -8521,39 +8622,49 @@ impl Operation for SiteReplicationRemoveHandler { let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationRemoveAction).await?; reject_site_replicator_on_public_admin(&cred)?; let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; + // The request body is read before the bucket-op guard and the state + // transaction: a client that stalls mid-body must hold neither the + // state-object lock nor the write half of the bucket-op RwLock (which + // would starve every bucket-operation hook in the meantime). + let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers); + let remove_req: SRRemoveReq = read_site_replication_json(req, "", false).await?; let (pending_remove, local_peer) = { let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.write().await; - let _state_guard = site_replication_state_process_guard().await; - let current_state = load_site_replication_state().await?; - if pending_endpoint_refresh(¤t_state).is_some() { - return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending")); - } - if current_state.pending_rotation.is_some() { - return Err(s3_error!(InvalidRequest, "service account rotation is pending")); - } - let local_peer = current_local_peer(&req, ¤t_state); - let remove_req: SRRemoveReq = read_site_replication_json(req, "", false).await?; + update_site_replication_state_when_changed(move |state| { + if pending_endpoint_refresh(state).is_some() { + return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending")); + } + if state.pending_rotation.is_some() { + return Err(s3_error!(InvalidRequest, "service account rotation is pending")); + } + let local_peer = local_peer_at_endpoint(local_endpoint, state); - if let Some(pending) = current_state.pending_remove.clone() { - (pending, local_peer) - } else { - validate_remove_sites_req(¤t_state, &remove_req)?; - let mut next_state = remove_sites(current_state.clone(), remove_req.clone()); - let mut peer_remove_req = remove_req; + // Resuming: the peers were already told about this pending + // removal, so re-persisting the same record buys nothing. + if let Some(pending) = state.pending_remove.clone() { + return Ok(StateCommit::Unchanged((pending, local_peer))); + } + + validate_remove_sites_req(state, &remove_req)?; + let service_account_access_key = state.service_account_access_key.clone(); + let secret_candidates = legacy_site_replicator_state_secret(state).into_iter().collect(); + let original_peers = state.peers.clone(); + let mut peer_remove_req = remove_req.clone(); peer_remove_req.requesting_dep_id = local_peer.deployment_id.clone(); + *state = remove_sites(std::mem::take(state), remove_req); let pending = PendingRemove { id: Uuid::new_v4().to_string(), req: peer_remove_req, - service_account_access_key: current_state.service_account_access_key.clone(), - secret_candidates: legacy_site_replicator_state_secret(¤t_state).into_iter().collect(), - original_peers: current_state.peers.clone(), + service_account_access_key, + secret_candidates, + original_peers, acked_deployment_ids: BTreeSet::new(), - updated_at: next_state.updated_at, + updated_at: state.updated_at, }; - next_state.pending_remove = Some(pending.clone()); - persist_site_replication_state(&next_state).await?; - (pending, local_peer) - } + state.pending_remove = Some(pending.clone()); + Ok(StateCommit::Changed((pending, local_peer))) + }) + .await? }; let mut peer_errors = Vec::new(); @@ -8724,102 +8835,186 @@ impl Operation for SiteReplicationNetPerfHandler { pub struct SRPeerJoinHandler {} +/// What the join admission decided about an incoming peer join. The verdict — +/// and the committed state the back-fill afterwards needs — travel out of +/// [`admit_peer_join`] instead of being answered where they are decided. +enum PeerJoinOutcome { + Applied(Box, PeerInfo), + /// A newer join already landed here; the sender is answered with the local + /// peer record and nothing is written. + Superseded(PeerInfo), +} + +/// The serialized half of an accepted peer join: staleness check, IAM apply, +/// state commit. +/// +/// Two locks, two scopes. The lifecycle guard (process-local) keeps the +/// admission mutually exclusive with this node's add / remove / rotate / +/// reconciler. The distributed join-admission lock then serializes the +/// admission CLUSTER-WIDE — the IAM write and the state commit cannot share +/// a transaction, so without it two joins accepted by different nodes of +/// this site interleave as "A checks for older T1, B applies secret B and +/// commits newer T2, A overwrites IAM with secret A, A's commit is refused +/// as superseded" — leaving the persisted state advertising B's contract +/// while IAM only accepts A's secret. Under the admission lock the +/// staleness check runs against a load taken INSIDE the lock, before +/// `apply_iam` changes anything, so a superseded join exits without +/// touching IAM at all. Crash safety is the lock subsystem's lease expiry +/// (same pattern as the repair execution lock); the closing transaction +/// still re-checks staleness for defence in depth and for old-version nodes +/// that do not take the admission lock during a rolling upgrade. +/// +/// Lock order: lifecycle -> join admission -> state object lock (the repair +/// path nests config-object locks the same way: repair execution -> state). +/// +/// `apply_iam` is injected so the interleaving regression tests can gate it +/// mid-flight; production passes the real service-account upsert. +async fn admit_peer_join( + local_endpoint: String, + join_req: SRPeerJoinReq, + defer_sync_state_enable: bool, + apply_iam: F, +) -> S3Result +where + F: FnOnce(SRPeerJoinReq) -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, +{ + let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; + admit_peer_join_across_nodes(local_endpoint, join_req, defer_sync_state_enable, apply_iam).await +} + +/// [`admit_peer_join`] minus the process-local lifecycle guard: the +/// distributed admission lock plus the fenced sequence under it. This is +/// exactly what a second node of this site runs concurrently — the lifecycle +/// guard cannot reach it — so the separate-nodes regression test drives this +/// function directly, and removing the admission lock breaks it. +async fn admit_peer_join_across_nodes( + local_endpoint: String, + join_req: SRPeerJoinReq, + defer_sync_state_enable: bool, + apply_iam: F, +) -> S3Result +where + F: FnOnce(SRPeerJoinReq) -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, +{ + let store = + current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; + with_config_object_write_lock(store, SITE_REPLICATION_JOIN_ADMISSION_LOCK_PATH.to_string(), move || async move { + let fresh = load_site_replication_state().await?; + let fresh_local_peer = local_peer_at_endpoint(local_endpoint.clone(), &fresh); + if join_request_is_superseded(&fresh, join_req.updated_at) { + let peer = fresh + .peers + .get(&fresh_local_peer.deployment_id) + .cloned() + .unwrap_or(fresh_local_peer); + return Ok(PeerJoinOutcome::Superseded(peer)); + } + + apply_iam(join_req.clone()).await?; + + let incoming_updated_at = join_req.updated_at; + update_site_replication_state_when_changed(move |state| { + let local_peer = local_peer_at_endpoint(local_endpoint, state); + if join_request_is_superseded(state, incoming_updated_at) { + let peer = state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer); + return Ok(StateCommit::Unchanged(PeerJoinOutcome::Superseded(peer))); + } + apply_peer_join(state, &local_peer, join_req, defer_sync_state_enable); + Ok(StateCommit::Changed(PeerJoinOutcome::Applied(Box::new(state.clone()), local_peer))) + }) + .await + }) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock site replication join admission failed: {e}")))? +} + +/// Upsert the replication service account a peer join carries. No-op when the +/// join brings no credentials. +async fn apply_peer_join_service_account(join_req: SRPeerJoinReq) -> S3Result<()> { + if join_req.svc_acct_access_key.is_empty() || join_req.svc_acct_secret_key.is_empty() { + return Ok(()); + } + let Some(iam_sys) = current_iam_handle() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; + + if iam_sys.get_service_account(&join_req.svc_acct_access_key).await.is_ok() { + iam_sys + .update_service_account( + &join_req.svc_acct_access_key, + UpdateServiceAccountOpts { + session_policy: if join_req.svc_acct_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT { + Some(site_replicator_service_account_policy()?) + } else { + None + }, + secret_key: Some(join_req.svc_acct_secret_key.clone()), + name: None, + description: None, + expiration: None, + status: None, + parent_user: None, + allow_site_replicator_account: join_req.svc_acct_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT, + }, + ) + .await + .map_err(ApiError::from)?; + } else { + iam_sys + .new_service_account( + &join_req.svc_acct_parent, + None, + NewServiceAccountOpts { + session_policy: if join_req.svc_acct_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT { + Some(site_replicator_service_account_policy()?) + } else { + None + }, + access_key: join_req.svc_acct_access_key.clone(), + secret_key: join_req.svc_acct_secret_key.clone(), + name: None, + description: None, + expiration: None, + allow_site_replicator_account: join_req.svc_acct_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT, + claims: None, + }, + ) + .await + .map_err(ApiError::from)?; + } + Ok(()) +} + #[async_trait::async_trait] impl Operation for SRPeerJoinHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?; let bootstrap_token = site_replication_bootstrap_token(&req.uri); - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; - let local_peer = current_local_peer(&req, &state); + let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers); + // The body is fully read before the admission takes the lifecycle + // guard: a sender that stalls mid-body must not block this node's + // add/remove/rotate/reconciler. let join_envelope: SRPeerJoinEnvelope = read_site_replication_json(req, &cred.secret_key, true).await?; let defer_sync_state_enable = join_envelope.defer_sync_state_enable; let join_req = join_envelope.request; validate_join_peer_snapshot(&join_req.peers)?; - if let Some(current_updated_at) = state.updated_at { - let Some(incoming_updated_at) = join_req.updated_at else { + let committed = + admit_peer_join(local_endpoint, join_req, defer_sync_state_enable, apply_peer_join_service_account).await?; + // Committed; the reverse-reachability probe and the bucket back-fill + // run outside the transaction — their transport helpers' retry-event + // bookkeeping re-enters it (P1-15). + let (state, local_peer) = match committed { + PeerJoinOutcome::Applied(state, local_peer) => (*state, local_peer), + PeerJoinOutcome::Superseded(peer) => { return json_response(&SRPeerJoinResponse { - peer: state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer), - ..Default::default() - }); - }; - if incoming_updated_at <= current_updated_at { - return json_response(&SRPeerJoinResponse { - peer: state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer), + peer, ..Default::default() }); } - } - - if !join_req.svc_acct_access_key.is_empty() && !join_req.svc_acct_secret_key.is_empty() { - let Some(iam_sys) = current_iam_handle() else { - return Err(s3_error!(InvalidRequest, "iam not init")); - }; - - if iam_sys.get_service_account(&join_req.svc_acct_access_key).await.is_ok() { - iam_sys - .update_service_account( - &join_req.svc_acct_access_key, - UpdateServiceAccountOpts { - session_policy: if join_req.svc_acct_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT { - Some(site_replicator_service_account_policy()?) - } else { - None - }, - secret_key: Some(join_req.svc_acct_secret_key.clone()), - name: None, - description: None, - expiration: None, - status: None, - parent_user: None, - allow_site_replicator_account: join_req.svc_acct_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT, - }, - ) - .await - .map_err(ApiError::from)?; - } else { - iam_sys - .new_service_account( - &join_req.svc_acct_parent, - None, - NewServiceAccountOpts { - session_policy: if join_req.svc_acct_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT { - Some(site_replicator_service_account_policy()?) - } else { - None - }, - access_key: join_req.svc_acct_access_key.clone(), - secret_key: join_req.svc_acct_secret_key.clone(), - name: None, - description: None, - expiration: None, - allow_site_replicator_account: join_req.svc_acct_access_key == SITE_REPLICATOR_SERVICE_ACCOUNT, - claims: None, - }, - ) - .await - .map_err(ApiError::from)?; - } - } - - state.service_account_access_key = join_req.svc_acct_access_key; - state.service_account_parent = join_req.svc_acct_parent; - state.updated_at = join_req.updated_at.or_else(|| Some(OffsetDateTime::now_utc())); - state.peers = normalize_join_peers_for_local(&local_peer, join_req.peers); - initialize_join_peer_sync_state(&mut state.peers, defer_sync_state_enable); - state.sync_state_initialized = true; - state.name = state - .peers - .get(&local_peer.deployment_id) - .map(|peer| peer.name.clone()) - .filter(|name| !name.is_empty()) - .unwrap_or_else(|| local_peer.name.clone()); - persist_site_replication_state(&state).await?; - // Committed; release the state lock before the reverse-reachability - // probe and bucket back-fill — their transport helpers' retry-event - // bookkeeping re-enters the state transaction (P1-15). - drop(_state_guard); + }; // Fix 1 (receiving side): ensure the joining peer also sets up replication for any // buckets it already owns so the reverse direction works from the start. Per-bucket // failures are logged (BUG2) so a reverse-direction back-fill gap is observable. @@ -9005,7 +9200,10 @@ impl Operation for SiteReplicationEditHandler { let ilm_expiry_override = sr_edit_ilm_expiry_override(&req.uri); let body = read_site_replication_body(req, &cred.secret_key, true).await?; let (mut incoming, tls_presence) = parse_public_peer_edit(&body)?; - let mut state_guard = Some(site_replication_state_process_guard().await); + // Planning snapshot: every commit below re-loads the state inside its + // transaction and re-checks the `updated_at` / pending-refresh + // precondition there, because the peer probes and fan-outs in between + // must not run under the state-object lock. let current_state = load_site_replication_state().await?; apply_public_peer_edit_tls_presence(¤t_state, &mut incoming, tls_presence); if !incoming.deployment_id.is_empty() || !incoming.endpoint.is_empty() || !incoming.name.is_empty() { @@ -9027,6 +9225,11 @@ impl Operation for SiteReplicationEditHandler { acked_deployment_ids: BTreeSet::new(), }) }); + // The precondition every commit below re-checks: the topology this + // edit was planned against, and the endpoint refresh it either + // continues or requires the absence of. + let expected_updated_at = current_state.updated_at; + let expected_pending_id = persisted_pending.as_ref().map(|pending| pending.id.clone()); let local_peer = current_local_runtime_peer(¤t_state); let existing_peer = existing_peer_for_edit(¤t_state, &incoming); let tls_capability_required = edit_peer_tls_capability_required(existing_peer, &incoming); @@ -9037,8 +9240,6 @@ impl Operation for SiteReplicationEditHandler { return Err(s3_error!(InvalidRequest, "site replication service account is not configured")); } let secret = site_replicator_service_account_secret(¤t_state.service_account_access_key).await?; - let expected_updated_at = current_state.updated_at; - drop(state_guard.take()); if tls_capability_required { require_edit_peer_tls_capability( ¤t_state, @@ -9052,39 +9253,30 @@ impl Operation for SiteReplicationEditHandler { if tls_transport_probe_required { probe_proposed_peer_tls_transport(&incoming, ¤t_state.service_account_access_key, &secret).await?; } - state_guard = Some(site_replication_state_process_guard().await); + // Early exit on a state that moved under the probe. Advisory only: + // the binding check is the CAS inside whichever commit follows. let latest_state = load_site_replication_state().await?; - if latest_state.updated_at != expected_updated_at - || pending_endpoint_refresh(&latest_state).as_ref().map(|pending| &pending.id) - != persisted_pending.as_ref().map(|pending| &pending.id) - { - return Err(s3_error!(InvalidRequest, "site replication state changed during capability probe")); - } + ensure_edit_precondition(&latest_state, expected_updated_at, expected_pending_id.as_ref(), "capability probe")?; service_account_secret_key = Some(secret); } - let mut state = if endpoint_refresh_requested { - current_state.clone() - } else { - edit_state(current_state.clone(), incoming.clone(), ilm_expiry_override) - }; - if endpoint_refresh_requested && current_state.service_account_access_key.is_empty() { return Err(s3_error!(InvalidRequest, "site replication service account is not configured")); } if current_state.service_account_access_key.is_empty() { - save_site_replication_state(&state).await?; + // No peers to notify: the edit is the whole operation, so it is + // computed and committed in one transaction. + let incoming = incoming.clone(); + update_site_replication_state(move |state| { + ensure_edit_precondition(state, expected_updated_at, expected_pending_id.as_ref(), "the edit")?; + *state = edit_state(std::mem::take(state), incoming, ilm_expiry_override); + Ok(()) + }) + .await?; } else { let service_account_secret_key = match service_account_secret_key { Some(secret) => secret, None => site_replicator_service_account_secret(¤t_state.service_account_access_key).await?, }; - let peers_to_send: Vec = if let Some(pending) = pending.as_ref() { - vec![pending.peer.clone()] - } else if ilm_expiry_override.is_some() { - state.peers.values().cloned().collect() - } else { - vec![normalize_peer_info(incoming)] - }; let routing_peers = pending .as_ref() .map(|pending| &pending.remote_peers) @@ -9096,8 +9288,6 @@ impl Operation for SiteReplicationEditHandler { let pending = pending.clone().ok_or_else(|| { S3Error::with_message(S3ErrorCode::InternalError, "endpoint refresh state is missing".to_string()) })?; - let expected_updated_at = current_state.updated_at; - drop(state_guard.take()); let probes = futures::future::join_all(remote_targets.iter().map(|target| { send_endpoint_refresh_admin_request_raw( target, @@ -9124,24 +9314,23 @@ impl Operation for SiteReplicationEditHandler { } } - state_guard = Some(site_replication_state_process_guard().await); - let latest_state = load_site_replication_state().await?; - if latest_state.updated_at != expected_updated_at - || pending_endpoint_refresh(&latest_state).as_ref().map(|pending| &pending.id) - != persisted_pending.as_ref().map(|pending| &pending.id) - { - return Err(s3_error!(InvalidRequest, "site replication state changed during capability probe")); - } - state = latest_state; let pending_id = pending.id.clone(); let refresh_request = EndpointRefreshRequest { id: pending.id.clone(), peer: pending.peer.clone(), }; - let pending = merge_pending_endpoint_refresh(&state, &pending, std::iter::empty::())?; - set_pending_endpoint_refresh(&mut state, pending.clone())?; - save_site_replication_state(&state).await?; - drop(state_guard.take()); + // Announce the pending refresh. The CAS sits in the same + // transaction as the write it guards, so a topology change + // that landed during the capability probes above cannot be + // overwritten by this snapshot. + let expected_pending_id = expected_pending_id.clone(); + let pending = update_site_replication_state(move |state| { + ensure_edit_precondition(state, expected_updated_at, expected_pending_id.as_ref(), "capability probe")?; + let pending = merge_pending_endpoint_refresh(state, &pending, std::iter::empty::())?; + set_pending_endpoint_refresh(state, pending.clone())?; + Ok(pending) + }) + .await?; let responses = futures::future::join_all(remote_targets.iter().map(|target| async { if legacy_deployment_ids.contains(&target.deployment_id) { refresh_legacy_peer_bucket_targets( @@ -9177,51 +9366,63 @@ impl Operation for SiteReplicationEditHandler { } } - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; - let Some(pending) = pending_endpoint_refresh(&state).filter(|pending| pending.id == pending_id) else { - return Err(s3_error!(InvalidRequest, "endpoint target refresh state changed during update")); - }; - let pending = merge_pending_endpoint_refresh(&state, &pending, acked_deployment_ids)?; - set_pending_endpoint_refresh(&mut state, pending)?; - save_site_replication_state(&state).await?; + let acked_pending_id = pending_id.clone(); + let service_account_access_key = update_site_replication_state(move |state| { + let Some(pending) = pending_endpoint_refresh(state).filter(|pending| pending.id == acked_pending_id) else { + return Err(s3_error!(InvalidRequest, "endpoint target refresh state changed during update")); + }; + let pending = merge_pending_endpoint_refresh(state, &pending, acked_deployment_ids)?; + set_pending_endpoint_refresh(state, pending)?; + Ok(state.service_account_access_key.clone()) + }) + .await?; if let Some(err) = refresh_error { return Err(err); } - let service_account_secret_key = - site_replicator_service_account_secret(&state.service_account_access_key).await?; - drop(_state_guard); + let service_account_secret_key = site_replicator_service_account_secret(&service_account_access_key).await?; refresh_bucket_targets_after_endpoint_edit(&pending_id, &service_account_secret_key).await?; - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; - let Some(pending) = pending_endpoint_refresh(&state).filter(|pending| pending.id == pending_id) else { - return Err(s3_error!(InvalidRequest, "endpoint target refresh state changed during update")); - }; - state = edit_state(state, pending.peer, ilm_expiry_override); - clear_pending_endpoint_refresh(&mut state); - save_site_replication_state(&state).await?; + update_site_replication_state(move |state| { + let Some(pending) = pending_endpoint_refresh(state).filter(|pending| pending.id == pending_id) else { + return Err(s3_error!(InvalidRequest, "endpoint target refresh state changed during update")); + }; + *state = edit_state(std::mem::take(state), pending.peer, ilm_expiry_override); + clear_pending_endpoint_refresh(state); + Ok(()) + }) + .await?; } else { // Commit before the peer fan-out (mirrors the add/join // handlers): a failed notification is recorded as a retry // event and converges from the committed local state — // fanning out first meant the retry event pointed at a state - // the local site had not saved. The generation is allocated in - // the same commit, i.e. under the state-object lock, so it - // orders this edit against one another node accepts - // concurrently — the process guard below cannot. - let edit_generation = next_peer_edit_generation(&mut state); - save_site_replication_state(&state).await?; + // the local site had not saved. The edit itself is applied to + // the state the transaction loads, under the CAS, so a + // topology change that slipped past the planning snapshot + // fails the edit instead of being overwritten by it. The + // generation is allocated in that same commit, i.e. under the + // state-object lock, so it orders this edit against one + // another node of this site accepts concurrently. + let incoming = incoming.clone(); + let (edit_generation, peers_to_send) = update_site_replication_state(move |state| { + ensure_edit_precondition(state, expected_updated_at, expected_pending_id.as_ref(), "the edit")?; + *state = edit_state(std::mem::take(state), incoming.clone(), ilm_expiry_override); + let peers_to_send: Vec = if ilm_expiry_override.is_some() { + state.peers.values().cloned().collect() + } else { + vec![normalize_peer_info(incoming)] + }; + Ok((next_peer_edit_generation(state), peers_to_send)) + }) + .await?; let edit_path = peer_edit_path_with_fence(local_deployment_id.as_deref(), edit_generation); let delivery_fence = local_deployment_id.is_some().then_some(edit_generation); - // The fan-out stays UNDER the state guard so deliveries issued - // by THIS node keep their commit order; the generation fence - // above is what covers the cross-node case, where the peer - // rejects a delivery an older edit is still trying to make. - // The retry-event bookkeeping is what cannot run under the - // guard — it re-enters the state transaction (P1-15) — so - // deliver with the plain transport here and settle the retry - // queue after the guard is released. + // The fan-out runs outside the transaction — peer traffic + // under the state-object lock would stall every writer of this + // site, and the retry bookkeeping below re-enters it (P1-15). + // Ordering is the generation fence's job: a delivery this + // fan-out is still retrying is rejected by the receiver once a + // newer generation from this site has landed there. let mut delivered: Vec = Vec::new(); let mut failure: Option<(PeerInfo, S3Error)> = None; 'fanout: for target in remote_targets { @@ -9243,7 +9444,6 @@ impl Operation for SiteReplicationEditHandler { } delivered.push(target.clone()); } - drop(state_guard.take()); // Settle only what this generation is entitled to: a newer // edit that committed and failed its own delivery while this @@ -9293,6 +9493,20 @@ impl Operation for SRPeerEditCapabilitiesHandler { pub struct SRPeerEditHandler {} +/// What the peer-edit transaction decided about an incoming delivery. The +/// checks and the write share one transaction, so the verdict has to travel +/// out of the closure instead of being answered where it is taken. +enum PeerEditOutcome { + /// Applied; carries the service account access key the follow-up + /// endpoint-refresh work needs from the committed state. + Applied(String), + /// Nothing to do — a superseded delivery or one this site already + /// committed. Answered as success so the sender stops retrying. + Acked, + /// Refused, with the detail the sender is told. + Rejected(&'static str), +} + #[async_trait::async_trait] impl Operation for SRPeerEditHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { @@ -9300,101 +9514,107 @@ impl Operation for SRPeerEditHandler { let queries = query_pairs(&req.uri); let ilm_expiry_override = sr_edit_ilm_expiry_override(&req.uri); let endpoint_refresh_requested = queries.get("refresh-targets").is_some_and(|value| value == "true"); - let delivery_fence = peer_edit_fence(&queries); - let state_guard = site_replication_state_process_guard().await; - let state = load_site_replication_state().await?; - // Ordering fence: the sending site allocates the generation under its - // state-object lock, so a delivery that lost the race carries a - // generation this site has already passed. Applying it would roll the - // peer back to the older edit. Ack it — the newer edit already landed, - // so the sender has nothing to retry. - if let Some((origin, generation)) = delivery_fence.as_ref() - && peer_edit_delivery_is_stale(&state, origin, *generation) - { - return json_response(&ReplicateEditStatus { - success: true, - status: SITE_REPL_EDIT_SUCCESS.to_string(), - api_version: Some(SITE_REPL_API_VERSION.to_string()), - ..Default::default() - }); - } - if endpoint_refresh_requested && (state.pending_rotation.is_some() || state.pending_remove.is_some()) { - return json_response(&ReplicateEditStatus { - success: false, - status: SITE_REPL_EDIT_SUCCESS.to_string(), - err_detail: "another site replication operation is pending".to_string(), - api_version: Some(SITE_REPL_API_VERSION.to_string()), - }); - } - let local_peer = current_local_peer(&req, &state); - let (refresh_id, mut incoming) = if endpoint_refresh_requested { + let commit_fence = peer_edit_fence(&queries); + let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers); + let (refresh_id, incoming) = if endpoint_refresh_requested { let refresh: EndpointRefreshRequest = read_site_replication_json(req, "", false).await?; (Some(refresh.id), refresh.peer) } else { (None, read_site_replication_json(req, "", false).await?) }; - if same_identity_endpoint(&incoming.endpoint, &local_peer.endpoint) { - incoming.deployment_id = local_peer.deployment_id.clone(); - if incoming.name.is_empty() { - incoming.name = local_peer.name.clone(); - } - } - align_peer_edit_deployment_id(&state, &mut incoming); - if endpoint_refresh_requested - && pending_endpoint_refresh(&state).is_some_and(|pending| refresh_id.as_deref() != Some(&pending.id)) - { - return json_response(&ReplicateEditStatus { - success: false, - status: SITE_REPL_EDIT_SUCCESS.to_string(), - err_detail: "another endpoint target refresh is pending".to_string(), - api_version: Some(SITE_REPL_API_VERSION.to_string()), - }); - } - if endpoint_refresh_requested - && (refresh_id.as_ref().is_none_or(String::is_empty) || !peer_endpoint_edit_requested(&state, &incoming)) - { - return json_response(&ReplicateEditStatus { - success: false, - status: SITE_REPL_EDIT_SUCCESS.to_string(), - err_detail: "peer endpoint was not found".to_string(), - api_version: Some(SITE_REPL_API_VERSION.to_string()), - }); - } - if endpoint_refresh_requested && internal_endpoint_refresh_already_committed(&state, &incoming) { - return json_response(&ReplicateEditStatus { - success: true, - status: SITE_REPL_EDIT_SUCCESS.to_string(), - api_version: Some(SITE_REPL_API_VERSION.to_string()), - ..Default::default() - }); - } - let mut state = if endpoint_refresh_requested { - validate_proposed_peer(&incoming)?; - state - } else { - apply_internal_peer_edit(state, &local_peer, incoming.clone(), ilm_expiry_override)? + // Everything the delivery is checked against — the fence, the pending + // operations, the peer it names — is read inside the transaction that + // applies it. Checking against a state loaded before the lock would + // let the check pass on one snapshot and the write land on another. + let commit_endpoint = local_endpoint.clone(); + let commit_refresh_id = refresh_id.clone(); + let outcome = update_site_replication_state_when_changed(move |state| { + let mut incoming = incoming; + let local_peer = local_peer_at_endpoint(commit_endpoint, state); + // Ordering fence: the sending site allocates the generation under + // its state-object lock, so a delivery that lost the race carries + // a generation this site has already passed. Applying it would + // roll the peer back to the older edit. Ack it — the newer edit + // already landed, so the sender has nothing to retry. + if let Some((origin, generation)) = commit_fence.as_ref() + && peer_edit_delivery_is_stale(state, origin, *generation) + { + return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked)); + } + if endpoint_refresh_requested && (state.pending_rotation.is_some() || state.pending_remove.is_some()) { + return Ok(StateCommit::Unchanged(PeerEditOutcome::Rejected( + "another site replication operation is pending", + ))); + } + if same_identity_endpoint(&incoming.endpoint, &local_peer.endpoint) { + incoming.deployment_id = local_peer.deployment_id.clone(); + if incoming.name.is_empty() { + incoming.name = local_peer.name.clone(); + } + } + align_peer_edit_deployment_id(state, &mut incoming); + if endpoint_refresh_requested + && pending_endpoint_refresh(state).is_some_and(|pending| commit_refresh_id.as_deref() != Some(&pending.id)) + { + return Ok(StateCommit::Unchanged(PeerEditOutcome::Rejected( + "another endpoint target refresh is pending", + ))); + } + if endpoint_refresh_requested + && (commit_refresh_id.as_ref().is_none_or(String::is_empty) || !peer_endpoint_edit_requested(state, &incoming)) + { + return Ok(StateCommit::Unchanged(PeerEditOutcome::Rejected("peer endpoint was not found"))); + } + if endpoint_refresh_requested && internal_endpoint_refresh_already_committed(state, &incoming) { + return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked)); + } + + if endpoint_refresh_requested { + validate_proposed_peer(&incoming)?; + set_pending_endpoint_refresh( + state, + PendingEndpointRefresh { + id: commit_refresh_id.unwrap_or_default(), + peer: incoming, + remote_peers: BTreeMap::new(), + acked_deployment_ids: BTreeSet::new(), + }, + )?; + } else { + *state = apply_internal_peer_edit(std::mem::take(state), &local_peer, incoming, ilm_expiry_override)?; + } + // Raise the origin's high-water mark in the same commit as the + // edit it fences: a crash between the two would let the superseded + // delivery apply on the next attempt. + if let Some((origin, generation)) = commit_fence.as_ref() { + record_applied_peer_edit_generation(state, origin, *generation); + } + Ok(StateCommit::Changed(PeerEditOutcome::Applied(state.service_account_access_key.clone()))) + }) + .await?; + + let service_account_access_key = match outcome { + PeerEditOutcome::Applied(service_account_access_key) => service_account_access_key, + PeerEditOutcome::Acked => { + return json_response(&ReplicateEditStatus { + success: true, + status: SITE_REPL_EDIT_SUCCESS.to_string(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }); + } + PeerEditOutcome::Rejected(err_detail) => { + return json_response(&ReplicateEditStatus { + success: false, + status: SITE_REPL_EDIT_SUCCESS.to_string(), + err_detail: err_detail.to_string(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }); + } }; if endpoint_refresh_requested { - set_pending_endpoint_refresh( - &mut state, - PendingEndpointRefresh { - id: refresh_id.clone().unwrap_or_default(), - peer: incoming.clone(), - remote_peers: BTreeMap::new(), - acked_deployment_ids: BTreeSet::new(), - }, - )?; - } - // Raise the origin's high-water mark in the same commit as the edit it - // fences: a crash between the two would let the superseded delivery - // apply on the next attempt. - if let Some((origin, generation)) = delivery_fence.as_ref() { - record_applied_peer_edit_generation(&mut state, origin, *generation); - } - save_site_replication_state(&state).await?; - if endpoint_refresh_requested { - if state.service_account_access_key.is_empty() { + if service_account_access_key.is_empty() { return json_response(&ReplicateEditStatus { success: false, status: SITE_REPL_EDIT_SUCCESS.to_string(), @@ -9402,23 +9622,29 @@ impl Operation for SRPeerEditHandler { api_version: Some(SITE_REPL_API_VERSION.to_string()), }); } - let service_account_secret_key = site_replicator_service_account_secret(&state.service_account_access_key).await?; + let service_account_secret_key = site_replicator_service_account_secret(&service_account_access_key).await?; let pending_id = refresh_id.unwrap_or_default(); - drop(state_guard); + // The bucket-target rewrite talks to the store for every bucket; + // it runs between the two transactions, never inside one. refresh_bucket_targets_after_endpoint_edit(&pending_id, &service_account_secret_key).await?; - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; - let Some(pending) = pending_endpoint_refresh(&state).filter(|pending| pending.id == pending_id) else { + let committed = update_site_replication_state_when_changed(move |state| { + let local_peer = local_peer_at_endpoint(local_endpoint, state); + let Some(pending) = pending_endpoint_refresh(state).filter(|pending| pending.id == pending_id) else { + return Ok(StateCommit::Unchanged(false)); + }; + *state = apply_internal_peer_edit(std::mem::take(state), &local_peer, pending.peer, ilm_expiry_override)?; + clear_pending_endpoint_refresh(state); + Ok(StateCommit::Changed(true)) + }) + .await?; + if !committed { return json_response(&ReplicateEditStatus { success: false, status: SITE_REPL_EDIT_SUCCESS.to_string(), err_detail: "endpoint target refresh state changed during update".to_string(), api_version: Some(SITE_REPL_API_VERSION.to_string()), }); - }; - state = apply_internal_peer_edit(state, &local_peer, pending.peer, ilm_expiry_override)?; - clear_pending_endpoint_refresh(&mut state); - save_site_replication_state(&state).await?; + } return json_response(&ReplicateEditStatus { success: true, status: SITE_REPL_EDIT_SUCCESS.to_string(), @@ -9439,19 +9665,19 @@ impl Operation for SRPeerRemoveHandler { let remove_req: SRRemoveReq = read_site_replication_json(req, "", false).await?; let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.write().await; - let _state_guard = site_replication_state_process_guard().await; - let current_state = load_site_replication_state().await?; - if pending_endpoint_refresh(¤t_state).is_some() { - return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending")); - } - if current_state.pending_rotation.is_some() { - return Err(s3_error!(InvalidRequest, "service account rotation is pending")); - } + let removed_deployment_ids = update_site_replication_state(move |state| { + if pending_endpoint_refresh(state).is_some() { + return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending")); + } + if state.pending_rotation.is_some() { + return Err(s3_error!(InvalidRequest, "service account rotation is pending")); + } - let removed_deployment_ids = removed_deployment_ids_for_remove_req(¤t_state, &remove_req); - - let state = remove_sites(current_state, remove_req); - persist_site_replication_state(&state).await?; + let removed_deployment_ids = removed_deployment_ids_for_remove_req(state, &remove_req); + *state = remove_sites(std::mem::take(state), remove_req); + Ok(removed_deployment_ids) + }) + .await?; // Clean up bucket targets and replication rules that referenced removed peers. if !removed_deployment_ids.is_empty() @@ -9483,7 +9709,6 @@ impl Operation for SiteReplicationResyncOpHandler { let requested_peer: PeerInfo = read_site_replication_json(req, "", false).await?; let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; let (peer, existing_status) = { - let _state_guard = site_replication_state_process_guard().await; let state = load_site_replication_state().await?; let local_peer = current_local_runtime_peer(&state); let requested_peer = normalize_peer_info(requested_peer); @@ -9644,9 +9869,11 @@ impl Operation for SRStateEditHandler { let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationOperationAction).await?; reject_site_replicator_on_public_admin(&cred)?; let body: SRStateEditReq = read_site_replication_json(req, "", false).await?; - let _state_guard = site_replication_state_process_guard().await; - let state = apply_state_edit_req(load_site_replication_state().await?, body); - save_site_replication_state(&state).await?; + update_site_replication_state(move |state| { + *state = apply_state_edit_req(std::mem::take(state), body); + Ok(()) + }) + .await?; Ok(empty_response(StatusCode::OK)) } } @@ -9658,15 +9885,11 @@ impl Operation for SiteReplicationRepairHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationOperationAction).await?; reject_site_replicator_on_public_admin(&cred)?; - let (state, local_peer) = { - let _state_guard = site_replication_state_process_guard().await; - let state = load_site_replication_state().await?; - if !state.enabled() || state.service_account_access_key.is_empty() { - return Err(s3_error!(InvalidRequest, "site replication is not configured")); - } - let local_peer = current_local_peer(&req, &state); - (state, local_peer) - }; + let state = load_site_replication_state().await?; + if !state.enabled() || state.service_account_access_key.is_empty() { + return Err(s3_error!(InvalidRequest, "site replication is not configured")); + } + let local_peer = current_local_peer(&req, &state); let body: SiteReplicationRepairRequest = read_site_replication_json(req, "", false).await?; let info = build_sr_info(&state, &local_peer).await?; let plan = site_replication_bootstrap_plan(&info)?; @@ -9769,44 +9992,55 @@ impl Operation for SRRotateServiceAccountHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationOperationAction).await?; reject_site_replicator_on_public_admin(&cred)?; - let (pending_rotation, local_peer, previous_access_key) = { - let _state_guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await?; + // The lifecycle guard is what keeps the rotation's IAM writes and the + // background service-account reconciler apart: the reconciler runs + // its whole repair under a lifecycle try-acquire, and its + // pending-rotation precheck is only sound if a rotation cannot start + // mid-repair and race its own IAM write against the reconciler's + // stale one. (The removed process mutex used to provide this + // exclusion as a side effect.) + let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; + let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers); + let rotation_parent = cred.access_key.clone(); + let (pending_rotation, local_peer, previous_access_key) = update_site_replication_state_when_changed(move |state| { if !state.enabled() { return Err(s3_error!(InvalidRequest, "site replication is not configured")); } - if pending_endpoint_refresh(&state).is_some() { + if pending_endpoint_refresh(state).is_some() { return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending")); } if state.pending_remove.is_some() { return Err(s3_error!(InvalidRequest, "site replication remove is pending")); } - let local_peer = current_local_peer(&req, &state); + let local_peer = local_peer_at_endpoint(local_endpoint, state); let previous_access_key = state.service_account_access_key.clone(); + // Resuming a rotation another attempt already recorded must + // not rewrite the state: the pending record is the contract + // the peers were told about. if let Some(pending) = state.pending_rotation.clone() { - (pending, local_peer, previous_access_key) - } else { - let new_secret_key = rustfs_credentials::gen_secret_key(40) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("generate secret key failed: {e}")))?; - state.service_account_access_key = SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(); - state.service_account_parent = cred.access_key.clone(); - state.updated_at = Some(OffsetDateTime::now_utc()); - let pending = PendingRotation { - id: Uuid::new_v4().to_string(), - access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), - parent: cred.access_key.clone(), - new_secret_key, - secret_candidates: legacy_site_replicator_state_secret(&state).into_iter().collect(), - peers: state.peers.clone(), - acked_deployment_ids: BTreeSet::new(), - updated_at: state.updated_at, - }; - state.pending_rotation = Some(pending.clone()); - save_site_replication_state(&state).await?; - (pending, local_peer, previous_access_key) + return Ok(StateCommit::Unchanged((pending, local_peer, previous_access_key))); } - }; + + let new_secret_key = rustfs_credentials::gen_secret_key(40) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("generate secret key failed: {e}")))?; + state.service_account_access_key = SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(); + state.service_account_parent = rotation_parent.clone(); + state.updated_at = Some(OffsetDateTime::now_utc()); + let pending = PendingRotation { + id: Uuid::new_v4().to_string(), + access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + parent: rotation_parent, + new_secret_key, + secret_candidates: legacy_site_replicator_state_secret(state).into_iter().collect(), + peers: state.peers.clone(), + acked_deployment_ids: BTreeSet::new(), + updated_at: state.updated_at, + }; + state.pending_rotation = Some(pending.clone()); + Ok(StateCommit::Changed((pending, local_peer, previous_access_key))) + }) + .await?; if !previous_access_key.is_empty() && let Ok(previous_iam_secret) = site_replicator_service_account_secret(&previous_access_key).await @@ -11356,6 +11590,35 @@ mod tests { assert!(!peer_capability_response_supported(&remote, StatusCode::NOT_FOUND, b"").expect("legacy peer")); } + /// P1-15 PR2: the add's finalize fan-out delivers peer edits after its + /// state transaction has been released — nothing may hold the state-object + /// lock across peer traffic. Ordering therefore rests entirely on the + /// generation allocated in that commit: an unstamped delivery is applied + /// by the receiver in arrival order, which is what the removed process + /// guard used to paper over (and never could across two nodes). + #[test] + fn add_handler_fans_out_peer_edits_under_the_committed_generation() { + let src = include_str!("site_replication.rs"); + let add = src + .split("impl Operation for SiteReplicationAddHandler") + .nth(1) + .and_then(|rest| rest.split("pub struct SiteReplicationRemoveHandler").next()) + .expect("add handler block"); + + assert!( + add.contains("next_peer_edit_generation"), + "the add must allocate a fan-out generation (inside the committing transaction)" + ); + assert!( + add.contains("peer_edit_path_with_fence"), + "the add's finalize fan-out must carry the committed generation fence" + ); + assert!( + !add.contains("SITE_REPLICATION_PEER_EDIT_PATH"), + "the finalize fan-out must not fall back to the unstamped peer-edit path" + ); + } + #[test] fn test_tls_capability_gates_run_before_add_or_edit_state_side_effects() { let src = include_str!("site_replication.rs"); @@ -11384,8 +11647,8 @@ mod tests { ); assert!( edit.find("require_edit_peer_tls_capability").expect("edit capability gate") - < edit.find("save_site_replication_state").expect("state save"), - "edit capability gate must run before state is saved" + < edit.find("update_site_replication_state(").expect("state commit"), + "edit capability gate must run before the state is committed" ); } @@ -11643,13 +11906,25 @@ mod tests { // rejects against — dropping either half silently restores // last-writer-wins between two nodes of the sending site. assert!( - handler_block.contains("peer_edit_delivery_is_stale(&state, origin, *generation)"), + handler_block.contains("peer_edit_delivery_is_stale(state, origin, *generation)"), "SRPeerEditHandler must reject peer edits a newer generation already superseded" ); assert!( - handler_block.contains("record_applied_peer_edit_generation(&mut state, origin, *generation);"), + handler_block.contains("record_applied_peer_edit_generation(state, origin, *generation);"), "SRPeerEditHandler must record the applied generation so later stale deliveries are recognised" ); + // P1-15 PR2: both halves of the fence and the edit they fence share + // ONE transaction. Checking the fence against a state read outside the + // lock would let the check pass on one snapshot and the write land on + // another — which is the interleaving the fence exists to reject. + assert!( + handler_block.contains("update_site_replication_state_when_changed(move |state| {"), + "SRPeerEditHandler must take the fence decision inside the state transaction" + ); + assert!( + !handler_block.contains("save_site_replication_state("), + "SRPeerEditHandler must not write the state outside the transaction boundary" + ); let sender_block = src .split("impl Operation for SiteReplicationEditHandler") @@ -11657,7 +11932,7 @@ mod tests { .and_then(|rest| rest.split("pub struct SRPeerEditCapabilitiesHandler").next()) .expect("SiteReplicationEditHandler block should exist"); assert!( - sender_block.contains("let edit_generation = next_peer_edit_generation(&mut state);"), + sender_block.contains("Ok((next_peer_edit_generation(state), peers_to_send))"), "the edit handler must allocate the generation inside the committed state, not outside the lock" ); } @@ -11936,14 +12211,12 @@ mod tests { let lifecycle = SiteReplicationLifecycleGuard::acquire().await; let add_guard = SiteReplicationAddInProgressGuard::start(lifecycle, HashSet::new()).expect("start site replication add guard"); - let add_state = site_replication_state_process_guard().await; let (started_tx, started_rx) = tokio::sync::oneshot::channel(); let (entered_tx, mut entered_rx) = tokio::sync::oneshot::channel(); let remove = tokio::spawn(async move { let _ = started_tx.send(()); let _lifecycle = SiteReplicationLifecycleGuard::acquire().await; let _bucket_op = SITE_REPLICATION_BUCKET_OP_LOCK.write().await; - let _state = site_replication_state_process_guard().await; let _ = entered_tx.send(()); }); started_rx.await.expect("remove task started"); @@ -11954,7 +12227,6 @@ mod tests { assert!(matches!(entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty))); drop(callback); - drop(add_state); drop(add_guard); tokio::time::timeout(Duration::from_millis(500), remove) .await @@ -12785,11 +13057,10 @@ mod tests { assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, iam_path), 1); } - /// P1-15 review follow-up: the receiving side of the ordering fence. The - /// sender's process mutex is per node, so two nodes of the same site can - /// fan out in the opposite order to their commits; the receiver decides - /// ordering from the generation the sender allocated under the distributed - /// state lock. + /// P1-15 review follow-up: the receiving side of the ordering fence. Two + /// nodes of the sending site can fan out in the opposite order to their + /// commits; the receiver decides ordering from the generation the sender + /// allocated under the distributed state lock. #[test] fn peer_edit_fence_rejects_a_delivery_the_newer_edit_already_passed() { let mut state = SiteReplicationState::default(); @@ -12821,6 +13092,62 @@ mod tests { assert!(peer_edit_fence(&HashMap::new()).is_none()); } + /// P1-15 PR2: an accepted join must PRESERVE the peer-edit high-water + /// marks of peers that stayed. Join fan-outs are routine (every add and + /// every service-account rotation delivers `SRPeerJoin` to existing + /// peers), so a blanket reset here would let any stalled older edit from + /// a peer that never left land after the join and roll its record back — + /// exactly the interleaving the fence exists to reject. + #[test] + fn peer_join_preserves_live_edit_generation_marks() { + let local = PeerInfo { + deployment_id: "site-b".to_string(), + ..peer("site-b", "https://site-b.example.com") + }; + let remote = PeerInfo { + deployment_id: "site-a".to_string(), + ..peer("site-a", "https://site-a.example.com") + }; + // `remote` is already a peer and has delivered edits up to generation + // 12; the incoming join (say, a rotation fan-out) keeps both sites. + let mut state = SiteReplicationState { + peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + applied_edit_generations: BTreeMap::from([(remote.deployment_id.clone(), 12)]), + ..Default::default() + }; + + apply_peer_join( + &mut state, + &local, + SRPeerJoinReq { + svc_acct_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + svc_acct_secret_key: "svc-secret".to_string(), + svc_acct_parent: "root".to_string(), + peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + updated_at: Some(OffsetDateTime::now_utc()), + }, + true, + ); + + assert_eq!( + state.applied_edit_generations.get(&remote.deployment_id), + Some(&12), + "the join must keep the live mark for a peer that stayed: {:?}", + state.applied_edit_generations + ); + assert!( + peer_edit_delivery_is_stale(&state, &remote.deployment_id, 11), + "a stalled pre-join delivery must still be fenced out after the join" + ); + assert_eq!(state.peers.len(), 2, "the join snapshot replaces the local topology"); + } + /// One edit fans out one delivery per peer record under a single /// generation (the ILM-expiry edit sends every peer's record). The /// receiver's fenced sequence — staleness check, apply, raise the @@ -15283,71 +15610,353 @@ mod tests { assert!(parse_site_resync_page(&query, &newer).is_err()); } - /// P1-15 review follow-up: isolates the PROCESS guard. One writer uses - /// the legacy shape that the not-yet-migrated call sites still use (take - /// the process mutex, then load / mutate / save through the plain - /// helpers, which take their own per-IO object locks); the other runs the - /// full transaction. They only stay serialized because the transaction - /// also takes the process mutex — drop it there and this test loses one - /// of the two updates. - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + /// P1-15 PR2: the persist-or-skip side of the transaction. A miss + /// (`StateCommit::Unchanged`) must not write at all: the shared persist + /// helper clears the whole object once a state has ≤1 peer and no pending + /// rotation/removal, so a no-op ack or clear that "harmlessly" persisted + /// would delete the retry queue and every other field along with it. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] - async fn test_transaction_serializes_against_a_process_only_legacy_writer() { + async fn test_missed_pending_clear_must_not_rewrite_the_state_object() { publish_ready_iam_context().await; + // One peer, no pending records: exactly the shape the persist helper's clear + // branch fires on. Only the test-only seeder can write it. let seed = SiteReplicationState { + peers: BTreeMap::from([( + "site-solo".to_string(), + PeerInfo { + deployment_id: "site-solo".to_string(), + ..peer("site-solo", "https://solo.example:9000") + }, + )]), + retry_queue: vec![SiteReplicationRetryEvent { + id: "evt-1".to_string(), + peer_deployment_id: "site-gone".to_string(), + peer_endpoint: "https://gone.example:9000".to_string(), + path: "/rustfs/admin/v3/site-replication/peer/iam-item".to_string(), + retry_count: 2, + failed: false, + last_error: "peer offline".to_string(), + updated_at: Some(OffsetDateTime::now_utc()), + edit_generation: None, + }], + ..Default::default() + }; + save_site_replication_state(&seed).await.expect("seed state"); + + clear_pending_remove("no-such-remove").await.expect("no-op clear"); + mark_pending_rotation_peer_acked("no-such-rotation", "site-x") + .await + .expect("no-op ack"); + record_pending_remove_secret_candidate("no-such-remove", "secret".to_string()) + .await + .expect("no-op candidate"); + + let reloaded = load_site_replication_state().await.expect("reload"); + assert_eq!( + reloaded.retry_queue.len(), + 1, + "a missed pending lookup persisted (and therefore cleared) the state object" + ); + assert_eq!(reloaded.peers.len(), 1, "the peer record must survive the no-op calls"); + } + + /// Review follow-up on P1-15 PR2 (overtrue): two joins accepted by the + /// same node must not interleave their IAM writes with each other's + /// commits. Join A loads a stale snapshot and pauses before its IAM + /// write; join B (newer) applies secret B and commits; A resumes, its + /// IAM write would overwrite secret B, and its commit is then refused as + /// superseded — the persisted state advertises B's contract while IAM + /// holds A's secret. `admit_peer_join` closes this by serializing the + /// whole admission under the lifecycle guard and re-checking staleness + /// BEFORE the IAM step: with the guard, B cannot even start while A is + /// gated mid-IAM. Remove the guard (or move the IAM step ahead of the + /// fresh staleness check) and this test deadlocks or records B's IAM + /// write before A finishes. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial] + async fn test_peer_join_admission_serializes_iam_apply_against_a_newer_join() { + publish_ready_iam_context().await; + + // Whole-second timestamps so the RFC3339 round trip through the state + // object cannot lose sub-second precision under the equality asserts. + let now = OffsetDateTime::now_utc().replace_nanosecond(0).expect("truncate nanos"); + let local = PeerInfo { + deployment_id: "site-local".to_string(), + ..peer("site-local", "https://local.example:9000") + }; + let remote = PeerInfo { + deployment_id: "site-remote".to_string(), + ..peer("site-remote", "https://remote.example:9000") + }; + let seed = SiteReplicationState { + peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + updated_at: Some(now - Duration::from_secs(60)), + ..Default::default() + }; + save_site_replication_state(&seed).await.expect("seed state"); + + let join_peers = BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]); + let join_req = |updated_at: OffsetDateTime, secret: &str| SRPeerJoinReq { + svc_acct_access_key: "svc-join".to_string(), + svc_acct_secret_key: secret.to_string(), + svc_acct_parent: "root".to_string(), + peers: join_peers.clone(), + updated_at: Some(updated_at), + }; + + let iam_log: Arc>> = Arc::new(StdMutex::new(Vec::new())); + let (a_entered_tx, a_entered_rx) = tokio::sync::oneshot::channel(); + let (a_gate_tx, a_gate_rx) = tokio::sync::oneshot::channel::<()>(); + + // Join A (older, T1): pauses inside its IAM step. + let log_a = iam_log.clone(); + let endpoint_a = "https://local.example:9000".to_string(); + let req_a = join_req(now - Duration::from_secs(30), "secret-a"); + let join_a = tokio::spawn(async move { + admit_peer_join(endpoint_a, req_a, true, move |_req| async move { + let _ = a_entered_tx.send(()); + let _ = a_gate_rx.await; + log_a.lock().expect("iam log").push("iam-a"); + Ok(()) + }) + .await + }); + a_entered_rx.await.expect("join A reached its IAM step"); + + // Join B (newer, T2) arrives while A is gated mid-IAM. The lifecycle + // guard must hold it at the door. + let log_b = iam_log.clone(); + let endpoint_b = "https://local.example:9000".to_string(); + let req_b = join_req(now, "secret-b"); + let join_b = tokio::spawn(async move { + admit_peer_join(endpoint_b, req_b, true, move |_req| async move { + log_b.lock().expect("iam log").push("iam-b"); + Ok(()) + }) + .await + }); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + iam_log.lock().expect("iam log").is_empty(), + "join B ran its IAM step while join A was still mid-admission: {:?}", + iam_log.lock().expect("iam log") + ); + + a_gate_tx.send(()).expect("release join A"); + let outcome_a = join_a.await.expect("join A task").expect("join A admission"); + let outcome_b = join_b.await.expect("join B task").expect("join B admission"); + assert!(matches!(outcome_a, PeerJoinOutcome::Applied(..)), "join A must commit first"); + assert!( + matches!(outcome_b, PeerJoinOutcome::Applied(..)), + "the newer join B must still apply after A" + ); + assert_eq!( + *iam_log.lock().expect("iam log"), + vec!["iam-a", "iam-b"], + "IAM writes must land in admission order, ending on the committed join's secret" + ); + assert_eq!( + load_site_replication_state().await.expect("reload").updated_at, + Some(now), + "the persisted state must end on join B, matching the last IAM write" + ); + } + + /// Review follow-up on P1-15 PR2 (overtrue, round 2): the same + /// interleaving driven by two SEPARATE NODES, which the process-local + /// lifecycle guard cannot reach. Both admissions run + /// `admit_peer_join_across_nodes` — the production path minus the + /// process-local guard, exactly what a second node executes — so only + /// the distributed join-admission lock keeps join B out while join A is + /// gated mid-IAM. Remove that lock and B's IAM write lands during A's + /// admission: the assertion on the empty IAM log turns red. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial] + async fn test_peer_join_admission_serializes_across_separate_nodes() { + publish_ready_iam_context().await; + + let now = OffsetDateTime::now_utc().replace_nanosecond(0).expect("truncate nanos"); + let local = PeerInfo { + deployment_id: "site-local".to_string(), + ..peer("site-local", "https://local.example:9000") + }; + let remote = PeerInfo { + deployment_id: "site-remote".to_string(), + ..peer("site-remote", "https://remote.example:9000") + }; + let seed = SiteReplicationState { + peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + updated_at: Some(now - Duration::from_secs(60)), + ..Default::default() + }; + save_site_replication_state(&seed).await.expect("seed state"); + + let join_peers = BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]); + let join_req = |updated_at: OffsetDateTime, secret: &str| SRPeerJoinReq { + svc_acct_access_key: "svc-join".to_string(), + svc_acct_secret_key: secret.to_string(), + svc_acct_parent: "root".to_string(), + peers: join_peers.clone(), + updated_at: Some(updated_at), + }; + + let iam_log: Arc>> = Arc::new(StdMutex::new(Vec::new())); + let (a_entered_tx, a_entered_rx) = tokio::sync::oneshot::channel(); + let (a_gate_tx, a_gate_rx) = tokio::sync::oneshot::channel::<()>(); + + // Node A (older join, T1) pauses inside its IAM step while holding + // only the distributed admission lock. + let log_a = iam_log.clone(); + let req_a = join_req(now - Duration::from_secs(30), "secret-a"); + let join_a = tokio::spawn(async move { + admit_peer_join_across_nodes("https://local.example:9000".to_string(), req_a, true, move |_req| async move { + let _ = a_entered_tx.send(()); + let _ = a_gate_rx.await; + log_a.lock().expect("iam log").push("iam-a"); + Ok(()) + }) + .await + }); + a_entered_rx.await.expect("node A reached its IAM step"); + + // Node B (newer join, T2) arrives on "another node": no process-local + // guard applies. The distributed admission lock must hold it. + let log_b = iam_log.clone(); + let req_b = join_req(now, "secret-b"); + let join_b = tokio::spawn(async move { + admit_peer_join_across_nodes("https://local.example:9000".to_string(), req_b, true, move |_req| async move { + log_b.lock().expect("iam log").push("iam-b"); + Ok(()) + }) + .await + }); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + iam_log.lock().expect("iam log").is_empty(), + "node B ran its IAM step while node A was still mid-admission: {:?}", + iam_log.lock().expect("iam log") + ); + + a_gate_tx.send(()).expect("release node A"); + let outcome_a = join_a.await.expect("node A task").expect("node A admission"); + let outcome_b = join_b.await.expect("node B task").expect("node B admission"); + assert!(matches!(outcome_a, PeerJoinOutcome::Applied(..)), "node A must commit first"); + assert!( + matches!(outcome_b, PeerJoinOutcome::Applied(..)), + "the newer join B must still apply after A" + ); + assert_eq!( + *iam_log.lock().expect("iam log"), + vec!["iam-a", "iam-b"], + "IAM writes must land in admission order, ending on the committed join's secret" + ); + assert_eq!( + load_site_replication_state().await.expect("reload").updated_at, + Some(now), + "the persisted state must end on join B, matching the last IAM write" + ); + } + + /// P1-15 PR2: the three-way contract of `finalize_pending_rotation_if_complete` + /// — no pending means "already finalized" (true, nothing written), a + /// different or incomplete rotation is left alone (false), and a fully + /// acked rotation is cleared in the same transaction that reports true. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial] + async fn test_finalize_pending_rotation_three_way_contract() { + publish_ready_iam_context().await; + + let local_peer = PeerInfo { + deployment_id: "site-local".to_string(), + ..peer("site-local", "https://local.example:9000") + }; + let remote_peer = PeerInfo { + deployment_id: "site-remote".to_string(), + ..peer("site-remote", "https://remote.example:9000") + }; + let seed = SiteReplicationState { + peers: BTreeMap::from([ + (local_peer.deployment_id.clone(), local_peer.clone()), + (remote_peer.deployment_id.clone(), remote_peer.clone()), + ]), pending_rotation: Some(PendingRotation { - id: "rot-legacy".to_string(), + id: "rot-final".to_string(), access_key: "svc-account".to_string(), + peers: BTreeMap::from([ + (local_peer.deployment_id.clone(), local_peer.clone()), + (remote_peer.deployment_id.clone(), remote_peer.clone()), + ]), ..Default::default() }), ..Default::default() }; save_site_replication_state(&seed).await.expect("seed state"); - const ROUNDS: usize = 8; - for round in 0..ROUNDS { - let legacy_id = format!("legacy-{round}"); - let legacy = tokio::spawn(async move { - // Exactly what an unmigrated call site does today. - let _guard = site_replication_state_process_guard().await; - let mut state = load_site_replication_state().await.expect("legacy load"); - if let Some(pending) = state.pending_rotation.as_mut() { - pending.secret_candidates.push(legacy_id); - } - save_site_replication_state(&state).await.expect("legacy save"); - }); - let ack_id = format!("ack-{round}"); - let migrated = tokio::spawn(async move { - mark_pending_rotation_peer_acked("rot-legacy", &ack_id) - .await - .expect("transaction writer"); - }); - legacy.await.expect("legacy task"); - migrated.await.expect("transaction task"); - } + assert!( + !finalize_pending_rotation_if_complete("other-rotation", &local_peer) + .await + .expect("mismatched id"), + "a different rotation id must not finalize" + ); + assert!( + !finalize_pending_rotation_if_complete("rot-final", &local_peer) + .await + .expect("incomplete acks"), + "an un-acked remote peer must block finalization" + ); + assert!( + load_site_replication_state() + .await + .expect("reload") + .pending_rotation + .is_some(), + "the pending rotation must survive both refusals" + ); - let final_state = load_site_replication_state().await.expect("reload"); - let pending = final_state.pending_rotation.expect("pending rotation survives"); - for round in 0..ROUNDS { - assert!( - pending.secret_candidates.contains(&format!("legacy-{round}")), - "legacy writer update {round} was lost; candidates: {:?}", - pending.secret_candidates - ); - assert!( - pending.acked_deployment_ids.contains(&format!("ack-{round}")), - "transaction writer update {round} was lost; acks: {:?}", - pending.acked_deployment_ids - ); - } + mark_pending_rotation_peer_acked("rot-final", &remote_peer.deployment_id) + .await + .expect("ack remote"); + assert!( + finalize_pending_rotation_if_complete("rot-final", &local_peer) + .await + .expect("finalize"), + "a fully acked rotation must finalize" + ); + assert!( + load_site_replication_state() + .await + .expect("reload") + .pending_rotation + .is_none(), + "finalization must clear the pending rotation" + ); + assert!( + finalize_pending_rotation_if_complete("rot-final", &local_peer) + .await + .expect("idempotent"), + "no pending rotation means already finalized" + ); } - /// P1-15 review follow-up: isolates the DISTRIBUTED guard. Both writers - /// bypass the process mutex, which is what two separate nodes do — the - /// mutex is per process and cannot serialize them. Only the state-object - /// write lock keeps their read-modify-write sequences apart; drop it and - /// this test loses an update. + /// P1-15 review follow-up: isolates the DISTRIBUTED guard, which is now + /// the whole boundary. Two "nodes" run the production transaction + /// concurrently; only the state-object write lock keeps their + /// read-modify-write sequences apart — drop it and this test loses an + /// update. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn test_state_object_lock_serializes_writers_from_separate_nodes() { @@ -15363,11 +15972,10 @@ mod tests { }; save_site_replication_state(&seed).await.expect("seed state"); - // Each "node" runs the production transaction minus the process - // mutex — the distributed state-object lock is the only thing left - // to keep them apart. + // Each "node" runs the production transaction; the distributed + // state-object lock is the only thing keeping them apart. fn node_local_update(candidate: String) -> impl std::future::Future> { - update_site_replication_state_as_separate_node(move |state| { + update_site_replication_state(move |state| { if let Some(pending) = state.pending_rotation.as_mut() { pending.secret_candidates.push(candidate); } @@ -15397,10 +16005,9 @@ mod tests { } /// P1-15 review follow-up: the sending side of the peer-edit ordering - /// fence, driven by two separate nodes. Both bypass the process mutex — - /// which is exactly what two nodes of one site do — so the generation is - /// only unique because it is allocated inside the state transaction, under - /// the distributed state-object lock. Two nodes sharing a generation would + /// fence, driven by two separate nodes. The generation is unique only + /// because it is allocated inside the state transaction, under the + /// distributed state-object lock. Two nodes sharing a generation would /// leave the receiver unable to tell which edit is newer. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] @@ -15419,7 +16026,7 @@ mod tests { save_site_replication_state(&seed).await.expect("seed state"); fn node_local_allocate() -> impl std::future::Future> { - update_site_replication_state_as_separate_node(|state| Ok(next_peer_edit_generation(state))) + update_site_replication_state(|state| Ok(next_peer_edit_generation(state))) } const ROUNDS: usize = 8; diff --git a/rustfs/src/admin/site_replication_state.rs b/rustfs/src/admin/site_replication_state.rs index eb0414665..d39618a2b 100644 --- a/rustfs/src/admin/site_replication_state.rs +++ b/rustfs/src/admin/site_replication_state.rs @@ -18,28 +18,21 @@ //! `config/site-replication/state.json` is mutated by read-modify-write //! sequences spread over many call sites: admin handlers, the retry-event //! writers on every hook broadcast path, and the service-side reload driven -//! over node RPC. Historically only some of them held the process-local -//! mutex and none held a distributed lock across the whole RMW, so -//! concurrent writers overwrote each other (single-process for the unlocked -//! writers, cross-node for everyone). +//! over node RPC. //! //! `with_site_replication_state_lock` is the single transaction boundary: -//! it holds the process-local mutex AND the distributed config-object write -//! lock (the pattern proven by the repair state, -//! `update_site_replication_repair_state`) for the duration of the caller's -//! closure. All IO inside the closure must use the `*_no_lock` config -//! helpers — the locked variants would self-deadlock on the same object -//! lock. Do not perform peer network calls or take other config locks +//! it holds the distributed config-object write lock (the pattern proven by +//! the repair state, `update_site_replication_repair_state`) for the +//! duration of the caller's closure. The object lock is the sole mechanism — +//! it is the only thing that can serialize two nodes of the same site, so a +//! process-local lock must never be reintroduced in front of it as if it +//! added protection. All IO inside the closure must use the `*_no_lock` +//! config helpers — the locked variants would self-deadlock on the same +//! object lock. Do not perform peer network calls or take other config locks //! inside the closure. //! -//! The process-local mutex is transitional: call sites still outside this -//! primitive serialize against migrated ones through it. Once every RMW -//! call site goes through here (P1-15 PR2) it will be removed, leaving the -//! object lock as the only mechanism. -//! -//! Lock order (unchanged from the historical comment next to the mutex): -//! lifecycle -> bucket operation -> repair admission -> state (process -//! mutex, then state object lock) -> per-bucket metadata. +//! Lock order: lifecycle -> bucket operation -> repair admission +//! -> state object lock -> per-bucket metadata. use crate::admin::storage_api::runtime::ECStore; use crate::admin::storage_api::s3::{S3Error, S3ErrorCode, S3Result}; @@ -53,24 +46,8 @@ use super::runtime_sources::current_object_store_handle; /// byte-level tolerant reload on the service side. pub(crate) const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json"; -/// Transitional process-local mutex — see the module docs. Stays private to -/// this module (owner-local static, enforced by -/// `scripts/check_architecture_migration_rules.sh`); callers go through -/// [`site_replication_state_process_guard`]. -static SITE_REPLICATION_STATE_LOCK: std::sync::LazyLock> = - std::sync::LazyLock::new(|| tokio::sync::Mutex::new(())); - -/// Owner helper for the transitional process mutex: the RMW call sites in -/// `handlers::site_replication` that PR2 has not migrated to -/// [`with_site_replication_state_lock`] yet hold this guard so they stay -/// mutually exclusive with the migrated ones. Removed together with the -/// mutex once every call site runs inside the transaction boundary. -pub(crate) async fn site_replication_state_process_guard() -> tokio::sync::MutexGuard<'static, ()> { - SITE_REPLICATION_STATE_LOCK.lock().await -} - /// Run `operation` under the site-replication state transaction boundary: -/// process mutex first, then the distributed state-object write lock. +/// the distributed state-object write lock. pub(crate) async fn with_site_replication_state_lock(operation: F) -> S3Result where T: Send + 'static, @@ -83,21 +60,11 @@ where /// Context-store variant for callers that resolve their store from an /// explicit [`AppContext`] (the service-side reload driven over node RPC). +/// +/// This is the whole boundary: a state-object write lock serializes writers +/// in *different* processes, which is what two nodes of one site are and +/// what a process mutex could never cover. pub(crate) async fn with_site_replication_state_lock_on(store: Arc, operation: F) -> S3Result -where - T: Send + 'static, - F: FnOnce() -> Fut + Send + 'static, - Fut: std::future::Future> + Send + 'static, -{ - let _process_guard = SITE_REPLICATION_STATE_LOCK.lock().await; - with_site_replication_state_object_lock(store, operation).await -} - -/// The distributed half of the boundary on its own: the state-object write -/// lock, without the process mutex. This is the only thing that serializes -/// writers in *different* processes (the mutex cannot), so it is also what -/// the separate-nodes regression test drives. -pub(crate) async fn with_site_replication_state_object_lock(store: Arc, operation: F) -> S3Result where T: Send + 'static, F: FnOnce() -> Fut + Send + 'static, diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index dba51a333..476fbce90 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -4033,7 +4033,7 @@ if [[ -s "$ECSTORE_REMOTE_TIER_DELETE_STATE_BYPASS_HITS_FILE" ]]; then report_failure "remote tier delete state access must stay behind ECStore tier sweeper owner helpers: $(paste -sd '; ' "$ECSTORE_REMOTE_TIER_DELETE_STATE_BYPASS_HITS_FILE")" fi -RUSTFS_OWNER_LOCAL_STATIC_NAMES='(KEYSTONE_AUTH|KEYSTONE_MAPPER|KEYSTONE_CONFIG|LICENSE_STATE|LICENSE_VERIFIER|CPU_CONT_GUARD|PROFILING_CANCEL_TOKEN|MEMORY_SYSTEM|DIAL9_TELEMETRY_GUARD|DISPLAY_CONFIG_SNAPSHOT|GLOBAL_CONFIG_SNAPSHOT|BUFFER_CONFIG_SINGLETON|BUFFER_PROFILE_ENABLED|LEGACY_CREDENTIAL_WARNED_KEYS|CONSOLE_CONFIG|ACTIVE_HTTP_REQUESTS|USE_STARSHARD_CACHE|BUCKET_CACHE_SMALL|BUCKET_CACHE_LARGE|GLOBAL_SSE_DEK_PROVIDER|SSE_TEST_LOCK|AUTH_FS|LOCK_STATS|DEADLOCK_DETECTOR|GET_OBJECT_BUFFER_THRESHOLD_WARNED|GET_READER_STREAM_BUFFER_SIZE_OVERRIDE|OBJECT_SEEK_SUPPORT_THRESHOLD|OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS|SUPPORTED_HEADERS|SITE_REPLICATION_PEER_CLIENT|SITE_REPLICATION_STATE_LOCK|AUDIT_MODULE_ENABLED|NOTIFY_MODULE_ENABLED|PERSISTED_NOTIFY_MODULE_ENABLED|PERSISTED_AUDIT_MODULE_ENABLED|PERSISTED_MODULE_SWITCH_CONFIGURED|DELETE_TAIL_TOTAL|DELETE_CLEANUP_TOTAL|DELETE_REPLICATION_TOTAL|DELETE_NOTIFY_TOTAL|EMBEDDED_SERVER_STARTED|TEST_OUTBOUND_TLS_GENERATION|TEST_REMAINING_FAILURES|CAPACITY_DIRTY_SCOPE_ENV|CAPACITY_DIRTY_SCOPE_INIT|GLOBAL_ENV)' +RUSTFS_OWNER_LOCAL_STATIC_NAMES='(KEYSTONE_AUTH|KEYSTONE_MAPPER|KEYSTONE_CONFIG|LICENSE_STATE|LICENSE_VERIFIER|CPU_CONT_GUARD|PROFILING_CANCEL_TOKEN|MEMORY_SYSTEM|DIAL9_TELEMETRY_GUARD|DISPLAY_CONFIG_SNAPSHOT|GLOBAL_CONFIG_SNAPSHOT|BUFFER_CONFIG_SINGLETON|BUFFER_PROFILE_ENABLED|LEGACY_CREDENTIAL_WARNED_KEYS|CONSOLE_CONFIG|ACTIVE_HTTP_REQUESTS|USE_STARSHARD_CACHE|BUCKET_CACHE_SMALL|BUCKET_CACHE_LARGE|GLOBAL_SSE_DEK_PROVIDER|SSE_TEST_LOCK|AUTH_FS|LOCK_STATS|DEADLOCK_DETECTOR|GET_OBJECT_BUFFER_THRESHOLD_WARNED|GET_READER_STREAM_BUFFER_SIZE_OVERRIDE|OBJECT_SEEK_SUPPORT_THRESHOLD|OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS|SUPPORTED_HEADERS|SITE_REPLICATION_PEER_CLIENT|AUDIT_MODULE_ENABLED|NOTIFY_MODULE_ENABLED|PERSISTED_NOTIFY_MODULE_ENABLED|PERSISTED_AUDIT_MODULE_ENABLED|PERSISTED_MODULE_SWITCH_CONFIGURED|DELETE_TAIL_TOTAL|DELETE_CLEANUP_TOTAL|DELETE_REPLICATION_TOTAL|DELETE_NOTIFY_TOTAL|EMBEDDED_SERVER_STARTED|TEST_OUTBOUND_TLS_GENERATION|TEST_REMAINING_FAILURES|CAPACITY_DIRTY_SCOPE_ENV|CAPACITY_DIRTY_SCOPE_INIT|GLOBAL_ENV)' ( cd "$ROOT_DIR" From ffe889ad59b9707cfb73d292637780c57facb510 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 14 Aug 2026 22:14:26 +0800 Subject: [PATCH 16/71] fix(storage): restore multipart disk compression and make the legacy decompressor resumable (#6044) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(storage): restore multipart disk compression and make the legacy decompressor resumable Multipart uploads have bypassed disk compression since #5169 removed the session marker as a stopgap for mid-stream GET failures. The actual root cause was never the multipart layout: the legacy DecompressReader reset its payload consumption state on every poll re-entry, so a Poll::Pending in the middle of a block payload (routine under the erasure duplex) desynchronized the block framing and surfaced as LZ4 frameType errors. This rewrites the decoder as a resumable state machine, restores the multipart session compression marker, reports logical part sizes in ListParts, and makes the rebalance migration read raw stored bytes so compressed and encrypted objects survive migration verbatim. Fixes #5957. Internal tracking: backlog#1848, backlog#1850. * feat(storage): stage multipart compression behind RUSTFS_COMPRESSION_MULTIPART_ENABLED Review follow-up: a rolling-upgrade window must not create new compressed multipart objects while pre-fix nodes (whose decompressor is not resumable) may still serve reads. The session marker is now additionally gated on RUSTFS_COMPRESSION_MULTIPART_ENABLED, default off, so the restored capability stays dark until the operator confirms fleet convergence. The default flips per the multipart-compression-default-off-window entry in docs/architecture/compat-cleanup-register.md once the minimum supported direct-upgrade release ships the resumable decoder. * chore(compat): satisfy the cleanup-register guard for the multipart compression switch The architecture guard requires every backticked identifier in a register entry to carry a RUSTFS_COMPAT_TODO source marker: keep only the entry slug in backticks, and add the marker (with its literal Remove-after condition) at the switch definition. * chore(rio): drop a dead store in the poison guard and note the end-block branch Review follow-up: the poison gate re-assigned an already-true flag, and the COMPRESS_TYPE_END branch reads as dead without stating that the writer never emits an end block — that absence is exactly what lets concatenated per-part streams decode as one. * fix(s3): report empty compressed multipart part size * fix(s3): report empty encrypted multipart part size --- .config/nextest.toml | 8 +- crates/e2e_test/src/common.rs | 5 +- crates/e2e_test/src/compression_test.rs | 666 +++++++++++++++++- .../src/inline_fast_path_cluster_test.rs | 31 +- crates/ecstore/src/api/mod.rs | 4 +- crates/ecstore/src/io_support/compress.rs | 22 + crates/ecstore/src/object_api/readers.rs | 417 +++++++++++ crates/rio/src/compress_reader.rs | 412 +++++++++-- docs/architecture/compat-cleanup-register.md | 1 + rustfs/src/app/multipart_usecase.rs | 63 +- rustfs/src/app/storage_api.rs | 6 +- rustfs/src/storage/s3_api/multipart.rs | 124 +++- rustfs/src/storage/storage_api.rs | 4 +- scripts/run.sh | 1 + 14 files changed, 1692 insertions(+), 72 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index d543d0c4a..ade76a58c 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -252,10 +252,16 @@ test-group = 'ecstore-serial-flaky' # cluster, so it keeps the lane's parallel-safe / no-external-dependency # properties. The RustFS warm backend has no loopback guard (that guard is # replication-only), so it needs no opt-in env for its 127.0.0.1 tier target. +# +# Disk compression (backlog#1848): the `compression` module joins the smoke +# lane so the multipart disk-compression roundtrips (restored after +# rustfs/rustfs#5169 disabled them) have PR-lane signal, not just merge-gate. +# Single-node servers on random ports with isolated temp dirs — meets the +# admission criteria unchanged. [profile.e2e-smoke] default-filter = """ package(e2e_test) & ( - test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/) + test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/) | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/) | test(/^reliant::lifecycle::/) | test(/^reliant::tiering::/) diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index a4f4dfedb..f1fcaa20a 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -67,7 +67,10 @@ fn configured_capture_log_path(temp_dir: &str) -> Option { capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned()) } -fn capture_command_logs(command: &mut Command, log_path: Option<&str>) -> Result<(), Box> { +pub(crate) fn capture_command_logs( + command: &mut Command, + log_path: Option<&str>, +) -> Result<(), Box> { let Some(log_path) = log_path else { return Ok(()); }; diff --git a/crates/e2e_test/src/compression_test.rs b/crates/e2e_test/src/compression_test.rs index f01e78320..775decfa6 100644 --- a/crates/e2e_test/src/compression_test.rs +++ b/crates/e2e_test/src/compression_test.rs @@ -2,6 +2,7 @@ use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path}; use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; use serial_test::serial; use std::fs; use std::path::PathBuf; @@ -25,6 +26,15 @@ fn generate_compressible_data(size: usize) -> Vec { data } +/// Deterministic 2048-byte-period binary pattern that compresses extremely well: every part +/// yields many compressed blocks, which is exactly the shape that reproduced the mid-payload +/// Pending truncation (rustfs/rustfs#5957). +fn generate_high_ratio_binary_data(size: usize, seed: u8) -> Vec { + (0..size) + .map(|i| ((i as u64).wrapping_mul(2_654_435_761).wrapping_add(seed as u64) >> 3) as u8) + .collect() +} + fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec { let bucket_path = PathBuf::from(temp_dir).join(bucket); let mut part_files = Vec::new(); @@ -55,9 +65,14 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul env.cleanup_existing_processes().await?; let binary_path = rustfs_binary_path(); - let process = Command::new(&binary_path) + // Route the child's stdout/stderr through the shared RUSTFS_E2E_LOG_DIR + // capture (survives the temp-dir cleanup on Drop and is uploaded as a CI + // artifact); without the env var the child inherits stdio as before. + let mut command = Command::new(&binary_path); + command .env("RUSTFS_CONSOLE_ENABLE", "false") .env("RUSTFS_COMPRESSION_ENABLED", "true") + .env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true") .args([ "--address", &env.address, @@ -66,8 +81,9 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul "--secret-key", &env.secret_key, &env.temp_dir, - ]) - .spawn()?; + ]); + crate::common::capture_command_logs(&mut command, env.capture_log_path.as_deref())?; + let process = command.spawn()?; env.process = Some(process); @@ -154,3 +170,647 @@ async fn test_compression_roundtrip() -> Result<(), Box Result<(), Box> { + let create = client.create_multipart_upload().bucket(bucket).key(key).send().await?; + let upload_id = create.upload_id().ok_or("missing upload id")?.to_string(); + + let mut completed_parts = Vec::with_capacity(parts.len()); + for (i, part) in parts.iter().enumerate() { + let part_number = (i + 1) as i32; + let upload = client + .upload_part() + .bucket(bucket) + .key(key) + .upload_id(&upload_id) + .part_number(part_number) + .body(ByteStream::from(part.to_vec())) + .send() + .await?; + completed_parts.push( + CompletedPart::builder() + .part_number(part_number) + .e_tag(upload.e_tag().unwrap_or_default()) + .build(), + ); + } + + client + .complete_multipart_upload() + .bucket(bucket) + .key(key) + .upload_id(&upload_id) + .multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build()) + .send() + .await?; + Ok(()) +} + +async fn fetch_range( + client: &aws_sdk_s3::Client, + bucket: &str, + key: &str, + range: &str, +) -> Result, Box> { + let response = client.get_object().bucket(bucket).key(key).range(range).send().await?; + Ok(response.body.collect().await?.into_bytes().to_vec()) +} + +/// Multipart disk compression roundtrip: parts are written as independent +/// compressed streams and every GET shape must reassemble the original bytes +/// (rustfs/rustfs#5957: multipart uploads previously bypassed disk compression +/// entirely). +#[tokio::test] +#[serial] +async fn test_compression_multipart_roundtrip() -> Result<(), Box> { + init_logging(); + info!("Starting multipart compression roundtrip test"); + + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_with_compression(&mut env).await?; + + let client = env.create_s3_client(); + env.create_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?; + + let object_key = "multipart-compressible.txt"; + let part1 = generate_compressible_data(MPU_PART1_SIZE); + let part2 = generate_compressible_data(MPU_PART2_SIZE); + let mut original_data = part1.clone(); + original_data.extend_from_slice(&part2); + let total_size = original_data.len(); + + multipart_upload(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &[&part1, &part2]).await?; + + let head_response = client + .head_object() + .bucket(MULTIPART_COMPRESSION_BUCKET) + .key(object_key) + .send() + .await?; + assert_eq!( + head_response.content_length().unwrap_or(0) as usize, + total_size, + "Content-Length should be the logical object size" + ); + + let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key); + assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object"); + let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum(); + assert!( + total_physical_size < (total_size / 2) as u64, + "Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)" + ); + info!("Multipart physical storage size: {total_physical_size} bytes (compressed from {total_size} bytes)"); + + // Full GET must reassemble both independently compressed parts. + let get_response = client + .get_object() + .bucket(MULTIPART_COMPRESSION_BUCKET) + .key(object_key) + .send() + .await?; + let downloaded = get_response.body.collect().await?.into_bytes(); + assert_eq!(downloaded.len(), total_size); + assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch"); + + // Range fully inside part 1. + let range_inside_part1 = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, "bytes=1024-999423").await?; + assert_eq!(&range_inside_part1[..], &original_data[1024..999424], "part-1 range mismatch"); + + // Range crossing the part boundary. + let boundary_start = MPU_PART1_SIZE - 128 * 1024; + let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1; + let range_crossing = fetch_range( + &client, + MULTIPART_COMPRESSION_BUCKET, + object_key, + &format!("bytes={boundary_start}-{boundary_end}"), + ) + .await?; + assert_eq!( + &range_crossing[..], + &original_data[boundary_start..boundary_end + 1], + "boundary-crossing range mismatch" + ); + + // Range fully inside part 2. + let part2_start = MPU_PART1_SIZE + 4096; + let part2_end = MPU_PART1_SIZE + 256 * 1024 - 1; + let range_inside_part2 = fetch_range( + &client, + MULTIPART_COMPRESSION_BUCKET, + object_key, + &format!("bytes={part2_start}-{part2_end}"), + ) + .await?; + assert_eq!( + &range_inside_part2[..], + &original_data[part2_start..part2_end + 1], + "part-2 range mismatch" + ); + + // Suffix range (last 128 KiB, entirely in part 2). + let suffix_len = 128 * 1024; + let suffix = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?; + assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch"); + + // partNumber GETs must return each original part. + for (part_number, expected) in [(1, &part1), (2, &part2)] { + let response = client + .get_object() + .bucket(MULTIPART_COMPRESSION_BUCKET) + .key(object_key) + .part_number(part_number) + .send() + .await?; + let body = response.body.collect().await?.into_bytes(); + assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch"); + } + + info!("Multipart compression roundtrip test passed"); + env.delete_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?; + env.stop_server(); + Ok(()) +} + +const MPU_HIGH_RATIO_BUCKET: &str = "compression-mpu-high-ratio-bucket"; + +/// High-ratio binary multipart payload: the object key is on the compression allow-list, so the +/// disk-compression path runs and each part is stored as many compressed blocks — the shape that +/// reproduced the mid-payload Pending truncation (rustfs/rustfs#5957). Every GET shape must return +/// the exact original bytes, and the stored size must show the data really was compressed. +#[tokio::test] +#[serial] +async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(), Box> { + init_logging(); + info!("Starting multipart high-ratio binary compression roundtrip test"); + + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_with_compression(&mut env).await?; + + let client = env.create_s3_client(); + env.create_test_bucket(MPU_HIGH_RATIO_BUCKET).await?; + + let object_key = "multipart-high-ratio.txt"; + let part1 = generate_high_ratio_binary_data(MPU_PART1_SIZE, 7); + let part2 = generate_high_ratio_binary_data(MPU_PART2_SIZE, 61); + let mut original_data = part1.clone(); + original_data.extend_from_slice(&part2); + let total_size = original_data.len(); + + multipart_upload(&client, MPU_HIGH_RATIO_BUCKET, object_key, &[&part1, &part2]).await?; + + let head_response = client + .head_object() + .bucket(MPU_HIGH_RATIO_BUCKET) + .key(object_key) + .send() + .await?; + assert_eq!( + head_response.content_length().unwrap_or(0) as usize, + total_size, + "Content-Length should be the logical object size" + ); + + // This pattern compresses to roughly 1/50 of its logical size, so a comfortably loose 2x + // margin still proves the parts were stored compressed rather than raw or double-encoded. + let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key); + assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object"); + let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum(); + assert!( + total_physical_size < (total_size as u64) / 2, + "Physical size {total_physical_size} should be far below the logical size {total_size} for high-ratio data" + ); + info!("High-ratio multipart physical storage size: {total_physical_size} bytes (logical {total_size} bytes)"); + + info!("step: full GET"); + let get_response = client + .get_object() + .bucket(MPU_HIGH_RATIO_BUCKET) + .key(object_key) + .send() + .await?; + let downloaded = get_response.body.collect().await?.into_bytes(); + assert_eq!(downloaded.len(), total_size); + assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch"); + + // Range crossing the part boundary. + info!("step: boundary range GET"); + let boundary_start = MPU_PART1_SIZE - 128 * 1024; + let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1; + let range_crossing = fetch_range( + &client, + MPU_HIGH_RATIO_BUCKET, + object_key, + &format!("bytes={boundary_start}-{boundary_end}"), + ) + .await?; + assert_eq!( + &range_crossing[..], + &original_data[boundary_start..boundary_end + 1], + "boundary-crossing range mismatch" + ); + + // partNumber GET for the trailing part. + info!("step: partNumber GET"); + let part2_response = client + .get_object() + .bucket(MPU_HIGH_RATIO_BUCKET) + .key(object_key) + .part_number(2) + .send() + .await?; + let part2_body = part2_response.body.collect().await?.into_bytes(); + assert_eq!(&part2_body[..], &part2[..], "partNumber=2 GET mismatch"); + + info!("Multipart high-ratio binary compression roundtrip test passed"); + env.delete_test_bucket(MPU_HIGH_RATIO_BUCKET).await?; + env.stop_server(); + Ok(()) +} + +const MPU_COPY_COMPRESSION_BUCKET: &str = "compression-mpu-copy-bucket"; +const MPU_COPY_SOURCE_SIZE: usize = 6 * 1024 * 1024; +const MPU_COPY_RANGE_LEN: usize = 5 * 1024 * 1024; + +/// UploadPartCopy feeds a part from an already stored (and already compressed) object. The copied +/// range must be decompressed on read and re-compressed into the destination part, so the final +/// object has to match "source prefix + uploaded tail" byte for byte. +#[tokio::test] +#[serial] +async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), Box> { + init_logging(); + info!("Starting multipart upload-part-copy compression roundtrip test"); + + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_with_compression(&mut env).await?; + + let client = env.create_s3_client(); + env.create_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?; + + // Source object: a plain PUT that goes through the single-stream compression path. + let source_key = "copy-source.txt"; + let source_data = generate_compressible_data(MPU_COPY_SOURCE_SIZE); + client + .put_object() + .bucket(MPU_COPY_COMPRESSION_BUCKET) + .key(source_key) + .body(ByteStream::from(source_data.clone())) + .send() + .await?; + + // Destination object: part 1 copied from the source, part 2 uploaded directly. + let target_key = "copy-target.txt"; + let part2 = generate_compressible_data(MPU_PART2_SIZE); + let mut expected_data = source_data[..MPU_COPY_RANGE_LEN].to_vec(); + expected_data.extend_from_slice(&part2); + let total_size = expected_data.len(); + + let create = client + .create_multipart_upload() + .bucket(MPU_COPY_COMPRESSION_BUCKET) + .key(target_key) + .send() + .await?; + let upload_id = create.upload_id().ok_or("missing upload id")?.to_string(); + + let copy_part = client + .upload_part_copy() + .bucket(MPU_COPY_COMPRESSION_BUCKET) + .key(target_key) + .upload_id(&upload_id) + .part_number(1) + .copy_source(format!("{MPU_COPY_COMPRESSION_BUCKET}/{source_key}")) + .copy_source_range(format!("bytes=0-{}", MPU_COPY_RANGE_LEN - 1)) + .send() + .await?; + let copy_etag = copy_part + .copy_part_result() + .and_then(|r| r.e_tag()) + .ok_or("missing copy part etag")? + .to_string(); + + let uploaded_part = client + .upload_part() + .bucket(MPU_COPY_COMPRESSION_BUCKET) + .key(target_key) + .upload_id(&upload_id) + .part_number(2) + .body(ByteStream::from(part2.clone())) + .send() + .await?; + + client + .complete_multipart_upload() + .bucket(MPU_COPY_COMPRESSION_BUCKET) + .key(target_key) + .upload_id(&upload_id) + .multipart_upload( + CompletedMultipartUpload::builder() + .parts(CompletedPart::builder().part_number(1).e_tag(copy_etag).build()) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(uploaded_part.e_tag().unwrap_or_default()) + .build(), + ) + .build(), + ) + .send() + .await?; + + let head_response = client + .head_object() + .bucket(MPU_COPY_COMPRESSION_BUCKET) + .key(target_key) + .send() + .await?; + assert_eq!( + head_response.content_length().unwrap_or(0) as usize, + total_size, + "Content-Length should be the logical object size" + ); + + let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key); + assert!(!part_files.is_empty(), "expected on-disk part files for the copied object"); + let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum(); + assert!( + total_physical_size < (total_size / 2) as u64, + "Physical size {total_physical_size} should be well below original size {total_size} (copied part compression applied)" + ); + + let get_response = client + .get_object() + .bucket(MPU_COPY_COMPRESSION_BUCKET) + .key(target_key) + .send() + .await?; + let downloaded = get_response.body.collect().await?.into_bytes(); + assert_eq!(downloaded.len(), total_size); + assert_eq!(&downloaded[..], &expected_data[..], "copied multipart GET data mismatch"); + + info!("Multipart upload-part-copy compression roundtrip test passed"); + env.delete_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?; + env.stop_server(); + Ok(()) +} + +const MPU_THREE_PARTS_BUCKET: &str = "compression-mpu-three-parts-bucket"; +const MPU_THREE_PARTS_TAIL_SIZE: usize = 512 * 1024; + +/// Three-part upload with uneven part sizes: each partNumber GET must map back to exactly one +/// compressed part stream, and a suffix range must resolve inside the trailing part. +#[tokio::test] +#[serial] +async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(), Box> { + init_logging(); + info!("Starting three-part multipart compression partNumber test"); + + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_with_compression(&mut env).await?; + + let client = env.create_s3_client(); + env.create_test_bucket(MPU_THREE_PARTS_BUCKET).await?; + + let object_key = "multipart-three-parts.txt"; + let part1 = generate_compressible_data(MPU_PART1_SIZE); + let part2 = generate_compressible_data(MPU_PART1_SIZE); + let part3 = generate_compressible_data(MPU_THREE_PARTS_TAIL_SIZE); + let mut original_data = part1.clone(); + original_data.extend_from_slice(&part2); + original_data.extend_from_slice(&part3); + let total_size = original_data.len(); + + multipart_upload(&client, MPU_THREE_PARTS_BUCKET, object_key, &[&part1, &part2, &part3]).await?; + + let head_response = client + .head_object() + .bucket(MPU_THREE_PARTS_BUCKET) + .key(object_key) + .send() + .await?; + assert_eq!( + head_response.content_length().unwrap_or(0) as usize, + total_size, + "Content-Length should be the logical object size" + ); + + let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key); + assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object"); + let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum(); + assert!( + total_physical_size < (total_size / 2) as u64, + "Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)" + ); + + // Every partNumber GET must return exactly the bytes of the corresponding uploaded part. + for (part_number, expected) in [(1, &part1), (2, &part2), (3, &part3)] { + let response = client + .get_object() + .bucket(MPU_THREE_PARTS_BUCKET) + .key(object_key) + .part_number(part_number) + .send() + .await?; + let body = response.body.collect().await?.into_bytes(); + assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch"); + } + + // Suffix range (last 64 KiB) resolves inside the trailing part. + let suffix_len = 64 * 1024; + let suffix = fetch_range(&client, MPU_THREE_PARTS_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?; + assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch"); + + info!("Three-part multipart compression partNumber test passed"); + env.delete_test_bucket(MPU_THREE_PARTS_BUCKET).await?; + env.stop_server(); + Ok(()) +} + +const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket"; + +async fn start_rustfs_with_compression_and_sse( + env: &mut RustFSTestEnvironment, +) -> Result<(), Box> { + use base64::Engine; + env.cleanup_existing_processes().await?; + + let binary_path = rustfs_binary_path(); + let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); + // Server output goes to a file inside the per-test temp dir so a failing + // run can be diagnosed from the child's logs. + let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?; + let server_log_err = server_log.try_clone()?; + let process = Command::new(&binary_path) + .env("RUSTFS_CONSOLE_ENABLE", "false") + .env("RUSTFS_COMPRESSION_ENABLED", "true") + .env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true") + .env("RUSTFS_SSE_S3_MASTER_KEY", master_key) + .env("RUST_LOG", "rustfs=info,rustfs_ecstore=info") + .stdout(std::process::Stdio::from(server_log)) + .stderr(std::process::Stdio::from(server_log_err)) + .args([ + "--address", + &env.address, + "--access-key", + &env.access_key, + "--secret-key", + &env.secret_key, + &env.temp_dir, + ]) + .spawn()?; + + env.process = Some(process); + + info!("Waiting for RustFS server with compression + SSE-S3 enabled on {}", env.address); + for i in 0..30 { + if TcpStream::connect(&env.address).await.is_ok() { + info!("RustFS server is ready after {} attempts", i + 1); + return Ok(()); + } + if i == 29 { + return Err("RustFS server failed to become ready".into()); + } + sleep(Duration::from_secs(1)).await; + } + Ok(()) +} + +/// SSE-S3 + disk compression multipart: each part is compressed and then encrypted, and every GET +/// shape must still return the original plaintext bytes. Physical size must shrink because the +/// compression runs before encryption. +#[tokio::test] +#[serial] +async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box> { + use aws_sdk_s3::types::ServerSideEncryption; + + init_logging(); + info!("Starting SSE-S3 multipart compression roundtrip test"); + + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_with_compression_and_sse(&mut env).await?; + + let client = env.create_s3_client(); + env.create_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?; + + let object_key = "multipart-sse-compressible.txt"; + let part1 = generate_compressible_data(MPU_PART1_SIZE); + let part2 = generate_compressible_data(MPU_PART2_SIZE); + let mut original_data = part1.clone(); + original_data.extend_from_slice(&part2); + let total_size = original_data.len(); + + let create = client + .create_multipart_upload() + .bucket(MPU_SSE_COMPRESSION_BUCKET) + .key(object_key) + .server_side_encryption(ServerSideEncryption::Aes256) + .send() + .await?; + let upload_id = create.upload_id().ok_or("missing upload id")?.to_string(); + + let mut completed_parts = Vec::new(); + for (i, part) in [&part1, &part2].into_iter().enumerate() { + let part_number = (i + 1) as i32; + let upload = client + .upload_part() + .bucket(MPU_SSE_COMPRESSION_BUCKET) + .key(object_key) + .upload_id(&upload_id) + .part_number(part_number) + .body(ByteStream::from(part.clone())) + .send() + .await?; + completed_parts.push( + CompletedPart::builder() + .part_number(part_number) + .e_tag(upload.e_tag().unwrap_or_default()) + .build(), + ); + } + + client + .complete_multipart_upload() + .bucket(MPU_SSE_COMPRESSION_BUCKET) + .key(object_key) + .upload_id(&upload_id) + .multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build()) + .send() + .await?; + + let head_response = client + .head_object() + .bucket(MPU_SSE_COMPRESSION_BUCKET) + .key(object_key) + .send() + .await?; + assert_eq!( + head_response.content_length().unwrap_or(0) as usize, + total_size, + "Content-Length should be the logical object size" + ); + assert_eq!( + head_response.server_side_encryption(), + Some(&ServerSideEncryption::Aes256), + "HEAD must report SSE-S3" + ); + + let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key); + assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object"); + let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum(); + assert!( + total_physical_size < (total_size / 2) as u64, + "Physical size {total_physical_size} should be well below original size {total_size} (compress-then-encrypt applied)" + ); + + let get_response = client + .get_object() + .bucket(MPU_SSE_COMPRESSION_BUCKET) + .key(object_key) + .send() + .await?; + let downloaded = get_response.body.collect().await?.into_bytes(); + assert_eq!(downloaded.len(), total_size); + assert_eq!(&downloaded[..], &original_data[..], "SSE-S3 multipart full GET data mismatch"); + + // Range crossing the part boundary must decrypt and decompress across parts. + let boundary_start = MPU_PART1_SIZE - 64 * 1024; + let boundary_end = MPU_PART1_SIZE + 64 * 1024 - 1; + let range_crossing = fetch_range( + &client, + MPU_SSE_COMPRESSION_BUCKET, + object_key, + &format!("bytes={boundary_start}-{boundary_end}"), + ) + .await?; + assert_eq!( + &range_crossing[..], + &original_data[boundary_start..boundary_end + 1], + "SSE-S3 boundary-crossing range mismatch" + ); + + // partNumber GET for the trailing part. + let part2_response = client + .get_object() + .bucket(MPU_SSE_COMPRESSION_BUCKET) + .key(object_key) + .part_number(2) + .send() + .await?; + let part2_body = part2_response.body.collect().await?.into_bytes(); + assert_eq!(&part2_body[..], &part2[..], "SSE-S3 partNumber=2 GET mismatch"); + + info!("SSE-S3 multipart compression roundtrip test passed"); + env.delete_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?; + env.stop_server(); + Ok(()) +} diff --git a/crates/e2e_test/src/inline_fast_path_cluster_test.rs b/crates/e2e_test/src/inline_fast_path_cluster_test.rs index 68f37b3a2..e2890c4a8 100644 --- a/crates/e2e_test/src/inline_fast_path_cluster_test.rs +++ b/crates/e2e_test/src/inline_fast_path_cluster_test.rs @@ -1828,33 +1828,36 @@ async fn four_node_compressed_inline_fallback() -> TestResult { Ok(()) } +/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes. +/// Reverting the multipart compression fix must fail this test. #[tokio::test] #[serial] -async fn four_node_multipart_ignores_disk_compression_fallback() -> TestResult { +async fn four_node_multipart_disk_compression_roundtrip() -> TestResult { init_logging(); let collector = OtlpMetricCollector::start().await?; let mut cluster = RustFSTestClusterEnvironment::new(4).await?; configure_reader_metric_cluster(&mut cluster, &collector); cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true"); + cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true"); cluster.start().await?; - let bucket = "inline-multipart-compression-fallback"; + let bucket = "inline-multipart-compression-roundtrip"; cluster.create_test_bucket(bucket).await?; let client = cluster.create_s3_client(0)?; - let key = "multipart/compression-disabled.txt"; + let key = "multipart/compressed.txt"; let (body, second_part, etag) = put_two_part_multipart(&client, bucket, key).await?; assert_reader_path( &collector, &client, - ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, MULTIPART), + ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED), ) .await?; assert_part_number_reader_path( &collector, &client, - PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), MULTIPART, LEGACY_DUPLEX), + PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), COMPRESSED, LEGACY_DUPLEX), ) .await?; @@ -1871,6 +1874,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key); cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true"); + cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true"); configure_mixed_msgpack_cluster(&mut cluster, &collector)?; cluster.start().await?; @@ -1890,14 +1894,21 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te ReaderPathExpectation::for_class( ReaderObject::new(bucket, multipart_key, &multipart_body, multipart_etag.as_deref(), None), LEGACY_DUPLEX, - MULTIPART, + COMPRESSED, ), ) .await?; assert_part_number_reader_path( &collector, &client, - PartNumberReaderPathExpectation::new(bucket, multipart_key, &second_part, multipart_body.len(), MULTIPART, LEGACY_DUPLEX), + PartNumberReaderPathExpectation::new( + bucket, + multipart_key, + &second_part, + multipart_body.len(), + COMPRESSED, + LEGACY_DUPLEX, + ), ) .await?; assert_msgpack_decode_observed(&collector, &decode_before).await?; @@ -2353,7 +2364,11 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_ hot_client.create_bucket().bucket(bucket).send().await?; put_lifecycle_with_transition_retry(&hot_client, bucket, &tier_name).await?; - let key = "transition/mixed-multipart.bin"; + // `.zip` sits on the disk-compression exclusion list: this test pins + // msgpack compat controls across ILM transition, and a compressed object + // would classify as `compressed` instead of `remote` (and the warm-tier + // read path does not decode compression — tracked separately). + let key = "transition/mixed-multipart.zip"; let (body, second_part, etag) = put_two_part_multipart(&hot_client, bucket, key).await?; wait_for_transition(&hot_client, bucket, key, &tier_name).await?; assert!( diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index ffc175659..d93b717bb 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -278,7 +278,9 @@ pub mod cluster { } pub mod compression { - pub use crate::io_support::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled}; + pub use crate::io_support::compress::{ + MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled, is_multipart_disk_compression_enabled, + }; } pub mod config { diff --git a/crates/ecstore/src/io_support/compress.rs b/crates/ecstore/src/io_support/compress.rs index 356974afe..6d197bfde 100644 --- a/crates/ecstore/src/io_support/compress.rs +++ b/crates/ecstore/src/io_support/compress.rs @@ -31,6 +31,13 @@ pub const ENV_DISK_COMPRESSION_MIME_TYPES: &str = "RUSTFS_COMPRESSION_MIME_TYPES // Environment variable for additional extensions to exclude from compression (comma-separated, e.g. ".foo,.bar") pub const ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS: &str = "RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS"; +// Environment variable to additionally enable disk compression for multipart uploads. +// Default off: nodes from before the resumable decompressor fix fail transient reads of +// compressed objects, so multipart compression stays dark until the operator confirms the +// fleet has converged on a fixed build. +// RUSTFS_COMPAT_TODO(multipart-compression-default-off-window): staged rollout switch for restored multipart compression, flipping the default to enabled on retirement. Remove after the minimum supported direct-upgrade release ships the resumable DecompressReader. +pub const ENV_DISK_COMPRESSION_MULTIPART_ENABLED: &str = "RUSTFS_COMPRESSION_MULTIPART_ENABLED"; + pub const DEFAULT_DISK_COMPRESS_EXTENSIONS: &str = ".txt,.log,.csv,.json,.tar,.xml,.bin"; pub const DEFAULT_DISK_COMPRESS_MIME_TYPES: &str = "text/*,application/json,application/xml,binary/octet-stream"; @@ -171,6 +178,21 @@ pub fn is_disk_compression_enabled() -> bool { DISK_COMPRESSION_CONFIG.get_or_init(parse_disk_compression_config).enabled } +// Parsed once at first use, mirroring DISK_COMPRESSION_CONFIG. +static MULTIPART_DISK_COMPRESSION_ENABLED: OnceLock = OnceLock::new(); + +/// Whether multipart uploads may advertise disk compression. Requires the +/// regular disk-compression gates to pass as well; this is the staged-rollout +/// switch that keeps multipart compression dark during rolling upgrades from +/// builds whose decompressor was not yet resumable. +pub fn is_multipart_disk_compression_enabled() -> bool { + *MULTIPART_DISK_COMPRESSION_ENABLED.get_or_init(|| { + env::var(ENV_DISK_COMPRESSION_MULTIPART_ENABLED) + .map(|s| matches!(s.to_ascii_lowercase().as_str(), "true" | "on" | "1")) + .unwrap_or(false) + }) +} + fn is_disk_compressible_with_config(headers: &http::HeaderMap, object_name: &str, config: &DiskCompressionConfig) -> bool { // Check if disk compression is enabled (read once at first use, then fixed for process lifetime) if !config.enabled { diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index b1731b466..6f43a340c 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -1781,6 +1781,423 @@ mod tests { assert_eq!(actual, b"fghijkl"); } + /// Compresses one multipart part exactly like the write path does + /// (`WritePlan::with_compression` wraps each part in its own + /// `compression_reader`), returning the on-disk bytes and the storage-format + /// compression index. + async fn compressed_part_fixture(data: &[u8]) -> (Vec, Option) { + use crate::io_support::rio::TryGetIndex as _; + let mut compressor = + crate::io_support::rio::compression_reader(Cursor::new(data.to_vec()), CompressionAlgorithm::default(), false); + let mut compressed = Vec::new(); + compressor.read_to_end(&mut compressed).await.expect("compress part stream"); + let index = compressor + .try_get_index() + .map(crate::io_support::rio::compression_index_storage_bytes); + (compressed, index) + } + + struct CompressedMultipartFixture { + object_info: ObjectInfo, + stored: Vec, + plaintext: Vec, + } + + /// Builds the on-disk representation of a compressed multipart object: each + /// part is an independent compressed stream and the storage layer serves + /// their concatenation. + async fn compressed_multipart_fixture(part_sizes: &[usize]) -> CompressedMultipartFixture { + let pattern = b"compressed multipart read path fixture data "; + let mut plaintext = Vec::new(); + let mut stored = Vec::new(); + let mut parts = Vec::with_capacity(part_sizes.len()); + + for (i, part_size) in part_sizes.iter().enumerate() { + let mut part_plaintext = Vec::with_capacity(*part_size); + while part_plaintext.len() < *part_size { + part_plaintext.extend_from_slice(pattern); + part_plaintext.push(i as u8); + } + part_plaintext.truncate(*part_size); + + let (compressed, index) = compressed_part_fixture(&part_plaintext).await; + parts.push(ObjectPartInfo { + number: i + 1, + size: compressed.len(), + actual_size: *part_size as i64, + index, + ..Default::default() + }); + stored.extend_from_slice(&compressed); + plaintext.extend_from_slice(&part_plaintext); + } + + let mut user_defined = HashMap::new(); + rustfs_utils::http::insert_str( + &mut user_defined, + rustfs_utils::http::SUFFIX_COMPRESSION, + crate::io_support::rio::compression_metadata_value(CompressionAlgorithm::default()), + ); + rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string()); + + let object_info = ObjectInfo { + bucket: "test-bucket".to_string(), + name: "compressed-multipart".to_string(), + size: stored.len() as i64, + etag: Some(format!("6bcf86bed8807b8e78f0fc6e0a53079d-{}", part_sizes.len())), + parts: Arc::new(parts), + user_defined: Arc::new(user_defined), + ..Default::default() + }; + + CompressedMultipartFixture { + object_info, + stored, + plaintext, + } + } + + /// Plans the read once to learn the storage window, then serves exactly that + /// window — mirroring how `set_disk` feeds the erasure read into the + /// returned reader. + async fn read_compressed_multipart( + fixture: &CompressedMultipartFixture, + rs: Option, + opts: &ObjectOptions, + ) -> Vec { + let headers = HeaderMap::new(); + let (_, offset, length) = + GetObjectReader::new(Box::new(Cursor::new(Vec::new())), rs.clone(), &fixture.object_info, opts, &headers) + .await + .expect("plan compressed multipart read"); + + let end = offset + usize::try_from(length).expect("storage window length must be non-negative"); + assert!( + end <= fixture.stored.len(), + "planned storage window {offset}..{end} exceeds stored stream of {} bytes", + fixture.stored.len() + ); + let window = fixture.stored[offset..end].to_vec(); + + let (mut reader, replay_offset, replay_length) = + GetObjectReader::new(Box::new(Cursor::new(window)), rs, &fixture.object_info, opts, &headers) + .await + .expect("build compressed multipart reader"); + assert_eq!((replay_offset, replay_length), (offset, length), "read plan must be deterministic"); + + reader.read_all().await.expect("read compressed multipart stream") + } + + /// Byte pattern with a 2 KiB period: it compresses extremely well while + /// looking nothing like ASCII fixtures. Mirrors the e2e generator that + /// exposed a truncated full GET on high-ratio multipart payloads. + fn high_ratio_binary_payload(size: usize, seed: u8) -> Vec { + (0..size) + .map(|i| ((i as u64).wrapping_mul(2_654_435_761).wrapping_add(seed as u64) >> 3) as u8) + .collect() + } + + #[tokio::test] + async fn compressed_multipart_full_get_handles_high_ratio_binary_payload() { + let part_sizes = [5 * 1024 * 1024_usize, 1024 * 1024]; + let mut plaintext = Vec::new(); + let mut stored = Vec::new(); + let mut parts = Vec::with_capacity(part_sizes.len()); + + for (i, part_size) in part_sizes.iter().enumerate() { + let part_plaintext = high_ratio_binary_payload(*part_size, if i == 0 { 7 } else { 61 }); + let (compressed, index) = compressed_part_fixture(&part_plaintext).await; + parts.push(ObjectPartInfo { + number: i + 1, + size: compressed.len(), + actual_size: *part_size as i64, + index, + ..Default::default() + }); + stored.extend_from_slice(&compressed); + plaintext.extend_from_slice(&part_plaintext); + } + + let mut user_defined = HashMap::new(); + rustfs_utils::http::insert_str( + &mut user_defined, + rustfs_utils::http::SUFFIX_COMPRESSION, + crate::io_support::rio::compression_metadata_value(CompressionAlgorithm::default()), + ); + rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string()); + let fixture = CompressedMultipartFixture { + object_info: ObjectInfo { + bucket: "test-bucket".to_string(), + name: "high-ratio-multipart".to_string(), + size: stored.len() as i64, + etag: Some("6bcf86bed8807b8e78f0fc6e0a53079d-2".to_string()), + parts: Arc::new(parts), + user_defined: Arc::new(user_defined), + ..Default::default() + }, + stored, + plaintext, + }; + + let read = read_compressed_multipart(&fixture, None, &ObjectOptions::default()).await; + + assert_eq!(read.len(), fixture.plaintext.len(), "full GET must return the logical size"); + assert_eq!(read, fixture.plaintext, "high-ratio multipart payload must survive the roundtrip"); + } + + /// Full GET over a compressed multipart object must decode across part + /// boundaries: every part is an independent compressed stream (this is also + /// the on-disk shape written by builds before rustfs/rustfs#5169 disabled + /// multipart compression, so this pins legacy-object readability). + #[tokio::test] + async fn compressed_multipart_full_get_decodes_across_part_boundaries() { + let fixture = compressed_multipart_fixture(&[3 * 1024 * 1024, 2 * 1024 * 1024, 512 * 1024]).await; + + let read = read_compressed_multipart(&fixture, None, &ObjectOptions::default()).await; + + assert_eq!(read.len(), fixture.plaintext.len(), "full GET must return the logical size"); + assert_eq!(read, fixture.plaintext, "full GET must reassemble all parts"); + } + + #[tokio::test] + async fn compressed_multipart_range_get_crosses_part_boundary() { + let fixture = compressed_multipart_fixture(&[3 * 1024 * 1024, 2 * 1024 * 1024]).await; + let boundary = 3 * 1024 * 1024_i64; + let rs = HTTPRangeSpec { + is_suffix_length: false, + start: boundary - 100_000, + end: boundary + 100_000 - 1, + }; + + let read = read_compressed_multipart(&fixture, Some(rs), &ObjectOptions::default()).await; + + let expected = &fixture.plaintext[(boundary - 100_000) as usize..(boundary + 100_000) as usize]; + assert_eq!(read, expected, "boundary-crossing range must splice both parts"); + } + + #[tokio::test] + async fn compressed_multipart_range_get_seeks_into_later_part() { + let fixture = compressed_multipart_fixture(&[3 * 1024 * 1024, 4 * 1024 * 1024]).await; + // Deep inside part 2 so the plan skips part 1 entirely and (when the + // part carries an index) seeks within part 2. + let start = 3 * 1024 * 1024_i64 + 2 * 1024 * 1024_i64 + 137; + let rs = HTTPRangeSpec { + is_suffix_length: false, + start, + end: start + 64 * 1024 - 1, + }; + + let read = read_compressed_multipart(&fixture, Some(rs), &ObjectOptions::default()).await; + + let expected = &fixture.plaintext[start as usize..(start + 64 * 1024) as usize]; + assert_eq!(read, expected, "range inside a later part must decode from that part"); + } + + /// Parts written without a compression index (small parts skip the index in + /// the rio-v2 backend) must still be rangeable: the plan starts at the part + /// boundary and skips decompressed bytes. + #[tokio::test] + async fn compressed_multipart_range_get_works_without_part_indexes() { + let mut fixture = compressed_multipart_fixture(&[1024 * 1024, 1024 * 1024]).await; + let parts = fixture + .object_info + .parts + .iter() + .map(|part| ObjectPartInfo { + index: None, + ..part.clone() + }) + .collect::>(); + fixture.object_info.parts = Arc::new(parts); + + let start = 1024 * 1024_i64 + 4096; + let rs = HTTPRangeSpec { + is_suffix_length: false, + start, + end: start + 32 * 1024 - 1, + }; + + let read = read_compressed_multipart(&fixture, Some(rs), &ObjectOptions::default()).await; + + let expected = &fixture.plaintext[start as usize..(start + 32 * 1024) as usize]; + assert_eq!(read, expected, "index-less parts must fall back to part-boundary skip"); + } + + #[tokio::test] + async fn compressed_multipart_part_number_get_returns_single_part() { + let part_sizes = [3 * 1024 * 1024, 2 * 1024 * 1024, 512 * 1024]; + let fixture = compressed_multipart_fixture(&part_sizes).await; + + let mut logical_offset = 0_usize; + for (i, part_size) in part_sizes.iter().enumerate() { + let opts = ObjectOptions { + part_number: Some(i + 1), + ..Default::default() + }; + + let read = read_compressed_multipart(&fixture, None, &opts).await; + + let expected = &fixture.plaintext[logical_offset..logical_offset + part_size]; + assert_eq!(read.len(), *part_size, "partNumber={} GET must return the part's logical size", i + 1); + assert_eq!(read, expected, "partNumber={} GET must return the original part bytes", i + 1); + logical_offset += part_size; + } + } + + #[tokio::test] + async fn compressed_multipart_suffix_range_reads_tail() { + let fixture = compressed_multipart_fixture(&[3 * 1024 * 1024, 1024 * 1024]).await; + let suffix_len = 128 * 1024_i64; + let rs = HTTPRangeSpec { + is_suffix_length: true, + start: suffix_len, + end: -1, + }; + + let read = read_compressed_multipart(&fixture, Some(rs), &ObjectOptions::default()).await; + + let expected = &fixture.plaintext[fixture.plaintext.len() - suffix_len as usize..]; + assert_eq!(read, expected, "suffix range must return the tail of the last part"); + } + + /// Builds an SSE-C + disk-compression multipart object exactly like the + /// write path: each part is compressed into its own stream and then + /// encrypted with the per-part key schedule. The fixture is + /// legacy-encryption-specific (`rustfs_rio::EncryptReader`), matching the + /// pre-existing `build_legacy_ssec_multipart_fixture` shape, while the + /// compression layer follows the active backend feature. + async fn compressed_encrypted_multipart_fixture(key_bytes: [u8; 32], part_sizes: &[usize]) -> CompressedMultipartFixture { + let pattern = b"compressed encrypted multipart fixture data "; + let mut plaintext = Vec::new(); + let mut stored = Vec::new(); + let mut parts = Vec::with_capacity(part_sizes.len()); + + for (i, part_size) in part_sizes.iter().enumerate() { + let part_number = i + 1; + let mut part_plaintext = Vec::with_capacity(*part_size); + while part_plaintext.len() < *part_size { + part_plaintext.extend_from_slice(pattern); + part_plaintext.push(part_number as u8); + } + part_plaintext.truncate(*part_size); + + let (compressed, index) = compressed_part_fixture(&part_plaintext).await; + let mut part_cipher = Vec::new(); + rustfs_rio::EncryptReader::new_multipart(Cursor::new(compressed), key_bytes, LEGACY_FIXTURE_BASE_NONCE, part_number) + .read_to_end(&mut part_cipher) + .await + .expect("encrypt compressed fixture part"); + + parts.push(ObjectPartInfo { + number: part_number, + size: part_cipher.len(), + actual_size: *part_size as i64, + index, + ..Default::default() + }); + stored.extend_from_slice(&part_cipher); + plaintext.extend_from_slice(&part_plaintext); + } + + let mut user_defined = legacy_ssec_multipart_metadata(key_bytes, plaintext.len()); + rustfs_utils::http::insert_str( + &mut user_defined, + rustfs_utils::http::SUFFIX_COMPRESSION, + crate::io_support::rio::compression_metadata_value(CompressionAlgorithm::default()), + ); + rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string()); + + let object_info = ObjectInfo { + bucket: "test-bucket".to_string(), + name: "compressed-encrypted-multipart".to_string(), + size: stored.len() as i64, + etag: Some(format!("6bcf86bed8807b8e78f0fc6e0a53079d-{}", part_sizes.len())), + parts: Arc::new(parts), + user_defined: Arc::new(user_defined), + ..Default::default() + }; + + CompressedMultipartFixture { + object_info, + stored, + plaintext, + } + } + + async fn read_compressed_encrypted_multipart( + fixture: &CompressedMultipartFixture, + key_bytes: [u8; 32], + rs: Option, + opts: &ObjectOptions, + ) -> Vec { + let headers = ssec_headers_from_key(key_bytes); + let (_, offset, length) = + GetObjectReader::new(Box::new(Cursor::new(Vec::new())), rs.clone(), &fixture.object_info, opts, &headers) + .await + .expect("plan compressed encrypted multipart read"); + + let end = offset + usize::try_from(length).expect("storage window length must be non-negative"); + assert!( + end <= fixture.stored.len(), + "planned storage window {offset}..{end} exceeds stored stream of {} bytes", + fixture.stored.len() + ); + let window = fixture.stored[offset..end].to_vec(); + + let (mut reader, replay_offset, replay_length) = + GetObjectReader::new(Box::new(Cursor::new(window)), rs, &fixture.object_info, opts, &headers) + .await + .expect("build compressed encrypted multipart reader"); + assert_eq!((replay_offset, replay_length), (offset, length), "read plan must be deterministic"); + + reader.read_all().await.expect("read compressed encrypted multipart stream") + } + + #[tokio::test] + async fn compressed_encrypted_multipart_full_get_roundtrip() { + let key_bytes = [0x6Eu8; 32]; + let fixture = compressed_encrypted_multipart_fixture(key_bytes, &[3 * 1024 * 1024, 1024 * 1024]).await; + + let read = read_compressed_encrypted_multipart(&fixture, key_bytes, None, &ObjectOptions::default()).await; + + assert_eq!(read.len(), fixture.plaintext.len(), "full GET must return the logical size"); + assert_eq!(read, fixture.plaintext, "SSE-C + compression full GET must reassemble all parts"); + } + + #[tokio::test] + async fn compressed_encrypted_multipart_range_crosses_part_boundary() { + let key_bytes = [0x6Eu8; 32]; + let fixture = compressed_encrypted_multipart_fixture(key_bytes, &[3 * 1024 * 1024, 1024 * 1024]).await; + let boundary = 3 * 1024 * 1024_i64; + let rs = HTTPRangeSpec { + is_suffix_length: false, + start: boundary - 65_536, + end: boundary + 65_536 - 1, + }; + + let read = read_compressed_encrypted_multipart(&fixture, key_bytes, Some(rs), &ObjectOptions::default()).await; + + let expected = &fixture.plaintext[(boundary - 65_536) as usize..(boundary + 65_536) as usize]; + assert_eq!(read, expected, "SSE-C + compression boundary-crossing range must splice both parts"); + } + + #[tokio::test] + async fn compressed_encrypted_multipart_part_number_get_returns_single_part() { + let key_bytes = [0x6Eu8; 32]; + let part_sizes = [3 * 1024 * 1024, 1024 * 1024]; + let fixture = compressed_encrypted_multipart_fixture(key_bytes, &part_sizes).await; + + let opts = ObjectOptions { + part_number: Some(2), + ..Default::default() + }; + let read = read_compressed_encrypted_multipart(&fixture, key_bytes, None, &opts).await; + + let expected = &fixture.plaintext[part_sizes[0]..]; + assert_eq!(read.len(), part_sizes[1], "partNumber=2 GET must return the part's logical size"); + assert_eq!(read, expected, "partNumber=2 GET must return the original part bytes"); + } + #[tokio::test] async fn test_get_object_reader_rejects_ssec_read_without_headers() { let object_info = ObjectInfo { diff --git a/crates/rio/src/compress_reader.rs b/crates/rio/src/compress_reader.rs index b8b1c985a..fb6ffeeba 100644 --- a/crates/rio/src/compress_reader.rs +++ b/crates/rio/src/compress_reader.rs @@ -71,6 +71,7 @@ where /// Optional: allow users to customize block_size pub fn with_block_size(inner: R, block_size: usize, compression_algorithm: CompressionAlgorithm) -> Self { + debug_assert!(block_size > 0, "CompressReader block_size must be non-zero"); Self { inner, buffer: Vec::new(), @@ -183,11 +184,21 @@ pin_project! { buffer: Vec, buffer_pos: usize, finished: bool, + // A previously surfaced stream error is sticky: without this, a caller + // that polls again after an error would restart at the header phase and + // read a truncated tail as a clean EOF, converting the error into a + // silently short body. + poisoned: bool, // Fields for saving header read progress across polls header_buf: [u8; 8], header_read: usize, - header_done: bool, - // Fields for saving compressed block read progress across polls + // Fields for saving compressed block read progress across polls. + // `compressed_len > 0` means a block payload is in flight: the header has + // been fully parsed and `compressed_read` bytes of the payload are already + // consumed from the inner stream. The header phase must not run again (and + // must not reset `compressed_read`) until this block completes, or a + // `Poll::Pending` in the middle of a payload would silently drop the bytes + // read so far and desynchronize the block framing. compressed_buf: Vec, compressed_read: usize, compressed_len: usize, @@ -205,9 +216,9 @@ where buffer: Vec::new(), buffer_pos: 0, finished: false, + poisoned: false, header_buf: [0u8; 8], header_read: 0, - header_done: false, compressed_buf: Vec::new(), compressed_read: 0, compressed_len: 0, @@ -236,54 +247,74 @@ where if *this.finished { return Poll::Ready(Ok(())); } - // Read header - while !*this.header_done && *this.header_read < HEADER_LEN { - let mut temp = [0u8; HEADER_LEN]; - let mut temp_buf = ReadBuf::new(&mut temp[0..HEADER_LEN - *this.header_read]); - match this.inner.as_mut().poll_read(cx, &mut temp_buf) { - Poll::Pending => return Poll::Pending, - Poll::Ready(Ok(())) => { - let n = temp_buf.filled().len(); - if n == 0 { - break; + if *this.poisoned { + return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "decompress reader previously failed"))); + } + + if *this.compressed_len == 0 { + // Read the 8-byte block header, resuming across polls via `header_read`. + while *this.header_read < HEADER_LEN { + let mut temp = [0u8; HEADER_LEN]; + let mut temp_buf = ReadBuf::new(&mut temp[0..HEADER_LEN - *this.header_read]); + match this.inner.as_mut().poll_read(cx, &mut temp_buf) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Ok(())) => { + let n = temp_buf.filled().len(); + if n == 0 { + if *this.header_read == 0 { + // Clean EOF on a block boundary. + *this.finished = true; + return Poll::Ready(Ok(())); + } + *this.poisoned = true; + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected EOF while reading compressed block header", + ))); + } + this.header_buf[*this.header_read..*this.header_read + n].copy_from_slice(&temp_buf.filled()[..n]); + *this.header_read += n; + } + Poll::Ready(Err(e)) => { + // error!("DecompressReader poll_read: read header error: {e}"); + *this.poisoned = true; + return Poll::Ready(Err(e)); } - this.header_buf[*this.header_read..*this.header_read + n].copy_from_slice(&temp_buf.filled()[..n]); - *this.header_read += n; - } - Poll::Ready(Err(e)) => { - // error!("DecompressReader poll_read: read header error: {e}"); - return Poll::Ready(Err(e)); } } - if *this.header_read < HEADER_LEN { - return Poll::Pending; - } - } - if !*this.header_done && *this.header_read == 0 { - return Poll::Ready(Ok(())); - } - let typ = this.header_buf[0]; - let len = (this.header_buf[1] as usize) | ((this.header_buf[2] as usize) << 8) | ((this.header_buf[3] as usize) << 16); - let crc = (this.header_buf[4] as u32) - | ((this.header_buf[5] as u32) << 8) - | ((this.header_buf[6] as u32) << 16) - | ((this.header_buf[7] as u32) << 24); - *this.header_read = 0; - *this.header_done = true; - if typ == COMPRESS_TYPE_END { + let typ = this.header_buf[0]; + let len = + (this.header_buf[1] as usize) | ((this.header_buf[2] as usize) << 8) | ((this.header_buf[3] as usize) << 16); + *this.header_read = 0; + + // `CompressReader` never emits an end block — a stream terminates on + // inner EOF, which is what lets concatenated per-part streams decode as + // one. This branch is kept for streams that do carry the marker. + if typ == COMPRESS_TYPE_END { + *this.compressed_read = 0; + *this.compressed_len = 0; + *this.finished = true; + return Poll::Ready(Ok(())); + } + if typ != COMPRESS_TYPE_COMPRESSED && typ != COMPRESS_TYPE_UNCOMPRESSED { + // error!("DecompressReader unknown compression type: {typ}"); + *this.poisoned = true; + return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Unknown compression type"))); + } + if len == 0 { + *this.poisoned = true; + return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid compressed block length"))); + } + + if this.compressed_buf.len() < len { + this.compressed_buf.resize(len, 0); + } + *this.compressed_len = len; *this.compressed_read = 0; - *this.compressed_len = 0; - *this.finished = true; - return Poll::Ready(Ok(())); } - if this.compressed_buf.len() < len { - this.compressed_buf.resize(len, 0); - } - *this.compressed_len = len; - *this.compressed_read = 0; - + // Fill the in-flight block payload, resuming across polls via `compressed_read`. while *this.compressed_read < *this.compressed_len { let mut temp_buf = ReadBuf::new(&mut this.compressed_buf[*this.compressed_read..*this.compressed_len]); match this.inner.as_mut().poll_read(cx, &mut temp_buf) { @@ -291,7 +322,13 @@ where Poll::Ready(Ok(())) => { let n = temp_buf.filled().len(); if n == 0 { - break; + *this.compressed_read = 0; + *this.compressed_len = 0; + *this.poisoned = true; + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected EOF while reading compressed block payload", + ))); } *this.compressed_read += n; } @@ -299,10 +336,17 @@ where // error!("DecompressReader poll_read: read compressed block error: {e}"); *this.compressed_read = 0; *this.compressed_len = 0; + *this.poisoned = true; return Poll::Ready(Err(e)); } } } + + let typ = this.header_buf[0]; + let crc = (this.header_buf[4] as u32) + | ((this.header_buf[5] as u32) << 8) + | ((this.header_buf[6] as u32) << 16) + | ((this.header_buf[7] as u32) << 24); let compressed_buf = &this.compressed_buf[..*this.compressed_len]; // `compressed_buf`'s length comes from the untrusted 24-bit header length field, so it // can be shorter than 16 bytes. `uvarint` is safe on any slice length (reads at most 10 @@ -316,6 +360,7 @@ where if uvarint <= 0 || uvarint as usize > compressed_buf.len() { *this.compressed_read = 0; *this.compressed_len = 0; + *this.poisoned = true; return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid compressed block length prefix"))); } let compressed_data = &compressed_buf[uvarint as usize..]; @@ -326,21 +371,29 @@ where // error!("DecompressReader decompress_block error: {e}"); *this.compressed_read = 0; *this.compressed_len = 0; + *this.poisoned = true; return Poll::Ready(Err(e)); } } - } else if typ == COMPRESS_TYPE_UNCOMPRESSED { - compressed_data.to_vec() } else { - // error!("DecompressReader unknown compression type: {typ}"); + // The header phase already rejected every type other than + // COMPRESS_TYPE_COMPRESSED / COMPRESS_TYPE_UNCOMPRESSED. + compressed_data.to_vec() + }; + if decompressed.is_empty() { + // The writer never emits zero-length plaintext blocks; an empty + // decode surfacing as Ready(Ok) with no bytes would read as EOF and + // silently truncate the stream. + *this.poisoned = true; *this.compressed_read = 0; *this.compressed_len = 0; - return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Unknown compression type"))); - }; + return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Empty compressed block"))); + } if decompressed.len() != uncompress_len as usize { // error!("DecompressReader decompressed length mismatch: {} != {}", decompressed.len(), uncompress_len); *this.compressed_read = 0; *this.compressed_len = 0; + *this.poisoned = true; return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Decompressed length mismatch"))); } let actual_crc = { @@ -352,13 +405,13 @@ where // error!("DecompressReader CRC32 mismatch: actual {actual_crc} != expected {crc}"); *this.compressed_read = 0; *this.compressed_len = 0; + *this.poisoned = true; return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "CRC32 mismatch"))); } *this.buffer = decompressed; *this.buffer_pos = 0; *this.compressed_read = 0; *this.compressed_len = 0; - *this.header_done = false; let to_copy = min(buf.remaining(), this.buffer.len()); buf.put_slice(&this.buffer[..to_copy]); *this.buffer_pos += to_copy; @@ -493,6 +546,184 @@ mod tests { assert_eq!(&decompressed, &data); } + /// Wraps a reader so every other poll returns `Poll::Pending` and every + /// `Ready` poll serves at most `chunk` bytes. This is the shape a duplex + /// pipe produces when the erasure writer is slower than the decoder, which + /// is exactly what desynchronized the block framing before the resumable + /// payload state was added (rustfs/rustfs#5957 multipart GET truncation). + struct PendingChunkReader { + inner: R, + chunk: usize, + pending_next: bool, + } + + impl PendingChunkReader { + fn new(inner: R, chunk: usize) -> Self { + Self { + inner, + chunk, + pending_next: true, + } + } + } + + impl AsyncRead for PendingChunkReader { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + if self.pending_next { + self.pending_next = false; + cx.waker().wake_by_ref(); + return std::task::Poll::Pending; + } + self.pending_next = true; + let cap = self.chunk.min(buf.remaining()); + let mut scratch = vec![0u8; cap]; + let mut inner_buf = tokio::io::ReadBuf::new(&mut scratch); + match std::pin::Pin::new(&mut self.inner).poll_read(cx, &mut inner_buf) { + std::task::Poll::Ready(Ok(())) => { + buf.put_slice(inner_buf.filled()); + std::task::Poll::Ready(Ok(())) + } + other => other, + } + } + } + + fn patterned_payload(size: usize, seed: u8) -> Vec { + (0..size) + .map(|i| ((i as u64).wrapping_mul(2_654_435_761).wrapping_add(seed as u64) >> 3) as u8) + .collect() + } + + /// Root-cause regression for the multipart compressed GET truncation: a + /// `Poll::Pending` in the middle of a block payload must not drop the bytes + /// already consumed. Before the resumable payload state, the decoder reset + /// `compressed_read` on every re-poll and surfaced + /// `LZ4 error: ERROR_frameType_unknown` mid-stream. + #[tokio::test] + async fn test_decompress_reader_survives_pending_mid_payload() { + let data = patterned_payload(100 * 1024, 7); + let mut compress_reader = + CompressReader::with_block_size(Cursor::new(data.clone()), 8192, CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + compress_reader.read_to_end(&mut compressed).await.unwrap(); + + for chunk in [1usize, 3, 7, 8, 17, 1000, 8192] { + let inner = PendingChunkReader::new(Cursor::new(compressed.clone()), chunk); + let mut decompress_reader = DecompressReader::new(inner, CompressionAlgorithm::default()); + let mut decompressed = Vec::new(); + decompress_reader.read_to_end(&mut decompressed).await.unwrap(); + assert_eq!(decompressed, data, "pending-chunked decode must be byte-exact for chunk={chunk}"); + } + } + + /// Two independently compressed streams concatenated back to back — the + /// on-disk shape of a compressed multipart object — must decode across the + /// stream boundary even when every poll can suspend mid-block. + #[tokio::test] + async fn test_decompress_reader_survives_pending_across_concatenated_streams() { + let part1 = patterned_payload(64 * 1024, 7); + let part2 = patterned_payload(24 * 1024, 61); + + let mut stored = Vec::new(); + for part in [&part1, &part2] { + let mut compress_reader = + CompressReader::with_block_size(Cursor::new(part.clone()), 8192, CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + compress_reader.read_to_end(&mut compressed).await.unwrap(); + stored.extend_from_slice(&compressed); + } + + let mut expected = part1; + expected.extend_from_slice(&part2); + + for chunk in [1usize, 5, 8, 13, 4096] { + let inner = PendingChunkReader::new(Cursor::new(stored.clone()), chunk); + let mut decompress_reader = DecompressReader::new(inner, CompressionAlgorithm::default()); + let mut decompressed = Vec::new(); + decompress_reader.read_to_end(&mut decompressed).await.unwrap(); + assert_eq!( + decompressed, expected, + "concatenated part streams must decode byte-exact for chunk={chunk}" + ); + } + } + + /// After the first stream error, every further poll must keep failing. + /// Without the sticky poison a retrying caller would restart at the header + /// phase and read the truncated tail as a clean EOF — converting a hard + /// error into a silently short body. + #[tokio::test] + async fn test_decompress_reader_error_is_sticky() { + let data = patterned_payload(32 * 1024, 7); + let mut compress_reader = CompressReader::with_block_size(Cursor::new(data), 8192, CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + compress_reader.read_to_end(&mut compressed).await.unwrap(); + compressed.truncate(compressed.len() - 3); + + let mut decompress_reader = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut out = Vec::new(); + let first = decompress_reader + .read_to_end(&mut out) + .await + .expect_err("truncated payload must error"); + assert_eq!(first.kind(), std::io::ErrorKind::UnexpectedEof); + + let mut retry = Vec::new(); + let second = decompress_reader + .read_to_end(&mut retry) + .await + .expect_err("a poll after the first error must not turn into a clean EOF"); + assert_eq!(second.kind(), std::io::ErrorKind::InvalidData); + assert!(retry.is_empty(), "no bytes may be produced after the stream failed"); + } + + /// A stream cut off in the middle of a block payload must fail with a clean + /// UnexpectedEof instead of decoding a short buffer. + #[tokio::test] + async fn test_decompress_reader_truncated_payload_is_unexpected_eof() { + let data = patterned_payload(32 * 1024, 7); + let mut compress_reader = CompressReader::with_block_size(Cursor::new(data), 8192, CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + compress_reader.read_to_end(&mut compressed).await.unwrap(); + + compressed.truncate(compressed.len() - 3); + let mut decompress_reader = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut out = Vec::new(); + let err = decompress_reader + .read_to_end(&mut out) + .await + .expect_err("truncated payload must error"); + assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof); + } + + /// A stream cut off in the middle of a block header must fail with a clean + /// UnexpectedEof instead of parsing a garbage header. + #[tokio::test] + async fn test_decompress_reader_truncated_header_is_unexpected_eof() { + let data = patterned_payload(12 * 1024, 7); + let mut compress_reader = CompressReader::with_block_size(Cursor::new(data), 8192, CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + compress_reader.read_to_end(&mut compressed).await.unwrap(); + + // Keep the first full block plus 3 bytes of the next header. + let ln = (compressed[1] as usize) | ((compressed[2] as usize) << 8) | ((compressed[3] as usize) << 16); + let first_block_end = 8 + ln; + assert!(compressed.len() > first_block_end, "fixture must contain more than one block"); + compressed.truncate(first_block_end + 3); + + let mut decompress_reader = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut out = Vec::new(); + let err = decompress_reader + .read_to_end(&mut out) + .await + .expect_err("truncated header must error"); + assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof); + } + // Regression: a corrupted block whose 24-bit length field is < 16 must not panic. // Header layout (HEADER_LEN = 8): [type, len_lo, len_mid, len_hi, crc0..crc3], then `len` // bytes of block body. Pre-fix, poll_read sliced `compressed_buf[0..16]` unconditionally, @@ -518,6 +749,85 @@ mod tests { assert_eq!(res.unwrap_err().kind(), std::io::ErrorKind::InvalidData); } + // Header-level fail-closed matrix, built by hand so the decoder is exercised against bytes no + // encoder in this crate can produce. Header layout (HEADER_LEN = 8): + // [type, len_lo, len_mid, len_hi, crc0..crc3], then `len` body bytes = uvarint(plain_len) + data. + #[tokio::test] + async fn test_decompress_reader_header_validation_matrix() { + // Build a block whose body is `uvarint(plain.len()) + plain` (i.e. the + // COMPRESS_TYPE_UNCOMPRESSED shape), with the header CRC taken over the plaintext exactly + // like the production writer does. + fn build_raw_block(typ: u8, plain: &[u8], len_override: Option) -> Vec { + let crc = { + let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + hasher.update(plain); + hasher.finalize() as u32 + }; + let mut uvarint_buf = [0u8; 10]; + let int_len = put_uvarint(&mut uvarint_buf[..], plain.len() as u64); + let body_len = int_len + plain.len(); + let len = len_override.unwrap_or(body_len); + + let mut out = Vec::with_capacity(HEADER_LEN + body_len); + out.push(typ); + out.push((len & 0xFF) as u8); + out.push(((len >> 8) & 0xFF) as u8); + out.push(((len >> 16) & 0xFF) as u8); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&uvarint_buf[..int_len]); + out.extend_from_slice(plain); + out + } + + let plain = b"uncompressed passthrough payload"; + + // (a) A well-formed uncompressed block decodes to the plaintext verbatim. + let mut out = Vec::new(); + DecompressReader::new( + Cursor::new(build_raw_block(COMPRESS_TYPE_UNCOMPRESSED, plain, None)), + CompressionAlgorithm::default(), + ) + .read_to_end(&mut out) + .await + .expect("a well-formed uncompressed block must decode"); + assert_eq!(out.as_slice(), plain.as_slice()); + + // (b) An unknown block type must be rejected instead of being treated as passthrough. + let mut out = Vec::new(); + let err = DecompressReader::new(Cursor::new(build_raw_block(0x7E, plain, None)), CompressionAlgorithm::default()) + .read_to_end(&mut out) + .await + .expect_err("unknown compression type must error"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!(err.to_string().contains("Unknown compression type"), "got: {err}"); + + // (c) A zero-length block would stall the decoder, so it must be rejected up front. + let mut out = Vec::new(); + let err = DecompressReader::new( + Cursor::new(build_raw_block(COMPRESS_TYPE_UNCOMPRESSED, plain, Some(0))), + CompressionAlgorithm::default(), + ) + .read_to_end(&mut out) + .await + .expect_err("zero-length block must error"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!(err.to_string().contains("Invalid compressed block length"), "got: {err}"); + + // (d) A block that decodes to zero plaintext bytes must be rejected: the + // writer never emits empty blocks, and an empty decode surfacing as + // Ready(Ok) with no bytes would read as EOF and silently truncate. + let mut out = Vec::new(); + let err = DecompressReader::new( + Cursor::new(build_raw_block(COMPRESS_TYPE_UNCOMPRESSED, b"", None)), + CompressionAlgorithm::default(), + ) + .read_to_end(&mut out) + .await + .expect_err("empty block must error"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!(err.to_string().contains("Empty compressed block"), "got: {err}"); + } + // Directly exercises the length-prefix guard: an unterminated varint (all continuation bytes) // makes `uvarint` return 0, which must be rejected as an invalid length prefix. #[tokio::test] diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 9f721f17b..9904b41c9 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -33,6 +33,7 @@ for later deletion. - `tonic-013-status-render` peer RPC failure classification: internode failures that reach a node only as text (a peer's error_info payload, a status flattened through format!) are classified by matching the rendering of an Unavailable gRPC status. Releases up to 1.0.0-alpha.38 shipped tonic 0.13, which rendered that status as "status: Unavailable, message: ..."; tonic 0.14 renders it as "code: 'The service is currently unavailable', message: ...". Both forms are matched so an older peer's relayed text still marks an unreachable peer offline. Remove the tonic 0.13 form after the minimum supported RustFS peer version ships tonic 0.14 or later. - `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection. - `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object. +- `multipart-compression-default-off-window` staged multipart disk-compression rollout: releases before the resumable legacy decompressor fail transient reads of compressed objects under mid-payload suspension, so multipart uploads advertise the compression marker only when RUSTFS_COMPRESSION_MULTIPART_ENABLED is set in addition to RUSTFS_COMPRESSION_ENABLED, keeping rolling upgrades from creating new compressed multipart objects while pre-fix nodes may still serve reads. Flip the default to enabled (and retire the extra switch) after the minimum supported direct-upgrade release ships the resumable decompressor. ## Review Checklist diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 66ffa26cc..1b871a0df 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -27,6 +27,7 @@ use super::storage_api::multipart_usecase::bucket::{ replication::{must_replicate_object, schedule_object_replication}, versioning_sys::BucketVersioningSys, }; +use super::storage_api::multipart_usecase::compression::{is_disk_compressible, is_multipart_disk_compression_enabled}; #[cfg(test)] use super::storage_api::multipart_usecase::contract::http::HTTPPreconditions; use super::storage_api::multipart_usecase::contract::multipart::{CompletePart, MultipartOperations as _, MultipartUploadResult}; @@ -39,7 +40,7 @@ use super::storage_api::multipart_usecase::error::{StorageError, is_err_object_n use super::storage_api::multipart_usecase::helper::OperationHelper; #[cfg(test)] use super::storage_api::multipart_usecase::io::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader}; -use super::storage_api::multipart_usecase::io::{HashReader, WriteEncryption, WritePlan}; +use super::storage_api::multipart_usecase::io::{HashReader, WriteEncryption, WritePlan, compression_metadata_value}; use super::storage_api::multipart_usecase::object_utils::to_s3s_etag; use super::storage_api::multipart_usecase::options::{ copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization, @@ -210,6 +211,28 @@ fn create_multipart_upload_metadata( metadata } +/// A multipart session advertises disk compression only when the staged-rollout +/// switch (`RUSTFS_COMPRESSION_MULTIPART_ENABLED`) is on, the object key/headers +/// qualify, AND the session is not an SSE-C ciphertext-passthrough replication +/// session, which must preserve source bytes verbatim. +/// +/// The rollout switch defaults to off so a rolling upgrade never creates new +/// compressed multipart objects while pre-fix nodes (whose decompressor is not +/// resumable) may still serve reads. Enable it once the fleet has converged on a +/// fixed build; the default flips per the `multipart-compression-default-off-window` +/// entry in docs/architecture/compat-cleanup-register.md. +/// +/// Each part is compressed as an independent stream; the GET path decodes across part +/// boundaries (see `ReadTransform::Compressed`), so the session may advertise +/// object-level compression again. +/// +/// Unlike single PUT there is no `MIN_DISK_COMPRESSIBLE_SIZE` floor here: the total +/// object size is unknown at CreateMultipartUpload time, so tiny multipart objects pay +/// the (harmless) framing overhead. This is a deliberate trade-off, not a bug. +fn should_advertise_session_compression(multipart_enabled: bool, ciphertext_passthrough: bool, disk_compressible: bool) -> bool { + multipart_enabled && !ciphertext_passthrough && disk_compressible +} + async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> { table_catalog::validate_bucket_object_mutation(bucket, key) .await @@ -837,8 +860,17 @@ impl DefaultMultipartUsecase { None => (None, None), }; - // Multipart parts are independent physical streams. Advertising object-level - // compression here would make GET decode the completed object as one stream. + if should_advertise_session_compression( + is_multipart_disk_compression_enabled(), + ciphertext_passthrough, + is_disk_compressible(&req.headers, &key), + ) { + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + compression_metadata_value(CompressionAlgorithm::default()), + ); + } let mt2 = metadata.clone(); let mut opts: ObjectOptions = @@ -1632,6 +1664,31 @@ mod tests { DefaultMultipartUsecase::without_context() } + #[test] + fn session_compression_is_advertised_only_for_non_passthrough_compressible_uploads() { + // (multipart_enabled, ciphertext_passthrough, disk_compressible, expected) + let cases = [ + (true, false, false, false), + (true, false, true, true), + (true, true, false, false), + (true, true, true, false), + // The staged-rollout switch keeps multipart compression dark by + // default regardless of the other gates. + (false, false, true, false), + (false, false, false, false), + (false, true, true, false), + (false, true, false, false), + ]; + + for (multipart_enabled, ciphertext_passthrough, disk_compressible, expected) in cases { + assert_eq!( + should_advertise_session_compression(multipart_enabled, ciphertext_passthrough, disk_compressible), + expected, + "multipart_enabled={multipart_enabled} ciphertext_passthrough={ciphertext_passthrough} disk_compressible={disk_compressible}" + ); + } + } + #[test] fn quota_accounting_uses_logical_size_when_available() { let mut metadata = HashMap::new(); diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 20e937790..b97b16bcb 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -942,7 +942,9 @@ pub(crate) mod concurrency { } pub(crate) mod compression { - pub(crate) use crate::storage::storage_api::ecstore_compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; + pub(crate) use crate::storage::storage_api::ecstore_compression::{ + MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_multipart_disk_compression_enabled, + }; } pub(crate) mod deadlock_detector { @@ -1153,7 +1155,7 @@ pub(crate) mod multipart_usecase { } pub(crate) use super::{ - access, bucket, data_usage, error, helper, io, object_utils, options, request_context, s3_api, set_disk, sse, + access, bucket, compression, data_usage, error, helper, io, object_utils, options, request_context, s3_api, set_disk, sse, }; pub(crate) use crate::storage::storage_api::{ECStore, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader}; } diff --git a/rustfs/src/storage/s3_api/multipart.rs b/rustfs/src/storage/s3_api/multipart.rs index ffc1a4692..9acf31575 100644 --- a/rustfs/src/storage/s3_api/multipart.rs +++ b/rustfs/src/storage/s3_api/multipart.rs @@ -39,6 +39,11 @@ pub(crate) struct ListMultipartUploadsParams { pub(crate) fn build_list_parts_output(res: ListPartsInfo) -> ListPartsOutput { let owner = rustfs_owner(); let initiator = rustfs_initiator(); + let transformed_parts = rustfs_utils::http::contains_key_str(&res.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION) + || res + .user_defined + .keys() + .any(|key| rustfs_utils::http::is_object_encryption_marker(key)); ListPartsOutput { bucket: Some(res.bucket), @@ -51,7 +56,14 @@ pub(crate) fn build_list_parts_output(res: ListPartsInfo) -> ListPartsOutput { e_tag: p.etag.map(|etag| to_s3s_etag(&etag)), last_modified: p.last_mod.map(Timestamp::from), part_number: p.part_num.try_into().ok(), - size: p.size.try_into().ok(), + // Compressed parts store fewer bytes than the client sent; S3 + // semantics report the uploaded (logical) size, matching + // GetObjectAttributes ObjectParts. + size: if p.actual_size > 0 || (transformed_parts && p.actual_size == 0) { + Some(p.actual_size) + } else { + p.size.try_into().ok() + }, ..Default::default() }) .collect(), @@ -247,6 +259,116 @@ mod tests { assert_eq!(output.initiator, Some(rustfs_initiator())); } + #[test] + fn test_list_parts_output_reports_logical_size_for_compressed_parts() { + let input = ListPartsInfo { + bucket: "bucket-a".to_string(), + object: "obj-a".to_string(), + upload_id: "upload-a".to_string(), + parts: vec![PartInfo { + part_num: 1, + // Stored (compressed) bytes on disk vs. the logical size the client uploaded. + size: 1_024, + actual_size: 8_388_608, + ..Default::default() + }], + ..Default::default() + }; + + let output = build_list_parts_output(input); + let parts = output.parts.as_ref().expect("parts should be present"); + + assert_eq!(parts.len(), 1); + assert_eq!( + parts[0].size, + Some(8_388_608), + "compressed parts must report the uploaded logical size, not the stored size" + ); + } + + #[test] + fn test_list_parts_output_reports_zero_logical_size_for_compressed_parts() { + let mut user_defined = std::collections::HashMap::new(); + rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string()); + let input = ListPartsInfo { + user_defined, + parts: vec![PartInfo { + part_num: 1, + // Legacy SSE writes an 8-byte end record for an empty part. + size: 8, + actual_size: 0, + ..Default::default() + }], + ..Default::default() + }; + + let output = build_list_parts_output(input); + let parts = output.parts.as_ref().expect("parts should be present"); + + assert_eq!(parts[0].size, Some(0)); + } + + #[test] + fn test_list_parts_output_reports_zero_logical_size_for_encrypted_parts() { + let input = ListPartsInfo { + user_defined: std::collections::HashMap::from([( + rustfs_utils::http::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), + "AES256".to_string(), + )]), + parts: vec![ + PartInfo { + part_num: 1, + size: 8, + actual_size: 0, + ..Default::default() + }, + PartInfo { + part_num: 2, + size: 8, + actual_size: -1, + ..Default::default() + }, + ], + ..Default::default() + }; + + let output = build_list_parts_output(input); + let parts = output.parts.as_ref().expect("parts should be present"); + + assert_eq!(parts[0].size, Some(0)); + assert_eq!(parts[1].size, Some(8)); + } + + #[test] + fn test_list_parts_output_falls_back_to_stored_size_when_actual_size_unknown() { + let input = ListPartsInfo { + parts: vec![ + PartInfo { + part_num: 1, + size: 1_024, + // Uncompressed parts leave actual_size unset. + actual_size: 0, + ..Default::default() + }, + PartInfo { + part_num: 2, + size: 1_024, + // Legacy/unknown sentinel must not leak a negative size to clients. + actual_size: -1, + ..Default::default() + }, + ], + ..Default::default() + }; + + let output = build_list_parts_output(input); + let parts = output.parts.as_ref().expect("parts should be present"); + + assert_eq!(parts.len(), 2); + assert_eq!(parts[0].size, Some(1024)); + assert_eq!(parts[1].size, Some(1024)); + } + #[test] fn test_list_parts_output_normalizes_legacy_storage_class_and_handles_overflow_markers() { let input = ListPartsInfo { diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index aacd91f8a..f4f084a31 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -409,7 +409,9 @@ pub(crate) mod ecstore_client { } pub(crate) mod ecstore_compression { - pub(crate) use rustfs_ecstore::api::compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; + pub(crate) use rustfs_ecstore::api::compression::{ + MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_multipart_disk_compression_enabled, + }; } pub(crate) mod ecstore_cluster { diff --git a/scripts/run.sh b/scripts/run.sh index 55c6a3d6b..7954c2be1 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -209,6 +209,7 @@ export RUSTFS_NS_SCANNER_INTERVAL=60 # Object scanning interval in seconds # Storage level compression (compression at object storage level) # export RUSTFS_COMPRESSION_ENABLED=true # Whether to enable storage-level compression for objects +# export RUSTFS_COMPRESSION_MULTIPART_ENABLED=true # Additionally compress multipart uploads (staged rollout switch: enable only after the whole fleet runs a build with the resumable decompressor; see docs/architecture/compat-cleanup-register.md) # HTTP Response Compression (whitelist-based, aligned with MinIO) # By default, HTTP response compression is DISABLED (aligned with MinIO behavior) From 56509ead1fc73fb386e428512ffe7ea004cde081 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 14 Aug 2026 23:15:08 +0800 Subject: [PATCH 17/71] fix(ci): remove unused ecstore error conversion (#6117) --- crates/ecstore/src/error/mod.rs | 69 ------------------- .../src/services/tier/warm_backend_s3.rs | 2 - 2 files changed, 71 deletions(-) diff --git a/crates/ecstore/src/error/mod.rs b/crates/ecstore/src/error/mod.rs index 4e8ff791f..76dcc7a0c 100644 --- a/crates/ecstore/src/error/mod.rs +++ b/crates/ecstore/src/error/mod.rs @@ -1075,9 +1075,6 @@ pub struct GenericError { #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum ObjectApiError { - #[error("BackendDown")] - BackendDown(String), - #[error("The operation is not valid for the current state of the object {}/{}({})", .0.bucket, .0.object, .0.version_id)] InvalidObjectState(GenericError), } @@ -1094,72 +1091,6 @@ pub struct ErrorResponse { pub host_id: String, } -pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::io::Error { - let mut bucket = ""; - let mut object = ""; - let mut version_id = ""; - if !params.is_empty() { - bucket = params[0]; - } - if params.len() >= 2 { - object = params[1]; - } - if params.len() >= 3 { - version_id = params[2]; - } - - if is_network_or_host_down(&err.to_string(), false) { - return std::io::Error::other(ObjectApiError::BackendDown(format!("{err}"))); - } - - let err_ = std::io::Error::other(err.to_string()); - let r_err = err; - let err; - let bucket = bucket.to_string(); - let object = object.to_string(); - let version_id = version_id.to_string(); - - match r_err.code { - S3ErrorCode::BucketNotEmpty => { - err = std::io::Error::other(StorageError::BucketNotEmpty("".to_string()).to_string()); - } - S3ErrorCode::InvalidBucketName => { - err = std::io::Error::other(StorageError::BucketNameInvalid(bucket)); - } - S3ErrorCode::InvalidPart => { - err = std::io::Error::other(StorageError::InvalidPart(0, bucket, object /* , version_id */)); - } - S3ErrorCode::NoSuchBucket => { - err = std::io::Error::other(StorageError::BucketNotFound(bucket)); - } - S3ErrorCode::NoSuchKey => { - if !object.is_empty() { - err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object)); - } else { - err = std::io::Error::other(StorageError::BucketNotFound(bucket)); - } - } - S3ErrorCode::NoSuchVersion => { - if !object.is_empty() { - err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object)); //, version_id); - } else { - err = std::io::Error::other(StorageError::BucketNotFound(bucket)); - } - } - S3ErrorCode::AccessDenied => { - err = std::io::Error::other(StorageError::PrefixAccessDenied(bucket, object)); - } - S3ErrorCode::NoSuchUpload => { - err = std::io::Error::other(StorageError::InvalidUploadID(bucket, object, version_id)); - } - _ => { - err = err_; - } - } - - err -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/ecstore/src/services/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs index a5f5b5ee4..814947987 100644 --- a/crates/ecstore/src/services/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -33,8 +33,6 @@ use crate::client::{ transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore}, transition_api::{ReadCloser, ReaderImpl}, }; -use crate::error::ErrorResponse; -use crate::error::error_resp_to_object_err; use crate::services::tier::{ tier_config::TierS3, warm_backend::{ From 0b2a46b36fc0964f0d64e71f2683da47e7b50ff4 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Fri, 14 Aug 2026 23:49:31 +0800 Subject: [PATCH 18/71] fix(log-analyzer): remove stale heal manager anchor (#6100) --- crates/log-analyzer/src/rules/seed/heal.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/log-analyzer/src/rules/seed/heal.rs b/crates/log-analyzer/src/rules/seed/heal.rs index aceacc829..b5564f49e 100644 --- a/crates/log-analyzer/src/rules/seed/heal.rs +++ b/crates/log-analyzer/src/rules/seed/heal.rs @@ -103,11 +103,7 @@ pub(super) fn rules() -> Vec { P2Degraded, "heal", "heal 任务调度/执行失败", - any([ - prefix("Heal task timeout"), - prefix("Heal task execution failed"), - contains("Heal manager is not running"), - ]), + any([prefix("Heal task timeout"), prefix("Heal task execution failed")]), "heal 任务调度/执行层故障。", "检查 heal 后台服务状态与资源压力。", ) From 69e8ef9af52bdee2b386dcb085353d35239d295c Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 15 Aug 2026 00:44:36 +0800 Subject: [PATCH 19/71] test(sse): align KMS context error assertion (#6118) --- rustfs/src/storage/sse.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 87f5af497..a876783c9 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -4565,11 +4565,9 @@ mod tests { }) .await .expect_err("mismatched kms context should fail"); - assert!( - err.message.contains("context") || err.message.contains("Context"), - "unexpected error for mismatched kms context: {}", - err.message - ); + assert_eq!(err.code, S3ErrorCode::InternalError); + assert_eq!(err.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError)); + assert_eq!(super::kms_data_plane_error_class(&err), "context_mismatch"); manager.stop().await.expect("kms service should stop cleanly"); reset_sse_dek_provider(); From e9f53180273b32a9c2b0aa8b5a384047bc7e875c Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 15 Aug 2026 01:29:08 +0800 Subject: [PATCH 20/71] chore(release): prepare 1.0.0-rc.2 --- Cargo.lock | 94 +++++++++++++++++++++--------------------- Cargo.toml | 94 +++++++++++++++++++++--------------------- README.md | 2 +- README_ZH.md | 2 +- flake.nix | 2 +- helm/rustfs/Chart.yaml | 4 +- rustfs.spec | 7 +++- 7 files changed, 104 insertions(+), 101 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f783af5c6..4c7c5de82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3761,7 +3761,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "e2e_test" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "anyhow", "astral-tokio-tar", @@ -9090,7 +9090,7 @@ dependencies = [ [[package]] name = "rustfs" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "aes-gcm", "anyhow", @@ -9227,7 +9227,7 @@ dependencies = [ [[package]] name = "rustfs-audit" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "async-trait", "const-str", @@ -9250,7 +9250,7 @@ dependencies = [ [[package]] name = "rustfs-checksums" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "base64-simd", "bytes", @@ -9266,7 +9266,7 @@ dependencies = [ [[package]] name = "rustfs-common" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "chrono", "hotpath", @@ -9284,7 +9284,7 @@ dependencies = [ [[package]] name = "rustfs-concurrency" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "hotpath", "insta", @@ -9297,7 +9297,7 @@ dependencies = [ [[package]] name = "rustfs-config" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "const-str", "hotpath", @@ -9307,7 +9307,7 @@ dependencies = [ [[package]] name = "rustfs-credentials" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "base64-simd", "hmac 0.13.0", @@ -9321,7 +9321,7 @@ dependencies = [ [[package]] name = "rustfs-crypto" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "aes-gcm", "argon2", @@ -9342,7 +9342,7 @@ dependencies = [ [[package]] name = "rustfs-data-usage" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "hotpath", "rmp-serde", @@ -9352,7 +9352,7 @@ dependencies = [ [[package]] name = "rustfs-ecstore" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "arc-swap", "async-channel", @@ -9491,7 +9491,7 @@ dependencies = [ [[package]] name = "rustfs-extension-schema" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "hotpath", "serde", @@ -9501,7 +9501,7 @@ dependencies = [ [[package]] name = "rustfs-filemeta" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "arc-swap", "byteorder", @@ -9528,7 +9528,7 @@ dependencies = [ [[package]] name = "rustfs-heal" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "async-trait", "base64 0.23.1", @@ -9559,7 +9559,7 @@ dependencies = [ [[package]] name = "rustfs-iam" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "arc-swap", "async-trait", @@ -9600,7 +9600,7 @@ dependencies = [ [[package]] name = "rustfs-io-core" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "bytes", "hotpath", @@ -9613,7 +9613,7 @@ dependencies = [ [[package]] name = "rustfs-io-metrics" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "criterion", "hotpath", @@ -9677,7 +9677,7 @@ dependencies = [ [[package]] name = "rustfs-keystone" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "bytes", "futures", @@ -9704,7 +9704,7 @@ dependencies = [ [[package]] name = "rustfs-kms" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "aes-gcm", "anyhow", @@ -9753,7 +9753,7 @@ dependencies = [ [[package]] name = "rustfs-lifecycle" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "async-trait", "hotpath", @@ -9776,7 +9776,7 @@ dependencies = [ [[package]] name = "rustfs-lock" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "async-trait", "compact_str", @@ -9799,7 +9799,7 @@ dependencies = [ [[package]] name = "rustfs-log-analyzer" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "chrono", "flate2", @@ -9818,7 +9818,7 @@ dependencies = [ [[package]] name = "rustfs-madmin" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "hotpath", "humantime", @@ -9833,7 +9833,7 @@ dependencies = [ [[package]] name = "rustfs-notify" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "arc-swap", "async-trait", @@ -9868,7 +9868,7 @@ dependencies = [ [[package]] name = "rustfs-object-capacity" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "criterion", "futures", @@ -9888,7 +9888,7 @@ dependencies = [ [[package]] name = "rustfs-object-data-cache" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "bytes", "criterion", @@ -9905,7 +9905,7 @@ dependencies = [ [[package]] name = "rustfs-obs" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "chrono", "crossbeam-channel", @@ -9960,7 +9960,7 @@ dependencies = [ [[package]] name = "rustfs-policy" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "async-trait", "base64-simd", @@ -9991,7 +9991,7 @@ dependencies = [ [[package]] name = "rustfs-protocols" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "astral-tokio-tar", "async-compression", @@ -10053,7 +10053,7 @@ dependencies = [ [[package]] name = "rustfs-protos" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "flatbuffers", "hotpath", @@ -10077,7 +10077,7 @@ dependencies = [ [[package]] name = "rustfs-replication" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "byteorder", "bytes", @@ -10095,7 +10095,7 @@ dependencies = [ [[package]] name = "rustfs-rio" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "aes-gcm", "arc-swap", @@ -10133,7 +10133,7 @@ dependencies = [ [[package]] name = "rustfs-rio-v2" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "aes-gcm", "bytes", @@ -10156,7 +10156,7 @@ dependencies = [ [[package]] name = "rustfs-s3-ops" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "hotpath", "rustfs-s3-types", @@ -10164,7 +10164,7 @@ dependencies = [ [[package]] name = "rustfs-s3-types" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "hotpath", "serde", @@ -10173,7 +10173,7 @@ dependencies = [ [[package]] name = "rustfs-s3select-api" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "async-trait", "bytes", @@ -10203,7 +10203,7 @@ dependencies = [ [[package]] name = "rustfs-s3select-query" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "async-recursion", "async-trait", @@ -10222,7 +10222,7 @@ dependencies = [ [[package]] name = "rustfs-scanner" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "async-trait", "bytes", @@ -10262,7 +10262,7 @@ dependencies = [ [[package]] name = "rustfs-security-governance" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "hotpath", "thiserror 2.0.20", @@ -10270,7 +10270,7 @@ dependencies = [ [[package]] name = "rustfs-signer" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "base64-simd", "bytes", @@ -10288,7 +10288,7 @@ dependencies = [ [[package]] name = "rustfs-storage-api" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "async-trait", "hotpath", @@ -10303,7 +10303,7 @@ dependencies = [ [[package]] name = "rustfs-targets" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "arc-swap", "async-nats", @@ -10357,7 +10357,7 @@ dependencies = [ [[package]] name = "rustfs-test-utils" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "hotpath", "rustfs-data-usage", @@ -10373,7 +10373,7 @@ dependencies = [ [[package]] name = "rustfs-tls-runtime" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "arc-swap", "hotpath", @@ -10394,7 +10394,7 @@ dependencies = [ [[package]] name = "rustfs-trusted-proxies" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "async-trait", "axum", @@ -10431,7 +10431,7 @@ dependencies = [ [[package]] name = "rustfs-utils" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "base64-simd", "blake2", @@ -10473,7 +10473,7 @@ dependencies = [ [[package]] name = "rustfs-zip" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "astral-tokio-tar", "async-compression", diff --git a/Cargo.toml b/Cargo.toml index 3993e54aa..9e0524dd7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ edition = "2024" license = "Apache-2.0" repository = "https://github.com/rustfs/rustfs" rust-version = "1.97.1" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" homepage = "https://rustfs.com" description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. " keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"] @@ -86,52 +86,52 @@ redundant_clone = "warn" [workspace.dependencies] # RustFS Internal Crates -rustfs = { path = "./rustfs", version = "1.0.0-rc.1" } -rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.1" } -rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.1" } -rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.1" } -rustfs-common = { path = "crates/common", version = "1.0.0-rc.1" } -rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.1" } -rustfs-config = { path = "./crates/config", version = "1.0.0-rc.1" } -rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.1" } -rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.1" } -rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.1" } -rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.1" } -rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.1" } -rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.1" } -rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.1" } -rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.1" } -rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.1" } -rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.1" } -rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.1" } -rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.1" } -rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.1" } -rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.1" } -rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.1" } -rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.1", default-features = false } -rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.1" } -rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.1" } -rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.1" } -rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.1" } -rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.1" } -rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.1" } -rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.1" } -rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.1" } -rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.1" } -rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.1" } -rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.1" } -rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.1" } -rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.1" } -rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.1" } -rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.1" } -rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.1" } -rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.1" } -rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.1" } -rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.1" } -rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.1" } -rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.1" } -rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.1" } -rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.1" } +rustfs = { path = "./rustfs", version = "1.0.0-rc.2" } +rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.2" } +rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.2" } +rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.2" } +rustfs-common = { path = "crates/common", version = "1.0.0-rc.2" } +rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.2" } +rustfs-config = { path = "./crates/config", version = "1.0.0-rc.2" } +rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.2" } +rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.2" } +rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.2" } +rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.2" } +rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.2" } +rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.2" } +rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.2" } +rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.2" } +rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.2" } +rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.2" } +rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.2" } +rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.2" } +rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.2" } +rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.2" } +rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.2" } +rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.2", default-features = false } +rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.2" } +rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.2" } +rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.2" } +rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.2" } +rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.2" } +rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.2" } +rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.2" } +rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.2" } +rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.2" } +rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.2" } +rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.2" } +rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.2" } +rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.2" } +rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.2" } +rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.2" } +rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.2" } +rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.2" } +rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.2" } +rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.2" } +rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.2" } +rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.2" } +rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.2" } +rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.2" } # Async Runtime and Networking async-channel = "2.5.0" diff --git a/README.md b/README.md index 0699bf3ae..1d2f362fd 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ chown -R 10001:10001 data logs docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest # Using specific version -docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1 +docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2 ``` If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command diff --git a/README_ZH.md b/README_ZH.md index ab376ad01..052515236 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -113,7 +113,7 @@ chown -R 10001:10001 data logs docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest # 使用指定版本运行 -docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1 +docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2 ``` 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录： diff --git a/flake.nix b/flake.nix index 35cc5c246..25121b38f 100644 --- a/flake.nix +++ b/flake.nix @@ -59,7 +59,7 @@ { default = rustPlatform.buildRustPackage { pname = "rustfs"; - version = "1.0.0-rc.1"; + version = "1.0.0-rc.2"; src = ./.; diff --git a/helm/rustfs/Chart.yaml b/helm/rustfs/Chart.yaml index d047c0e69..d583fdb71 100644 --- a/helm/rustfs/Chart.yaml +++ b/helm/rustfs/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: rustfs description: RustFS helm chart to deploy RustFS on kubernetes cluster. type: application -version: "1.0.0-rc.1" -appVersion: "1.0.0-rc.1" +version: "1.0.0-rc.2" +appVersion: "1.0.0-rc.2" home: https://rustfs.com icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg maintainers: diff --git a/rustfs.spec b/rustfs.spec index 72fe4565c..dd4144a99 100644 --- a/rustfs.spec +++ b/rustfs.spec @@ -1,9 +1,9 @@ %global _enable_debug_packages 0 %global _empty_manifest_terminate_build 0 -%global prerelease rc.1 +%global prerelease rc.2 Name: rustfs Version: 1.0.0 -Release: rc.1 +Release: rc.2 Summary: High-performance distributed object storage for MinIO alternative License: Apache-2.0 @@ -58,6 +58,9 @@ install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown %_bindir/rustfs %changelog +* Fri Aug 14 2026 overtrue +- Update RPM package to RustFS 1.0.0-rc.2 + * Sat Aug 08 2026 overtrue - Update RPM package to RustFS 1.0.0-rc.1 From 9138c2457110ac18ec138575c301b72f00fb95b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sat, 15 Aug 2026 01:50:35 +0800 Subject: [PATCH 21/71] fix(site-replication): lift a rejoined site's restarted edit counter over stale marks (#6119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(site-replication): lift a rejoined site's restarted edit counter over stale fence marks A site removed while unreachable (unilateral removal: the receiver never dropped it from its peer map, so parse_site_replication_state's load-time mark pruning never fired) that later rejoins recreates its state object and restarts edit_generation at zero. The receiver's surviving high-water mark then silently fences out every stamped delivery from that origin — peer edits and the add finalize fan-out alike are acked without applying — until the restarted counter catches up. Allocate the generation as a hybrid logical clock instead: max(wall clock in unix nanoseconds, previous + 1), still inside the state transaction under the distributed state-object lock. Every value a lifetime hands out is capped by the wall clock at its own allocation, so a recreated lifetime's first allocation exceeds them all and clears the stale mark, while a pre-removal delivery still in flight stays below the new floor and remains correctly fenced. previous+1 keeps allocations strictly increasing across same-tick allocations and mid-lifetime clock regressions. Nothing changes on the wire or in the persisted schema: editGeneration stays the single fence param and edit_generation the single counter field, so pre-hybrid receivers get the fix as soon as the sender upgrades, old binaries preserve the field across rolling up/downgrades, and marks recorded by plain-counter receivers (small values) are cleared by any wall-clock allocation. A clock that regresses across a delete/recreate degrades to a fence that self-heals once real time passes the previous lifetime's last allocation, and introduces no rollback window beyond what the plain counter already had. An epoch-based design (editEpoch wire param + per-origin epoch marks) was built first and rejected under adversarial review: old binaries rewriting the state object drop the unknown epoch fields, which both disarms the fix mid-rolling-upgrade and — because epoch adoption lowers the generation mark — reopens the pre-restart rollback the fence exists to prevent; a backwards clock also fences an origin permanently instead of self-healing. The hybrid clock has none of these modes. --- rustfs/src/admin/handlers/site_replication.rs | 227 +++++++++++++++++- 1 file changed, 218 insertions(+), 9 deletions(-) diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index be2220d64..471a624d4 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -1067,9 +1067,13 @@ fn parse_site_replication_state(data: &[u8]) -> S3Result { state.peers = normalize_peer_map_by_identity(state.peers); // A peer-edit high-water mark only fences a CURRENT peer. A site that // leaves drops below two peers, which clears its own state object and - // restarts its generation counter at zero — a mark left over from the - // previous membership would then reject every edit it sends after it - // rejoins. Dropping departed origins on load also keeps the map bounded. + // restarts its generation counter — a mark left over from the previous + // membership must not reject the edits it sends after it rejoins. This + // pruning covers departures THIS site observed; an origin removed + // unilaterally elsewhere stays in this peer map with its mark, and the + // wall-clock floor in `next_peer_edit_generation` is what lifts its + // restarted counter over that mark. Dropping departed origins on load + // also keeps the map bounded. state .applied_edit_generations .retain(|origin, _| state.peers.contains_key(origin)); @@ -5935,11 +5939,51 @@ fn summarize_peer_error_detail(detail: &str) -> String { summary } -/// Allocate the next peer-edit generation. Called inside the state -/// transaction, so the counter is handed out under the distributed -/// state-object lock and two nodes of this site can never take the same one. +/// The wall clock in unix nanoseconds, clamped into u64. A pre-1970 (or +/// post-2554) clock yields 0, which makes the hybrid allocation below +/// degrade to the plain `previous + 1` counter — monotone, never panicking. +fn edit_generation_wall_clock() -> u64 { + u64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(0) +} + +/// Allocate the next peer-edit generation as a hybrid logical clock: +/// `max(wall clock in unix nanoseconds, previous + 1)`. Called inside the +/// state transaction, so the value is handed out under the distributed +/// state-object lock and two nodes of this site can never take the same one +/// (`previous + 1` keeps the sequence strictly increasing even when two +/// allocations land in one clock tick, and keeps it monotone on a node +/// whose clock stepped backwards mid-lifetime). +/// +/// The wall-clock floor is what survives the counter's death. A site +/// removed while unreachable — the receiver never dropped it from its peer +/// map, so the load-time mark pruning in `parse_site_replication_state` +/// never fired — that later rejoins recreates its state object with the +/// counter back at zero. A plain counter would then hand out generations +/// below the receiver's stale high-water mark and every delivery would be +/// silently fenced until the counter caught up. Jumping to wall time clears +/// that mark: every value the deleted lifetime handed out was capped by the +/// wall clock at its own allocation (or by a prior lifetime's cap, applied +/// inductively), so the recreated lifetime's first allocation exceeds them +/// all — while a pre-removal delivery still in flight stays below the new +/// floor and remains correctly fenced. Marks recorded by pre-hybrid +/// receivers (small plain-counter values) sit far below any wall-clock +/// value, so a restarted origin passes those too — the fix needs only the +/// sender upgraded, nothing on the wire or in the receiver changed. +/// +/// A wall clock that regresses across a delete/recreate (the recreating +/// node's clock behind the clock that fed the previous lifetime) mints +/// below the stale mark and the origin stays fenced — but only until real +/// time passes the previous lifetime's last allocation, because every later +/// allocation takes the wall-clock floor again. Bounded by the skew, +/// self-healing, and no rollback window beyond the plain counter's: a +/// delivery applies only at or above the receiver's mark, so the one +/// cross-lifetime interleaving that can apply stale content — a +/// pre-removal delivery whose generation lands above everything the +/// regressed new lifetime has minted — required the same straggler landing +/// above the mark under the plain counter, where the recreated counter's +/// low restart made it strictly easier to hit. fn next_peer_edit_generation(state: &mut SiteReplicationState) -> u64 { - state.edit_generation = state.edit_generation.saturating_add(1); + state.edit_generation = edit_generation_wall_clock().max(state.edit_generation.saturating_add(1)); state.edit_generation } @@ -13244,6 +13288,104 @@ mod tests { assert!(!peer_edit_delivery_is_stale(&reloaded, "origin-site", 1)); } + /// The unilateral-removal rejoin gap the hybrid clock closes. The origin + /// was removed while unreachable, but THIS site never dropped it from + /// its peer map, so the load-time mark pruning never fired and the mark + /// from the previous membership survives. The origin's recreated state + /// object restarts its counter, and with a plain `previous + 1` counter + /// every delivery it sent — generations 1, 2, … below the stale mark — + /// would be silently acked-and-dropped until the counter caught up. The + /// wall-clock floor in `next_peer_edit_generation` lifts the restarted + /// counter over every value the deleted lifetime handed out. Reverting + /// the allocation to the plain counter (dropping the wall-clock max) + /// turns the not-stale assertion red. + #[test] + fn hybrid_generation_unfences_a_rejoined_origin_whose_counter_restarted() { + // First lifetime of the origin's state object: two allocations, both + // capped by the wall clock at their own allocation. + let mut first_life = SiteReplicationState::default(); + let straggler = next_peer_edit_generation(&mut first_life); + let last_applied = next_peer_edit_generation(&mut first_life); + assert!(last_applied > straggler, "allocations must be strictly increasing"); + + // The receiver applied up to `last_applied` and keeps the origin in + // its peer map across the unilateral removal — reloading must keep + // the mark, which is exactly why pruning cannot cover this case. + let mut receiver = SiteReplicationState::default(); + receiver.peers.insert( + "origin-site".to_string(), + PeerInfo { + deployment_id: "origin-site".to_string(), + ..peer("origin", "https://origin.example:9000") + }, + ); + record_applied_peer_edit_generation(&mut receiver, "origin-site", last_applied); + let mut receiver = parse_site_replication_state(&serde_json::to_vec(&receiver).expect("serialize")).expect("reload"); + assert_eq!(receiver.applied_edit_generations.get("origin-site"), Some(&last_applied)); + + // The origin rejoins with a RECREATED state object: counter back at + // zero. The wall-clock floor must lift its first allocation over the + // previous lifetime's mark… + let mut second_life = SiteReplicationState::default(); + let restarted = next_peer_edit_generation(&mut second_life); + assert!( + !peer_edit_delivery_is_stale(&receiver, "origin-site", restarted), + "the recreated lifetime's first allocation ({restarted}) must not be fenced by the previous lifetime's mark ({last_applied})" + ); + record_applied_peer_edit_generation(&mut receiver, "origin-site", restarted); + + // …while a pre-removal delivery still in flight stays below the new + // floor and remains correctly fenced — the rollback the fence exists + // to reject. + assert!( + peer_edit_delivery_is_stale(&receiver, "origin-site", straggler), + "a pre-removal in-flight delivery ({straggler}) must stay fenced after the rejoin" + ); + } + + /// Marks recorded before the hybrid clock existed are small plain-counter + /// values, far below any wall-clock allocation: a restarted origin passes + /// them as soon as the SENDER runs the hybrid clock — nothing changes on + /// the wire or in the receiver, so pre-hybrid receivers get the fix too. + /// The other direction is unchanged: among plain-counter values the + /// generation order still fences the delivery that lost the race. + #[test] + fn hybrid_generation_passes_marks_recorded_by_plain_counter_receivers() { + let mut receiver = SiteReplicationState::default(); + record_applied_peer_edit_generation(&mut receiver, "origin-site", 57); + assert!(peer_edit_delivery_is_stale(&receiver, "origin-site", 56)); + assert!(!peer_edit_delivery_is_stale(&receiver, "origin-site", 57)); + + let mut rejoined = SiteReplicationState::default(); + let restarted = next_peer_edit_generation(&mut rejoined); + assert!( + !peer_edit_delivery_is_stale(&receiver, "origin-site", restarted), + "a wall-clock allocation ({restarted}) must clear a plain-counter mark (57)" + ); + } + + /// The `previous + 1` half of the hybrid clock: allocations stay strictly + /// increasing even when the wall clock cannot move them forward — two + /// allocations inside one clock tick, or a clock that stepped backwards + /// mid-lifetime (a counter already ahead of the wall clock advances by + /// exactly one per allocation instead of jumping back). Dropping the + /// `previous + 1` half (allocating bare wall time) turns this red. + #[test] + fn hybrid_generation_is_strictly_increasing_when_the_clock_stalls() { + let mut state = SiteReplicationState { + // A counter far ahead of any wall clock this test will see. + edit_generation: u64::MAX / 2, + ..Default::default() + }; + assert_eq!(next_peer_edit_generation(&mut state), u64::MAX / 2 + 1); + assert_eq!(next_peer_edit_generation(&mut state), u64::MAX / 2 + 2); + // Saturation pins at the ceiling instead of wrapping; the equal-value + // escape (`applied > generation` is false for equal) keeps deliveries + // applying rather than fencing the origin out. + state.edit_generation = u64::MAX; + assert_eq!(next_peer_edit_generation(&mut state), u64::MAX); + } + #[test] fn test_retry_stats_for_state_counts_pending_and_failed() { let state = SiteReplicationState { @@ -16044,10 +16186,77 @@ mod tests { generations.len(), "two nodes took the same edit generation, so their deliveries cannot be ordered: {generations:?}" ); + // The hybrid clock allocates `max(wall nanos, previous + 1)` — the + // persisted counter is the largest allocation, and the `+ 1` half + // keeps allocations distinct even inside one clock tick. + assert_eq!( + Some(&load_site_replication_state().await.expect("reload").edit_generation), + unique.last(), + "the persisted counter must be the largest allocation handed out" + ); + } + + /// The unilateral-removal rejoin, end to end across the state object's + /// real lifecycle: dropping below two peers clears the object (the + /// counter dies with it), and the recreated object's first allocation — + /// raced by two nodes — must clear the previous lifetime's values via + /// the wall-clock floor, so a receiver still holding the old mark + /// accepts the restarted counter instead of fencing it. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial] + async fn test_recreated_state_object_allocates_over_the_previous_lifetimes_mark() { + publish_ready_iam_context().await; + let seed = || SiteReplicationState { + peers: ["site-a", "site-b"] + .into_iter() + .map(|name| (name.to_string(), peer(name, &format!("https://{name}.example:9000")))) + .collect(), + ..Default::default() + }; + + save_site_replication_state(&seed()).await.expect("seed state"); + let straggler = update_site_replication_state(|state| Ok(next_peer_edit_generation(state))) + .await + .expect("first-life allocation"); + let last_applied = update_site_replication_state(|state| Ok(next_peer_edit_generation(state))) + .await + .expect("first-life allocation"); + // A receiver that never dropped this site from its peer map holds + // this mark across the removal. + let mut receiver = SiteReplicationState::default(); + record_applied_peer_edit_generation(&mut receiver, "origin-site", last_applied); + + // Unilateral removal: the site drops below two peers, which clears + // its state object and the counter with it. + let mut departed = seed(); + departed.peers.remove("site-b"); + save_site_replication_state(&departed).await.expect("clear state"); assert_eq!( load_site_replication_state().await.expect("reload").edit_generation, - generations.len() as u64, - "the persisted counter must account for every allocation" + 0, + "clearing the state object must take the counter with it" + ); + + // Rejoin recreates the state object; two nodes race the first + // allocation of the new life. + save_site_replication_state(&seed()).await.expect("recreate state"); + let node_a = tokio::spawn(update_site_replication_state(|state| Ok(next_peer_edit_generation(state)))); + let node_b = tokio::spawn(update_site_replication_state(|state| Ok(next_peer_edit_generation(state)))); + let generation_a = node_a.await.expect("node a task").expect("node a allocation"); + let generation_b = node_b.await.expect("node b task").expect("node b allocation"); + assert_ne!(generation_a, generation_b, "racing allocations must stay distinct"); + + // The receiver's stale mark must not fence the restarted counter… + let restarted = generation_a.min(generation_b); + assert!( + !peer_edit_delivery_is_stale(&receiver, "origin-site", restarted), + "the recreated life's first allocation ({restarted}) must clear the previous life's mark ({last_applied})" + ); + record_applied_peer_edit_generation(&mut receiver, "origin-site", restarted); + // …while the cleared life's in-flight leftovers stay fenced. + assert!( + peer_edit_delivery_is_stale(&receiver, "origin-site", straggler), + "a pre-removal in-flight delivery ({straggler}) must stay fenced after the rejoin" ); } From 71e83aeec401b337bc5fe0d6948fc4dad7e3cf08 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 15 Aug 2026 07:13:37 +0800 Subject: [PATCH 22/71] fix(ci): pin Docker images to release source (#6121) --- .github/workflows/docker.yml | 26 ++++++++++++++++++- .../check_preview_release_workflow.sh | 7 +++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4e9a72c0d..bf3aa5b54 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -94,6 +94,7 @@ jobs: short_sha: ${{ steps.check.outputs.short_sha }} is_prerelease: ${{ steps.check.outputs.is_prerelease }} create_latest: ${{ steps.check.outputs.create_latest }} + source_ref: ${{ steps.check.outputs.source_ref }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -118,6 +119,7 @@ jobs: short_sha="" is_prerelease=false create_latest=false + source_ref="$GITHUB_SHA" if [[ "${{ github.event_name }}" == "workflow_run" ]]; then # Triggered by build workflow completion @@ -137,6 +139,7 @@ jobs: # Extract version info from commit message or use commit SHA # Use Git to generate consistent short SHA (ensures uniqueness like build.yml) short_sha=$(git rev-parse --short "$HEAD_SHA") + source_ref="$HEAD_SHA" # Determine build type based on triggering workflow event and ref triggering_event="$TRIGGERING_EVENT" @@ -261,6 +264,23 @@ jobs: echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported" ;; esac + + if [[ "$should_build" == true && "$input_version" != "latest" ]]; then + tag_ref="refs/tags/$input_version" + if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then + if [[ "$input_version" == v* ]]; then + tag_ref="refs/tags/${input_version#v}" + else + tag_ref="refs/tags/v$input_version" + fi + fi + + if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then + echo "❌ Release tag not found for Docker build: $input_version" + exit 1 + fi + source_ref="$tag_ref" + fi fi { @@ -271,6 +291,7 @@ jobs: echo "short_sha=$short_sha" echo "is_prerelease=$is_prerelease" echo "create_latest=$create_latest" + echo "source_ref=$source_ref" } >> "$GITHUB_OUTPUT" echo "🐳 Docker Build Summary:" @@ -281,6 +302,7 @@ jobs: echo " - Short SHA: $short_sha" echo " - Is prerelease: $is_prerelease" echo " - Create latest: $create_latest" + echo " - Source ref: $source_ref" # Build multi-arch Docker images # Strategy: Build images using pre-built binaries from dl.rustfs.com @@ -308,6 +330,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false + ref: ${{ needs.build-check.outputs.source_ref }} - name: Login to Docker Hub uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 @@ -397,7 +420,8 @@ jobs: LABELS="org.opencontainers.image.title=RustFS" LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system" LABELS="$LABELS,org.opencontainers.image.version=$VERSION" - LABELS="$LABELS,org.opencontainers.image.revision=${{ github.sha }}" + SOURCE_REVISION="$(git rev-parse HEAD)" + LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION" LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}" LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE" diff --git a/scripts/security/check_preview_release_workflow.sh b/scripts/security/check_preview_release_workflow.sh index 638d68b7f..fcff4706c 100755 --- a/scripts/security/check_preview_release_workflow.sh +++ b/scripts/security/check_preview_release_workflow.sh @@ -195,6 +195,13 @@ IFS= read -r -d '' expected_docker_automatic_guard <<'EOF' || true EOF expected_docker_automatic_guard=${expected_docker_automatic_guard%$'\n'} require_job_if "$docker_workflow" "build-check" "$expected_docker_automatic_guard" +require_line "$docker_workflow" ' source_ref: ${{ steps.check.outputs.source_ref }}' "Docker source ref output" +require_line "$docker_workflow" ' source_ref="$HEAD_SHA"' "automatic Docker source ref" +require_line "$docker_workflow" ' source_ref="$tag_ref"' "manual Docker source ref" +require_line "$docker_workflow" ' ref: ${{ needs.build-check.outputs.source_ref }}' "Docker release source checkout" +require_line "$docker_workflow" ' SOURCE_REVISION="$(git rev-parse HEAD)"' "Docker source revision resolution" +require_line "$docker_workflow" ' LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"' "Docker revision label" +require_absent "$docker_workflow" 'org.opencontainers.image.revision=${{ github.sha }}' "Docker revision must not use the workflow branch SHA" docker_manual_guard=$(awk ' $0 == " *-preview*)" { in_preview = 1 } From 72fd7339c9cf73a8dbb8cfcc1dc8cd5ea5be8693 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 15 Aug 2026 08:32:10 +0800 Subject: [PATCH 23/71] test(utils): allow ephemeral port reuse (#6122) * test(utils): allow ephemeral port reuse * test(kms): allow any ciphertext prefix --- crates/kms/tests/behavior_backup.rs | 2 -- crates/utils/src/net.rs | 3 --- 2 files changed, 5 deletions(-) diff --git a/crates/kms/tests/behavior_backup.rs b/crates/kms/tests/behavior_backup.rs index 598ec4eca..65e58fed8 100644 --- a/crates/kms/tests/behavior_backup.rs +++ b/crates/kms/tests/behavior_backup.rs @@ -225,8 +225,6 @@ async fn nothing_readable_leaves_the_bundle_unwrapped() { "artifact {} carries the raw on-disk record", artifact.path ); - // A cheap structural check too: an encrypted payload is not JSON. - assert_ne!(payload.first(), Some(&b'{'), "artifact {} looks like plaintext JSON", artifact.path); } // The manifest itself is not encrypted, so assert directly that it carries diff --git a/crates/utils/src/net.rs b/crates/utils/src/net.rs index d9bf106f5..873a6d940 100644 --- a/crates/utils/src/net.rs +++ b/crates/utils/src/net.rs @@ -659,9 +659,6 @@ mod test { // Port should be in valid range (u16 max is always <= 65535) assert!(port1 > 0); assert!(port2 > 0); - - // Different calls should typically return different ports - assert_ne!(port1, port2); } #[test] From 1619c4be600a6998e074f62c267151242401577c Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Sat, 15 Aug 2026 21:32:05 +0800 Subject: [PATCH 24/71] fix(scanner): add context to corrupt metadata logs (#6099) Co-authored-by: Henry Guo --- crates/filemeta/src/filemeta/codec.rs | 5 +- crates/scanner/Cargo.toml | 2 +- crates/scanner/src/scanner_folder.rs | 123 +++++++++++++++++++++++--- crates/scanner/src/scanner_io.rs | 11 --- 4 files changed, 113 insertions(+), 28 deletions(-) diff --git a/crates/filemeta/src/filemeta/codec.rs b/crates/filemeta/src/filemeta/codec.rs index 7f2851581..86af49b97 100644 --- a/crates/filemeta/src/filemeta/codec.rs +++ b/crates/filemeta/src/filemeta/codec.rs @@ -135,10 +135,7 @@ impl FileMeta { let i = buf.len() as u64; // check version, buf = buf[8..] - let (buf, _, _) = Self::check_xl2_v1(buf).map_err(|e| { - error!("failed to check XL2 v1 format: {}", e); - e - })?; + let (buf, _, _) = Self::check_xl2_v1(buf)?; if buf.len() < 5 { error!( diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 9b5aa3342..16ff34a8e 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -102,7 +102,7 @@ bytes.workspace = true hex-simd.workspace = true [dev-dependencies] -tracing-subscriber = { workspace = true, features = ["env-filter", "time"] } +tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] } serial_test = { workspace = true } temp-env = { workspace = true } tempfile = { workspace = true } diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index ac6b86c32..7e353bf75 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -65,6 +65,7 @@ const LOG_SUBSYSTEM_FOLDER: &str = "folder"; const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; const LOG_SUBSYSTEM_HEAL: &str = "heal"; const EVENT_SCANNER_FOLDER_STATE: &str = "scanner_folder_state"; +const EVENT_SCANNER_METADATA_CORRUPT: &str = "scanner_metadata_corrupt"; const EVENT_SCANNER_LIFECYCLE_ACTION: &str = "scanner_lifecycle_action"; const EVENT_SCANNER_HEAL_ADMISSION: &str = "scanner_heal_admission"; const EVENT_SCANNER_ALERT_STATE: &str = "scanner_alert_state"; @@ -2154,17 +2155,34 @@ impl FolderScanner { self.record_failed(&item.path); if should_log_failed_object(into.failed_objects) { - warn!( - target: "rustfs::scanner::folder", - event = EVENT_SCANNER_FOLDER_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_FOLDER, - path = %item.path, - failed_objects = into.failed_objects, - state = "get_size_failed", - error = %e, - "Scanner folder failed to get object size" - ); + if let GetSizeFailureAction::HealMetadata { object } = &failure_action { + error!( + target: "rustfs::scanner::folder", + event = EVENT_SCANNER_METADATA_CORRUPT, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_FOLDER, + drive = %self.local_disk.path().display(), + bucket = %item.bucket, + object = %object, + metadata_path = %item.path, + failed_objects = into.failed_objects, + state = "metadata_corrupt", + error = %e, + "Scanner detected corrupt object metadata" + ); + } else { + warn!( + target: "rustfs::scanner::folder", + event = EVENT_SCANNER_FOLDER_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_FOLDER, + path = %item.path, + failed_objects = into.failed_objects, + state = "get_size_failed", + error = %e, + "Scanner folder failed to get object size" + ); + } } } @@ -3054,12 +3072,59 @@ mod tests { use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass}; use rustfs_filemeta::{FileInfo, FileMeta}; use serial_test::serial; + use std::io::Write; #[cfg(unix)] use std::os::unix::fs::{PermissionsExt, symlink}; + use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use temp_env::{with_var, with_var_unset}; + use tracing_subscriber::fmt::MakeWriter; use uuid::Uuid; + #[derive(Clone, Default)] + struct CapturedLogs { + buffer: Arc>>, + } + + struct CapturedLogWriter { + buffer: Arc>>, + } + + impl CapturedLogs { + fn contents(&self) -> String { + let buffer = self + .buffer + .lock() + .expect("captured logs mutex should not be poisoned") + .clone(); + String::from_utf8(buffer).expect("captured logs should be valid UTF-8") + } + } + + impl Write for CapturedLogWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.buffer + .lock() + .expect("captured logs mutex should not be poisoned") + .extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> MakeWriter<'a> for CapturedLogs { + type Writer = CapturedLogWriter; + + fn make_writer(&'a self) -> Self::Writer { + CapturedLogWriter { + buffer: Arc::clone(&self.buffer), + } + } + } + #[test] fn scanner_size_summary_application_saturates_usage_counters() { let target = "arn:minio:replication::target".to_string(); @@ -4542,9 +4607,19 @@ mod tests { assert!(budget.entries_visited() >= 1); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] #[serial] async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() { + let logs = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .json() + .with_max_level(tracing::Level::ERROR) + .with_writer(logs.clone()) + .with_ansi(false) + .without_time() + .finish(); + let _subscriber_guard = tracing::subscriber::set_default(subscriber); + let (mut scanner, temp_dir) = build_test_scanner().await; let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); @@ -4596,6 +4671,30 @@ mod tests { assert!(!budget.budget_elapsed()); assert_eq!(budget.reason(), None); + let captured = logs.contents(); + assert!( + !captured.contains("failed to check XL2 v1 format"), + "the context-free filemeta parser error must not be emitted" + ); + let events = captured + .lines() + .map(|line| serde_json::from_str::(line).expect("captured scanner log should be valid JSON")) + .filter(|line| line["fields"]["event"] == EVENT_SCANNER_METADATA_CORRUPT) + .collect::>(); + assert_eq!( + events.len(), + 1, + "one corrupt metadata observation must emit one scanner-owned diagnostic event" + ); + let fields = &events[0]["fields"]; + assert_eq!(fields["component"], LOG_COMPONENT_SCANNER); + assert_eq!(fields["subsystem"], LOG_SUBSYSTEM_FOLDER); + assert_eq!(fields["drive"], temp_dir.to_string_lossy().as_ref()); + assert_eq!(fields["bucket"], "bucket"); + assert_eq!(fields["object"], "object"); + assert_eq!(fields["metadata_path"], metadata_path.to_string_lossy().as_ref()); + assert_eq!(fields["state"], "metadata_corrupt"); + let retry_budget = ScannerCycleBudget::new_with_progress_tracking( &parent, crate::scanner_budget::ScannerCycleBudgetConfig { diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 2b234ac59..15ee9cca0 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -3849,17 +3849,6 @@ impl ScannerIODisk for Disk { let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) { Ok(versions) => versions, Err(e) => { - error!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_DISK_BUCKET_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - bucket = %item.bucket, - object = %item.object_path(), - state = "file_info_versions_failed", - error = %e, - "Scanner disk bucket failed to resolve file info versions" - ); return Err(scanner_metadata_corrupt_error( format!("failed to resolve file info versions: {e}"), &item.bucket, From 7f23a1ba91a8348ed193e7849da30783c5a1c12a Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 16 Aug 2026 01:18:26 +0800 Subject: [PATCH 25/71] feat(ecstore): report inline early-stop miss reasons (#6134) Co-authored-by: heihutu --- crates/ecstore/src/diagnostics/get.rs | 37 +++ .../src/set_disk/core/io_primitives.rs | 247 ++++++++++++++---- crates/ecstore/src/set_disk/mod.rs | 36 ++- 3 files changed, 265 insertions(+), 55 deletions(-) diff --git a/crates/ecstore/src/diagnostics/get.rs b/crates/ecstore/src/diagnostics/get.rs index c02e41161..4032773f5 100644 --- a/crates/ecstore/src/diagnostics/get.rs +++ b/crates/ecstore/src/diagnostics/get.rs @@ -190,6 +190,17 @@ pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_su pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY: &str = "data_read_inline_body_verify"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED: &str = "data_read_inline_deleted"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY: &str = "data_read_inline_geometry"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH: &str = "data_read_inline_identity_mismatch"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD: &str = "data_read_inline_missing_payload"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD: &str = "data_read_inline_missing_shard"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE: &str = "data_read_inline_not_inline"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE: &str = "data_read_inline_part_shape"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE: &str = "data_read_inline_remote"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE: &str = "data_read_inline_size"; +pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED: &str = "data_read_inline_transformed"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found"; @@ -551,6 +562,32 @@ mod tests { assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker"); + assert_eq!( + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, + "data_read_inline_body_verify" + ); + assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, "data_read_inline_deleted"); + assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, "data_read_inline_geometry"); + assert_eq!( + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH, + "data_read_inline_identity_mismatch" + ); + assert_eq!( + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD, + "data_read_inline_missing_payload" + ); + assert_eq!( + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, + "data_read_inline_missing_shard" + ); + assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, "data_read_inline_not_inline"); + assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, "data_read_inline_part_shape"); + assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, "data_read_inline_remote"); + assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, "data_read_inline_size"); + assert_eq!( + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, + "data_read_inline_transformed" + ); assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found"); diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 521cc1a33..fff99a3cc 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -32,15 +32,22 @@ use crate::diagnostics::get::{ GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER, GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED, - GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, - GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, - GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, - GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, - GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, - GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, - GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, - GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, - GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, + GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, + GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, + GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, + GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, + GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, + GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR, + GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID, + GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY, + GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, + GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, @@ -652,36 +659,48 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo && left.erasure.distribution == right.erasure.distribution } -pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified( +pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( bucket: &str, object: &str, candidate: &FileInfo, parts_metadata: &[FileInfo], disks: &[Option], -) -> bool { - if !candidate.inline_data() - || candidate.is_compressed() +) -> Option<&'static str> { + if !candidate.inline_data() { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE); + } + if candidate.is_compressed() || candidate .metadata .keys() .any(|key| rustfs_utils::http::is_object_encryption_marker(key)) - || candidate.is_remote() - || candidate.deleted - || candidate.size <= 0 - || candidate.parts.len() != 1 - || !candidate.has_valid_erasure_geometry() { - return false; + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED); + } + if candidate.is_remote() { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE); + } + if candidate.deleted { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED); + } + if candidate.size <= 0 { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); + } + if candidate.parts.len() != 1 { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); + } + if !candidate.has_valid_erasure_geometry() { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); } let Ok(object_size) = usize::try_from(candidate.size) else { - return false; + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); }; if candidate.parts.first().is_none_or(|part| part.size != object_size) { - return false; + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); } if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) { - return false; + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); } let Ok(erasure) = coding::Erasure::try_new_with_options( @@ -690,18 +709,18 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified( candidate.erasure.block_size, candidate.uses_legacy_checksum, ) else { - return false; + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); }; - let Some(data_files) = - collect_inline_data_shard_fileinfos_by_index(parts_metadata, candidate, erasure.data_shards, |index| { + let data_files = + match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| { disks.get(index).is_some_and(Option::is_some) - }) - else { - return false; - }; + }) { + Ok(data_files) => data_files, + Err(reason) => return Some(reason), + }; let Some(part) = candidate.parts.first() else { - return false; + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); }; let checksum_info = candidate.erasure.get_checksum_info(part.number); let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { @@ -721,12 +740,13 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified( let Ok(mut readers) = build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await else { - return false; + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY); }; - try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size) - .await - .is_some_and(|body| body.len() == object_size) + match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await { + Some(body) if body.len() == object_size => None, + _ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), + } } pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str { @@ -2469,6 +2489,7 @@ impl SetDisks { let mut next_fanout_index = 0usize; let mut scheduled_count = 0usize; let mut force_full_wait = false; + let mut final_miss_reason_override = None; let spawn_read_version = |join_set: &mut JoinSet<(usize, disk::error::Result, Duration)>, index: usize, disk: Option| { let task_opts = opts; @@ -2541,17 +2562,29 @@ impl SetDisks { .or_else(|| accumulator.version_early_stop_decision()) { let should_return_early = if read_data { - let allow_data_read_early_stop = match accumulator.candidate.as_ref() { - Some(candidate) => { - data_read_early_stop_inline_body_verified(bucket.as_ref(), object.as_ref(), candidate, &ress, disks) - .await + match accumulator.candidate.as_ref() { + Some(candidate) => match data_read_early_stop_inline_body_miss_reason( + bucket.as_ref(), + object.as_ref(), + candidate, + &ress, + disks, + ) + .await + { + None => true, + Some(reason) => { + force_full_wait = true; + final_miss_reason_override = Some(reason); + false + } + }, + None => { + force_full_wait = true; + final_miss_reason_override = Some(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM); + false } - None => false, - }; - if !allow_data_read_early_stop { - force_full_wait = true; } - allow_data_read_early_stop } else { true }; @@ -2613,7 +2646,12 @@ impl SetDisks { } } - rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, accumulator.final_miss_reason()); + let accumulator_miss_reason = accumulator.final_miss_reason(); + let final_miss_reason = match (final_miss_reason_override, accumulator_miss_reason) { + (Some(reason), GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM) => reason, + _ => accumulator_miss_reason, + }; + rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason); rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0); rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0); let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); @@ -6067,11 +6105,126 @@ mod tests { .clone(); assert!( - data_read_early_stop_inline_body_verified(bucket, object, &candidate, &parts_metadata, &disks).await, + data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks) + .await + .is_none(), "legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm" ); } + #[tokio::test] + async fn data_read_early_stop_reports_inline_miss_reasons() { + let bucket = "inline-data-get-miss-reason-bucket"; + let object = "inline-data-get-miss-reason-object"; + let payload = b"verified inline payload"; + let (_dirs, disks) = call_counter_local_disks(bucket, 4).await; + let files = inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, false).await; + let distribution = files + .first() + .map(|file| file.erasure.distribution.clone()) + .expect("fixture should include metadata"); + let order = bounded_metadata_fanout_order(bucket, object, 4, 2); + let mut parts_metadata = vec![FileInfo::default(); 4]; + for disk_index in order.into_iter().take(3) { + let block_index = distribution + .get(disk_index) + .copied() + .expect("fixture distribution should cover every disk"); + parts_metadata[disk_index] = files + .get(block_index.checked_sub(1).expect("erasure block indexes are one-based")) + .expect("fixture should include every distributed shard") + .clone(); + } + let candidate = parts_metadata + .iter() + .find(|file| file.name == object) + .expect("fixture should include observed metadata") + .clone(); + let data_disk = distribution + .iter() + .position(|block_index| *block_index == 1) + .expect("fixture distribution should include first data shard"); + + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks).await, + None + ); + + let mut not_inline = candidate.clone(); + rustfs_utils::http::remove_str(&mut not_inline.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA); + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, ¬_inline, &parts_metadata, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE) + ); + + let mut transformed = candidate.clone(); + rustfs_utils::http::insert_str(&mut transformed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &transformed, &parts_metadata, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED) + ); + + let mut deleted = candidate.clone(); + deleted.deleted = true; + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &deleted, &parts_metadata, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED) + ); + + let mut zero_size = candidate.clone(); + zero_size.size = 0; + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &zero_size, &parts_metadata, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE) + ); + + let mut multipart = candidate.clone(); + multipart.parts.push(multipart.parts[0].clone()); + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &multipart, &parts_metadata, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE) + ); + + let mut invalid_geometry = candidate.clone(); + invalid_geometry.erasure.data_blocks = 0; + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &invalid_geometry, &parts_metadata, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY) + ); + + let mut missing_shard = parts_metadata.clone(); + missing_shard[data_disk] = FileInfo::default(); + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_shard, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD) + ); + + let mut missing_payload = parts_metadata.clone(); + missing_payload[data_disk].data = None; + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_payload, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD) + ); + + let mut identity_mismatch = parts_metadata.clone(); + identity_mismatch[data_disk].version_id = Some(Uuid::new_v4()); + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &identity_mismatch, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH) + ); + + let mut corrupt = parts_metadata.clone(); + if let Some(data) = corrupt[data_disk].data.as_mut() { + let mut corrupt_data = data.to_vec(); + corrupt_data[0] ^= 0x01; + *data = Bytes::from(corrupt_data); + } + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &corrupt, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY) + ); + } + #[test] #[serial_test::serial] fn metadata_fanout_lifecycle_records_real_early_stop_abort() { @@ -6161,7 +6314,7 @@ mod tests { &[ ("path", GET_OBJECT_PATH_INTERNAL_META), ("decision", "miss"), - ("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM), + ("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), ], ), 1, @@ -6173,7 +6326,7 @@ mod tests { &[ ("path", GET_OBJECT_PATH_LEGACY_DUPLEX), ("decision", "miss"), - ("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM), + ("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), ], ), 0, diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 87f4624ef..7f3a37ea0 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -59,7 +59,10 @@ use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl}; use crate::cluster::rpc::heal_bucket_local_on_disks; use crate::data_usage::record_compression_total_memory; use crate::diagnostics::get::{ - GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING, + GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE, @@ -3866,8 +3869,17 @@ fn collect_inline_data_shard_fileinfos_by_index<'a>( parts_metadata: &'a [FileInfo], fi: &FileInfo, data_shards: usize, - mut disk_is_online: impl FnMut(usize) -> bool, + disk_is_online: impl FnMut(usize) -> bool, ) -> Option> { + collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, fi, data_shards, disk_is_online).ok() +} + +fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>( + parts_metadata: &'a [FileInfo], + fi: &FileInfo, + data_shards: usize, + mut disk_is_online: impl FnMut(usize) -> bool, +) -> std::result::Result, &'static str> { let distribution = &fi.erasure.distribution; let mut data_files = vec![None; data_shards]; @@ -3875,27 +3887,35 @@ fn collect_inline_data_shard_fileinfos_by_index<'a>( if !disk_is_online(disk_index) { continue; } - let block_index = *distribution.get(disk_index)?; + let Some(&block_index) = distribution.get(disk_index) else { + return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); + }; if block_index == 0 || block_index > data_shards { continue; } + if file_info.name.is_empty() { + return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD); + } if file_info.erasure.index != block_index { - continue; + return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH); } if !file_info.has_valid_erasure_geometry() { - continue; + return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); } if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) { - continue; + return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH); } if file_info.data.as_ref().is_none_or(|data| data.is_empty()) { - continue; + return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD); } data_files[block_index - 1] = Some(file_info); } - data_files.into_iter().collect() + data_files + .into_iter() + .collect::>>() + .ok_or(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD) } impl SetDisks { From db8f55cb97e60a8e91e185e3ce13c5a911a99341 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Sun, 16 Aug 2026 03:05:09 +0800 Subject: [PATCH 26/71] feat(table-catalog): finalize Iceberg REST behavior (#6072) * feat(table-catalog): finalize Iceberg REST behavior * fix(table-catalog): address REST finalization regressions * test(table-catalog): expect REST commit conflicts * test(table-catalog): avoid serialized view test deadlocks * fix(table-catalog): adapt shared test backend * fix(table-catalog): enforce Iceberg metadata invariants * fix(table-catalog): preserve manifest length in test * test(table-catalog): use valid metadata fixtures * test(table-catalog): seed manifests before manifest lists * fix(table-catalog): restore validation gates --------- Co-authored-by: Henry Guo Co-authored-by: overtrue --- Cargo.lock | 4 + Cargo.toml | 2 +- docs/architecture/s3-tables-support-matrix.md | 12 +- rustfs/Cargo.toml | 3 +- .../handlers/table_catalog/credentials.rs | 2 +- .../src/admin/handlers/table_catalog/mod.rs | 1559 ++++++--- .../src/admin/handlers/table_catalog/table.rs | 14 +- .../src/admin/handlers/table_catalog/tests.rs | 2943 +++++++++++++++-- .../src/admin/handlers/table_catalog/view.rs | 14 +- rustfs/src/storage/access.rs | 2 +- rustfs/src/table_catalog/iceberg/manifest.rs | 255 +- .../src/table_catalog/iceberg/validation.rs | 2036 +++++++++++- rustfs/src/table_catalog/mod.rs | 4 + rustfs/src/table_catalog/store/migration.rs | 6 +- rustfs/src/table_catalog/store/mod.rs | 128 +- rustfs/src/table_catalog/store/object.rs | 194 +- rustfs/src/table_catalog/store/strong.rs | 148 +- rustfs/src/table_catalog/test_support.rs | 151 +- rustfs/src/table_catalog/tests.rs | 1839 +++++++++- scripts/table-catalog/failure_coverage.py | 16 +- .../table-catalog/test_failure_coverage.py | 4 + 21 files changed, 8165 insertions(+), 1171 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4c7c5de82..38637d173 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -278,6 +278,7 @@ checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" dependencies = [ "bigdecimal", "bon", + "crc32fast", "digest 0.11.3", "log", "miniz_oxide 0.9.1", @@ -289,9 +290,11 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "snap", "strum", "thiserror 2.0.20", "uuid", + "zstd", ] [[package]] @@ -9200,6 +9203,7 @@ dependencies = [ "serial_test", "sha2 0.11.0", "shadow-rs", + "snap", "socket2", "subtle", "sysinfo", diff --git a/Cargo.toml b/Cargo.toml index 9e0524dd7..6bbd3acd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,7 +171,7 @@ tower = { version = "0.5.3" } tower-http = { version = "0.7.0" } # Serialization and Data Formats -apache-avro = "0.22.0" +apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] } bytes = { version = "1.12.1" } bytesize = "2.7.0" byteorder = "1.5.0" diff --git a/docs/architecture/s3-tables-support-matrix.md b/docs/architecture/s3-tables-support-matrix.md index cb356350c..65eeb4be7 100644 --- a/docs/architecture/s3-tables-support-matrix.md +++ b/docs/architecture/s3-tables-support-matrix.md @@ -61,17 +61,17 @@ catalog extension. | Area | Status | Covered behavior | |---|---|---| -| Catalog config | Supported | `GET /v1/config` advertises RustFS catalog defaults and route capabilities. | +| Catalog config | Supported | `GET /v1/config` advertises RustFS catalog defaults and only the supported OpenAPI REST paths in `endpoints`. RustFS administration, maintenance, migration, diagnostics, refs, and metadata-location extensions remain available but are not presented as standard Iceberg REST endpoints. | | Table bucket discovery | Supported | `PUT` and `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. | | Namespaces | Supported | Create, list, load, existence check, and drop namespace routes are registered on both catalog prefixes. List responses support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. Namespace identifiers are limited to 512 ASCII characters so persisted paths and stateless continuation tokens remain bounded. | -| Tables | Supported | Create, register, list, load, existence check, commit, metadata-location get/update, and drop table routes are registered on both catalog prefixes. Table and view listings support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. | -| Commit CAS | Supported | Single-table commits validate base metadata, expected version token, referenced object existence, warehouse scope, and Iceberg commit requirements before advancing the current metadata pointer. Standard commits preserve the normal commit-token file name and use an immutable-table-scoped fallback when rename followed by source-name reuse would otherwise collide at the same generation and commit ID. | +| Tables | Supported | Create, register, list, load, existence check, commit, metadata-location get/update, and drop table routes are registered on both catalog prefixes. Table and view listings support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. Commit identifiers must match the URL resource; unknown requirements, updates, and snapshot operations fail as bad requests; staged create, register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. Standard statistics, partition statistics, and schema/spec cleanup updates are accepted. | +| Commit CAS | Supported | Single-table commits validate base metadata, expected version token, referenced object existence, warehouse scope, and Iceberg commit requirements before advancing the current metadata pointer. Externally supplied metadata transitions preserve monotonic column, partition, and sequence assignment watermarks and immutable definitions for retained schemas, partition specs, sort orders, and snapshots. Standard commits preserve the normal commit-token file name and use an immutable-table-scoped fallback when rename followed by source-name reuse would otherwise collide at the same generation and commit ID. The catalog does not advertise `idempotency-key-lifetime`; clients must treat standard mutation-wide `Idempotency-Key` semantics as unsupported. | | Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose staged/finalization gaps and repair safe idempotency gaps without moving the table pointer. | | Snapshot refs | Supported | Refs can be listed, created or replaced, and deleted through catalog commits. `main` is protected and refs with explicit retention require forced delete. | -| Iceberg views | Supported | Basic create, list, load, replace, existence check, and drop routes persist view metadata with view-scoped authorization. | -| Table credentials endpoint | Supported | Returns an empty `storage-credentials` list by default. Returns table-scoped temporary credentials only when credential vending is enabled. | +| Iceberg views | Supported | Basic create, list, load, replace, existence check, and drop routes persist view metadata with view-scoped authorization. Replace identifiers must match the URL resource, `schema-id: -1` resolves to the last added schema, one commit timestamp is used consistently, and only Iceberg view format version 1 is accepted. | +| Table credentials endpoint | Supported | Returns an empty `storage-credentials` list by default. Returns table-scoped temporary credentials only when credential vending is enabled. Credential responses set `Cache-Control: no-store, private`, `Pragma: no-cache`, and `Expires: 0`. | | Catalog diagnostics and export | Supported | Exposes recovery state, consistency state, backing manifest, recoverable commit-log WAL state, strong backing migration target, single-active-writer policy, and scale validation matrix. | -| Catalog import and rollback | Supported | Import/register and rollback use catalog validation and commit paths rather than direct pointer mutation. | +| Catalog import and rollback | Supported | Import/register and online rollback use catalog validation and commit paths rather than direct pointer mutation. Online rollback accepts only a forward-safe metadata target that preserves assignment watermarks and retained definitions. Restoring an older target that lowers those watermarks is an offline disaster-recovery operation and requires every writer to be stopped. | | External catalog bridge | Supported operator path | Operator-supplied metadata pointer sync/import is supported for external catalog identity boundaries. Online vendor SDK polling and policy mirroring are not claimed. | | Multi-table transactions | Not claimed | RustFS currently claims single-table commit atomicity only. | diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index b88db7aee..69916207e 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -278,6 +278,8 @@ rustfs-signer.workspace = true serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } serde_urlencoded = { workspace = true } +snap.workspace = true +zstd.workspace = true # Cryptography and Security rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] } @@ -355,7 +357,6 @@ rcgen = { workspace = true } rustfs-test-utils.workspace = true # diagnose_e2e fixtures (archives are generated in-test, never checked in) zip = { workspace = true } -zstd = { workspace = true } # Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via # the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6). rustfs-ecstore = { workspace = true, features = ["test-util"] } diff --git a/rustfs/src/admin/handlers/table_catalog/credentials.rs b/rustfs/src/admin/handlers/table_catalog/credentials.rs index e997e39d7..62efde2e5 100644 --- a/rustfs/src/admin/handlers/table_catalog/credentials.rs +++ b/rustfs/src/admin/handlers/table_catalog/credentials.rs @@ -29,6 +29,6 @@ impl Operation for RestLoadCredentialsHandler { let issuer = IamTableCredentialIssuer::from_request(&req)?; let response = load_credentials_response(&store, &warehouse, &namespace, &table, &issuer, Some(&principal.credentials)).await?; - build_json_response(StatusCode::OK, &response) + build_sensitive_json_response(StatusCode::OK, &response) } } diff --git a/rustfs/src/admin/handlers/table_catalog/mod.rs b/rustfs/src/admin/handlers/table_catalog/mod.rs index f083d252b..1dd03db42 100644 --- a/rustfs/src/admin/handlers/table_catalog/mod.rs +++ b/rustfs/src/admin/handlers/table_catalog/mod.rs @@ -25,6 +25,7 @@ use crate::auth::{check_key_valid_with_context, get_session_token}; use crate::error::ApiError; use crate::server::{RemoteAddr, TABLE_CATALOG_COMPAT_PREFIX, TABLE_CATALOG_PREFIX}; use crate::table_catalog::{DEFAULT_WAREHOUSE_ID, TableCatalogStore}; +use bytes::Bytes; use futures::{StreamExt, TryStreamExt, stream}; use http::{HeaderMap, HeaderValue, StatusCode}; use hyper::Method; @@ -74,6 +75,9 @@ const ENV_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: &str = "RUSTFS_TABLE_CATALOG_CRE const DEFAULT_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 15 * 60; const MIN_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60; const MAX_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60 * 60; +const TABLE_CATALOG_REQUEST_BODY_TIMEOUT: StdDuration = StdDuration::from_secs(30); +const TABLE_CATALOG_COMMIT_REQUIREMENT_MAX_COUNT: usize = 1_024; +const TABLE_CATALOG_COMMIT_UPDATE_MAX_COUNT: usize = 1_024; const NAMESPACE_REQUEST_BODY_MAX_SIZE: usize = MAX_ADMIN_REQUEST_BODY_SIZE; const NAMESPACE_REQUEST_BODY_TIMEOUT: StdDuration = StdDuration::from_secs(10); const RENAME_TABLE_BODY_MAX_SIZE: usize = 16 * 1024; @@ -96,6 +100,7 @@ const ICEBERG_ERROR_NO_SUCH_VIEW: &str = "NoSuchViewException"; const ICEBERG_ERROR_REST: &str = "RESTException"; const ICEBERG_ERROR_UNPROCESSABLE_ENTITY: &str = "UnprocessableEntityException"; const ICEBERG_ERROR_UNSUPPORTED_OPERATION: &str = "UnsupportedOperationException"; +const ICEBERG_VIEW_FORMAT_VERSION: i64 = 1; const REST_PAGE_TOKEN_VERSION: u8 = 1; const REST_PAGE_TOKEN_MAX_LENGTH: usize = 16 * 1024; const REST_DEFAULT_PAGE_SIZE: usize = 1000; @@ -159,52 +164,12 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[ "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials", "POST /v1/{prefix}/namespaces/{namespace}/tables/{table}", "DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "PUT /buckets/{warehouse}", - "GET /buckets/{warehouse}", - "GET /{warehouse}/catalog/migration", - "POST /{warehouse}/catalog/migration", - "DELETE /{warehouse}/catalog/migration", - "GET /{warehouse}/namespaces", - "POST /{warehouse}/namespaces", - "GET /{warehouse}/namespaces/{namespace}", - "HEAD /{warehouse}/namespaces/{namespace}", - "DELETE /{warehouse}/namespaces/{namespace}", - "GET /{warehouse}/namespaces/{namespace}/tables", - "POST /{warehouse}/namespaces/{namespace}/tables", - "POST /{warehouse}/namespaces/{namespace}/register", - "GET /{warehouse}/namespaces/{namespace}/views", - "POST /{warehouse}/namespaces/{namespace}/views", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}", - "HEAD /{warehouse}/namespaces/{namespace}/tables/{table}", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/credentials", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}", - "DELETE /{warehouse}/namespaces/{namespace}/tables/{table}", - "GET /{warehouse}/namespaces/{namespace}/views/{view}", - "HEAD /{warehouse}/namespaces/{namespace}/views/{view}", - "POST /{warehouse}/namespaces/{namespace}/views/{view}", - "DELETE /{warehouse}/namespaces/{namespace}/views/{view}", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/refs", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}", - "DELETE /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/worker/run", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/heartbeat", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/quarantine", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external", - "PUT /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external/sync", - "GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/recovery", - "POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback", + "GET /v1/{prefix}/namespaces/{namespace}/views", + "POST /v1/{prefix}/namespaces/{namespace}/views", + "GET /v1/{prefix}/namespaces/{namespace}/views/{view}", + "HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}", + "POST /v1/{prefix}/namespaces/{namespace}/views/{view}", + "DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}", ]; const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] = &[ "POST /v1/{prefix}/namespaces/{namespace}/properties", @@ -341,7 +306,7 @@ struct CreateViewRequest { #[serde(deny_unknown_fields)] struct RestCommitTableRequest { #[serde(default, rename = "identifier")] - _identifier: Option, + identifier: Option, #[serde(default, rename = "commit-id")] commit_id: Option, #[serde(default, rename = "idempotency-key")] @@ -365,8 +330,10 @@ struct RestCommitTableRequest { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct RestCommitViewRequest { + #[serde(default, rename = "identifier")] + identifier: Option, #[serde(default, rename = "commit-id")] - commit_id: Option, + _commit_id: Option, #[serde(default, rename = "expected-version-token")] expected_version_token: Option, #[serde(default, rename = "expected-metadata-location")] @@ -705,6 +672,12 @@ enum RestPagination { }, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RestTableSnapshotSelection { + All, + Refs, +} + impl RestPagination { fn page_request(&self) -> Option<(Option<&str>, NonZeroUsize)> { match self { @@ -984,6 +957,18 @@ fn build_json_response(status: StatusCode, body: &T) -> S3Result(status: StatusCode, body: &T) -> S3Result> { + let mut response = build_json_response(status, body)?; + response + .headers + .insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store, private")); + response + .headers + .insert(http::header::PRAGMA, HeaderValue::from_static("no-cache")); + response.headers.insert(http::header::EXPIRES, HeaderValue::from_static("0")); + Ok(response) +} + fn empty_response(status: StatusCode) -> S3Response<(StatusCode, Body)> { S3Response::new((status, Body::default())) } @@ -1267,7 +1252,7 @@ struct TableCommitPublicationState { bucket_fence: Option, table_fence: Option<(String, String, String)>, observed_objects: BTreeMap<(String, String), TableCommitObservedObject>, - guards: Vec>, + guards: Vec, } #[derive(Clone)] @@ -1730,7 +1715,7 @@ where &self, bucket: &str, object: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { + ) -> crate::table_catalog::TableCatalogStoreResult { self.backend.acquire_read_lock(bucket, object).await } @@ -1738,7 +1723,7 @@ where &self, bucket: &str, object: &str, - ) -> crate::table_catalog::TableCatalogStoreResult> { + ) -> crate::table_catalog::TableCatalogStoreResult { self.backend.acquire_write_lock(bucket, object).await } @@ -1750,7 +1735,8 @@ where } fn table_bucket_commit_publication_is_held(&self, table_bucket: &str) -> bool { - self.publication.lock().bucket_fence.as_deref() == Some(table_bucket) + let publication = self.publication.lock(); + publication.bucket_fence.as_deref() == Some(table_bucket) && publication.guards.iter().all(|guard| !guard.is_lock_lost()) } async fn prepare_table_commit_publication( @@ -1763,11 +1749,12 @@ where } fn table_commit_publication_is_held(&self, table_bucket: &str, namespace: &str, table: &str) -> bool { - self.publication - .lock() + let publication = self.publication.lock(); + publication .table_fence .as_ref() .is_some_and(|held| held.0 == table_bucket && held.1 == namespace && held.2 == table) + && publication.guards.iter().all(|guard| !guard.is_lock_lost()) } fn complete_table_commit_publication(&self) { @@ -1775,20 +1762,72 @@ where } } -async fn read_json_body(mut input: Body) -> S3Result { - let body = input - .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) +async fn read_limited_body(mut input: Body, max_size: usize, timeout: StdDuration, operation: Option<&str>) -> S3Result { + tokio::time::timeout(timeout, input.store_all_limited(max_size)) .await - .map_err(|err| s3_error!(InvalidRequest, "failed to read request body: {}", err))?; + .map_err(|_| { + operation.map_or_else( + || S3Error::from(ApiError::invalid_request("timed out reading request body")), + |operation| S3Error::from(ApiError::invalid_request(format!("timed out reading {operation} request body"))), + ) + })? + .map_err(|err| S3Error::from(ApiError::invalid_request(format!("failed to read request body: {err}")))) +} + +async fn read_json_body(input: Body) -> S3Result { + let body = read_limited_body(input, MAX_ADMIN_REQUEST_BODY_SIZE, TABLE_CATALOG_REQUEST_BODY_TIMEOUT, None).await?; if body.is_empty() { - return Err(s3_error!(InvalidRequest, "request body is required")); + return Err(S3Error::from(ApiError::invalid_request("request body is required"))); } - serde_json::from_slice(&body).map_err(|err| s3_error!(InvalidRequest, "invalid JSON: {}", err)) + serde_json::from_slice(&body).map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid JSON: {err}")))) +} + +fn validate_rest_commit_request_shape( + value: &serde_json::Value, + require_requirements: bool, + require_updates: bool, +) -> S3Result<()> { + let object = value + .as_object() + .ok_or_else(|| S3Error::from(ApiError::invalid_request("commit request must be a JSON object")))?; + if object.get("new-metadata-location").is_some_and(serde_json::Value::is_string) { + for field in ["requirements", "updates"] { + if object + .get(field) + .and_then(serde_json::Value::as_array) + .is_some_and(|values| !values.is_empty()) + { + return Err(S3Error::from(ApiError::invalid_request(format!( + "legacy metadata pointer commit must not include standard {field}" + )))); + } + } + return Ok(()); + } + if require_requirements && !object.contains_key("requirements") { + return Err(S3Error::from(ApiError::invalid_request("commit request requires requirements"))); + } + if require_updates && !object.contains_key("updates") { + return Err(S3Error::from(ApiError::invalid_request("commit request requires updates"))); + } + Ok(()) +} + +async fn read_rest_commit_table_request(input: Body) -> S3Result { + let value = read_json_body::(input).await?; + validate_rest_commit_request_shape(&value, true, true)?; + serde_json::from_value(value).map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid JSON: {err}")))) +} + +async fn read_rest_commit_view_request(input: Body) -> S3Result { + let value = read_json_body::(input).await?; + validate_rest_commit_request_shape(&value, false, true)?; + serde_json::from_value(value).map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid JSON: {err}")))) } async fn read_bounded_json_body( headers: &HeaderMap, - mut input: Body, + input: Body, max_size: usize, timeout: StdDuration, operation: &str, @@ -1803,34 +1842,28 @@ async fn read_bounded_json_body( return Err(S3Error::from(ApiError::invalid_request(format!("{operation} request body is too large")))); } } - let body = tokio::time::timeout(timeout, input.store_all_limited(max_size)) - .await - .map_err(|_| S3Error::from(ApiError::invalid_request(format!("timed out reading {operation} request body"))))? - .map_err(|err| S3Error::from(ApiError::invalid_request(format!("failed to read request body: {err}"))))?; + let body = read_limited_body(input, max_size, timeout, Some(operation)).await?; if body.is_empty() { return Err(S3Error::from(ApiError::invalid_request("request body is required"))); } serde_json::from_slice(&body).map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid JSON: {err}")))) } -async fn read_json_body_or_default(mut input: Body) -> S3Result +async fn read_json_body_or_default(input: Body) -> S3Result where T: Default + DeserializeOwned, { - let body = input - .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) - .await - .map_err(|err| s3_error!(InvalidRequest, "failed to read request body: {}", err))?; + let body = read_limited_body(input, MAX_ADMIN_REQUEST_BODY_SIZE, TABLE_CATALOG_REQUEST_BODY_TIMEOUT, None).await?; if body.is_empty() { return Ok(T::default()); } - serde_json::from_slice(&body).map_err(|err| s3_error!(InvalidRequest, "invalid JSON: {}", err)) + serde_json::from_slice(&body).map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid JSON: {err}")))) } fn warehouse_from_params(params: &Params<'_, '_>) -> S3Result { let warehouse = params.get("warehouse").unwrap_or(""); if warehouse.is_empty() { - return Err(s3_error!(InvalidRequest, "warehouse is required")); + return Err(S3Error::from(ApiError::invalid_request("warehouse is required"))); } Ok(warehouse.to_string()) } @@ -1855,6 +1888,95 @@ fn warehouse_from_config_query(uri: &http::Uri) -> S3Result> { Ok(warehouse) } +fn rest_purge_requested_from_query(uri: &http::Uri) -> S3Result { + let mut purge_requested = None; + if let Some(query) = uri.query() { + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + if key != "purgeRequested" { + continue; + } + if purge_requested.is_some() { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "purgeRequested query parameter must not be repeated", + )); + } + let value = if value.eq_ignore_ascii_case("true") { + true + } else if value.eq_ignore_ascii_case("false") { + false + } else { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "purgeRequested query parameter must be true or false", + )); + }; + purge_requested = Some(value); + } + } + Ok(purge_requested.unwrap_or(false)) +} + +fn rest_table_snapshot_selection_from_query(uri: &http::Uri) -> S3Result { + let mut selection = None; + if let Some(query) = uri.query() { + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + if key != "snapshots" { + continue; + } + if selection.is_some() { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "snapshots query parameter must not be repeated", + )); + } + selection = Some(match value.as_ref() { + "all" => RestTableSnapshotSelection::All, + "refs" => RestTableSnapshotSelection::Refs, + _ => { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "snapshots query parameter must be all or refs", + )); + } + }); + } + } + Ok(selection.unwrap_or(RestTableSnapshotSelection::All)) +} + +fn apply_rest_table_snapshot_selection(metadata: &mut serde_json::Value, selection: RestTableSnapshotSelection) { + if selection == RestTableSnapshotSelection::All { + return; + } + let mut referenced_snapshot_ids = metadata + .get("refs") + .and_then(serde_json::Value::as_object) + .into_iter() + .flat_map(|refs| refs.values()) + .filter_map(|reference| reference.get("snapshot-id").and_then(serde_json::Value::as_i64)) + .collect::>(); + if let Some(current_snapshot_id) = metadata + .get("current-snapshot-id") + .and_then(serde_json::Value::as_i64) + .filter(|snapshot_id| *snapshot_id != -1) + { + referenced_snapshot_ids.insert(current_snapshot_id); + } + if let Some(snapshots) = metadata.get_mut("snapshots").and_then(serde_json::Value::as_array_mut) { + snapshots.retain(|snapshot| { + snapshot + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_some_and(|snapshot_id| referenced_snapshot_ids.contains(&snapshot_id)) + }); + } +} + fn rest_pagination_from_query(uri: &http::Uri, context: RestPageContext<'_>) -> S3Result { let mut page_token = None; let mut page_token_seen = false; @@ -2242,6 +2364,23 @@ fn namespace_segments(namespace: &crate::table_catalog::Namespace) -> Vec, + namespace: &crate::table_catalog::Namespace, + name: &str, +) -> S3Result<()> { + if let Some(identifier) = identifier + && (identifier.namespace != namespace_segments(namespace) || identifier.name != name) + { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + "request identifier must match the resource URL", + )); + } + Ok(()) +} + fn namespace_from_segments(segments: &[String]) -> S3Result { crate::table_catalog::Namespace::from_segments(segments.to_vec()) .map_err(|err| s3_error!(InvalidRequest, "invalid namespace: {}", err)) @@ -2487,13 +2626,15 @@ fn load_table_response_from_entry(entry: crate::table_catalog::TableEntry, metad fn load_view_response_from_entry(entry: crate::table_catalog::ViewEntry, metadata: serde_json::Value) -> RestLoadViewResponse { let mut config = BTreeMap::new(); let warehouse_location = entry.warehouse_location.clone(); + let metadata_location = table_metadata_location_for_client(&entry.table_bucket, &entry.metadata_location); + let metadata = table_metadata_for_client(&entry.table_bucket, metadata); config.insert("warehouse-location".to_string(), warehouse_location.clone()); config.insert(CREDENTIAL_SCOPE_CONFIG_KEY.to_string(), CREDENTIAL_SCOPE_TABLE_PREFIX.to_string()); config.insert(CREDENTIAL_SCOPE_PREFIX_CONFIG_KEY.to_string(), warehouse_location); config.insert(CREDENTIAL_MODE_CONFIG_KEY.to_string(), CREDENTIAL_MODE_CLIENT_PROVIDED.to_string()); RestLoadViewResponse { - metadata_location: entry.metadata_location, + metadata_location, metadata, config, } @@ -2662,13 +2803,59 @@ fn validate_metadata_view_location_in_bucket(bucket: &str, metadata: &serde_json validate_view_location_in_bucket(bucket, location) } +fn validate_persisted_table_metadata( + entry: &crate::table_catalog::TableEntry, + metadata: &serde_json::Value, + require_current_warehouse: bool, +) -> S3Result<()> { + crate::table_catalog::validate_supported_table_metadata(metadata).map_err(|_| persisted_metadata_error("table"))?; + validate_metadata_table_location_in_bucket(&entry.table_bucket, metadata).map_err(|_| persisted_metadata_error("table"))?; + let metadata_uuid = metadata_table_uuid(metadata).map_err(|_| persisted_metadata_error("table"))?; + let metadata_location = metadata_table_location(metadata).map_err(|_| persisted_metadata_error("table"))?; + let format_version = metadata_format_version(metadata).map_err(|_| persisted_metadata_error("table"))?; + if metadata_uuid != entry.table_uuid + || (require_current_warehouse && format_version < entry.format_version) + || (require_current_warehouse && metadata_location != entry.warehouse_location) + { + return Err(persisted_metadata_error("table")); + } + Ok(()) +} + +fn validate_persisted_table_metadata_location(entry: &crate::table_catalog::TableEntry, metadata_location: &str) -> S3Result<()> { + if !crate::table_catalog::is_valid_table_metadata_location_for_entry(entry, metadata_location) { + return Err(persisted_metadata_error("table")); + } + Ok(()) +} + +fn validate_persisted_view_metadata(entry: &crate::table_catalog::ViewEntry, metadata: &serde_json::Value) -> S3Result<()> { + validate_persisted_view_metadata_identity(entry, metadata)?; + crate::table_catalog::validate_supported_view_metadata(metadata).map_err(|_| persisted_metadata_error("view")) +} + +fn validate_persisted_view_metadata_identity( + entry: &crate::table_catalog::ViewEntry, + metadata: &serde_json::Value, +) -> S3Result<()> { + validate_metadata_view_location_in_bucket(&entry.table_bucket, metadata).map_err(|_| persisted_metadata_error("view"))?; + let metadata_uuid = metadata_view_uuid(metadata).map_err(|_| persisted_metadata_error("view"))?; + let metadata_location = metadata_table_location(metadata).map_err(|_| persisted_metadata_error("view"))?; + let format_version = metadata_format_version(metadata).map_err(|_| persisted_metadata_error("view"))?; + if metadata_uuid != entry.view_uuid || metadata_location != entry.warehouse_location || format_version != entry.format_version + { + return Err(persisted_metadata_error("view")); + } + Ok(()) +} + fn validate_metadata_matches_current_metadata( current_metadata: &serde_json::Value, target_metadata: &serde_json::Value, ) -> S3Result<()> { - crate::table_catalog::validate_supported_table_metadata(current_metadata).map_err(catalog_store_error)?; crate::table_catalog::validate_supported_table_metadata(target_metadata).map_err(catalog_store_error)?; - validate_metadata_identity_matches_current_metadata(current_metadata, target_metadata) + validate_metadata_identity_matches_current_metadata(current_metadata, target_metadata)?; + crate::table_catalog::validate_table_metadata_transition(current_metadata, target_metadata).map_err(catalog_store_error) } fn validate_metadata_identity_matches_current_metadata( @@ -2676,15 +2863,20 @@ fn validate_metadata_identity_matches_current_metadata( target_metadata: &serde_json::Value, ) -> S3Result<()> { let expected_table_uuid = metadata_table_uuid(current_metadata)?; - metadata_format_version(current_metadata)?; + let expected_format_version = metadata_format_version(current_metadata)?; let target_table_uuid = metadata_table_uuid(target_metadata)?; - metadata_format_version(target_metadata)?; + let target_format_version = metadata_format_version(target_metadata)?; if target_table_uuid != expected_table_uuid { return Err(s3_error!( InvalidRequest, "table metadata table-uuid does not match current table metadata" )); } + if target_format_version < expected_format_version { + return Err(S3Error::from(ApiError::invalid_request( + "table metadata format-version cannot be downgraded", + ))); + } Ok(()) } @@ -2727,7 +2919,11 @@ fn table_entry_from_register_request( request: RegisterTableRequest, ) -> S3Result { if request.overwrite { - return Err(s3_error!(NotImplemented, "register table overwrite is not supported")); + return Err(iceberg_rest_error( + ICEBERG_ERROR_UNSUPPORTED_OPERATION, + StatusCode::NOT_ACCEPTABLE, + "register table overwrite is not supported", + )); } let table = crate::table_catalog::IdentifierSegment::parse(request.name) .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; @@ -2806,7 +3002,11 @@ fn table_entry_from_create_table_request( mut properties, } = request; if stage_create { - return Err(s3_error!(NotImplemented, "stage-create is not supported")); + return Err(iceberg_rest_error( + ICEBERG_ERROR_UNSUPPORTED_OPERATION, + StatusCode::NOT_ACCEPTABLE, + "stage-create is not supported", + )); } let table = crate::table_catalog::IdentifierSegment::parse(name) @@ -2899,13 +3099,7 @@ fn initial_table_metadata_json( let schema_object = schema .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "schema must be a JSON object"))?; - schema_object - .entry("schema-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); - let schema_id = schema_object - .get("schema-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "schema-id must be an integer"))?; + schema_object.insert("schema-id".to_string(), serde_json::Value::from(0)); let last_column_id = max_field_id(&schema); let mut spec = partition_spec.unwrap_or_else(|| { @@ -2917,17 +3111,11 @@ fn initial_table_metadata_json( let spec_object = spec .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "partition-spec must be a JSON object"))?; - spec_object - .entry("spec-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); + spec_object.insert("spec-id".to_string(), serde_json::Value::from(0)); spec_object .entry("fields".to_string()) .or_insert_with(|| serde_json::Value::Array(Vec::new())); - let spec_id = spec_object - .get("spec-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "partition spec-id must be an integer"))?; - let last_partition_id = max_partition_field_id(&spec); + let last_partition_id = assign_partition_field_ids(&mut spec, 999, &BTreeMap::new())?; let mut sort_order = write_order.unwrap_or_else(|| { serde_json::json!({ @@ -2938,17 +3126,19 @@ fn initial_table_metadata_json( let sort_order_object = sort_order .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "write-order must be a JSON object"))?; - sort_order_object - .entry("order-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); - sort_order_object + let sort_order_fields = sort_order_object .entry("fields".to_string()) .or_insert_with(|| serde_json::Value::Array(Vec::new())); - let sort_order_id = sort_order_object - .get("order-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "sort order-id must be an integer"))?; - + let sort_order_id = if sort_order_fields + .as_array() + .ok_or_else(|| S3Error::from(ApiError::invalid_request("write-order fields must be an array")))? + .is_empty() + { + 0 + } else { + 1 + }; + sort_order_object.insert("order-id".to_string(), serde_json::Value::from(sort_order_id)); let mut metadata = serde_json::json!({ "format-version": entry.format_version, "table-uuid": entry.table_uuid, @@ -2956,9 +3146,9 @@ fn initial_table_metadata_json( "last-updated-ms": current_time_millis(), "last-column-id": last_column_id, "schemas": [schema], - "current-schema-id": schema_id, + "current-schema-id": 0, "partition-specs": [spec], - "default-spec-id": spec_id, + "default-spec-id": 0, "last-partition-id": last_partition_id, "sort-orders": [sort_order], "default-sort-order-id": sort_order_id, @@ -2985,26 +3175,12 @@ fn initial_view_metadata_json( let schema_object = schema .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "schema must be a JSON object"))?; - schema_object - .entry("schema-id".to_string()) - .or_insert_with(|| serde_json::Value::from(0)); - let schema_id = schema_object - .get("schema-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "schema-id must be an integer"))?; + schema_object.insert("schema-id".to_string(), serde_json::Value::from(0)); let view_version_object = view_version .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))?; - view_version_object - .entry("version-id".to_string()) - .or_insert_with(|| serde_json::Value::from(1)); - view_version_object - .entry("schema-id".to_string()) - .or_insert_with(|| serde_json::Value::from(schema_id)); - view_version_object - .entry("timestamp-ms".to_string()) - .or_insert_with(|| serde_json::Value::from(current_time_millis())); + view_version_object.insert("schema-id".to_string(), serde_json::Value::from(0)); let version_id = view_version_object .get("version-id") .and_then(serde_json::Value::as_i64) @@ -3012,13 +3188,12 @@ fn initial_view_metadata_json( let timestamp_ms = view_version_object .get("timestamp-ms") .and_then(serde_json::Value::as_i64) - .unwrap_or_else(current_time_millis); + .ok_or_else(|| S3Error::from(ApiError::invalid_request("view-version timestamp-ms must be an integer")))?; - Ok(serde_json::json!({ + let metadata = serde_json::json!({ "format-version": entry.format_version, "view-uuid": entry.view_uuid, "location": entry.warehouse_location, - "last-updated-ms": current_time_millis(), "current-version-id": version_id, "schemas": [schema], "versions": [view_version], @@ -3026,9 +3201,10 @@ fn initial_view_metadata_json( "timestamp-ms": timestamp_ms, "version-id": version_id }], - "metadata-log": [], "properties": properties - })) + }); + validate_supported_view_metadata(&metadata)?; + Ok(metadata) } fn current_time_millis() -> i64 { @@ -3047,8 +3223,10 @@ fn max_field_id(value: &serde_json::Value) -> i64 { fn collect_max_field_id(value: &serde_json::Value, max_id: &mut i64) { match value { serde_json::Value::Object(object) => { - if let Some(id) = object.get("id").and_then(serde_json::Value::as_i64) { - *max_id = (*max_id).max(id); + for field in ["id", "element-id", "key-id", "value-id"] { + if let Some(id) = object.get(field).and_then(serde_json::Value::as_i64) { + *max_id = (*max_id).max(id); + } } for child in object.values() { collect_max_field_id(child, max_id); @@ -3063,19 +3241,6 @@ fn collect_max_field_id(value: &serde_json::Value, max_id: &mut i64) { } } -fn max_partition_field_id(value: &serde_json::Value) -> i64 { - let mut max_id = 999; - let Some(fields) = value.get("fields").and_then(serde_json::Value::as_array) else { - return max_id; - }; - for field in fields { - if let Some(field_id) = field.get("field-id").and_then(serde_json::Value::as_i64) { - max_id = max_id.max(field_id); - } - } - max_id -} - fn standard_commit_ids(commit_id: Option) -> (String, String) { match commit_id { Some(commit_id) => match Uuid::parse_str(&commit_id) { @@ -3163,7 +3328,7 @@ fn validate_table_commit_requirements(metadata: &serde_json::Value, requirements .ok_or_else(|| s3_error!(InvalidRequest, "commit requirement type is required"))?; match requirement_type { "assert-create" => { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: table already exists")); + return Err(commit_requirement_failed("commit requirement failed: table already exists")); } "assert-table-uuid" => { let expected = requirement @@ -3175,7 +3340,7 @@ fn validate_table_commit_requirements(metadata: &serde_json::Value, requirements .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing table-uuid"))?; if actual != expected { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: table uuid changed")); + return Err(commit_requirement_failed("commit requirement failed: table uuid changed")); } } "assert-current-schema-id" => { @@ -3206,8 +3371,11 @@ fn validate_table_commit_requirements(metadata: &serde_json::Value, requirements )?; } "assert-ref-snapshot-id" => validate_ref_snapshot_requirement(metadata, requirement)?, - "assert-current-snapshot-id" => validate_current_snapshot_requirement(metadata, requirement)?, - _ => return Err(s3_error!(NotImplemented, "unsupported commit requirement: {requirement_type}")), + _ => { + return Err(S3Error::from(ApiError::invalid_request(format!( + "unsupported commit requirement: {requirement_type}" + )))); + } } } Ok(()) @@ -3238,7 +3406,7 @@ fn validate_i64_requirement_with_metadata_key( .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing {metadata_key}"))?; if actual != expected { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: {label} changed")); + return Err(commit_requirement_failed(format!("commit requirement failed: {label} changed"))); } Ok(()) } @@ -3255,7 +3423,7 @@ fn validate_ref_snapshot_requirement(metadata: &serde_json::Value, requirement: .and_then(serde_json::Value::as_i64); if requirement.get("snapshot-id").is_some_and(serde_json::Value::is_null) { if actual.is_some() { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: snapshot ref exists")); + return Err(commit_requirement_failed("commit requirement failed: snapshot ref exists")); } return Ok(()); } @@ -3264,25 +3432,7 @@ fn validate_ref_snapshot_requirement(metadata: &serde_json::Value, requirement: .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "assert-ref-snapshot-id requires snapshot-id"))?; if actual != Some(expected) { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: snapshot ref changed")); - } - Ok(()) -} - -fn validate_current_snapshot_requirement(metadata: &serde_json::Value, requirement: &serde_json::Value) -> S3Result<()> { - let actual = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); - if requirement.get("snapshot-id").is_some_and(serde_json::Value::is_null) { - if actual.is_some() { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: current snapshot exists")); - } - return Ok(()); - } - let expected = requirement - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "assert-current-snapshot-id requires snapshot-id"))?; - if actual != Some(expected) { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: current snapshot changed")); + return Err(commit_requirement_failed("commit requirement failed: snapshot ref changed")); } Ok(()) } @@ -3304,6 +3454,16 @@ fn apply_table_commit_updates_at( if !metadata.is_object() { return Err(s3_error!(InvalidRequest, "current table metadata must be a JSON object")); } + if metadata.get("format-version").is_some() { + crate::table_catalog::synchronize_table_metadata_version_fields(&mut metadata).map_err(catalog_store_error)?; + } + let mut next_schema_id = next_catalog_id_for_updates(&metadata, updates, "add-schema", "schemas", "schema-id")?; + let mut next_spec_id = next_catalog_id_for_updates(&metadata, updates, "add-spec", "partition-specs", "spec-id")?; + let mut next_sort_order_id = next_catalog_id_for_updates(&metadata, updates, "add-sort-order", "sort-orders", "order-id")?; + let mut last_added_schema_id = None; + let mut last_added_spec_id = None; + let mut last_added_sort_order_id = None; + let mut added_snapshot_ids = BTreeSet::new(); for update in updates { let action = update @@ -3311,25 +3471,81 @@ fn apply_table_commit_updates_at( .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "table update action is required"))?; match action { - "assign-uuid" => apply_assign_uuid_update(&mut metadata, update)?, + "assign-uuid" => apply_assign_uuid_update(&mut metadata, update, "table-uuid", "table")?, "upgrade-format-version" => apply_upgrade_format_version_update(&mut metadata, update)?, - "add-schema" => apply_add_schema_update(&mut metadata, update)?, - "set-current-schema" => apply_set_current_schema_update(&mut metadata, update)?, - "add-spec" => apply_add_spec_update(&mut metadata, update)?, - "set-default-spec" => apply_set_default_spec_update(&mut metadata, update)?, - "add-sort-order" => apply_add_sort_order_update(&mut metadata, update)?, - "set-default-sort-order" => apply_set_default_sort_order_update(&mut metadata, update)?, - "add-snapshot" => apply_add_snapshot_update(&mut metadata, update)?, - "set-snapshot-ref" => apply_set_snapshot_ref_update(&mut metadata, update)?, + "add-schema" => { + let schema_id = take_catalog_assigned_id(&mut next_schema_id, "schema-id")?; + apply_add_table_schema_update(&mut metadata, update, schema_id)?; + last_added_schema_id = Some(schema_id); + } + "set-current-schema" => { + apply_set_current_schema_update(&mut metadata, update, last_added_schema_id)?; + } + "add-spec" => { + let spec_id = take_catalog_assigned_id(&mut next_spec_id, "spec-id")?; + apply_add_spec_update(&mut metadata, update, spec_id)?; + last_added_spec_id = Some(spec_id); + } + "set-default-spec" => { + apply_set_default_spec_update(&mut metadata, update, last_added_spec_id)?; + } + "add-sort-order" => { + let sort_order_id = take_catalog_assigned_id(&mut next_sort_order_id, "sort order-id")?; + last_added_sort_order_id = Some(apply_add_sort_order_update(&mut metadata, update, sort_order_id)?); + } + "set-default-sort-order" => { + apply_set_default_sort_order_update(&mut metadata, update, last_added_sort_order_id)?; + } + "add-snapshot" => { + added_snapshot_ids.insert(apply_add_snapshot_update(&mut metadata, update)?); + } + "set-snapshot-ref" => { + apply_set_snapshot_ref_update(&mut metadata, update, &added_snapshot_ids, commit_timestamp_ms)?; + } "remove-snapshots" => apply_remove_snapshots_update(&mut metadata, update)?, "remove-snapshot-ref" => apply_remove_snapshot_ref_update(&mut metadata, update)?, "set-location" => apply_set_location_update(&mut metadata, update)?, "set-properties" => apply_set_properties_update(&mut metadata, update)?, "remove-properties" => apply_remove_properties_update(&mut metadata, update)?, - _ => return Err(s3_error!(NotImplemented, "unsupported table update: {action}")), + "set-statistics" => apply_set_snapshot_file_update( + &mut metadata, + update, + "statistics", + "statistics", + crate::table_catalog::IcebergStatisticsFileKind::Table, + )?, + "remove-statistics" => apply_remove_snapshot_file_update(&mut metadata, update, "statistics")?, + "set-partition-statistics" => { + apply_set_snapshot_file_update( + &mut metadata, + update, + "partition-statistics", + "partition-statistics", + crate::table_catalog::IcebergStatisticsFileKind::Partition, + )?; + } + "remove-partition-statistics" => { + apply_remove_snapshot_file_update(&mut metadata, update, "partition-statistics")?; + } + "remove-partition-specs" => { + apply_remove_metadata_ids_update(&mut metadata, update, "partition-specs", "spec-id", "spec-ids")?; + } + "remove-schemas" => { + apply_remove_metadata_ids_update(&mut metadata, update, "schemas", "schema-id", "schema-ids")?; + } + "add-encryption-key" | "remove-encryption-key" => { + return Err(iceberg_rest_error( + ICEBERG_ERROR_UNSUPPORTED_OPERATION, + StatusCode::NOT_ACCEPTABLE, + "table encryption keys require Iceberg format-version 3", + )); + } + _ => return Err(S3Error::from(ApiError::invalid_request(format!("unsupported table update: {action}")))), } } + prune_intermediate_snapshot_log_entries(&mut metadata, &added_snapshot_ids)?; + if metadata.get("format-version").is_some() { crate::table_catalog::synchronize_table_metadata_version_fields(&mut metadata).map_err(catalog_store_error)?; } @@ -3338,6 +3554,27 @@ fn apply_table_commit_updates_at( Ok(metadata) } +fn prune_intermediate_snapshot_log_entries(metadata: &mut serde_json::Value, added_snapshot_ids: &BTreeSet) -> S3Result<()> { + if added_snapshot_ids.is_empty() { + return Ok(()); + } + let current_snapshot_id = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); + let snapshot_log = ensure_array_field(metadata, "snapshot-log")?; + for entry in snapshot_log.iter() { + entry + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| S3Error::from(ApiError::invalid_request("snapshot-log snapshot-id must be an integer")))?; + } + snapshot_log.retain(|entry| { + entry + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_none_or(|snapshot_id| !added_snapshot_ids.contains(&snapshot_id) || Some(snapshot_id) == current_snapshot_id) + }); + Ok(()) +} + fn validate_view_commit_requirements(metadata: &serde_json::Value, requirements: &[serde_json::Value]) -> S3Result<()> { for requirement in requirements { let requirement_type = requirement @@ -3355,32 +3592,35 @@ fn validate_view_commit_requirements(metadata: &serde_json::Value, requirements: .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "current view metadata is missing view-uuid"))?; if actual != expected { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: view uuid changed")); + return Err(commit_requirement_failed("commit requirement failed: view uuid changed")); } } - "assert-current-view-version-id" => { - validate_i64_requirement_with_metadata_key( - metadata, - requirement, - "current-view-version-id", - "current-version-id", - "current view version id", - )?; + _ => { + return Err(S3Error::from(ApiError::invalid_request(format!( + "unsupported view commit requirement: {requirement_type}" + )))); } - _ => return Err(s3_error!(NotImplemented, "unsupported view commit requirement: {requirement_type}")), } } Ok(()) } -fn apply_view_commit_updates( +fn validate_supported_view_metadata(metadata: &serde_json::Value) -> S3Result<()> { + crate::table_catalog::validate_supported_view_metadata(metadata).map_err(catalog_store_error) +} + +fn apply_view_commit_updates_at( mut metadata: serde_json::Value, updates: &[serde_json::Value], - previous_metadata_location: &str, + commit_timestamp_ms: i64, ) -> S3Result { if !metadata.is_object() { return Err(s3_error!(InvalidRequest, "current view metadata must be a JSON object")); } + let mut next_schema_id = next_catalog_id_for_updates(&metadata, updates, "add-schema", "schemas", "schema-id")?; + let mut last_added_schema_id = None; + let mut last_added_view_version_id = None; + let mut added_view_version_timestamps = BTreeMap::new(); for update in updates { let action = update @@ -3388,40 +3628,138 @@ fn apply_view_commit_updates( .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "view update action is required"))?; match action { - "assign-uuid" => apply_assign_uuid_update(&mut metadata, update)?, - "add-schema" => apply_add_schema_update(&mut metadata, update)?, - "set-current-schema" => apply_set_current_schema_update(&mut metadata, update)?, - "add-view-version" => apply_add_view_version_update(&mut metadata, update)?, - "set-current-view-version" => apply_set_current_view_version_update(&mut metadata, update)?, + "assign-uuid" => apply_assign_uuid_update(&mut metadata, update, "view-uuid", "view")?, + "upgrade-format-version" => apply_upgrade_view_format_version_update(update)?, + "add-schema" => { + let schema_id = take_catalog_assigned_id(&mut next_schema_id, "schema-id")?; + apply_add_view_schema_update(&mut metadata, update, schema_id)?; + last_added_schema_id = Some(schema_id); + } + "add-view-version" => { + let (version_id, timestamp_ms) = apply_add_view_version_update(&mut metadata, update, last_added_schema_id)?; + last_added_view_version_id = Some(version_id); + added_view_version_timestamps.insert(version_id, timestamp_ms); + } + "set-current-view-version" => { + apply_set_current_view_version_update( + &mut metadata, + update, + last_added_view_version_id, + &added_view_version_timestamps, + commit_timestamp_ms, + )?; + } "set-location" => apply_set_location_update(&mut metadata, update)?, "set-properties" => apply_set_properties_update(&mut metadata, update)?, "remove-properties" => apply_remove_properties_update(&mut metadata, update)?, - _ => return Err(s3_error!(NotImplemented, "unsupported view update: {action}")), + _ => return Err(S3Error::from(ApiError::invalid_request(format!("unsupported view update: {action}")))), } } - crate::table_catalog::validate_view_metadata_references(&metadata).map_err(catalog_store_error)?; - append_previous_metadata_log(&mut metadata, previous_metadata_location)?; - metadata_object_mut(&mut metadata)?.insert("last-updated-ms".to_string(), serde_json::Value::from(current_time_millis())); + validate_supported_view_metadata(&metadata)?; Ok(metadata) } -fn apply_assign_uuid_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_set_snapshot_file_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + metadata_field: &str, + update_field: &str, + kind: crate::table_catalog::IcebergStatisticsFileKind, +) -> S3Result<()> { + let value = update + .get(update_field) + .cloned() + .ok_or_else(|| S3Error::from(ApiError::invalid_request(format!("{update_field} is required"))))?; + let snapshot_id = + crate::table_catalog::validate_iceberg_statistics_file(&value, update_field, kind).map_err(catalog_store_error)?; + if let Some(deprecated_snapshot_id) = update.get("snapshot-id") { + let deprecated_snapshot_id = deprecated_snapshot_id.as_i64().ok_or_else(|| { + iceberg_rest_error(ICEBERG_ERROR_BAD_REQUEST, StatusCode::BAD_REQUEST, "snapshot-id must be an integer") + })?; + if deprecated_snapshot_id != snapshot_id { + return Err(iceberg_rest_error( + ICEBERG_ERROR_BAD_REQUEST, + StatusCode::BAD_REQUEST, + format!("{update_field}.snapshot-id does not match snapshot-id"), + )); + } + } + let values = ensure_array_field(metadata, metadata_field)?; + values.retain(|value| value.get("snapshot-id").and_then(serde_json::Value::as_i64) != Some(snapshot_id)); + values.push(value); + Ok(()) +} + +fn apply_remove_snapshot_file_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + metadata_field: &str, +) -> S3Result<()> { + let snapshot_id = update + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| S3Error::from(ApiError::invalid_request("remove update requires snapshot-id")))?; + if let Some(values) = metadata.get_mut(metadata_field).and_then(serde_json::Value::as_array_mut) { + values.retain(|value| value.get("snapshot-id").and_then(serde_json::Value::as_i64) != Some(snapshot_id)); + } + Ok(()) +} + +fn apply_remove_metadata_ids_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + metadata_field: &str, + id_field: &str, + update_field: &str, +) -> S3Result<()> { + let ids = update + .get(update_field) + .and_then(serde_json::Value::as_array) + .ok_or_else(|| S3Error::from(ApiError::invalid_request(format!("{update_field} must be an array"))))? + .iter() + .map(|value| { + value + .as_i64() + .ok_or_else(|| S3Error::from(ApiError::invalid_request(format!("{update_field} must contain integers")))) + }) + .collect::>>()?; + if let Some(values) = metadata.get_mut(metadata_field).and_then(serde_json::Value::as_array_mut) { + values.retain(|value| { + value + .get(id_field) + .and_then(serde_json::Value::as_i64) + .is_none_or(|id| !ids.contains(&id)) + }); + } + Ok(()) +} + +fn apply_assign_uuid_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + uuid_field: &str, + entity: &str, +) -> S3Result<()> { let uuid = update .get("uuid") .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "assign-uuid requires uuid"))?; let object = metadata_object_mut(metadata)?; - if let Some(existing) = object.get("table-uuid").and_then(serde_json::Value::as_str) + if let Some(existing) = object.get(uuid_field).and_then(serde_json::Value::as_str) && existing != uuid { - return Err(s3_error!(PreconditionFailed, "cannot reassign table uuid")); + return Err(commit_requirement_failed(format!("cannot reassign {entity} uuid"))); } - object.insert("table-uuid".to_string(), serde_json::Value::String(uuid.to_string())); + object.insert(uuid_field.to_string(), serde_json::Value::String(uuid.to_string())); Ok(()) } -fn apply_add_view_version_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_add_view_version_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + last_added_schema_id: Option, +) -> S3Result<(i64, i64)> { let mut view_version = update .get("view-version") .cloned() @@ -3429,35 +3767,48 @@ fn apply_add_view_version_update(metadata: &mut serde_json::Value, update: &serd if !view_version.is_object() { return Err(s3_error!(InvalidRequest, "view-version must be a JSON object")); } - if view_version.get("version-id").is_none() { - let next_id = next_array_object_i64(metadata, "versions", "version-id")?; + let version_id = view_version + .get("version-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "view-version version-id must be an integer"))?; + if view_version.get("schema-id").and_then(serde_json::Value::as_i64) == Some(-1) { + let schema_id = resolve_last_added_update_id(-1, last_added_schema_id, "add-view-version", "add-schema")?; view_version .as_object_mut() .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))? - .insert("version-id".to_string(), serde_json::Value::from(next_id)); + .insert("schema-id".to_string(), serde_json::Value::from(schema_id)); } - view_version - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "view-version must be a JSON object"))? - .entry("timestamp-ms".to_string()) - .or_insert_with(|| serde_json::Value::from(current_time_millis())); + let timestamp_ms = view_version + .get("timestamp-ms") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "view-version timestamp-ms must be an integer"))?; ensure_array_field(metadata, "versions")?.push(view_version); - Ok(()) + Ok((version_id, timestamp_ms)) } -fn apply_set_current_view_version_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_set_current_view_version_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + last_added_view_version_id: Option, + added_view_version_timestamps: &BTreeMap, + commit_timestamp_ms: i64, +) -> S3Result<()> { let requested_id = update .get("view-version-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "set-current-view-version requires view-version-id"))?; - let version_id = if requested_id == -1 { - last_array_object_i64(metadata, "versions", "version-id")? - } else { - requested_id - }; + let version_id = + resolve_last_added_update_id(requested_id, last_added_view_version_id, "set-current-view-version", "add-view-version")?; + if metadata.get("current-version-id").and_then(serde_json::Value::as_i64) == Some(version_id) { + return Ok(()); + } + let history_timestamp_ms = added_view_version_timestamps + .get(&version_id) + .copied() + .unwrap_or(commit_timestamp_ms); metadata_object_mut(metadata)?.insert("current-version-id".to_string(), serde_json::Value::from(version_id)); ensure_array_field(metadata, "version-log")?.push(serde_json::json!({ - "timestamp-ms": current_time_millis(), + "timestamp-ms": history_timestamp_ms, "version-id": version_id })); Ok(()) @@ -3498,148 +3849,300 @@ fn apply_upgrade_format_version_update(metadata: &mut serde_json::Value, update: Ok(()) } -fn apply_add_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_upgrade_view_format_version_update(update: &serde_json::Value) -> S3Result<()> { + let version = update + .get("format-version") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "upgrade-format-version requires format-version"))?; + if version != ICEBERG_VIEW_FORMAT_VERSION { + return Err(iceberg_rest_error( + ICEBERG_ERROR_UNSUPPORTED_OPERATION, + StatusCode::NOT_ACCEPTABLE, + format!("unsupported Iceberg view format-version: {version}"), + )); + } + Ok(()) +} + +fn catalog_assigned_schema(update: &serde_json::Value, schema_id: i64) -> S3Result { let mut schema = update .get("schema") .cloned() .ok_or_else(|| s3_error!(InvalidRequest, "add-schema requires schema"))?; - if !schema.is_object() { - return Err(s3_error!(InvalidRequest, "add-schema schema must be a JSON object")); - } - if schema.get("schema-id").is_none() { - let next_id = next_array_object_i64(metadata, "schemas", "schema-id")?; - schema - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "add-schema schema must be a JSON object"))? - .insert("schema-id".to_string(), serde_json::Value::from(next_id)); - } + let schema_object = schema + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "add-schema schema must be a JSON object"))?; + schema_object.insert("schema-id".to_string(), serde_json::Value::from(schema_id)); + Ok(schema) +} + +fn apply_add_table_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value, schema_id: i64) -> S3Result<()> { + let schema = catalog_assigned_schema(update, schema_id)?; let last_column_id = max_field_id(&schema); ensure_array_field(metadata, "schemas")?.push(schema); let object = metadata_object_mut(metadata)?; let current_last = object .get("last-column-id") .and_then(serde_json::Value::as_i64) - .unwrap_or_default(); + .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing last-column-id"))?; object.insert("last-column-id".to_string(), serde_json::Value::from(current_last.max(last_column_id))); Ok(()) } -fn apply_set_current_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_add_view_schema_update(metadata: &mut serde_json::Value, update: &serde_json::Value, schema_id: i64) -> S3Result<()> { + let schema = catalog_assigned_schema(update, schema_id)?; + ensure_array_field(metadata, "schemas")?.push(schema); + Ok(()) +} + +fn apply_set_current_schema_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + last_added_schema_id: Option, +) -> S3Result<()> { let requested_id = update .get("schema-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "set-current-schema requires schema-id"))?; - let schema_id = if requested_id == -1 { - last_array_object_i64(metadata, "schemas", "schema-id")? - } else { - requested_id - }; + let schema_id = resolve_last_added_update_id(requested_id, last_added_schema_id, "set-current-schema", "add-schema")?; metadata_object_mut(metadata)?.insert("current-schema-id".to_string(), serde_json::Value::from(schema_id)); Ok(()) } -fn apply_add_spec_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_add_spec_update(metadata: &mut serde_json::Value, update: &serde_json::Value, spec_id: i64) -> S3Result<()> { let mut spec = update .get("spec") .cloned() .ok_or_else(|| s3_error!(InvalidRequest, "add-spec requires spec"))?; - if !spec.is_object() { - return Err(s3_error!(InvalidRequest, "add-spec spec must be a JSON object")); - } - if spec.get("spec-id").is_none() { - let next_id = next_array_object_i64(metadata, "partition-specs", "spec-id")?; - spec.as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "add-spec spec must be a JSON object"))? - .insert("spec-id".to_string(), serde_json::Value::from(next_id)); - } - let last_partition_id = max_partition_field_id(&spec); - ensure_array_field(metadata, "partition-specs")?.push(spec); - let object = metadata_object_mut(metadata)?; - let current_last = object + let spec_object = spec + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "add-spec spec must be a JSON object"))?; + spec_object.insert("spec-id".to_string(), serde_json::Value::from(spec_id)); + let current_last = metadata .get("last-partition-id") .and_then(serde_json::Value::as_i64) .unwrap_or(999); - object.insert( - "last-partition-id".to_string(), - serde_json::Value::from(current_last.max(last_partition_id)), - ); + let existing_fields = existing_partition_field_ids(metadata)?; + let last_partition_id = assign_partition_field_ids(&mut spec, current_last, &existing_fields)?; + crate::table_catalog::validate_partition_spec_sources_against_current_schema(metadata, &spec).map_err(catalog_store_error)?; + ensure_array_field(metadata, "partition-specs")?.push(spec); + let object = metadata_object_mut(metadata)?; + object.insert("last-partition-id".to_string(), serde_json::Value::from(last_partition_id)); Ok(()) } -fn apply_set_default_spec_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn existing_partition_field_ids(metadata: &serde_json::Value) -> S3Result> { + let mut existing = BTreeMap::new(); + for spec in metadata + .get("partition-specs") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + { + for field in spec.get("fields").and_then(serde_json::Value::as_array).into_iter().flatten() { + let source_id = field + .get("source-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "partition source-id must be an integer"))?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "partition transform must be a string"))?; + let field_id = field + .get("field-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "partition field-id must be an integer"))?; + match existing.insert((source_id, transform.to_string()), field_id) { + Some(previous) if previous != field_id => { + return Err(s3_error!(InvalidRequest, "equivalent partition fields must reuse the same field-id")); + } + _ => {} + } + } + } + Ok(existing) +} + +fn assign_partition_field_ids( + spec: &mut serde_json::Value, + current_last: i64, + existing_fields: &BTreeMap<(i64, String), i64>, +) -> S3Result { + let fields = spec + .get_mut("fields") + .and_then(serde_json::Value::as_array_mut) + .ok_or_else(|| s3_error!(InvalidRequest, "partition spec fields must be an array"))?; + let mut assigned_ids = BTreeSet::new(); + let mut last_partition_id = current_last; + for field in fields.iter() { + let field = field + .as_object() + .ok_or_else(|| s3_error!(InvalidRequest, "partition spec fields must be JSON objects"))?; + let Some(field_id) = field.get("field-id") else { + continue; + }; + let field_id = field_id + .as_i64() + .ok_or_else(|| s3_error!(InvalidRequest, "partition field-id must be an integer"))?; + if i32::try_from(field_id).is_err() || !assigned_ids.insert(field_id) { + return Err(s3_error!(InvalidRequest, "partition field-id must be a unique signed 32-bit integer")); + } + let source_id = field + .get("source-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "partition source-id must be an integer"))?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "partition transform must be a string"))?; + if existing_fields + .get(&(source_id, transform.to_string())) + .is_some_and(|existing_id| *existing_id != field_id) + { + return Err(s3_error!(InvalidRequest, "equivalent partition fields must reuse the same field-id")); + } + last_partition_id = last_partition_id.max(field_id); + } + for field in fields.iter_mut().filter(|field| field.get("field-id").is_none()) { + let source_id = field + .get("source-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "partition source-id must be an integer"))?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| s3_error!(InvalidRequest, "partition transform must be a string"))?; + let field_id = match existing_fields.get(&(source_id, transform.to_string())) { + Some(field_id) => *field_id, + None => { + last_partition_id = last_partition_id + .checked_add(1) + .filter(|field_id| i32::try_from(*field_id).is_ok()) + .ok_or_else(|| s3_error!(InvalidRequest, "partition field-id exceeds the signed 32-bit range"))?; + last_partition_id + } + }; + if !assigned_ids.insert(field_id) { + return Err(s3_error!(InvalidRequest, "partition field-id must be unique within a partition spec")); + } + field + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "partition spec fields must be JSON objects"))? + .insert("field-id".to_string(), serde_json::Value::from(field_id)); + } + Ok(last_partition_id) +} + +fn apply_set_default_spec_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + last_added_spec_id: Option, +) -> S3Result<()> { let requested_id = update .get("spec-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "set-default-spec requires spec-id"))?; - let spec_id = if requested_id == -1 { - last_array_object_i64(metadata, "partition-specs", "spec-id")? - } else { - requested_id - }; + let spec_id = resolve_last_added_update_id(requested_id, last_added_spec_id, "set-default-spec", "add-spec")?; metadata_object_mut(metadata)?.insert("default-spec-id".to_string(), serde_json::Value::from(spec_id)); Ok(()) } -fn apply_add_sort_order_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_add_sort_order_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + sort_order_id: i64, +) -> S3Result { let mut sort_order = update .get("sort-order") .cloned() .ok_or_else(|| s3_error!(InvalidRequest, "add-sort-order requires sort-order"))?; - if !sort_order.is_object() { - return Err(s3_error!(InvalidRequest, "add-sort-order sort-order must be a JSON object")); + let sort_order_object = sort_order + .as_object_mut() + .ok_or_else(|| s3_error!(InvalidRequest, "add-sort-order sort-order must be a JSON object"))?; + let fields_are_empty = sort_order_object + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| s3_error!(InvalidRequest, "sort-order fields must be an array"))? + .is_empty(); + let assigned_id = if fields_are_empty { 0 } else { sort_order_id }; + sort_order_object.insert("order-id".to_string(), serde_json::Value::from(assigned_id)); + crate::table_catalog::validate_sort_order_sources_against_current_schema(metadata, &sort_order) + .map_err(catalog_store_error)?; + let sort_orders = ensure_array_field(metadata, "sort-orders")?; + if assigned_id == 0 { + sort_orders.retain(|order| order.get("order-id").and_then(serde_json::Value::as_i64) != Some(0)); } - if sort_order.get("order-id").is_none() { - let next_id = next_array_object_i64(metadata, "sort-orders", "order-id")?; - sort_order - .as_object_mut() - .ok_or_else(|| s3_error!(InvalidRequest, "add-sort-order sort-order must be a JSON object"))? - .insert("order-id".to_string(), serde_json::Value::from(next_id)); - } - ensure_array_field(metadata, "sort-orders")?.push(sort_order); - Ok(()) + sort_orders.push(sort_order); + Ok(assigned_id) } -fn apply_set_default_sort_order_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_set_default_sort_order_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + last_added_sort_order_id: Option, +) -> S3Result<()> { let requested_id = update .get("sort-order-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "set-default-sort-order requires sort-order-id"))?; - let sort_order_id = if requested_id == -1 { - last_array_object_i64(metadata, "sort-orders", "order-id")? - } else { - requested_id - }; + let sort_order_id = + resolve_last_added_update_id(requested_id, last_added_sort_order_id, "set-default-sort-order", "add-sort-order")?; metadata_object_mut(metadata)?.insert("default-sort-order-id".to_string(), serde_json::Value::from(sort_order_id)); Ok(()) } -fn apply_add_snapshot_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_add_snapshot_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result { let snapshot = update .get("snapshot") .cloned() .ok_or_else(|| s3_error!(InvalidRequest, "add-snapshot requires snapshot"))?; + let format_version = metadata + .get("format-version") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing format-version"))?; + if format_version == 2 && snapshot.get("manifests").is_some() { + return Err(s3_error!(InvalidRequest, "Iceberg v2 snapshots require manifest-list")); + } let snapshot_id = snapshot .get("snapshot-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-id must be an integer"))?; - let sequence_number = snapshot - .get("sequence-number") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot sequence-number must be an integer"))?; - let timestamp_ms = snapshot + let sequence_number = snapshot_sequence_number(&snapshot, format_version)?; + snapshot .get("timestamp-ms") .and_then(serde_json::Value::as_i64) - .unwrap_or_else(current_time_millis); - validate_added_snapshot(metadata, &snapshot, snapshot_id, sequence_number)?; + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot timestamp-ms must be an integer"))?; + validate_added_snapshot(metadata, &snapshot, snapshot_id, sequence_number, format_version)?; ensure_array_field(metadata, "snapshots")?.push(snapshot); - let object = metadata_object_mut(metadata)?; - object.insert("last-sequence-number".to_string(), serde_json::Value::from(sequence_number)); - object.insert("current-snapshot-id".to_string(), serde_json::Value::from(snapshot_id)); - ensure_array_field(metadata, "snapshot-log")?.push(serde_json::json!({ - "timestamp-ms": timestamp_ms, - "snapshot-id": snapshot_id - })); - Ok(()) + if format_version > 1 { + metadata_object_mut(metadata)?.insert("last-sequence-number".to_string(), serde_json::Value::from(sequence_number)); + } + Ok(snapshot_id) +} + +fn snapshot_sequence_number(snapshot: &serde_json::Value, format_version: i64) -> S3Result { + let sequence_number = match snapshot.get("sequence-number") { + Some(sequence_number) => sequence_number + .as_i64() + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot sequence-number must be an integer")), + None if format_version == 1 => Ok(0), + None => Err(s3_error!(InvalidRequest, "Iceberg v2 snapshot sequence-number is required")), + }?; + if format_version == 1 && sequence_number != 0 { + return Err(s3_error!(InvalidRequest, "Iceberg v1 snapshot sequence-number must be zero")); + } + Ok(sequence_number) +} + +fn snapshot_parent_id(snapshot: &serde_json::Value) -> S3Result> { + snapshot + .get("parent-snapshot-id") + .map(|parent_snapshot_id| { + parent_snapshot_id + .as_i64() + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot parent-snapshot-id must be an integer")) + }) + .transpose() } fn validate_added_snapshot( @@ -3647,6 +4150,7 @@ fn validate_added_snapshot( snapshot: &serde_json::Value, snapshot_id: i64, sequence_number: i64, + format_version: i64, ) -> S3Result<()> { if metadata .get("snapshots") @@ -3657,22 +4161,31 @@ fn validate_added_snapshot( .any(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) }) { - return Err(s3_error!(PreconditionFailed, "snapshot id already exists")); + return Err(commit_requirement_failed("snapshot id already exists")); } - let current_snapshot_id = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); - if let Some(parent_snapshot_id) = snapshot.get("parent-snapshot-id").and_then(serde_json::Value::as_i64) - && Some(parent_snapshot_id) != current_snapshot_id + let parent_snapshot_id = snapshot_parent_id(snapshot)?; + if let Some(parent_snapshot_id) = parent_snapshot_id + && !metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .is_some_and(|snapshots| { + snapshots + .iter() + .any(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(parent_snapshot_id)) + }) { - return Err(s3_error!(PreconditionFailed, "snapshot parent no longer matches current snapshot")); + return Err(commit_requirement_failed("snapshot parent does not exist")); } - let current_sequence_number = metadata - .get("last-sequence-number") - .and_then(serde_json::Value::as_i64) - .unwrap_or_default(); - if sequence_number <= current_sequence_number { - return Err(s3_error!(PreconditionFailed, "snapshot sequence number must advance")); + if format_version > 1 { + let current_sequence_number = metadata + .get("last-sequence-number") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "current Iceberg v2 metadata is missing last-sequence-number"))?; + if sequence_number <= current_sequence_number { + return Err(commit_requirement_failed("snapshot sequence number must advance")); + } } if !snapshot_has_manifest_references(snapshot) { @@ -3685,7 +4198,7 @@ fn validate_added_snapshot( .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "snapshot summary.operation is required"))?; if !matches!(operation, "append" | "overwrite" | "delete" | "replace") { - return Err(s3_error!(NotImplemented, "unsupported snapshot operation: {operation}")); + return Err(s3_error!(InvalidRequest, "unsupported snapshot operation: {operation}")); } Ok(()) @@ -3785,31 +4298,56 @@ async fn validate_table_snapshot_commit_conflicts( where B: crate::table_catalog::TableCatalogObjectBackend, { - let Some(snapshot) = added_snapshot_update(updates)? else { - return Ok(()); - }; + let mut snapshot_state = current_metadata.clone(); + for update in updates { + match update.get("action").and_then(serde_json::Value::as_str) { + Some("add-snapshot") => { + let snapshot = update + .get("snapshot") + .ok_or_else(|| s3_error!(InvalidRequest, "add-snapshot requires snapshot"))?; + validate_snapshot_file_conflicts(metadata_backend, bucket, entry, &snapshot_state, snapshot).await?; + apply_add_snapshot_update(&mut snapshot_state, update)?; + } + Some("remove-snapshots") => apply_remove_snapshots_update(&mut snapshot_state, update)?, + _ => {} + } + } + Ok(()) +} + +async fn validate_snapshot_file_conflicts( + metadata_backend: &B, + bucket: &str, + entry: &crate::table_catalog::TableEntry, + snapshot_state: &serde_json::Value, + snapshot: &serde_json::Value, +) -> S3Result<()> +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ let snapshot_id = snapshot .get("snapshot-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-id must be an integer"))?; - let sequence_number = snapshot - .get("sequence-number") + let format_version = snapshot_state + .get("format-version") .and_then(serde_json::Value::as_i64) - .ok_or_else(|| s3_error!(InvalidRequest, "snapshot sequence-number must be an integer"))?; + .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing format-version"))?; + let sequence_number = snapshot_sequence_number(snapshot, format_version)?; let operation = snapshot .get("summary") .and_then(|summary| summary.get("operation")) .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "snapshot summary.operation is required"))?; - - let current_live_files = load_current_snapshot_live_files(metadata_backend, bucket, entry, current_metadata).await?; + let parent_snapshot_id = snapshot_parent_id(snapshot)?; + let parent_live_files = load_snapshot_live_files(metadata_backend, bucket, entry, snapshot_state, parent_snapshot_id).await?; let changes = load_snapshot_file_changes( metadata_backend, bucket, entry, snapshot, SnapshotChangeContext { - current_live_files: ¤t_live_files, + current_live_files: &parent_live_files, snapshot_id, sequence_number, }, @@ -3817,10 +4355,9 @@ where .await?; for location in changes.added_data_files.iter().chain(changes.added_delete_files.iter()) { - if current_live_files.contains(location) { - return Err(s3_error!( - PreconditionFailed, - "commit requirement failed: added file already exists in current snapshot" + if parent_live_files.contains(location) { + return Err(commit_requirement_failed( + "commit requirement failed: added file already exists in parent snapshot", )); } } @@ -3832,12 +4369,8 @@ where } } "overwrite" | "delete" | "replace" => { - if current_metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_none() - { - return Err(s3_error!(InvalidRequest, "row-level snapshot operation requires a current snapshot")); + if parent_snapshot_id.is_none() { + return Err(s3_error!(InvalidRequest, "row-level snapshot operation requires a parent snapshot")); } if operation == "overwrite" { if !changes.has_any_change() { @@ -3850,48 +4383,30 @@ where )); } for location in changes.deleted_data_files.iter().chain(changes.deleted_delete_files.iter()) { - if !current_live_files.contains(location) { - return Err(s3_error!(PreconditionFailed, "commit requirement failed: deleted file is not current")); + if !parent_live_files.contains(location) { + return Err(commit_requirement_failed( + "commit requirement failed: deleted file is not in the parent snapshot", + )); } } } - _ => return Err(s3_error!(NotImplemented, "unsupported snapshot operation: {operation}")), + _ => return Err(s3_error!(InvalidRequest, "unsupported snapshot operation: {operation}")), } Ok(()) } -fn added_snapshot_update(updates: &[serde_json::Value]) -> S3Result> { - let mut snapshot = None; - for update in updates { - if update.get("action").and_then(serde_json::Value::as_str) != Some("add-snapshot") { - continue; - } - if snapshot.is_some() { - return Err(s3_error!(InvalidRequest, "standard commit supports one add-snapshot update")); - } - snapshot = Some( - update - .get("snapshot") - .ok_or_else(|| s3_error!(InvalidRequest, "add-snapshot requires snapshot"))?, - ); - } - Ok(snapshot) -} - -async fn load_current_snapshot_live_files( +async fn load_snapshot_live_files( metadata_backend: &B, bucket: &str, entry: &crate::table_catalog::TableEntry, current_metadata: &serde_json::Value, + snapshot_id: Option, ) -> S3Result where B: crate::table_catalog::TableCatalogObjectBackend, { - let Some(current_snapshot_id) = current_metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64) - else { + let Some(snapshot_id) = snapshot_id else { return Ok(SnapshotLiveFiles::default()); }; let snapshot = current_metadata @@ -3900,9 +4415,9 @@ where .and_then(|snapshots| { snapshots .iter() - .find(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(current_snapshot_id)) + .find(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) }) - .ok_or_else(|| s3_error!(InvalidRequest, "current snapshot metadata is missing"))?; + .ok_or_else(|| commit_requirement_failed("commit requirement failed: parent snapshot no longer exists"))?; let mut live_files = SnapshotLiveFiles::default(); for manifest in read_snapshot_manifest_references(metadata_backend, bucket, entry, snapshot).await? { @@ -4204,7 +4719,12 @@ fn table_commit_object_key( Ok(object_key) } -fn apply_set_snapshot_ref_update(metadata: &mut serde_json::Value, update: &serde_json::Value) -> S3Result<()> { +fn apply_set_snapshot_ref_update( + metadata: &mut serde_json::Value, + update: &serde_json::Value, + added_snapshot_ids: &BTreeSet, + commit_timestamp_ms: i64, +) -> S3Result<()> { let ref_name = update .get("ref-name") .and_then(serde_json::Value::as_str) @@ -4220,9 +4740,47 @@ fn apply_set_snapshot_ref_update(metadata: &mut serde_json::Value, update: &serd .filter(|(key, _)| key.as_str() != "action" && key.as_str() != "ref-name") .map(|(key, value)| (key.clone(), value.clone())) .collect::>(); - ensure_object_field(metadata, "refs")?.insert(ref_name.to_string(), serde_json::Value::Object(reference)); + if !metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .is_some_and(|snapshots| { + snapshots + .iter() + .any(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) + }) + { + return Err(s3_error!(InvalidRequest, "set-snapshot-ref targets an unknown snapshot")); + } + let next_reference = serde_json::Value::Object(reference); + let unchanged = metadata + .get("refs") + .and_then(serde_json::Value::as_object) + .and_then(|refs| refs.get(ref_name)) + == Some(&next_reference); + ensure_object_field(metadata, "refs")?.insert(ref_name.to_string(), next_reference); if ref_name == "main" { metadata_object_mut(metadata)?.insert("current-snapshot-id".to_string(), serde_json::Value::from(snapshot_id)); + if !unchanged { + let timestamp_ms = if added_snapshot_ids.contains(&snapshot_id) { + metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .and_then(|snapshots| { + snapshots + .iter() + .find(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) + }) + .and_then(|snapshot| snapshot.get("timestamp-ms")) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot timestamp-ms must be an integer"))? + } else { + commit_timestamp_ms + }; + ensure_array_field(metadata, "snapshot-log")?.push(serde_json::json!({ + "timestamp-ms": timestamp_ms, + "snapshot-id": snapshot_id + })); + } } Ok(()) } @@ -4233,19 +4791,75 @@ fn apply_remove_snapshots_update(metadata: &mut serde_json::Value, update: &serd .and_then(serde_json::Value::as_array) .ok_or_else(|| s3_error!(InvalidRequest, "remove-snapshots requires snapshot-ids"))? .iter() - .filter_map(serde_json::Value::as_i64) - .collect::>(); - ensure_array_field(metadata, "snapshots")?.retain(|snapshot| { + .map(|snapshot_id| { + snapshot_id + .as_i64() + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-ids must contain integers")) + }) + .collect::>>()?; + let snapshots = ensure_array_field(metadata, "snapshots")?; + let snapshot_count = snapshots.len(); + snapshots.retain(|snapshot| { snapshot .get("snapshot-id") .and_then(serde_json::Value::as_i64) .is_none_or(|snapshot_id| !ids.contains(&snapshot_id)) }); - ensure_array_field(metadata, "snapshot-log")?.retain(|log| { - log.get("snapshot-id") + let removed_snapshot = snapshots.len() != snapshot_count; + if removed_snapshot { + let remaining_snapshot_ids = snapshots + .iter() + .filter_map(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64)) + .collect::>(); + let snapshot_log = ensure_array_field(metadata, "snapshot-log")?; + let previous_log = std::mem::take(snapshot_log); + for log in previous_log { + let snapshot_id = log + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "snapshot-log snapshot-id must be an integer"))?; + if remaining_snapshot_ids.contains(&snapshot_id) { + snapshot_log.push(log); + } else { + snapshot_log.clear(); + } + } + } + let dangling_refs = metadata + .get("refs") + .and_then(serde_json::Value::as_object) + .into_iter() + .flat_map(|refs| refs.iter()) + .filter_map(|(name, reference)| { + reference + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .filter(|snapshot_id| ids.contains(snapshot_id)) + .map(|_| name.clone()) + }) + .collect::>(); + let removed_main = dangling_refs.iter().any(|name| name == "main") + || metadata + .get("current-snapshot-id") .and_then(serde_json::Value::as_i64) - .is_none_or(|snapshot_id| !ids.contains(&snapshot_id)) - }); + .is_some_and(|snapshot_id| ids.contains(&snapshot_id)); + let refs = ensure_object_field(metadata, "refs")?; + for name in dangling_refs { + refs.remove(&name); + } + if removed_main { + metadata_object_mut(metadata)?.insert("current-snapshot-id".to_string(), serde_json::Value::from(-1)); + } + for field in ["statistics", "partition-statistics"] { + if let Some(values) = metadata.get_mut(field).and_then(serde_json::Value::as_array_mut) { + values.retain(|value| { + value + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .is_none_or(|snapshot_id| !ids.contains(&snapshot_id)) + }); + } + } Ok(()) } @@ -4254,7 +4868,10 @@ fn apply_remove_snapshot_ref_update(metadata: &mut serde_json::Value, update: &s .get("ref-name") .and_then(serde_json::Value::as_str) .ok_or_else(|| s3_error!(InvalidRequest, "remove-snapshot-ref requires ref-name"))?; - ensure_object_field(metadata, "refs")?.remove(ref_name); + let removed = ensure_object_field(metadata, "refs")?.remove(ref_name).is_some(); + if removed && ref_name == "main" { + metadata_object_mut(metadata)?.insert("current-snapshot-id".to_string(), serde_json::Value::from(-1)); + } Ok(()) } @@ -4340,8 +4957,47 @@ fn ensure_object_field<'a>( .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {key} must be an object")) } +fn validate_commit_item_count(label: &str, count: usize, max_count: usize) -> S3Result<()> { + if count > max_count { + return Err(s3_error!(InvalidRequest, "{label} exceeds the maximum count of {max_count}")); + } + Ok(()) +} + +fn validate_rest_commit_item_counts(requirements: &[serde_json::Value], updates: &[serde_json::Value]) -> S3Result<()> { + validate_commit_item_count("commit requirements", requirements.len(), TABLE_CATALOG_COMMIT_REQUIREMENT_MAX_COUNT)?; + validate_commit_item_count("commit updates", updates.len(), TABLE_CATALOG_COMMIT_UPDATE_MAX_COUNT) +} + +fn next_catalog_id_for_updates( + metadata: &serde_json::Value, + updates: &[serde_json::Value], + action: &str, + array_key: &str, + id_key: &str, +) -> S3Result> { + updates + .iter() + .any(|update| update.get("action").and_then(serde_json::Value::as_str) == Some(action)) + .then(|| next_array_object_i64(metadata, array_key, id_key)) + .transpose() +} + +fn take_catalog_assigned_id(next_id: &mut Option, label: &str) -> S3Result { + let next_id = next_id + .as_mut() + .ok_or_else(|| s3_error!(InternalError, "catalog-assigned {label} state is missing"))?; + let assigned_id = *next_id; + *next_id = next_id + .checked_add(1) + .ok_or_else(|| s3_error!(InvalidRequest, "catalog-assigned {label} exceeds the signed 64-bit range"))?; + Ok(assigned_id) +} + fn next_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result { - Ok(last_array_object_i64(metadata, array_key, id_key)?.saturating_add(1)) + last_array_object_i64(metadata, array_key, id_key)? + .checked_add(1) + .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} {id_key} exceeds the signed 64-bit range")) } fn last_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result { @@ -4356,6 +5012,18 @@ fn last_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: .ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} has no {id_key}")) } +fn resolve_last_added_update_id( + requested_id: i64, + last_added_id: Option, + update_action: &str, + required_action: &str, +) -> S3Result { + if requested_id != -1 { + return Ok(requested_id); + } + last_added_id.ok_or_else(|| s3_error!(InvalidRequest, "{update_action} id -1 requires a preceding {required_action} update")) +} + fn table_commit_operation(metadata: &serde_json::Value) -> String { metadata .get("snapshots") @@ -4392,6 +5060,18 @@ fn iceberg_rest_error(error_type: &str, status: StatusCode, message: impl Into) -> S3Error { + iceberg_rest_error(ICEBERG_ERROR_COMMIT_FAILED, StatusCode::CONFLICT, message) +} + +fn persisted_metadata_error(entity: &str) -> S3Error { + iceberg_rest_error( + ICEBERG_ERROR_REST, + StatusCode::INTERNAL_SERVER_ERROR, + format!("persisted {entity} metadata is invalid"), + ) +} + fn catalog_store_error(err: crate::table_catalog::TableCatalogStoreError) -> S3Error { match err { crate::table_catalog::TableCatalogStoreError::NotFound(message) => { @@ -4687,6 +5367,15 @@ where { let (entry, metadata) = view_entry_from_create_view_request(bucket, namespace, request)?; ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; + crate::table_catalog::TableCommitPublication::begin_table_bucket(metadata_backend, bucket) + .await + .map_err(catalog_store_error)?; + if !crate::table_catalog::TableCommitPublication::holds_table_bucket(metadata_backend, bucket) { + return Err(catalog_store_error(crate::table_catalog::TableCatalogStoreError::Internal( + "view creation requires a table-bucket publication fence".to_string(), + ))); + } + let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend); let metadata_data = serde_json::to_vec(&metadata) .map_err(|err| s3_error!(InternalError, "failed to serialize initial view metadata: {}", err))?; metadata_backend @@ -4699,7 +5388,7 @@ where .await .map_err(catalog_store_already_exists_error)?; store - .create_view(entry.clone()) + .create_view_with_publication(entry.clone(), metadata_backend) .await .map_err(catalog_store_already_exists_error)?; Ok(load_view_response_from_entry(entry, metadata)) @@ -4719,6 +5408,17 @@ async fn read_table_metadata_json( Ok(metadata) } +async fn read_persisted_metadata_json( + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + metadata_location: &str, + entity: &str, +) -> S3Result { + read_table_metadata_json(metadata_backend, bucket, metadata_location) + .await + .map_err(|_| persisted_metadata_error(entity)) +} + async fn read_generated_table_metadata_json( metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, bucket: &str, @@ -4815,7 +5515,7 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + let metadata = read_persisted_table_metadata_for_entry(metadata_backend, &entry, &entry.metadata_location, true).await?; Ok(load_table_response_from_entry(entry, metadata)) } @@ -4866,7 +5566,13 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_VIEW, StatusCode::NOT_FOUND, "view not found")); }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + let view_name = + crate::table_catalog::IdentifierSegment::parse(view.to_string()).map_err(|_| persisted_metadata_error("view"))?; + if !crate::table_catalog::is_valid_view_metadata_location(namespace, &view_name, &entry.metadata_location) { + return Err(persisted_metadata_error("view")); + } + let metadata = read_persisted_metadata_json(metadata_backend, bucket, &entry.metadata_location, "view").await?; + validate_persisted_view_metadata(&entry, &metadata)?; Ok(load_view_response_from_entry(entry, metadata)) } @@ -4898,6 +5604,8 @@ async fn replace_view_response( where S: crate::table_catalog::TableCatalogStore + ?Sized, { + validate_rest_commit_item_counts(&request.requirements, &request.updates)?; + validate_rest_commit_identifier(request.identifier.as_ref(), namespace, view)?; let Some(current) = store .load_view(bucket, &namespace.public_name(), view) .await @@ -4905,23 +5613,33 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_VIEW, StatusCode::NOT_FOUND, "view not found")); }; - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_view_commit_requirements(¤t_metadata, &request.requirements)?; let view_name = crate::table_catalog::IdentifierSegment::parse(view.to_string()) .map_err(|err| s3_error!(InvalidRequest, "invalid view name: {}", err))?; + if !crate::table_catalog::is_valid_view_metadata_location(namespace, &view_name, ¤t.metadata_location) { + return Err(persisted_metadata_error("view")); + } + let current_metadata = read_persisted_metadata_json(metadata_backend, bucket, ¤t.metadata_location, "view").await?; + if request.new_metadata_location.is_some() { + validate_persisted_view_metadata_identity(¤t, ¤t_metadata)?; + } else { + validate_persisted_view_metadata(¤t, ¤t_metadata)?; + } + validate_view_commit_requirements(¤t_metadata, &request.requirements)?; let (next_metadata_location, next_metadata) = if let Some(new_metadata_location) = request.new_metadata_location { + let new_metadata_location = table_metadata_location_for_catalog(bucket, &new_metadata_location)?; if !crate::table_catalog::is_valid_view_metadata_location(namespace, &view_name, &new_metadata_location) { return Err(s3_error!(InvalidRequest, "metadata location must be inside the view metadata directory")); } let target_metadata = read_table_metadata_json(metadata_backend, bucket, &new_metadata_location).await?; + validate_supported_view_metadata(&target_metadata)?; validate_metadata_view_location_in_bucket(bucket, &target_metadata)?; validate_metadata_matches_current_view_metadata(¤t_metadata, &target_metadata)?; (new_metadata_location, target_metadata) } else { - let next_metadata = apply_view_commit_updates(current_metadata.clone(), &request.updates, ¤t.metadata_location)?; + let next_metadata = apply_view_commit_updates_at(current_metadata.clone(), &request.updates, current_time_millis())?; validate_metadata_view_location_in_bucket(bucket, &next_metadata)?; validate_metadata_matches_current_view_metadata(¤t_metadata, &next_metadata)?; - let (_, metadata_file_token) = standard_commit_ids(request.commit_id); + let (_, metadata_file_token) = standard_commit_ids(None); let next_generation = current.generation.saturating_add(1); let next_metadata_location = crate::table_catalog::default_view_metadata_file_path( namespace, @@ -4942,19 +5660,29 @@ where (next_metadata_location, next_metadata) }; + let expected_metadata_location = request + .expected_metadata_location + .as_deref() + .map(|location| table_metadata_location_for_catalog(bucket, location)) + .transpose()? + .unwrap_or_else(|| current.metadata_location.clone()); + let table_bucket_fence_required = metadata_table_location(&next_metadata)? != current.warehouse_location; + let result = store - .replace_view(crate::table_catalog::ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - view: view.to_string(), - expected_version_token: request - .expected_version_token - .unwrap_or_else(|| current.version_token.clone()), - expected_metadata_location: request - .expected_metadata_location - .unwrap_or_else(|| current.metadata_location.clone()), - new_metadata_location: next_metadata_location, - }) + .replace_view_with_publication( + crate::table_catalog::ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.to_string(), + expected_version_token: request + .expected_version_token + .unwrap_or_else(|| current.version_token.clone()), + expected_metadata_location, + new_metadata_location: next_metadata_location, + }, + table_bucket_fence_required, + metadata_backend, + ) .await .map_err(catalog_store_error)?; Ok(load_view_response_from_entry(result.view, next_metadata)) @@ -5074,8 +5802,14 @@ where let previous_metadata_location = existing_commit .as_ref() .map_or_else(|| current.metadata_location.clone(), |commit| commit.previous_metadata_location.clone()); - let previous_metadata = read_table_metadata_json(metadata_backend, bucket, &previous_metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &previous_metadata)?; + let require_current_warehouse = existing_commit.is_none(); + let previous_metadata = read_persisted_table_metadata_for_entry( + metadata_backend, + ¤t, + &previous_metadata_location, + require_current_warehouse, + ) + .await?; let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; let table_bucket_fence_required = table_warehouse_location_changes(¤t, &target_metadata)?; @@ -5129,6 +5863,8 @@ async fn commit_table_response( where S: crate::table_catalog::TableCatalogStore + ?Sized, { + validate_rest_commit_item_counts(&request.requirements, &request.updates)?; + validate_rest_commit_identifier(request.identifier.as_ref(), namespace, table)?; if request.new_metadata_location.is_none() { return standard_commit_table_response(store, metadata_backend, bucket, namespace, table, request).await; } @@ -5151,8 +5887,8 @@ where if !crate::table_catalog::is_valid_table_metadata_location_for_entry(¤t, &request.new_metadata_location) { return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + let current_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, ¤t, ¤t.metadata_location, true).await?; let target_metadata = read_table_metadata_json(metadata_backend, bucket, &request.new_metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; let table_bucket_fence_required = table_warehouse_location_changes(¤t, &target_metadata)?; @@ -5165,9 +5901,13 @@ where "commit retry does not match the original request", )); } - let previous_metadata = - read_table_metadata_json(metadata_backend, bucket, &existing_commit.previous_metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &previous_metadata)?; + let previous_metadata = read_persisted_table_metadata_for_entry( + metadata_backend, + ¤t, + &existing_commit.previous_metadata_location, + false, + ) + .await?; validate_table_commit_requirements(&previous_metadata, &client_requirements)?; validate_metadata_matches_current_metadata(&previous_metadata, &target_metadata)?; validate_table_metadata_snapshot_graph(metadata_backend, bucket, ¤t, Some(&previous_metadata), &target_metadata) @@ -5207,7 +5947,8 @@ where { return Ok(response); } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + let current_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, ¤t, ¤t.metadata_location, true).await?; validate_table_commit_requirements(¤t_metadata, &request.requirements)?; let expected_metadata = current_metadata.clone(); let previous_metadata_location = table_metadata_location_for_client(bucket, ¤t.metadata_location); @@ -5395,6 +6136,7 @@ async fn commit_table_replay_response( } read_table_metadata_json(metadata_backend, bucket, &result.table.metadata_location).await? }; + validate_persisted_table_metadata(&result.table, &metadata, true)?; Ok(commit_table_response_from_result(result, metadata)) } @@ -5427,8 +6169,8 @@ where )); } - let previous_metadata = read_table_metadata_json(metadata_backend, bucket, &commit.previous_metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, &previous_metadata)?; + let previous_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, current, &commit.previous_metadata_location, false).await?; validate_table_commit_requirements(&previous_metadata, &request.requirements)?; let target_metadata = read_table_metadata_json(metadata_backend, bucket, &commit.new_metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; @@ -5692,7 +6434,8 @@ where return Ok(report); } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; + let current_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, ¤t, ¤t.metadata_location, true).await?; let updates = [serde_json::json!({ "action": "remove-snapshots", "snapshot-ids": expired_snapshot_ids.clone() @@ -5758,7 +6501,7 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + let metadata = read_persisted_table_metadata_for_entry(metadata_backend, &entry, &entry.metadata_location, true).await?; let current_snapshot_id = metadata.get("current-snapshot-id").and_then(serde_json::Value::as_i64); let refs = metadata .get("refs") @@ -5835,7 +6578,7 @@ where namespace, table, RestCommitTableRequest { - _identifier: None, + identifier: None, commit_id: request.commit_id, idempotency_key: request.idempotency_key, operation: Some("set-snapshot-ref".to_string()), @@ -5872,7 +6615,7 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); }; - let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; + let metadata = read_persisted_table_metadata_for_entry(metadata_backend, &entry, &entry.metadata_location, true).await?; let reference = metadata .get("refs") .and_then(serde_json::Value::as_object) @@ -5895,7 +6638,7 @@ where namespace, table, RestCommitTableRequest { - _identifier: None, + identifier: None, commit_id: request.commit_id, idempotency_key: request.idempotency_key, operation: Some("remove-snapshot-ref".to_string()), @@ -5960,6 +6703,18 @@ fn external_catalog_bridge_response_from_entry( } } +async fn read_persisted_table_metadata_for_entry( + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + entry: &crate::table_catalog::TableEntry, + metadata_location: &str, + require_current_warehouse: bool, +) -> S3Result { + validate_persisted_table_metadata_location(entry, metadata_location)?; + let metadata = read_persisted_metadata_json(metadata_backend, &entry.table_bucket, metadata_location, "table").await?; + validate_persisted_table_metadata(entry, &metadata, require_current_warehouse)?; + Ok(metadata) +} + fn external_catalog_bridge_capabilities() -> Vec { EXTERNAL_CATALOG_BRIDGE_CAPABILITIES .iter() @@ -6208,8 +6963,8 @@ where .expected_metadata_location .clone() .ok_or_else(|| s3_error!(InvalidRequest, "external catalog sync requires expected-metadata-location"))?; - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + let current_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, ¤t, ¤t.metadata_location, true).await?; validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; validate_table_metadata_snapshot_graph(metadata_backend, bucket, ¤t, Some(¤t_metadata), &target_metadata) .await?; @@ -6349,8 +7104,8 @@ where if !crate::table_catalog::is_valid_table_metadata_location_for_entry(¤t, &metadata_location) { return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + let current_metadata = + read_persisted_table_metadata_for_entry(metadata_backend, ¤t, ¤t.metadata_location, true).await?; let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; diff --git a/rustfs/src/admin/handlers/table_catalog/table.rs b/rustfs/src/admin/handlers/table_catalog/table.rs index 21376fd81..b98b2fbfd 100644 --- a/rustfs/src/admin/handlers/table_catalog/table.rs +++ b/rustfs/src/admin/handlers/table_catalog/table.rs @@ -125,7 +125,9 @@ impl Operation for RestLoadTableHandler { ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = load_table_response(&store, &metadata_backend, &warehouse, &namespace, &table).await?; + let snapshot_selection = rest_table_snapshot_selection_from_query(&req.uri)?; + let mut response = load_table_response(&store, &metadata_backend, &warehouse, &namespace, &table).await?; + apply_rest_table_snapshot_selection(&mut response.metadata, snapshot_selection); build_json_response(StatusCode::OK, &response) } } @@ -158,7 +160,7 @@ impl Operation for RestCommitTableHandler { let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; install_table_catalog_s3_request_info(&mut req, &principal)?; ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; - let request = read_json_body::(std::mem::take(&mut req.input)).await?; + let request = read_rest_commit_table_request(std::mem::take(&mut req.input)).await?; let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); @@ -178,6 +180,14 @@ impl Operation for RestDropTableHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableAction).await?; + let purge_requested = rest_purge_requested_from_query(&req.uri)?; + if purge_requested { + return Err(iceberg_rest_error( + ICEBERG_ERROR_UNSUPPORTED_OPERATION, + StatusCode::NOT_ACCEPTABLE, + "purgeRequested=true is not supported", + )); + } ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let store = table_catalog_store_from_extensions(&req.extensions)?; drop_table_in_store(&store, &warehouse, &namespace, &table).await?; diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index 454d12bad..013ae9304 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -166,106 +166,8 @@ fn catalog_config_response_lists_standard_rest_endpoints() { assert_eq!(response.admin_discovery.runtime_capabilities, "/rustfs/admin/v4/runtime/capabilities"); assert_eq!(response.admin_discovery.cluster_snapshot, "/rustfs/admin/v4/cluster/snapshot"); assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog"); - assert!(response.endpoints.contains(&"GET /v1/{prefix}/namespaces")); - assert!(response.endpoints.contains(&"GET /{warehouse}/catalog/migration")); - assert!(response.endpoints.contains(&"POST /{warehouse}/catalog/migration")); - assert!(response.endpoints.contains(&"DELETE /{warehouse}/catalog/migration")); - assert!(response.endpoints.contains(&"HEAD /v1/{prefix}/namespaces/{namespace}")); - assert!( - response - .endpoints - .contains(&"GET /v1/{prefix}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials") - ); - assert!(response.endpoints.contains(&"GET /{warehouse}/namespaces")); - assert!(response.endpoints.contains(&"POST /{warehouse}/namespaces")); - assert!(response.endpoints.contains(&"HEAD /{warehouse}/namespaces/{namespace}")); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/register") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables") - ); - assert!(response.endpoints.contains(&"GET /{warehouse}/namespaces/{namespace}/views")); - assert!(response.endpoints.contains(&"POST /{warehouse}/namespaces/{namespace}/views")); - assert!( - response - .endpoints - .contains(&"HEAD /{warehouse}/namespaces/{namespace}/views/{view}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"HEAD /{warehouse}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/credentials") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/views/{view}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/refs") - ); - assert!( - response - .endpoints - .contains(&"PUT /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}") - ); - assert!( - response - .endpoints - .contains(&"DELETE /{warehouse}/namespaces/{namespace}/tables/{table}/refs/{ref}") - ); - assert!( - response - .endpoints - .contains(&"GET /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external") - ); - assert!( - response - .endpoints - .contains(&"PUT /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/external/sync") - ); - assert!( - response - .endpoints - .contains(&"POST /{warehouse}/namespaces/{namespace}/tables/{table}/catalog/recovery") - ); + assert_eq!(response.endpoints.as_slice(), TABLE_CATALOG_ENDPOINTS); + assert!(response.endpoints.iter().all(|endpoint| endpoint.contains("/v1/{prefix}/"))); } #[test] @@ -313,6 +215,31 @@ fn warehouse_config_query_rejects_empty_and_repeated_values() { assert!(warehouse_from_config_query(&uri).is_err()); } +#[test] +fn drop_table_purge_query_is_explicit_and_strict() { + for (uri, expected) in [ + ("/iceberg/v1/analytics/namespaces/sales/tables/orders", false), + ("/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=false", false), + ("/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=true", true), + ("/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=False", false), + ("/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=True", true), + ] { + assert_eq!( + rest_purge_requested_from_query(&uri.parse().expect("URI")).expect("purge query should parse"), + expected + ); + } + + for uri in [ + "/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=1", + "/iceberg/v1/analytics/namespaces/sales/tables/orders?purgeRequested=true&purgeRequested=false", + ] { + let error = rest_purge_requested_from_query(&uri.parse().expect("URI")).expect_err("invalid purge query should fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); + assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); + } +} + #[test] fn catalog_conflicts_use_operation_specific_iceberg_errors() { let already_exists = catalog_store_already_exists_error(crate::table_catalog::TableCatalogStoreError::Conflict( @@ -521,6 +448,22 @@ fn table_catalog_handlers_require_table_admin_actions() { } } +#[test] +fn standard_rest_handlers_wire_strict_response_guards() { + let src = table_catalog_handler_source(); + let drop_table = operation_block(&src, "RestDropTableHandler"); + assert!(drop_table.contains("rest_purge_requested_from_query(&req.uri)?")); + assert!(drop_table.contains("if purge_requested")); + assert!(drop_table.contains("StatusCode::NOT_ACCEPTABLE")); + + let load_table = operation_block(&src, "RestLoadTableHandler"); + assert!(load_table.contains("rest_table_snapshot_selection_from_query(&req.uri)?")); + assert!(load_table.contains("apply_rest_table_snapshot_selection(&mut response.metadata, snapshot_selection);")); + + let credentials = operation_block(&src, "RestLoadCredentialsHandler"); + assert!(credentials.contains("build_sensitive_json_response(StatusCode::OK, &response)")); +} + #[test] fn table_bucket_handlers_resolve_state_from_the_request_context() { let src = table_catalog_handler_source(); @@ -580,6 +523,7 @@ fn table_pointer_write_handlers_install_commit_publication_guard() { "ImportTableCatalogHandler", "PutTableRefHandler", "DeleteTableRefHandler", + "RestReplaceViewHandler", "RollbackTableCatalogHandler", "SyncExternalCatalogBridgeHandler", ] { @@ -1345,6 +1289,32 @@ async fn namespace_property_update_body_is_bounded_and_required() { .expect("maximum valid namespace properties should remain within the domain limit"); } +#[tokio::test(start_paused = true)] +async fn generic_json_body_readers_time_out_stalled_streams() { + let required_stream = futures::stream::pending::, std::io::Error>>(); + let required = tokio::spawn(read_json_body::(Body::http_body(http_body_util::StreamBody::new( + required_stream, + )))); + let optional_stream = futures::stream::pending::, std::io::Error>>(); + let optional = tokio::spawn(read_json_body_or_default::(Body::http_body( + http_body_util::StreamBody::new(optional_stream), + ))); + tokio::task::yield_now().await; + tokio::time::advance(TABLE_CATALOG_REQUEST_BODY_TIMEOUT).await; + + let required_error = required + .await + .expect("required body task should complete") + .expect_err("stalled required body should time out"); + let optional_error = optional + .await + .expect("optional body task should complete") + .expect_err("stalled optional body should time out"); + + assert_eq!(required_error.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(optional_error.code(), &S3ErrorCode::InvalidRequest); +} + #[test] fn list_tables_response_uses_rest_identifier_shape() { let namespace = crate::table_catalog::Namespace::parse("analytics.daily_events").expect("namespace should parse"); @@ -1930,9 +1900,356 @@ fn create_table_request_honors_supported_format_version_property() { assert_eq!(metadata["current-schema-id"], 0); assert!(metadata.get("partition-specs").is_some()); assert!(metadata.get("sort-orders").is_some()); + assert_eq!(metadata["sort-orders"][0]["order-id"], 0); + assert_eq!(metadata["default-sort-order-id"], 0); assert!(metadata.get("last-sequence-number").is_none()); } +#[test] +fn catalog_assigns_read_only_schema_spec_and_sort_order_ids() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 41, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "partition-spec": { + "spec-id": 42, + "fields": [{"source-id": 1, "name": "id", "transform": "identity"}] + }, + "write-order": { + "order-id": 43, + "fields": [{ + "source-id": 1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + }, + "properties": {} + })) + .expect("create table request should parse"); + let (_, metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("table metadata should be created"); + + assert_eq!(metadata["schemas"][0]["schema-id"], 0); + assert_eq!(metadata["current-schema-id"], 0); + assert_eq!(metadata["partition-specs"][0]["spec-id"], 0); + assert_eq!(metadata["partition-specs"][0]["fields"][0]["field-id"], 1000); + assert_eq!(metadata["default-spec-id"], 0); + assert_eq!(metadata["last-partition-id"], 1000); + assert_eq!(metadata["sort-orders"][0]["order-id"], 1); + assert_eq!(metadata["default-sort-order-id"], 1); + + let updated = apply_table_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "add-schema", + "schema": { + "type": "struct", + "schema-id": 41, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + } + }), + serde_json::json!({"action": "set-current-schema", "schema-id": -1}), + serde_json::json!({ + "action": "add-spec", + "spec": { + "spec-id": 42, + "fields": [ + {"source-id": 1, "name": "id", "transform": "identity"}, + {"source-id": 1, "name": "id_bucket", "transform": "bucket[16]"} + ] + } + }), + serde_json::json!({"action": "set-default-spec", "spec-id": -1}), + serde_json::json!({ + "action": "add-sort-order", + "sort-order": {"order-id": 43, "fields": []} + }), + serde_json::json!({"action": "set-default-sort-order", "sort-order-id": -1}), + ], + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect("catalog-assigned metadata IDs should apply"); + + assert_eq!(updated["schemas"][1]["schema-id"], 1); + assert_eq!(updated["current-schema-id"], 1); + assert_eq!(updated["partition-specs"][1]["spec-id"], 1); + assert_eq!(updated["partition-specs"][1]["fields"][0]["field-id"], 1000); + assert_eq!(updated["partition-specs"][1]["fields"][1]["field-id"], 1001); + assert_eq!(updated["default-spec-id"], 1); + assert_eq!(updated["last-partition-id"], 1001); + assert_eq!(updated["sort-orders"].as_array().map(Vec::len), Some(2)); + assert_eq!(updated["sort-orders"][0]["order-id"], 1); + assert_eq!(updated["sort-orders"][1]["order-id"], 0); + assert_eq!(updated["default-sort-order-id"], 0); +} + +#[test] +fn standard_commit_binds_new_specs_and_sort_orders_to_current_schema() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "properties": {} + })) + .expect("create table request should parse"); + let (_, metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("table metadata should be created"); + let schema_updates = [ + serde_json::json!({ + "action": "add-schema", + "schema": {"type": "struct", "schema-id": 1, "fields": []} + }), + serde_json::json!({"action": "set-current-schema", "schema-id": -1}), + ]; + + let mut spec_updates = schema_updates.to_vec(); + spec_updates.push(serde_json::json!({ + "action": "add-spec", + "spec": { + "spec-id": 1, + "fields": [{"source-id": 1, "name": "id", "transform": "identity"}] + } + })); + apply_table_commit_updates_at( + metadata.clone(), + &spec_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect_err("a new partition spec must bind to the current schema"); + spec_updates[2]["spec"]["fields"][0]["transform"] = serde_json::Value::from("void"); + apply_table_commit_updates_at( + metadata.clone(), + &spec_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect("a void partition field may retain a source removed from the current schema"); + + let mut sort_updates = schema_updates.to_vec(); + sort_updates.push(serde_json::json!({ + "action": "add-sort-order", + "sort-order": { + "order-id": 1, + "fields": [{ + "source-id": 1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + } + })); + apply_table_commit_updates_at(metadata, &sort_updates, "s3://warehouse/tables/table-id/metadata/v1.metadata.json", 2) + .expect_err("a new sort order must bind to the current schema"); + + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events_v1", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "properties": {"format-version": "1"} + })) + .expect("v1 create table request should parse"); + let (_, v1_metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("v1 table metadata should be created"); + spec_updates[2]["spec"]["fields"][0]["transform"] = serde_json::Value::from("identity"); + apply_table_commit_updates_at( + v1_metadata.clone(), + &spec_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect_err("a new v1 partition spec must bind to the updated current schema"); + apply_table_commit_updates_at( + v1_metadata.clone(), + &sort_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect_err("a new v1 sort order must bind to the updated current schema"); + + let mut singular_v1_metadata = v1_metadata.clone(); + let singular_v1_object = singular_v1_metadata + .as_object_mut() + .expect("v1 table metadata should be an object"); + for field in [ + "schemas", + "current-schema-id", + "partition-specs", + "default-spec-id", + "last-partition-id", + "sort-orders", + "default-sort-order-id", + ] { + singular_v1_object.remove(field); + } + let singular_v1_updates = [ + serde_json::json!({ + "action": "add-schema", + "schema": { + "type": "struct", + "schema-id": 1, + "fields": [{"id": 2, "name": "category", "required": true, "type": "string"}] + } + }), + serde_json::json!({ + "action": "add-spec", + "spec": { + "spec-id": 1, + "fields": [{"source-id": 1, "name": "id", "transform": "identity"}] + } + }), + ]; + let singular_v1_updated = apply_table_commit_updates_at( + singular_v1_metadata, + &singular_v1_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect("a singular v1 table may add schema history without changing its current schema"); + assert_eq!(singular_v1_updated["current-schema-id"], 0); + assert_eq!(singular_v1_updated["schema"]["schema-id"], 0); + + let valid_v1_updates = [ + serde_json::json!({ + "action": "add-schema", + "schema": { + "type": "struct", + "schema-id": 1, + "fields": [{"id": 2, "name": "category", "required": true, "type": "string"}] + } + }), + serde_json::json!({"action": "set-current-schema", "schema-id": -1}), + serde_json::json!({ + "action": "add-spec", + "spec": { + "spec-id": 1, + "fields": [{"source-id": 2, "name": "category", "transform": "identity"}] + } + }), + ]; + apply_table_commit_updates_at( + v1_metadata, + &valid_v1_updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect("a new v1 partition spec may bind to a field in the updated current schema"); +} + +#[test] +fn last_added_table_ids_require_a_preceding_add_update() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "properties": {} + })) + .expect("create table request should parse"); + let (_, metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("table metadata should be created"); + let invalid_update_sequences = [ + vec![ + serde_json::json!({"action": "set-current-schema", "schema-id": -1}), + serde_json::json!({ + "action": "add-schema", + "schema": {"type": "struct", "schema-id": 1, "fields": []} + }), + ], + vec![ + serde_json::json!({"action": "set-default-spec", "spec-id": -1}), + serde_json::json!({ + "action": "add-spec", + "spec": {"spec-id": 1, "fields": []} + }), + ], + vec![ + serde_json::json!({"action": "set-default-sort-order", "sort-order-id": -1}), + serde_json::json!({ + "action": "add-sort-order", + "sort-order": {"order-id": 1, "fields": []} + }), + ], + ]; + + for updates in invalid_update_sequences { + let error = apply_table_commit_updates_at( + metadata.clone(), + &updates, + "s3://warehouse/tables/table-id/metadata/v1.metadata.json", + 2, + ) + .expect_err("a later add update must not satisfy an earlier -1 reference"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + assert!( + error + .message() + .is_some_and(|message| message.contains("requires a preceding")) + ); + } +} + +#[test] +fn create_table_counts_collection_ids_in_last_column_id() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "items", + "required": true, + "type": { + "type": "list", + "element-id": 7, + "element-required": true, + "element": "long" + } + }, + { + "id": 2, + "name": "lookup", + "required": true, + "type": { + "type": "map", + "key-id": 8, + "key": "string", + "value-id": 9, + "value-required": false, + "value": "string" + } + } + ] + } + })) + .expect("create table request should parse"); + + let (_, metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("table metadata should be created"); + + assert_eq!(metadata["last-column-id"], 9); +} + #[test] fn commit_table_request_accepts_standard_iceberg_rest_shape() { let request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ @@ -1956,6 +2273,86 @@ fn commit_table_request_accepts_standard_iceberg_rest_shape() { assert_eq!(request.requirements.len(), 1); } +#[tokio::test] +async fn commit_request_readers_require_standard_arrays_and_preserve_legacy_pointer_shapes() { + let table_error = read_rest_commit_table_request(Body::from("{}".to_string())) + .await + .expect_err("standard table commits must include requirements and updates"); + assert_eq!(table_error.code(), &S3ErrorCode::InvalidRequest); + + let table = read_rest_commit_table_request(Body::from(r#"{"requirements":[],"updates":[]}"#.to_string())) + .await + .expect("standard table commits may provide empty requirements and updates"); + assert!(table.requirements.is_empty()); + assert!(table.updates.is_empty()); + + let legacy_table = read_rest_commit_table_request(Body::from( + r#"{"expected-version-token":"token-v1","expected-metadata-location":"s3://warehouse/tables/table-id/metadata/v1.metadata.json","new-metadata-location":"s3://warehouse/tables/table-id/metadata/v2.metadata.json"}"# + .to_string(), + )) + .await + .expect("legacy pointer commits may omit standard arrays"); + assert_eq!( + legacy_table.new_metadata_location.as_deref(), + Some("s3://warehouse/tables/table-id/metadata/v2.metadata.json") + ); + + let mixed_table = read_rest_commit_table_request(Body::from( + r#"{"expected-version-token":"token-v1","new-metadata-location":"s3://warehouse/tables/table-id/metadata/v2.metadata.json","requirements":[],"updates":[{"action":"set-properties","updates":{"owner":"lakehouse"}}]}"# + .to_string(), + )) + .await + .expect_err("legacy pointer commits must not silently discard standard updates"); + assert_eq!(mixed_table.code(), &S3ErrorCode::InvalidRequest); + + let view_error = read_rest_commit_view_request(Body::from("{}".to_string())) + .await + .expect_err("standard view commits must include updates"); + assert_eq!(view_error.code(), &S3ErrorCode::InvalidRequest); + + let view = read_rest_commit_view_request(Body::from(r#"{"updates":[]}"#.to_string())) + .await + .expect("standard view commits may omit requirements"); + assert!(view.requirements.is_empty()); + assert!(view.updates.is_empty()); + + let legacy_view = read_rest_commit_view_request(Body::from( + r#"{"commit-id":"legacy-view-update","new-metadata-location":"s3://warehouse/views/view-id/metadata/v2.metadata.json"}"# + .to_string(), + )) + .await + .expect("legacy view pointer commits may omit standard updates"); + assert_eq!(legacy_view._commit_id.as_deref(), Some("legacy-view-update")); +} + +#[test] +fn unsupported_create_and_register_modes_return_iceberg_errors() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let register_error = table_entry_from_register_request( + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: "s3://warehouse/metadata/00001.metadata.json".to_string(), + overwrite: true, + }, + ) + .expect_err("register overwrite should remain unsupported"); + assert_eq!(register_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_UNSUPPORTED_OPERATION.into())); + assert_eq!(register_error.status_code(), Some(StatusCode::NOT_ACCEPTABLE)); + + let create_request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": {"type": "struct", "schema-id": 0, "fields": []}, + "stage-create": true + })) + .expect("stage-create request should parse"); + let create_error = table_entry_from_create_table_request("warehouse", &namespace, create_request) + .expect_err("staged create should remain unsupported"); + assert_eq!(create_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_UNSUPPORTED_OPERATION.into())); + assert_eq!(create_error.status_code(), Some(StatusCode::NOT_ACCEPTABLE)); +} + #[test] fn standard_commit_ids_use_uuid_for_metadata_file_when_provided() { let commit_id = "11111111-1111-4111-8111-111111111111"; @@ -1987,7 +2384,7 @@ fn format_upgrade_assigns_v1_snapshot_sequences_and_rejects_v3() { "schema-id": 0, "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] }, - "partition-spec": [], + "partition-spec": [{"source-id": 1, "name": "id", "transform": "identity"}], "snapshots": [{ "snapshot-id": 10, "timestamp-ms": 1, @@ -2005,6 +2402,8 @@ fn format_upgrade_assigns_v1_snapshot_sequences_and_rejects_v3() { .expect("upgraded metadata fields should synchronize"); assert_eq!(metadata["snapshots"][0]["sequence-number"], 0); + assert_eq!(metadata["partition-specs"][0]["fields"][0]["field-id"], 1000); + assert_eq!(metadata["last-partition-id"], 1000); crate::table_catalog::validate_supported_table_metadata(&metadata).expect("upgraded metadata should satisfy the v2 contract"); let error = apply_upgrade_format_version_update( @@ -2171,6 +2570,120 @@ async fn create_table_holds_bucket_fence_from_metadata_write_through_registratio ); } +#[tokio::test] +async fn create_view_holds_publication_fences_from_metadata_write_through_registration() { + let pause = TestCatalogPublishPause::default(); + let store = Arc::new(TestTableCatalogStore { + create_view_pause: Some(pause.clone()), + ..Default::default() + }); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let metadata_backend = TestTableCatalogObjectBackend { + put_object_barrier: Some(Arc::clone(&barrier)), + ..Default::default() + }; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(store.as_ref(), "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + store.as_ref(), + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let request = serde_json::from_value::(serde_json::json!({ + "name": "recent_events", + "schema": {"type": "struct", "fields": []}, + "view-version": { + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + } + })) + .expect("create view request should parse"); + + let create_store = Arc::clone(&store); + let create_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let create_namespace = namespace.clone(); + let create = tokio::spawn(async move { + create_view_response(create_store.as_ref(), &create_backend, "warehouse", &create_namespace, request, true).await + }); + tokio::time::timeout(StdDuration::from_secs(2), async { + while metadata_backend.state.lock().await.objects.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("initial view metadata write should reach its publication pause"); + + let bucket_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + assert!( + metadata_backend.write_lock_is_held("warehouse", &bucket_lock).await, + "view publication must fence data-plane writers before initial metadata is visible" + ); + barrier.wait().await; + tokio::time::timeout(StdDuration::from_secs(2), pause.wait_started()) + .await + .expect("view creation should reach catalog publication"); + + let view_name = crate::table_catalog::IdentifierSegment::parse("recent_events").expect("view should parse"); + let view_lock = crate::table_catalog::default_table_publication_lock_path(&namespace, &view_name); + assert!( + metadata_backend.write_lock_is_held("warehouse", &bucket_lock).await, + "view creation must retain the bucket fence until catalog publication" + ); + assert!( + metadata_backend.write_lock_is_held("warehouse", &view_lock).await, + "view creation must hold the view publication fence before registration" + ); + assert!( + store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .is_none(), + "the view must remain invisible before catalog publication" + ); + + metadata_backend.lock_attempts.lock().await.clear(); + let writer_backend = metadata_backend.clone(); + let writer_lock = bucket_lock.clone(); + let writer = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::acquire_read_lock(&writer_backend, "warehouse", &writer_lock).await + }); + metadata_backend.wait_for_lock_attempts(1).await; + assert!(!writer.is_finished(), "a data-plane writer must wait for view registration"); + + pause.release(); + tokio::time::timeout(StdDuration::from_secs(2), create) + .await + .expect("view creation should complete") + .expect("view creation task should join") + .expect("view creation should succeed"); + tokio::time::timeout(StdDuration::from_secs(2), writer) + .await + .expect("writer should continue after view registration") + .expect("writer task should join") + .expect("writer lock acquisition should succeed"); + assert!( + store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .is_some(), + "the view must become visible after catalog publication" + ); +} + #[tokio::test] async fn create_table_response_recreates_dropped_identifier_without_overwriting_retained_metadata() { let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); @@ -2189,6 +2702,7 @@ async fn create_table_response_recreates_dropped_identifier_without_overwriting_ .expect("first metadata should exist"); let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": "11111111-1111-4111-8111-111111111111", "updates": [ { @@ -3199,6 +3713,7 @@ async fn standard_commit_uses_client_uuid_commit_id_in_metadata_file_name() { let commit_id = "11111111-1111-4111-8111-111111111111"; let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": commit_id, "updates": [ { @@ -3247,6 +3762,7 @@ async fn standard_commit_accepts_non_uuid_client_commit_id_without_using_it_in_m create_standard_events_table(&store, &metadata_backend, &namespace).await; let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": "commit-1", "updates": [ { @@ -3620,7 +4136,25 @@ async fn commit_publication_authorizes_referenced_objects() { let manifest_list = format!("{table_location}/metadata/snap-10.avro"); let manifest = format!("{table_location}/metadata/manifest-snap-10.avro"); let data_file = format!("{table_location}/data/part-10.parquet"); + let statistics_file = format!("{table_location}/metadata/stats-10.puffin"); + let partition_statistics_file = format!("{table_location}/metadata/partition-stats-10.parquet"); + let statistics_bytes = b"PFA1PFA1".to_vec(); + let partition_statistics_bytes = test_parquet_i32_bytes(&[1]); seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; + metadata_backend + .put_bytes( + "warehouse", + &test_snapshot_object_key("warehouse", &statistics_file), + statistics_bytes.clone(), + ) + .await; + metadata_backend + .put_bytes( + "warehouse", + &test_snapshot_object_key("warehouse", &partition_statistics_file), + partition_statistics_bytes.clone(), + ) + .await; let request = serde_json::from_value(serde_json::json!({ "commit-id": "22222222-2222-4222-8222-222222222222", "requirements": [], @@ -3640,6 +4174,24 @@ async fn commit_publication_authorizes_referenced_objects() { "ref-name": "main", "snapshot-id": 10, "type": "branch" + }, + { + "action": "set-statistics", + "statistics": { + "snapshot-id": 10, + "statistics-path": statistics_file, + "file-size-in-bytes": statistics_bytes.len(), + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + } + }, + { + "action": "set-partition-statistics", + "partition-statistics": { + "snapshot-id": 10, + "statistics-path": partition_statistics_file, + "file-size-in-bytes": partition_statistics_bytes.len() + } } ] })) @@ -3659,6 +4211,8 @@ async fn commit_publication_authorizes_referenced_objects() { test_snapshot_object_key("warehouse", &manifest_list), test_snapshot_object_key("warehouse", &manifest), test_snapshot_object_key("warehouse", &data_file), + test_snapshot_object_key("warehouse", &statistics_file), + test_snapshot_object_key("warehouse", &partition_statistics_file), ]; for object in expected_reads { assert!( @@ -4316,19 +4870,16 @@ async fn standard_commit_publishes_more_than_ten_thousand_live_files() { .map(|index| format!("{table_location}/data/part-{index:05}.parquet")) .collect::>(); let manifest_files = data_files.iter().map(|file| (file.as_str(), 0, 1, 20, 1)).collect::>(); + let manifest_bytes = test_manifest_avro_bytes(&manifest_files); metadata_backend .put_bytes( "warehouse", &test_snapshot_object_key("warehouse", &manifest_list), - test_manifest_list_avro_bytes(&[&manifest], 1, 20), + test_manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())], 1, 20), ) .await; metadata_backend - .put_bytes( - "warehouse", - &test_snapshot_object_key("warehouse", &manifest), - test_manifest_avro_bytes(&manifest_files), - ) + .put_bytes("warehouse", &test_snapshot_object_key("warehouse", &manifest), manifest_bytes) .await; { let mut state = metadata_backend.state.lock().await; @@ -4437,6 +4988,7 @@ async fn standard_commit_ignores_generation_only_orphan_metadata_file() { let commit_id = "22222222-2222-4222-8222-222222222222"; let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": commit_id, "updates": [ { @@ -4482,6 +5034,7 @@ async fn concurrent_standard_commits_write_distinct_metadata_files_before_pointe let first_commit_id = "33333333-3333-4333-8333-333333333333"; let second_commit_id = "44444444-4444-4444-8444-444444444444"; let first_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": first_commit_id, "updates": [ { @@ -4494,6 +5047,7 @@ async fn concurrent_standard_commits_write_distinct_metadata_files_before_pointe })) .expect("first standard commit table request should parse"); let second_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "commit-id": second_commit_id, "updates": [ { @@ -4536,7 +5090,7 @@ async fn concurrent_standard_commits_write_distinct_metadata_files_before_pointe } #[tokio::test] -async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_matches() { +async fn standard_commit_rejects_unbound_legacy_catalog_identity() { let store = TestTableCatalogStore::default(); let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -4579,6 +5133,7 @@ async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_match .await; let commit_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "set-properties", @@ -4589,7 +5144,7 @@ async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_match ] })) .expect("standard commit table request should parse"); - let committed = commit_table_response( + let error = commit_table_response( &store, &trusted_table_commit_backend(&metadata_backend), "warehouse", @@ -4598,15 +5153,15 @@ async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_match commit_request, ) .await - .expect("legacy catalog uuid should not block standard commit"); + .expect_err("a legacy catalog identity that does not match persisted metadata must fail closed"); - assert_eq!(committed.metadata["table-uuid"], "metadata-table-uuid"); - assert_eq!(committed.metadata["properties"]["owner"], "lakehouse"); - assert_eq!(committed.generation, legacy_entry.generation + 1); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_REST.into())); + assert_eq!(error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR)); + assert_events_table_entry_unchanged(&store, &legacy_entry).await; } #[tokio::test] -async fn metadata_location_api_accepts_legacy_catalog_uuid_when_target_matches_current_metadata() { +async fn metadata_location_api_rejects_unbound_legacy_catalog_identity() { let store = TestTableCatalogStore::default(); let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -4652,7 +5207,7 @@ async fn metadata_location_api_accepts_legacy_catalog_uuid_when_target_matches_c next_metadata["last-sequence-number"] = serde_json::Value::from(2); metadata_backend.put_json("warehouse", next_location, next_metadata).await; - let updated = update_table_metadata_location_response( + let error = update_table_metadata_location_response( &store, &trusted_table_commit_backend(&metadata_backend), "warehouse", @@ -4660,16 +5215,17 @@ async fn metadata_location_api_accepts_legacy_catalog_uuid_when_target_matches_c "events", UpdateTableMetadataLocationRequest { metadata_location: next_location.to_string(), - version_token: legacy_entry.version_token, + version_token: legacy_entry.version_token.clone(), commit_id: Some("commit-1".to_string()), idempotency_key: None, }, ) .await - .expect("legacy catalog uuid should not block metadata-location update"); + .expect_err("metadata-location updates must reject unbound legacy catalog identity"); - assert_eq!(updated.metadata_location, table_metadata_location_for_client("warehouse", next_location)); - assert_eq!(updated.generation, legacy_entry.generation + 1); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_REST.into())); + assert_eq!(error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR)); + assert_events_table_entry_unchanged(&store, &legacy_entry).await; } #[tokio::test] @@ -5016,18 +5572,17 @@ async fn table_metadata_maintenance_helper_commits_compaction_through_publicatio let manifest = format!("{metadata_dir}/manifest-20.avro"); let left_data = format!("{data_dir}/part-left.parquet"); let right_data = format!("{data_dir}/part-right.parquet"); + let manifest_bytes = test_manifest_avro_bytes(&[(&left_data, 0, 0, 20, 7), (&right_data, 0, 0, 20, 7)]); seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; - backend - .put_bytes(bucket, &manifest_list, test_manifest_list_avro_bytes(&[&manifest], 7, 20)) - .await; backend .put_bytes( bucket, - &manifest, - test_manifest_avro_bytes(&[(&left_data, 0, 0, 20, 7), (&right_data, 0, 0, 20, 7)]), + &manifest_list, + test_manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())], 7, 20), ) .await; + backend.put_bytes(bucket, &manifest, manifest_bytes).await; backend.put_bytes(bucket, &left_data, test_parquet_i32_bytes(&[1, 2])).await; backend.put_bytes(bucket, &right_data, test_parquet_i32_bytes(&[3, 4])).await; backend @@ -5294,25 +5849,36 @@ async fn table_refs_response_reports_current_and_user_defined_refs() { let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; + let mut metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + metadata["last-sequence-number"] = serde_json::Value::from(2); + metadata["snapshots"] = serde_json::json!([ + { + "snapshot-id": 9, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-9.avro", + "summary": {"operation": "append"} + }, + { + "snapshot-id": 10, + "parent-snapshot-id": 9, + "sequence-number": 2, + "timestamp-ms": 2, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + } + ]); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["snapshot-log"] = serde_json::json!([ + {"timestamp-ms": 1, "snapshot-id": 9}, + {"timestamp-ms": 2, "snapshot-id": 10} + ]); + metadata["refs"] = serde_json::json!({ + "main": {"snapshot-id": 10, "type": "branch"}, + "audit": {"snapshot-id": 9, "type": "tag"} + }); backend - .put_json_with_mod_time( - bucket, - ¤t, - serde_json::json!({ - "current-snapshot-id": 10, - "refs": { - "main": { - "snapshot-id": 10, - "type": "branch" - }, - "audit": { - "snapshot-id": 9, - "type": "tag" - } - } - }), - Some(OffsetDateTime::UNIX_EPOCH), - ) + .put_json_with_mod_time(bucket, ¤t, metadata, Some(OffsetDateTime::UNIX_EPOCH)) .await; let response = table_refs_response(&store, &backend, bucket, &namespace, "events") @@ -5695,39 +6261,30 @@ async fn external_catalog_bridge_sync_conflicts_leave_pointer_unchanged() { } #[test] -fn commit_requirements_reject_mismatched_table_uuid() { +fn snapshot_conflict_requirements_validate_snapshot_ref_id() { let metadata = serde_json::json!({ - "table-uuid": "actual-table-uuid" - }); - let requirements = vec![serde_json::json!({ - "type": "assert-table-uuid", - "uuid": "stale-table-uuid" - })]; - - assert!(validate_table_commit_requirements(&metadata, &requirements).is_err()); -} - -#[test] -fn snapshot_conflict_requirements_validate_current_snapshot_id() { - let metadata = serde_json::json!({ - "current-snapshot-id": 10 + "current-snapshot-id": 10, + "refs": {"main": {"type": "branch", "snapshot-id": 10}} }); let matching = vec![serde_json::json!({ - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 })]; validate_table_commit_requirements(&metadata, &matching).expect("matching current snapshot should pass"); let stale = vec![serde_json::json!({ - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 9 })]; assert!(validate_table_commit_requirements(&metadata, &stale).is_err()); let no_snapshot_metadata = serde_json::json!({}); let create_like = vec![serde_json::json!({ - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": null })]; validate_table_commit_requirements(&no_snapshot_metadata, &create_like) @@ -5735,26 +6292,20 @@ fn snapshot_conflict_requirements_validate_current_snapshot_id() { } #[test] -fn snapshot_conflict_rejects_stale_parent_or_sequence_number() { - let metadata = serde_json::json!({ - "current-snapshot-id": 10, - "last-sequence-number": 4, - "snapshots": [ - { - "snapshot-id": 10, - "sequence-number": 4, - "timestamp-ms": 1234, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", - "summary": { - "operation": "append" - } - } - ], - "snapshot-log": [], - "metadata-log": [] - }); +fn snapshot_conflict_rejects_unknown_parent_or_stale_sequence_number() { + let mut metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["last-sequence-number"] = serde_json::Value::from(4); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 4, + "timestamp-ms": 1234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + metadata["refs"] = serde_json::json!({"main": {"snapshot-id": 10, "type": "branch"}}); - let stale_parent = vec![serde_json::json!({ + let unknown_parent = vec![serde_json::json!({ "action": "add-snapshot", "snapshot": { "snapshot-id": 11, @@ -5767,7 +6318,11 @@ fn snapshot_conflict_rejects_stale_parent_or_sequence_number() { } } })]; - assert!(apply_table_commit_updates(metadata.clone(), &stale_parent, "metadata/00001.metadata.json").is_err()); + let error = apply_table_commit_updates(metadata.clone(), &unknown_parent, "metadata/00001.metadata.json") + .expect_err("unknown snapshot parents must fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_COMMIT_FAILED.into())); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT)); + assert_eq!(error.message(), Some("snapshot parent does not exist")); let stale_sequence = vec![serde_json::json!({ "action": "add-snapshot", @@ -5782,28 +6337,341 @@ fn snapshot_conflict_rejects_stale_parent_or_sequence_number() { } } })]; - assert!(apply_table_commit_updates(metadata, &stale_sequence, "metadata/00001.metadata.json").is_err()); + let error = apply_table_commit_updates(metadata.clone(), &stale_sequence, "metadata/00001.metadata.json") + .expect_err("snapshot sequence numbers must advance"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_COMMIT_FAILED.into())); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT)); + assert_eq!(error.message(), Some("snapshot sequence number must advance")); + + let stale_root_sequence = vec![serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "sequence-number": 4, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", + "summary": { + "operation": "append" + } + } + })]; + let error = apply_table_commit_updates(metadata, &stale_root_sequence, "metadata/00001.metadata.json") + .expect_err("root snapshot sequence numbers must advance"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_COMMIT_FAILED.into())); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT)); + assert_eq!(error.message(), Some("snapshot sequence number must advance")); +} + +#[test] +fn snapshot_updates_move_only_the_declared_reference() { + let mut metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["last-sequence-number"] = serde_json::Value::from(4); + metadata["snapshots"] = serde_json::json!([ + { + "snapshot-id": 9, + "sequence-number": 3, + "timestamp-ms": 1000, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-9.avro", + "summary": {"operation": "append"} + }, + { + "snapshot-id": 10, + "parent-snapshot-id": 9, + "sequence-number": 4, + "timestamp-ms": 1234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + } + ]); + metadata["refs"] = serde_json::json!({"main": {"snapshot-id": 10, "type": "branch"}}); + metadata["snapshot-log"] = serde_json::json!([{"timestamp-ms": 1234, "snapshot-id": 10}]); + let add_snapshot = serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 9, + "sequence-number": 5, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", + "summary": {"operation": "append"} + } + }); + + let added = + apply_table_commit_updates_at(metadata, std::slice::from_ref(&add_snapshot), "metadata/00001.metadata.json", 3000) + .expect("a snapshot may branch from any retained parent"); + assert_eq!(added["current-snapshot-id"], 10); + assert_eq!(added["refs"]["main"]["snapshot-id"], 10); + assert_eq!(added["snapshot-log"].as_array().map(Vec::len), Some(1)); + + let branch = apply_table_commit_updates_at( + added, + &[serde_json::json!({ + "action": "set-snapshot-ref", + "ref-name": "audit", + "snapshot-id": 11, + "type": "branch" + })], + "metadata/00002.metadata.json", + 3001, + ) + .expect("a non-main branch should be updated"); + assert_eq!(branch["current-snapshot-id"], 10); + assert_eq!(branch["refs"]["main"]["snapshot-id"], 10); + assert_eq!(branch["refs"]["audit"]["snapshot-id"], 11); + assert_eq!(branch["snapshot-log"].as_array().map(Vec::len), Some(1)); + + let main = apply_table_commit_updates_at( + branch, + &[serde_json::json!({ + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + })], + "metadata/00003.metadata.json", + 3002, + ) + .expect("main should move to a retained snapshot"); + assert_eq!(main["current-snapshot-id"], 11); + assert_eq!(main["snapshot-log"].as_array().map(Vec::len), Some(2)); + assert_eq!(main["snapshot-log"][1], serde_json::json!({"timestamp-ms": 3002, "snapshot-id": 11})); + + let unchanged = apply_table_commit_updates_at( + main, + &[serde_json::json!({ + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + })], + "metadata/00004.metadata.json", + 3003, + ) + .expect("replaying an unchanged main reference should be a no-op for snapshot history"); + assert_eq!(unchanged["snapshot-log"].as_array().map(Vec::len), Some(2)); +} + +#[test] +fn newly_added_main_snapshot_uses_its_snapshot_timestamp_in_history() { + let mut metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + metadata["current-snapshot-id"] = serde_json::Value::from(-1); + let updated = apply_table_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "sequence-number": 1, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-11.avro", + "summary": {"operation": "append"} + } + }), + serde_json::json!({ + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + }), + ], + "metadata/00001.metadata.json", + 3000, + ) + .expect("new snapshot and main reference should apply"); + + assert_eq!(updated["current-snapshot-id"], 11); + assert_eq!(updated["snapshot-log"], serde_json::json!([{"timestamp-ms": 2234, "snapshot-id": 11}])); +} + +#[test] +fn only_v1_snapshots_may_omit_sequence_number() { + let v1 = serde_json::json!({ + "format-version": 1, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-updated-ms": 1, + "last-column-id": 1, + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "partition-spec": [], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [] + }); + let v1_updated = apply_table_commit_updates_at( + v1, + &[serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "timestamp-ms": 2234, + "manifests": ["s3://warehouse/tables/table-id/metadata/manifest-10.avro"], + "summary": {"operation": "append"} + } + })], + "metadata/00001.metadata.json", + 3000, + ) + .expect("an Iceberg v1 zero sequence snapshot may omit sequence-number"); + assert!(v1_updated["snapshots"][0].get("sequence-number").is_none()); + assert!(v1_updated.get("last-sequence-number").is_none()); + + let v2 = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + let v2_error = apply_table_commit_updates_at( + v2, + &[serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + } + })], + "metadata/00001.metadata.json", + 3000, + ) + .expect_err("new Iceberg v2 snapshots must include sequence-number"); + assert_eq!(v2_error.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(v2_error.message(), Some("Iceberg v2 snapshot sequence-number is required")); +} + +#[test] +fn snapshot_updates_reject_non_integer_parent_ids() { + let metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + let error = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({ + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "parent-snapshot-id": "invalid", + "sequence-number": 1, + "timestamp-ms": 2234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + } + })], + "metadata/00001.metadata.json", + 3000, + ) + .expect_err("snapshot parent IDs must be integers"); + + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); +} + +#[tokio::test] +async fn standard_commit_accepts_multiple_ordered_snapshots() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let first_manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let first_manifest = format!("{table_location}/metadata/manifest-snap-10.avro"); + let first_data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest( + &metadata_backend, + "warehouse", + &first_manifest_list, + 10, + 1, + &[(&first_data_file, 0, 1, 10, 1)], + ) + .await; + let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); + let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); + let second_data_file = format!("{table_location}/data/part-11.parquet"); + seed_test_manifest(&metadata_backend, "warehouse", &second_manifest, &[(&second_data_file, 0, 1, 11, 2)]).await; + seed_test_manifest_list_entries( + &metadata_backend, + "warehouse", + &second_manifest_list, + &[(&first_manifest, 1, 10), (&second_manifest, 2, 11)], + ) + .await; + let request = serde_json::from_value(serde_json::json!({ + "requirements": [], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": first_manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + }, + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 11, + "parent-snapshot-id": 10, + "sequence-number": 2, + "timestamp-ms": 2234, + "manifest-list": second_manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 11, + "type": "branch" + } + ] + })) + .expect("multi-snapshot commit request should parse"); + + let committed = standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + request, + ) + .await + .expect("ordered intermediate snapshots should commit"); + + assert_eq!(committed.metadata["snapshots"].as_array().map(Vec::len), Some(2)); + assert_eq!(committed.metadata["current-snapshot-id"], 11); + assert_eq!(committed.metadata["last-sequence-number"], 2); + assert_eq!( + committed.metadata["snapshot-log"], + serde_json::json!([{"timestamp-ms": 2234, "snapshot-id": 11}]) + ); } #[test] fn snapshot_conflict_rejects_unknown_snapshot_operations() { - let metadata = serde_json::json!({ - "current-snapshot-id": 10, - "last-sequence-number": 4, - "snapshots": [ - { - "snapshot-id": 10, - "sequence-number": 4, - "timestamp-ms": 1234, - "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", - "summary": { - "operation": "append" - } - } - ], - "snapshot-log": [], - "metadata-log": [] - }); + let mut metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["last-sequence-number"] = serde_json::Value::from(4); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 4, + "timestamp-ms": 1234, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + metadata["refs"] = serde_json::json!({"main": {"snapshot-id": 10, "type": "branch"}}); let updates = vec![serde_json::json!({ "action": "add-snapshot", @@ -5818,7 +6686,10 @@ fn snapshot_conflict_rejects_unknown_snapshot_operations() { } } })]; - assert!(apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json").is_err()); + let error = apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json") + .expect_err("unknown snapshot operations must fail"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(error.message(), Some("unsupported snapshot operation: unknown")); } #[tokio::test] @@ -5842,6 +6713,7 @@ async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { ) .await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -5889,7 +6761,8 @@ async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { let overwrite_request_json = serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -5958,7 +6831,7 @@ async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { } #[tokio::test] -async fn row_level_conflict_allows_v1_manifest_snapshot() { +async fn row_level_conflict_rejects_embedded_manifests_for_v2_snapshot() { let store = TestTableCatalogStore::default(); let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -5967,9 +6840,13 @@ async fn row_level_conflict_allows_v1_manifest_snapshot() { .as_str() .expect("created metadata should have table location"); let manifest = format!("{table_location}/metadata/manifest-10.avro"); - let data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, 1)]).await; + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -5995,7 +6872,7 @@ async fn row_level_conflict_allows_v1_manifest_snapshot() { })) .expect("append request should parse"); - let commit = commit_table_response( + let error = commit_table_response( &store, &trusted_table_commit_backend(&metadata_backend), "warehouse", @@ -6004,67 +6881,17 @@ async fn row_level_conflict_allows_v1_manifest_snapshot() { append_request, ) .await - .expect("v1 manifests snapshot should commit"); + .expect_err("new v2 snapshots must use a manifest list"); - assert_eq!(commit.metadata["current-snapshot-id"], 10); - assert_eq!(commit.metadata["last-sequence-number"], 1); - - let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); - let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); - let second_data_file = format!("{table_location}/data/part-11.parquet"); - let second_manifest_list_key = test_snapshot_object_key("warehouse", &second_manifest_list); - metadata_backend - .put_bytes( - "warehouse", - &second_manifest_list_key, - test_manifest_list_avro_entries(&[(&manifest, 1, 10), (&second_manifest, 2, 11)]), - ) - .await; - seed_test_manifest(&metadata_backend, "warehouse", &second_manifest, &[(&second_data_file, 0, 1, 11, 2)]).await; - let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 10 - } - ], - "updates": [ - { - "action": "add-snapshot", - "snapshot": { - "snapshot-id": 11, - "parent-snapshot-id": 10, - "sequence-number": 2, - "timestamp-ms": 2234, - "manifest-list": second_manifest_list, - "summary": { - "operation": "append" - } - } - }, - { - "action": "set-snapshot-ref", - "ref-name": "main", - "snapshot-id": 11, - "type": "branch" - } - ] - })) - .expect("second append request should parse"); - - let upgraded = commit_table_response( - &store, - &trusted_table_commit_backend(&metadata_backend), - "warehouse", - &namespace, - "events", - second_append, - ) - .await - .expect("manifest-list snapshot should inherit a legacy manifest with unknown provenance"); - - assert_eq!(upgraded.metadata["current-snapshot-id"], 11); - assert_eq!(upgraded.metadata["last-sequence-number"], 2); + assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, current.metadata_location); + assert_eq!(unchanged.version_token, current.version_token); + assert_eq!(unchanged.generation, current.generation); } #[tokio::test] @@ -6079,9 +6906,10 @@ async fn row_level_conflict_inherits_manifest_list_sequence_numbers() { let manifest_list = format!("{table_location}/metadata/snap-10.avro"); let manifest = format!("{table_location}/metadata/manifest-10.avro"); let data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 1, 10).await; seed_test_manifest_with_nullable_sequences(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, 10, None)]).await; + seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], 1, 10).await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6132,9 +6960,10 @@ async fn row_level_conflict_allows_inherited_manifests_on_append() { let first_manifest_list = format!("{table_location}/metadata/snap-10.avro"); let first_manifest = format!("{table_location}/metadata/manifest-10.avro"); let first_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; seed_test_manifest(&metadata_backend, "warehouse", &first_manifest, &[(&first_data_file, 0, 1, 10, 1)]).await; + seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; let first_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6171,19 +7000,19 @@ async fn row_level_conflict_allows_inherited_manifests_on_append() { let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); let second_data_file = format!("{table_location}/data/part-11.parquet"); - let second_manifest_list_key = test_snapshot_object_key("warehouse", &second_manifest_list); - metadata_backend - .put_bytes( - "warehouse", - &second_manifest_list_key, - test_manifest_list_avro_entries(&[(&first_manifest, 1, 10), (&second_manifest, 2, 11)]), - ) - .await; seed_test_manifest(&metadata_backend, "warehouse", &second_manifest, &[(&second_data_file, 0, 1, 11, 2)]).await; + seed_test_manifest_list_entries( + &metadata_backend, + "warehouse", + &second_manifest_list, + &[(&first_manifest, 1, 10), (&second_manifest, 2, 11)], + ) + .await; let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6238,9 +7067,10 @@ async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { let first_manifest_list = format!("{table_location}/metadata/snap-10.avro"); let first_manifest = format!("{table_location}/metadata/manifest-10.avro"); let first_data_file = format!("{table_location}/data/part-10.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; seed_test_manifest(&metadata_backend, "warehouse", &first_manifest, &[(&first_data_file, 0, 1, 10, 1)]).await; + seed_test_manifest_list(&metadata_backend, "warehouse", &first_manifest_list, &[&first_manifest], 1, 10).await; let first_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6284,7 +7114,8 @@ async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { let second_append: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6328,10 +7159,7 @@ async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { assert_eq!(unchanged.generation, current.generation); } -/// Table-driven fold of the three commit-rejection cases whose bodies were -/// identical apart from four literals (backlog#1837 PR3). Each row keeps its -/// original manifest-list sequence, data-file name, manifest-entry snapshot -/// id, and failure message, so no poison combination is lost. +/// Table-driven coverage for stale or historical manifest sequence failures. #[tokio::test] async fn row_level_conflict_rejects_stale_or_historical_manifest_sequences() { // (case, manifest-list sequence, data-file suffix, manifest-entry snapshot id, expected failure) @@ -6375,8 +7203,8 @@ async fn row_level_conflict_rejects_stale_or_historical_manifest_sequences() { let manifest_list = format!("{table_location}/metadata/snap-11.avro"); let manifest = format!("{table_location}/metadata/manifest-11.avro"); let data_file = format!("{table_location}/data/part-{data_file_suffix}.parquet"); - seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], *manifest_list_sequence, 11).await; seed_test_manifest(&metadata_backend, "warehouse", &manifest, &[(&data_file, 0, 1, *entry_snapshot_id, 1)]).await; + seed_test_manifest_list(&metadata_backend, "warehouse", &manifest_list, &[&manifest], *manifest_list_sequence, 11).await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "updates": [ { @@ -6441,6 +7269,7 @@ async fn row_level_conflict_allows_add_only_overwrite_snapshot() { ) .await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6488,7 +7317,8 @@ async fn row_level_conflict_allows_add_only_overwrite_snapshot() { let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6552,6 +7382,7 @@ async fn row_level_conflict_rejects_delete_of_non_current_file() { ) .await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6606,7 +7437,8 @@ async fn row_level_conflict_rejects_delete_of_non_current_file() { let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6639,7 +7471,8 @@ async fn row_level_conflict_rejects_delete_of_non_current_file() { .await .expect_err("stale row-level delete should conflict"); - assert_eq!(error.code(), &s3s::S3ErrorCode::PreconditionFailed); + assert_eq!(error.code(), &s3s::S3ErrorCode::Custom(ICEBERG_ERROR_COMMIT_FAILED.into())); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT)); let unchanged = store .load_table("warehouse", "analytics", "events") .await @@ -6668,6 +7501,7 @@ async fn row_level_conflict_rejects_append_with_delete_files() { let delete_file = format!("{table_location}/delete/delete-10.parquet"); seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&delete_file, 1, 1, 10, 1)]).await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6728,6 +7562,7 @@ async fn row_level_conflict_rejects_missing_manifest_before_pointer_update() { ) .await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6769,7 +7604,8 @@ async fn row_level_conflict_rejects_missing_manifest_before_pointer_update() { let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6834,6 +7670,7 @@ async fn row_level_conflict_rejects_manifest_outside_table_warehouse() { ) .await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -6875,7 +7712,8 @@ async fn row_level_conflict_rejects_manifest_outside_table_warehouse() { let overwrite_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], @@ -6919,6 +7757,110 @@ async fn row_level_conflict_rejects_manifest_outside_table_warehouse() { assert_eq!(unchanged.generation, committed.generation); } +#[tokio::test] +async fn statistics_updates_reject_unpublished_objects_before_pointer_update() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; + let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("append request should parse"); + commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect("append commit should succeed"); + let committed = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let outside_statistics_file = "s3://warehouse/tables/other-table/metadata/stats-10.puffin"; + metadata_backend + .put_bytes( + "warehouse", + &test_snapshot_object_key("warehouse", outside_statistics_file), + b"outside-stats".to_vec(), + ) + .await; + + for (commit_id, statistics_file) in [ + ( + "55555555-5555-4555-8555-555555555551", + format!("{table_location}/metadata/missing-stats-10.puffin"), + ), + ("55555555-5555-4555-8555-555555555552", outside_statistics_file.to_string()), + ] { + let request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "commit-id": commit_id, + "requirements": [{"type": "assert-ref-snapshot-id", "ref": "main", "snapshot-id": 10}], + "updates": [{ + "action": "set-statistics", + "statistics": { + "snapshot-id": 10, + "statistics-path": statistics_file, + "file-size-in-bytes": 5, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + } + }] + })) + .expect("statistics request should parse"); + + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + request, + ) + .await + .expect_err("unpublished statistics object should fail before pointer update"); + + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should still exist"); + assert_eq!(unchanged.metadata_location, committed.metadata_location); + assert_eq!(unchanged.version_token, committed.version_token); + assert_eq!(unchanged.generation, committed.generation); + } +} + #[tokio::test] async fn bodyless_ref_delete_uses_default_request_options() { let request: DeleteTableRefRequest = read_json_body_or_default(Body::empty()) @@ -6933,15 +7875,443 @@ async fn bodyless_ref_delete_uses_default_request_options() { } #[test] -fn table_updates_reject_unknown_actions() { +fn unknown_commit_requirements_and_updates_are_bad_requests() { + let unknown_requirement = vec![serde_json::json!({"type": "unknown-requirement"})]; + let nonstandard_table_requirement = vec![serde_json::json!({"type": "assert-current-snapshot-id", "snapshot-id": 10})]; + let unknown_update = vec![serde_json::json!({"action": "unknown-update"})]; + let table_requirement_error = validate_table_commit_requirements(&serde_json::json!({}), &unknown_requirement) + .expect_err("unknown table requirement should fail"); + let nonstandard_table_requirement_error = + validate_table_commit_requirements(&serde_json::json!({"current-snapshot-id": 10}), &nonstandard_table_requirement) + .expect_err("nonstandard table requirement should fail"); + let table_update_error = apply_table_commit_updates(serde_json::json!({}), &unknown_update, "metadata/00001.metadata.json") + .expect_err("unknown table update should fail"); + let view_requirement_error = validate_view_commit_requirements(&serde_json::json!({}), &unknown_requirement) + .expect_err("unknown view requirement should fail"); + let nonstandard_view_requirement_error = validate_view_commit_requirements( + &serde_json::json!({"current-version-id": 1}), + &[serde_json::json!({"type": "assert-current-view-version-id", "current-view-version-id": 1})], + ) + .expect_err("nonstandard view requirement should fail"); + let nonstandard_view_update_error = apply_view_commit_updates_at( + serde_json::json!({}), + &[serde_json::json!({"action": "set-current-schema", "schema-id": 1})], + 0, + ) + .expect_err("nonstandard view update should fail"); + assert_eq!( + nonstandard_view_update_error.message(), + Some("unsupported view update: set-current-schema") + ); + let view_update_error = + apply_view_commit_updates_at(serde_json::json!({}), &unknown_update, 0).expect_err("unknown view update should fail"); + + for error in [ + table_requirement_error, + nonstandard_table_requirement_error, + table_update_error, + view_requirement_error, + nonstandard_view_requirement_error, + nonstandard_view_update_error, + view_update_error, + ] { + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + } +} + +#[test] +fn failed_commit_requirements_use_iceberg_conflict_errors() { + let table_error = validate_table_commit_requirements( + &serde_json::json!({"table-uuid": "current"}), + &[serde_json::json!({"type": "assert-table-uuid", "uuid": "stale"})], + ) + .expect_err("stale table requirement should fail"); + let view_error = validate_view_commit_requirements( + &serde_json::json!({"view-uuid": "current"}), + &[serde_json::json!({"type": "assert-view-uuid", "uuid": "stale"})], + ) + .expect_err("stale view requirement should fail"); + + for error in [table_error, view_error] { + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_COMMIT_FAILED.into())); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT)); + } +} + +#[test] +fn commit_identifier_must_match_the_resource_url() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let matching = RestTableIdentifier { + namespace: vec!["analytics".to_string()], + name: "events".to_string(), + }; + validate_rest_commit_identifier(Some(&matching), &namespace, "events").expect("matching identifier should be accepted"); + + for identifier in [ + RestTableIdentifier { + namespace: vec!["staging".to_string()], + name: "events".to_string(), + }, + RestTableIdentifier { + namespace: vec!["analytics".to_string()], + name: "other".to_string(), + }, + ] { + assert_eq!( + validate_rest_commit_identifier(Some(&identifier), &namespace, "events") + .expect_err("identifier mismatch should fail") + .code(), + &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into()) + ); + } +} + +#[tokio::test] +async fn mismatched_commit_identifiers_leave_catalog_pointers_unchanged() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_before = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let table_error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + RestCommitTableRequest { + identifier: Some(RestTableIdentifier { + namespace: vec!["analytics".to_string()], + name: "other".to_string(), + }), + commit_id: None, + idempotency_key: None, + operation: None, + expected_version_token: None, + expected_metadata_location: None, + new_metadata_location: None, + requirements: Vec::new(), + updates: vec![serde_json::json!({"action": "set-properties", "updates": {"owner": "bad"}})], + writer: None, + }, + ) + .await + .expect_err("mismatched table identifier should fail"); + assert_eq!(table_error.status_code(), Some(StatusCode::BAD_REQUEST)); + assert_eq!( + store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"), + table_before + ); + + create_standard_recent_events_view(&store, &metadata_backend, &namespace).await; + let view_before = store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .expect("view should exist"); + let view_error = replace_view_response( + &store, + &metadata_backend, + "warehouse", + &namespace, + "recent_events", + RestCommitViewRequest { + identifier: Some(RestTableIdentifier { + namespace: vec!["analytics".to_string()], + name: "other".to_string(), + }), + _commit_id: None, + expected_version_token: None, + expected_metadata_location: None, + new_metadata_location: None, + requirements: Vec::new(), + updates: vec![serde_json::json!({"action": "set-properties", "updates": {"owner": "bad"}})], + }, + ) + .await + .expect_err("mismatched view identifier should fail"); + assert_eq!(view_error.status_code(), Some(StatusCode::BAD_REQUEST)); + assert_eq!( + store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .expect("view should exist"), + view_before + ); +} + +#[test] +fn table_updates_apply_standard_statistics_and_metadata_cleanup_actions() { let metadata = serde_json::json!({ + "last-updated-ms": 1, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": []}, + {"type": "struct", "schema-id": 1, "fields": []} + ], + "current-schema-id": 0, + "partition-specs": [ + {"spec-id": 0, "fields": []}, + {"spec-id": 1, "fields": []} + ], + "default-spec-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "default-sort-order-id": 0, + "snapshots": [{"snapshot-id": 10, "schema-id": 0}], + "current-snapshot-id": 10, + "refs": {"main": {"type": "branch", "snapshot-id": 10}}, "metadata-log": [] }); - let updates = vec![serde_json::json!({ - "action": "rewrite-everything" - })]; + let updated = apply_table_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "set-statistics", + "statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/stats.puffin", + "file-size-in-bytes": 128, + "file-footer-size-in-bytes": 16, + "blob-metadata": [{ + "type": "apache-datasketches-theta-v1", + "snapshot-id": 10, + "sequence-number": 1, + "fields": [1], + "properties": {"compression-codec": "zstd"} + }] + } + }), + serde_json::json!({ + "action": "set-partition-statistics", + "partition-statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/partition-stats.parquet", + "file-size-in-bytes": 64 + } + }), + serde_json::json!({"action": "remove-partition-specs", "spec-ids": [1]}), + serde_json::json!({"action": "remove-schemas", "schema-ids": [1]}), + ], + "metadata/00001.metadata.json", + 100, + ) + .expect("standard table updates should apply"); + assert_eq!(updated["statistics"][0]["snapshot-id"], 10); + assert_eq!(updated["partition-statistics"][0]["snapshot-id"], 10); + assert_eq!(updated["partition-specs"].as_array().map(Vec::len), Some(1)); + assert_eq!(updated["schemas"].as_array().map(Vec::len), Some(1)); + let removed = apply_table_commit_updates_at( + updated, + &[ + serde_json::json!({"action": "remove-statistics", "snapshot-id": 10}), + serde_json::json!({"action": "remove-partition-statistics", "snapshot-id": 10}), + ], + "metadata/00002.metadata.json", + 101, + ) + .expect("standard table removals should apply"); + assert!(removed["statistics"].as_array().is_some_and(Vec::is_empty)); + assert!(removed["partition-statistics"].as_array().is_some_and(Vec::is_empty)); +} - assert!(apply_table_commit_updates(metadata, &updates, "metadata/00001.metadata.json").is_err()); +#[test] +fn remove_snapshots_rejects_mixed_snapshot_id_types() { + let metadata = serde_json::json!({ + "snapshots": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "snapshot-log": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "metadata-log": [] + }); + let error = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({"action": "remove-snapshots", "snapshot-ids": [10, "bad"]})], + "metadata/00001.metadata.json", + 100, + ) + .expect_err("mixed snapshot id types must fail before removing snapshots"); + + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); +} + +#[test] +fn remove_snapshots_removes_associated_statistics_entries() { + let metadata = serde_json::json!({ + "last-updated-ms": 1, + "snapshots": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "snapshot-log": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "statistics": [ + {"snapshot-id": 10, "statistics-path": "s3://warehouse/stats-10.puffin"}, + {"snapshot-id": 11, "statistics-path": "s3://warehouse/stats-11.puffin"} + ], + "partition-statistics": [ + {"snapshot-id": 10, "statistics-path": "s3://warehouse/partition-stats-10.parquet"}, + {"snapshot-id": 11, "statistics-path": "s3://warehouse/partition-stats-11.parquet"} + ], + "metadata-log": [] + }); + let updated = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({"action": "remove-snapshots", "snapshot-ids": [10]})], + "metadata/00001.metadata.json", + 100, + ) + .expect("snapshot expiration should remove associated statistics entries"); + + for field in ["snapshots", "snapshot-log", "statistics", "partition-statistics"] { + assert_eq!(updated[field].as_array().map(Vec::len), Some(1)); + assert_eq!(updated[field][0]["snapshot-id"], 11); + } +} + +#[test] +fn removing_snapshots_clears_dangling_references_and_current_snapshot() { + let metadata = serde_json::json!({ + "last-updated-ms": 1, + "current-snapshot-id": 10, + "snapshots": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "refs": { + "main": {"snapshot-id": 10, "type": "branch"}, + "release": {"snapshot-id": 10, "type": "tag"}, + "audit": {"snapshot-id": 11, "type": "branch"} + }, + "snapshot-log": [{"snapshot-id": 10}, {"snapshot-id": 11}], + "metadata-log": [] + }); + let updated = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({"action": "remove-snapshots", "snapshot-ids": [10]})], + "metadata/00001.metadata.json", + 100, + ) + .expect("snapshot removal should clean references"); + + assert_eq!(updated["current-snapshot-id"], -1); + assert!(updated["refs"].get("main").is_none()); + assert!(updated["refs"].get("release").is_none()); + assert_eq!(updated["refs"]["audit"]["snapshot-id"], 11); +} + +#[test] +fn removing_an_intermediate_snapshot_truncates_earlier_history() { + let metadata = serde_json::json!({ + "last-updated-ms": 1, + "current-snapshot-id": 12, + "snapshots": [{"snapshot-id": 10}, {"snapshot-id": 11}, {"snapshot-id": 12}], + "refs": {"main": {"snapshot-id": 12, "type": "branch"}}, + "snapshot-log": [ + {"timestamp-ms": 10, "snapshot-id": 10}, + {"timestamp-ms": 11, "snapshot-id": 11}, + {"timestamp-ms": 12, "snapshot-id": 12} + ], + "metadata-log": [] + }); + let updated = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({"action": "remove-snapshots", "snapshot-ids": [11]})], + "metadata/00001.metadata.json", + 100, + ) + .expect("snapshot removal should preserve valid time-travel history only"); + + assert_eq!(updated["snapshot-log"], serde_json::json!([{"timestamp-ms": 12, "snapshot-id": 12}])); +} + +#[test] +fn removing_main_snapshot_reference_clears_current_snapshot() { + let metadata = serde_json::json!({ + "last-updated-ms": 1, + "current-snapshot-id": 10, + "snapshots": [{"snapshot-id": 10}], + "refs": {"main": {"snapshot-id": 10, "type": "branch"}}, + "snapshot-log": [{"snapshot-id": 10}], + "metadata-log": [] + }); + let updated = apply_table_commit_updates_at( + metadata, + &[serde_json::json!({"action": "remove-snapshot-ref", "ref-name": "main"})], + "metadata/00001.metadata.json", + 100, + ) + .expect("main reference removal should succeed"); + + assert_eq!(updated["current-snapshot-id"], -1); + assert!(updated["refs"].get("main").is_none()); +} + +#[test] +fn table_statistics_updates_reject_malformed_standard_files() { + let metadata = serde_json::json!({"metadata-log": []}); + for update in [ + serde_json::json!({ + "action": "set-statistics", + "statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/stats.puffin", + "file-footer-size-in-bytes": 1, + "blob-metadata": [] + } + }), + serde_json::json!({ + "action": "set-statistics", + "statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/stats.puffin", + "file-size-in-bytes": 1, + "file-footer-size-in-bytes": 2, + "blob-metadata": [] + } + }), + serde_json::json!({ + "action": "set-partition-statistics", + "partition-statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/partition-stats.parquet" + } + }), + serde_json::json!({ + "action": "set-statistics", + "snapshot-id": 11, + "statistics": { + "snapshot-id": 10, + "statistics-path": "s3://warehouse/stats.puffin", + "file-size-in-bytes": 1, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + } + }), + ] { + let error = apply_table_commit_updates_at(metadata.clone(), &[update], "metadata/00001.metadata.json", 100) + .expect_err("malformed statistics updates must fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); + assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); + } +} + +#[test] +fn table_encryption_key_updates_require_format_version_three() { + for update in [ + serde_json::json!({ + "action": "add-encryption-key", + "encryption-key": {"key-id": "key-1", "encrypted-key-metadata": "AQID"} + }), + serde_json::json!({"action": "remove-encryption-key", "key-id": "key-1"}), + ] { + let error = apply_table_commit_updates_at( + test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"), + &[update], + "metadata/00001.metadata.json", + 100, + ) + .expect_err("Iceberg v2 tables must reject v3 encryption-key updates"); + assert_eq!(error.status_code(), Some(StatusCode::NOT_ACCEPTABLE)); + } } #[test] @@ -6980,6 +8350,7 @@ fn create_view_request_accepts_standard_iceberg_rest_shape() { }, "view-version": { "version-id": 1, + "timestamp-ms": 1, "schema-id": 0, "summary": { "engine-name": "spark", @@ -7005,6 +8376,395 @@ fn create_view_request_accepts_standard_iceberg_rest_shape() { assert_eq!(request.properties.get("comment").map(String::as_str), Some("recent event ids")); } +#[test] +fn view_versions_use_the_created_schema_and_resolve_minus_one() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateViewRequest = serde_json::from_value(serde_json::json!({ + "name": "recent_events", + "schema": {"type": "struct", "schema-id": 3, "fields": []}, + "view-version": { + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 99, + "summary": {"engine-name": "spark"}, + "default-catalog": "warehouse", + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }, + "properties": {} + })) + .expect("view request should parse"); + let (_, metadata) = view_entry_from_create_view_request("warehouse", &namespace, request) + .expect("create should resolve the current schema placeholder"); + assert_eq!(metadata["schemas"][0]["schema-id"], 0); + assert_eq!(metadata["versions"][0]["schema-id"], 0); + + let updated = apply_view_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "add-schema", + "schema": {"type": "struct", "schema-id": 99, "fields": []} + }), + serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 2, + "timestamp-ms": 2, + "schema-id": -1, + "summary": {"engine-name": "spark"}, + "default-catalog": "warehouse", + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 2", "dialect": "spark"}] + } + }), + ], + 2, + ) + .expect("view commit should resolve the current schema placeholder"); + assert_eq!(updated["schemas"][1]["schema-id"], 1); + assert_eq!(updated["versions"][1]["schema-id"], 1); + assert_eq!(updated["versions"][1]["timestamp-ms"], 2); + assert!(updated.get("last-updated-ms").is_none()); + assert!(updated.get("metadata-log").is_none()); + assert!(updated.get("last-column-id").is_none()); +} + +#[test] +fn current_view_version_minus_one_selects_the_last_added_version() { + let metadata = serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://warehouse/views/view-id", + "current-version-id": 5, + "schemas": [{"type": "struct", "schema-id": 0, "fields": []}], + "versions": [{ + "version-id": 5, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 5", "dialect": "spark"}] + }], + "version-log": [{"version-id": 5, "timestamp-ms": 1}], + "properties": {} + }); + let updated = apply_view_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 3, + "timestamp-ms": 2, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 3", "dialect": "spark"}] + } + }), + serde_json::json!({"action": "set-current-view-version", "view-version-id": -1}), + ], + 2, + ) + .expect("minus one should resolve to the last added view version"); + + assert_eq!(updated["current-version-id"], 3); + assert_eq!( + updated["version-log"] + .as_array() + .and_then(|log| log.last()) + .map(|entry| &entry["version-id"]), + Some(&serde_json::Value::from(3)) + ); + assert_eq!( + updated["version-log"] + .as_array() + .and_then(|log| log.last()) + .map(|entry| &entry["timestamp-ms"]), + Some(&serde_json::Value::from(2)) + ); +} + +#[test] +fn last_added_view_ids_require_a_preceding_add_update() { + let metadata = serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://warehouse/views/view-id", + "current-version-id": 1, + "schemas": [{"type": "struct", "schema-id": 0, "fields": []}], + "versions": [{ + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }], + "version-log": [{"version-id": 1, "timestamp-ms": 1}], + "properties": {} + }); + let version = serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 2, + "timestamp-ms": 2, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 2", "dialect": "spark"}] + } + }); + let invalid_update_sequences = [ + vec![serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 2, + "timestamp-ms": 2, + "schema-id": -1, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 2", "dialect": "spark"}] + } + })], + vec![serde_json::json!({"action": "set-current-view-version", "view-version-id": -1})], + vec![ + serde_json::json!({"action": "set-current-view-version", "view-version-id": -1}), + version, + ], + ]; + + for updates in invalid_update_sequences { + let error = apply_view_commit_updates_at(metadata.clone(), &updates, 3) + .expect_err("historical or later view objects must not satisfy a -1 reference"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + assert!( + error + .message() + .is_some_and(|message| message.contains("requires a preceding")) + ); + } +} + +#[test] +fn view_history_uses_added_version_time_and_skips_current_version_noops() { + let metadata = serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://warehouse/views/view-id", + "current-version-id": 1, + "schemas": [{"type": "struct", "schema-id": 0, "fields": []}], + "versions": [{ + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }], + "version-log": [{"version-id": 1, "timestamp-ms": 1}], + "properties": {} + }); + let unchanged = apply_view_commit_updates_at( + metadata.clone(), + &[serde_json::json!({"action": "set-current-view-version", "view-version-id": 1})], + 100, + ) + .expect("setting the current view version again should be a no-op"); + assert_eq!(unchanged["version-log"], metadata["version-log"]); + + let updated = apply_view_commit_updates_at( + metadata, + &[ + serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 2, + "timestamp-ms": 20, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 2", "dialect": "spark"}] + } + }), + serde_json::json!({ + "action": "add-view-version", + "view-version": { + "version-id": 3, + "timestamp-ms": 30, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 3", "dialect": "spark"}] + } + }), + serde_json::json!({"action": "set-current-view-version", "view-version-id": 2}), + ], + 100, + ) + .expect("an explicitly selected version added in this commit should use its own timestamp"); + assert_eq!(updated["current-version-id"], 2); + assert_eq!( + updated["version-log"].as_array().and_then(|log| log.last()), + Some(&serde_json::json!({ + "timestamp-ms": 20, + "version-id": 2 + })) + ); +} + +#[test] +fn view_requests_and_metadata_require_standard_fields() { + let request_without_properties = serde_json::from_value::(serde_json::json!({ + "name": "recent_events", + "schema": {"type": "struct", "schema-id": 0, "fields": []}, + "view-version": { + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + } + })) + .expect("the Java REST serializer omits empty view properties"); + assert!(request_without_properties.properties.is_empty()); + + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request = serde_json::from_value::(serde_json::json!({ + "name": "recent_events", + "schema": {"type": "struct", "schema-id": 0, "fields": []}, + "view-version": { + "version-id": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }, + "properties": {} + })) + .expect("request shape should parse before metadata validation"); + let error = view_entry_from_create_view_request("warehouse", &namespace, request) + .expect_err("view-version timestamp-ms must be required"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + + let malformed_schema_request = serde_json::from_value::(serde_json::json!({ + "name": "recent_events", + "schema": {}, + "view-version": { + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }, + "properties": {} + })) + .expect("request shape should parse before schema validation"); + view_entry_from_create_view_request("warehouse", &namespace, malformed_schema_request) + .expect_err("create view must reject schemas missing type and fields"); + + let metadata = serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://warehouse/views/view-id", + "current-version-id": 1, + "schemas": [{"type": "struct", "schema-id": 0, "fields": []}], + "versions": [{ + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }], + "version-log": [{"version-id": 1, "timestamp-ms": 1}], + "properties": {} + }); + let mut missing_current = metadata.clone(); + missing_current + .as_object_mut() + .expect("metadata should be an object") + .remove("current-version-id"); + validate_supported_view_metadata(&missing_current).expect_err("current-version-id must be required"); + + let mut unsupported_representation = metadata.clone(); + unsupported_representation["versions"][0]["representations"][0]["type"] = serde_json::Value::from("python"); + validate_supported_view_metadata(&unsupported_representation).expect_err("non-SQL view representations must be rejected"); + + let mut duplicate_dialect = metadata.clone(); + duplicate_dialect["versions"][0]["representations"] = serde_json::json!([ + {"type": "sql", "sql": "SELECT 1", "dialect": "spark"}, + {"type": "sql", "sql": "SELECT 2", "dialect": "SPARK"} + ]); + validate_supported_view_metadata(&duplicate_dialect).expect_err("a view version must not contain duplicate SQL dialects"); + + let mut malformed_schema = metadata.clone(); + malformed_schema["schemas"] = serde_json::json!([{"schema-id": 0}]); + validate_supported_view_metadata(&malformed_schema).expect_err("view schemas must include type and fields"); + + let updated = apply_view_commit_updates_at( + metadata.clone(), + &[serde_json::json!({"action": "set-properties", "updates": {"owner": "analytics"}})], + 2, + ) + .expect("standard view metadata without table-only timestamps must remain mutable"); + assert_eq!(updated["properties"]["owner"], "analytics"); + assert!(updated.get("last-updated-ms").is_none()); + assert!(updated.get("metadata-log").is_none()); + + let missing_version_id = apply_view_commit_updates_at( + metadata, + &[serde_json::json!({ + "action": "add-view-version", + "view-version": { + "timestamp-ms": 2, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 2", "dialect": "spark"}] + } + })], + 2, + ) + .expect_err("add-view-version must not synthesize version-id"); + assert_eq!(missing_version_id.code(), &S3ErrorCode::InvalidRequest); +} + +#[test] +fn view_commit_rejects_unsupported_format_version_upgrade() { + let metadata = serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://warehouse/views/view-id", + "schemas": [{"schema-id": 0, "type": "struct", "fields": []}], + "versions": [{ + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {"engine-name": "spark"}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }], + "current-version-id": 1, + "version-log": [], + "metadata-log": [], + "properties": {} + }); + + let error = apply_view_commit_updates_at( + metadata, + &[serde_json::json!({"action": "upgrade-format-version", "format-version": 2})], + 0, + ) + .expect_err("Iceberg view format-version 2 is unsupported"); + + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_UNSUPPORTED_OPERATION.into())); + assert_eq!(error.status_code(), Some(StatusCode::NOT_ACCEPTABLE)); +} + #[test] fn create_view_request_accepts_deep_warehouse_location() { let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -7021,6 +8781,7 @@ fn create_view_request_accepts_deep_warehouse_location() { }, "view-version": { "version-id": 1, + "timestamp-ms": 1, "schema-id": 0, "summary": { "engine-name": "spark" @@ -7034,7 +8795,8 @@ fn create_view_request_accepts_deep_warehouse_location() { "dialect": "spark" } ] - } + }, + "properties": {} })) .expect("deep create view request should parse"); @@ -7082,6 +8844,7 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { }, "view-version": { "version-id": 1, + "timestamp-ms": 1, "schema-id": 0, "summary": { "engine-name": "spark" @@ -7095,19 +8858,23 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { "dialect": "spark" } ] - } + }, + "properties": {} })) .expect("standard create view request should parse"); - let created = create_view_response(&store, &metadata_backend, "warehouse", &namespace, create_request, true) + let create_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let created = create_view_response(&store, &create_backend, "warehouse", &namespace, create_request, true) .await .expect("view should be created"); assert_eq!(created.metadata["format-version"], 1); assert_eq!(created.metadata["current-version-id"], 1); assert_eq!(created.metadata["versions"][0]["representations"][0]["dialect"], "spark"); + let created_metadata_key = table_metadata_location_for_catalog("warehouse", &created.metadata_location) + .expect("created metadata location should normalize for the object backend"); assert!( metadata_backend - .object_exists("warehouse", &created.metadata_location) + .object_exists("warehouse", &created_metadata_key) .await .expect("view metadata object lookup should succeed") ); @@ -7123,12 +8890,51 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { .await .expect("view should load"); assert_eq!(loaded.metadata_location, created.metadata_location); + + let metadata_directory = created_metadata_key + .rsplit_once('/') + .map(|(directory, _)| directory) + .expect("created metadata key should have a directory"); + let invalid_target_key = format!("{metadata_directory}/invalid.metadata.json"); + let mut invalid_target = created.metadata.clone(); + invalid_target["format-version"] = serde_json::Value::from(2); + metadata_backend + .put_json("warehouse", &invalid_target_key, invalid_target) + .await; + let invalid_replace = serde_json::from_value::(serde_json::json!({ + "new-metadata-location": table_metadata_location_for_client("warehouse", &invalid_target_key), + "updates": [] + })) + .expect("external replace request should parse"); + let invalid_replace_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let error = replace_view_response( + &store, + &invalid_replace_backend, + "warehouse", + &namespace, + "recent_events", + invalid_replace, + ) + .await + .expect_err("unsupported external view metadata must fail before pointer publication"); + assert_eq!(error.status_code(), Some(StatusCode::NOT_ACCEPTABLE)); + assert_eq!( + store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .expect("view should remain registered") + .metadata_location, + created_metadata_key + ); + let replace_request: RestCommitViewRequest = serde_json::from_value(serde_json::json!({ "updates": [ { "action": "add-view-version", "view-version": { "version-id": 2, + "timestamp-ms": 2, "schema-id": 0, "summary": { "engine-name": "spark" @@ -7151,7 +8957,8 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { ] })) .expect("replace view request should parse"); - let replaced = replace_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events", replace_request) + let replace_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let replaced = replace_view_response(&store, &replace_backend, "warehouse", &namespace, "recent_events", replace_request) .await .expect("view should replace"); assert_ne!(replaced.metadata_location, created.metadata_location); @@ -7188,6 +8995,7 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { }, "view-version": { "version-id": 1, + "timestamp-ms": 1, "schema-id": 0, "summary": { "engine-name": "spark" @@ -7201,15 +9009,217 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { "dialect": "spark" } ] - } + }, + "properties": {} })) .expect("standard recreate view request should parse"); - let recreated = create_view_response(&store, &metadata_backend, "warehouse", &namespace, recreate_request, true) + let recreate_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let recreated = create_view_response(&store, &recreate_backend, "warehouse", &namespace, recreate_request, true) .await .expect("dropped view name should be reusable"); assert_ne!(recreated.metadata_location, created.metadata_location); } +#[tokio::test] +async fn replace_view_holds_target_metadata_and_view_fences_until_pointer_publish() { + let pause = TestCatalogPublishPause::default(); + let store = Arc::new(TestTableCatalogStore { + replace_view_pause: Some(pause.clone()), + ..Default::default() + }); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_recent_events_view(store.as_ref(), &metadata_backend, &namespace).await; + let current_metadata_key = table_metadata_location_for_catalog("warehouse", &created.metadata_location) + .expect("created metadata location should normalize"); + let metadata_directory = current_metadata_key + .rsplit_once('/') + .map(|(directory, _)| directory) + .expect("created metadata key should have a directory"); + let target_metadata_key = format!("{metadata_directory}/external.metadata.json"); + metadata_backend + .put_json("warehouse", &target_metadata_key, created.metadata.clone()) + .await; + let request = serde_json::from_value::(serde_json::json!({ + "new-metadata-location": table_metadata_location_for_client("warehouse", &target_metadata_key), + "requirements": [], + "updates": [] + })) + .expect("replace view request should parse"); + + let replace_store = Arc::clone(&store); + let replace_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let replace_namespace = namespace.clone(); + let replace = tokio::spawn(async move { + let result = replace_view_response( + replace_store.as_ref(), + &replace_backend, + "warehouse", + &replace_namespace, + "recent_events", + request, + ) + .await; + replace_backend.finish(result).await + }); + tokio::time::timeout(StdDuration::from_secs(2), pause.wait_started()) + .await + .expect("view replacement should reach catalog publication"); + + let view_name = crate::table_catalog::IdentifierSegment::parse("recent_events").expect("view should parse"); + let view_lock = crate::table_catalog::default_table_publication_lock_path(&namespace, &view_name); + let bucket_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + assert!( + !metadata_backend.write_lock_is_held("warehouse", &bucket_lock).await, + "view replacement without warehouse relocation must not serialize the table bucket" + ); + assert!( + metadata_backend.write_lock_is_held("warehouse", &view_lock).await, + "view replacement must retain its publication fence until pointer publication" + ); + assert!( + metadata_backend.write_lock_is_held("warehouse", &target_metadata_key).await, + "target view metadata must remain stable until pointer publication" + ); + + metadata_backend.lock_attempts.lock().await.clear(); + let writer_backend = metadata_backend.clone(); + let writer_target = target_metadata_key.clone(); + let writer = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock(&writer_backend, "warehouse", &writer_target).await + }); + metadata_backend.wait_for_lock_attempts(1).await; + assert!(!writer.is_finished(), "a target metadata writer must wait for pointer publication"); + + pause.release(); + let replaced = tokio::time::timeout(StdDuration::from_secs(2), replace) + .await + .expect("view replacement should complete") + .expect("view replacement task should join") + .expect("view replacement should succeed"); + tokio::time::timeout(StdDuration::from_secs(2), writer) + .await + .expect("target metadata writer should continue after publication") + .expect("target metadata writer task should join") + .expect("target metadata writer lock acquisition should succeed"); + assert_eq!( + table_metadata_location_for_catalog("warehouse", &replaced.metadata_location) + .expect("replaced metadata location should normalize"), + target_metadata_key + ); +} + +#[tokio::test] +async fn replace_view_holds_table_bucket_fence_for_warehouse_relocation() { + let pause = TestCatalogPublishPause::default(); + let store = Arc::new(TestTableCatalogStore { + replace_view_pause: Some(pause.clone()), + ..Default::default() + }); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_recent_events_view(store.as_ref(), &metadata_backend, &namespace).await; + let current_metadata_key = table_metadata_location_for_catalog("warehouse", &created.metadata_location) + .expect("created metadata location should normalize"); + let metadata_directory = current_metadata_key + .rsplit_once('/') + .map(|(directory, _)| directory) + .expect("created metadata key should have a directory"); + let target_metadata_key = format!("{metadata_directory}/relocated.metadata.json"); + let mut target_metadata = created.metadata; + target_metadata["location"] = serde_json::Value::String("s3://warehouse/views/relocated".to_string()); + metadata_backend + .put_json("warehouse", &target_metadata_key, target_metadata) + .await; + let request = serde_json::from_value::(serde_json::json!({ + "new-metadata-location": table_metadata_location_for_client("warehouse", &target_metadata_key), + "requirements": [], + "updates": [] + })) + .expect("replace view request should parse"); + + let replace_store = Arc::clone(&store); + let replace_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let replace_namespace = namespace.clone(); + let replace = tokio::spawn(async move { + let result = replace_view_response( + replace_store.as_ref(), + &replace_backend, + "warehouse", + &replace_namespace, + "recent_events", + request, + ) + .await; + replace_backend.finish(result).await + }); + tokio::time::timeout(StdDuration::from_secs(2), pause.wait_started()) + .await + .expect("view replacement should reach catalog publication"); + + let bucket_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + assert!( + metadata_backend.write_lock_is_held("warehouse", &bucket_lock).await, + "view warehouse relocation must retain the table-bucket publication fence" + ); + + pause.release(); + tokio::time::timeout(StdDuration::from_secs(2), replace) + .await + .expect("view replacement should complete") + .expect("view replacement task should join") + .expect("view replacement should succeed"); +} + +#[tokio::test] +async fn external_view_metadata_replacement_repairs_legacy_incomplete_metadata() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_recent_events_view(&store, &metadata_backend, &namespace).await; + let current = store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .expect("view should exist"); + let mut incomplete = created.metadata.clone(); + incomplete + .as_object_mut() + .expect("view metadata should be an object") + .remove("versions"); + metadata_backend + .put_json("warehouse", ¤t.metadata_location, incomplete) + .await; + load_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events") + .await + .expect_err("legacy incomplete view metadata must not be served"); + + let metadata_directory = current + .metadata_location + .rsplit_once('/') + .map(|(directory, _)| directory) + .expect("current metadata location should have a directory"); + let target = format!("{metadata_directory}/repaired.metadata.json"); + metadata_backend + .put_json("warehouse", &target, created.metadata.clone()) + .await; + let request = serde_json::from_value::(serde_json::json!({ + "new-metadata-location": table_metadata_location_for_client("warehouse", &target), + "requirements": [{"type": "assert-view-uuid", "uuid": current.view_uuid}], + "updates": [] + })) + .expect("view repair request should parse"); + let publication_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let repaired = replace_view_response(&store, &publication_backend, "warehouse", &namespace, "recent_events", request) + .await + .expect("a valid external metadata target should repair legacy incomplete metadata"); + + assert_eq!(repaired.metadata, created.metadata); + load_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events") + .await + .expect("the repaired view should load"); +} + #[tokio::test] async fn table_ref_write_responses_use_commit_guard_and_protect_deletes() { let store = TestTableCatalogStore::default(); @@ -7224,6 +9234,7 @@ async fn table_ref_write_responses_use_commit_guard_and_protect_deletes() { seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; let append_request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "requirements": [], "updates": [ { "action": "add-snapshot", @@ -7436,6 +9447,75 @@ fn load_table_response_preserves_format_v4_relative_metadata_log() { ); } +#[test] +fn load_table_snapshot_selection_validates_and_filters_refs() { + assert_eq!( + rest_table_snapshot_selection_from_query(&"/".parse().expect("URI should parse")) + .expect("omitted snapshots selection should parse"), + RestTableSnapshotSelection::All + ); + assert_eq!( + rest_table_snapshot_selection_from_query(&"/?snapshots=all".parse().expect("URI should parse")) + .expect("all snapshots selection should parse"), + RestTableSnapshotSelection::All + ); + assert_eq!( + rest_table_snapshot_selection_from_query(&"/?snapshots=refs".parse().expect("URI should parse")) + .expect("referenced snapshots selection should parse"), + RestTableSnapshotSelection::Refs + ); + for uri in ["/?snapshots=unknown", "/?snapshots=all&snapshots=refs"] { + let error = rest_table_snapshot_selection_from_query(&uri.parse().expect("URI should parse")) + .expect_err("invalid snapshots selections must fail"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); + } + + let metadata = serde_json::json!({ + "snapshots": [ + {"snapshot-id": 1}, + {"snapshot-id": 2}, + {"snapshot-id": 3} + ], + "refs": { + "audit": {"type": "tag", "snapshot-id": 1}, + "main": {"type": "branch", "snapshot-id": 3} + } + }); + let mut all = metadata.clone(); + apply_rest_table_snapshot_selection(&mut all, RestTableSnapshotSelection::All); + assert_eq!(all["snapshots"].as_array().map(Vec::len), Some(3)); + + let mut referenced = metadata; + apply_rest_table_snapshot_selection(&mut referenced, RestTableSnapshotSelection::Refs); + assert_eq!( + referenced["snapshots"] + .as_array() + .expect("snapshots should remain an array") + .iter() + .filter_map(|snapshot| snapshot["snapshot-id"].as_i64()) + .collect::>(), + vec![1, 3] + ); + + let mut implicit_main = serde_json::json!({ + "current-snapshot-id": 2, + "snapshots": [ + {"snapshot-id": 1}, + {"snapshot-id": 2} + ] + }); + apply_rest_table_snapshot_selection(&mut implicit_main, RestTableSnapshotSelection::Refs); + assert_eq!( + implicit_main["snapshots"] + .as_array() + .expect("snapshots should remain an array") + .iter() + .filter_map(|snapshot| snapshot["snapshot-id"].as_i64()) + .collect::>(), + vec![2] + ); +} + #[test] fn table_metadata_location_for_catalog_accepts_only_the_table_bucket() { let object_key = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json"; @@ -7640,6 +9720,19 @@ async fn credential_response_serializes_sensitive_config_only_inside_storage_cre ); } +#[test] +fn credential_http_response_disables_caching() { + let response = build_sensitive_json_response(StatusCode::OK, &serde_json::json!({"storage-credentials": []})) + .expect("sensitive response should build"); + + assert_eq!( + response.headers.get(http::header::CACHE_CONTROL), + Some(&HeaderValue::from_static("no-store, private")) + ); + assert_eq!(response.headers.get(http::header::PRAGMA), Some(&HeaderValue::from_static("no-cache"))); + assert_eq!(response.headers.get(http::header::EXPIRES), Some(&HeaderValue::from_static("0"))); +} + #[test] fn table_credentials_do_not_snapshot_parent_groups() { let principal = rustfs_credentials::Credentials { @@ -7817,10 +9910,12 @@ fn commit_table_request_uses_rest_commit_fields() { "new-metadata-location": ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json", "requirements": [ { - "type": "assert-current-snapshot-id", + "type": "assert-ref-snapshot-id", + "ref": "main", "snapshot-id": 10 } ], + "updates": [], "writer": "pyiceberg" })) .expect("commit request should parse"); @@ -7837,6 +9932,26 @@ fn commit_table_request_uses_rest_commit_fields() { assert_eq!(request.writer.as_deref(), Some("pyiceberg")); } +#[test] +fn rest_commit_item_counts_are_bounded_before_processing() { + let allowed = vec![serde_json::Value::Null; TABLE_CATALOG_COMMIT_REQUIREMENT_MAX_COUNT]; + validate_rest_commit_item_counts(&allowed, &[]).expect("the documented requirement limit should be accepted"); + let too_many_requirements = vec![serde_json::Value::Null; TABLE_CATALOG_COMMIT_REQUIREMENT_MAX_COUNT + 1]; + assert_eq!( + validate_rest_commit_item_counts(&too_many_requirements, &[]) + .expect_err("excess requirements must be rejected") + .code(), + &S3ErrorCode::InvalidRequest + ); + let too_many_updates = vec![serde_json::Value::Null; TABLE_CATALOG_COMMIT_UPDATE_MAX_COUNT + 1]; + assert_eq!( + validate_rest_commit_item_counts(&[], &too_many_updates) + .expect_err("excess updates must be rejected") + .code(), + &S3ErrorCode::InvalidRequest + ); +} + fn trusted_table_commit_backend( backend: &TestTableCatalogObjectBackend, ) -> TableCommitObjectBackend { @@ -7870,15 +9985,53 @@ async fn seed_test_manifest_list( snapshot_id: i64, ) { let manifest_list_key = test_snapshot_object_key(bucket, manifest_list_location); + let mut manifests = Vec::with_capacity(manifest_locations.len()); + for manifest_location in manifest_locations { + let manifest_key = test_snapshot_object_key(bucket, manifest_location); + let manifest_length = backend + .state + .lock() + .await + .objects + .get(&(bucket.to_string(), manifest_key)) + .map(|object| object.data.len()) + .expect("test manifest must be seeded before its manifest list"); + manifests.push((*manifest_location, manifest_length)); + } backend .put_bytes( bucket, &manifest_list_key, - test_manifest_list_avro_bytes(manifest_locations, sequence_number, snapshot_id), + test_manifest_list_avro_bytes(&manifests, sequence_number, snapshot_id), ) .await; } +async fn seed_test_manifest_list_entries( + backend: &TestTableCatalogObjectBackend, + bucket: &str, + manifest_list_location: &str, + manifest_entries: &[(&str, i64, i64)], +) { + let manifest_list_key = test_snapshot_object_key(bucket, manifest_list_location); + let mut manifests = Vec::with_capacity(manifest_entries.len()); + for (manifest_location, sequence_number, snapshot_id) in manifest_entries { + let manifest_key = test_snapshot_object_key(bucket, manifest_location); + let manifest_length = backend + .state + .lock() + .await + .objects + .get(&(bucket.to_string(), manifest_key)) + .map(|object| object.data.len()) + .expect("test manifest must be seeded before its manifest list"); + manifests.push((*manifest_location, manifest_length, *sequence_number, *snapshot_id)); + } + backend + .put_bytes(bucket, &manifest_list_key, test_manifest_list_avro_entries(&manifests)) + .await; +} + async fn seed_test_snapshot_manifest( backend: &TestTableCatalogObjectBackend, bucket: &str, @@ -7893,16 +10046,15 @@ async fn seed_test_snapshot_manifest( .expect("manifest list location should include a file name"); let manifest_key = test_snapshot_object_key(bucket, &manifest_location); let manifest_list_key = test_snapshot_object_key(bucket, manifest_list_location); + let manifest_bytes = test_manifest_avro_bytes(files); backend .put_bytes( bucket, &manifest_list_key, - test_manifest_list_avro_bytes(&[&manifest_location], sequence_number, snapshot_id), + test_manifest_list_avro_bytes(&[(&manifest_location, manifest_bytes.len())], sequence_number, snapshot_id), ) .await; - backend - .put_bytes(bucket, &manifest_key, test_manifest_avro_bytes(files)) - .await; + backend.put_bytes(bucket, &manifest_key, manifest_bytes).await; seed_test_manifest_data_files(backend, bucket, files).await; } @@ -7995,6 +10147,55 @@ where .expect("table should be created") } +async fn create_standard_recent_events_view( + store: &S, + metadata_backend: &TestTableCatalogObjectBackend, + namespace: &crate::table_catalog::Namespace, +) -> RestLoadViewResponse +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + ensure_table_bucket_entry(store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + if store + .get_namespace("warehouse", &namespace.public_name()) + .await + .expect("namespace lookup should succeed") + .is_none() + { + create_namespace_response( + store, + "warehouse", + CreateNamespaceRequest { + namespace: namespace.public_name().split('.').map(str::to_string).collect(), + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + } + let request: CreateViewRequest = serde_json::from_value(serde_json::json!({ + "name": "recent_events", + "schema": {"type": "struct", "fields": []}, + "view-version": { + "version-id": 1, + "timestamp-ms": 1, + "schema-id": 0, + "summary": {}, + "default-namespace": ["analytics"], + "representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}] + }, + "properties": {} + })) + .expect("standard create view request should parse"); + let publication_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + create_view_response(store, &publication_backend, "warehouse", namespace, request, true) + .await + .expect("view should be created") +} + fn standard_property_commit_request(commit_id: &str, table_uuid: &str, owner: &str) -> RestCommitTableRequest { serde_json::from_value(serde_json::json!({ "commit-id": commit_id, @@ -8403,7 +10604,7 @@ async fn table_helpers_call_catalog_store() { new_metadata_location: Some(table_metadata_location_for_client("warehouse", next_metadata_location)), requirements: client_requirements.clone(), updates: Vec::new(), - _identifier: None, + identifier: None, writer: Some("pyiceberg".to_string()), }, ) @@ -8445,6 +10646,121 @@ async fn table_helpers_call_catalog_store() { ); } +#[tokio::test] +async fn load_table_rejects_invalid_persisted_metadata() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let metadata_key = table_metadata_location_for_catalog("warehouse", &created.metadata_location) + .expect("created metadata location should normalize"); + let mut invalid_metadata = created.metadata; + invalid_metadata["location"] = serde_json::Value::from("s3://other-warehouse/tables/table-id"); + metadata_backend.put_json("warehouse", &metadata_key, invalid_metadata).await; + + let error = load_table_response(&store, &metadata_backend, "warehouse", &namespace, "events") + .await + .expect_err("load table must reject persisted metadata outside the table bucket"); + assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_REST.into())); + assert_eq!(error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR)); +} + +#[test] +fn table_format_upgrade_accepts_historical_v1_metadata_and_rejects_downgrade() { + let entry = crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: "analytics".to_string(), + table: "events".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://warehouse/tables/table-id".to_string(), + metadata_location: "tables/table-id/metadata/00002.metadata.json".to_string(), + version_token: "token-v2".to_string(), + generation: 2, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + let historical_v1 = serde_json::json!({ + "format-version": 1, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-updated-ms": 1, + "last-column-id": 1, + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "partition-spec": [], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [] + }); + let current_v2 = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); + + validate_persisted_table_metadata(&entry, &historical_v1, false) + .expect("historical v1 metadata should remain readable after a v2 upgrade"); + validate_persisted_table_metadata(&entry, &historical_v1, true) + .expect_err("the current pointer must match the catalog format version"); + let mut legacy_entry = entry; + legacy_entry.format_version = 1; + validate_persisted_table_metadata(&legacy_entry, ¤t_v2, true) + .expect("a current v2 metadata file committed before format persistence must remain readable"); + validate_persisted_table_metadata(&legacy_entry, ¤t_v2, false) + .expect("post-upgrade v2 metadata must remain readable as a historical commit base"); + validate_metadata_identity_matches_current_metadata(&historical_v1, ¤t_v2).expect("a v1 table may upgrade to v2"); + validate_metadata_identity_matches_current_metadata(¤t_v2, &historical_v1) + .expect_err("a v2 table must not downgrade to v1"); +} + +#[tokio::test] +async fn load_responses_reject_persisted_metadata_for_another_catalog_identity() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_entry = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let mut foreign_table_metadata = table.metadata; + foreign_table_metadata["table-uuid"] = serde_json::Value::from("foreign-table-uuid"); + metadata_backend + .put_json("warehouse", &table_entry.metadata_location, foreign_table_metadata) + .await; + + let table_error = load_table_response(&store, &metadata_backend, "warehouse", &namespace, "events") + .await + .expect_err("load table must bind persisted metadata to the catalog identity"); + assert_eq!(table_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_REST.into())); + assert_eq!(table_error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR)); + + let view = create_standard_recent_events_view(&store, &metadata_backend, &namespace).await; + let view_entry = store + .load_view("warehouse", "analytics", "recent_events") + .await + .expect("view lookup should succeed") + .expect("view should exist"); + let mut foreign_view_metadata = view.metadata; + foreign_view_metadata["view-uuid"] = serde_json::Value::from("foreign-view-uuid"); + metadata_backend + .put_json("warehouse", &view_entry.metadata_location, foreign_view_metadata) + .await; + + let view_error = load_view_response(&store, &metadata_backend, "warehouse", &namespace, "recent_events") + .await + .expect_err("load view must bind persisted metadata to the catalog identity"); + assert_eq!(view_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_REST.into())); + assert_eq!(view_error.status_code(), Some(StatusCode::INTERNAL_SERVER_ERROR)); +} + #[tokio::test] async fn register_table_response_adopts_metadata_table_uuid() { let store = TestTableCatalogStore::default(); @@ -8805,13 +11121,10 @@ async fn metadata_location_api_loads_and_updates_current_pointer() { ) .expect("table entry should build"); let table_uuid = entry.table_uuid.clone(); + let warehouse_location = entry.warehouse_location.clone(); store.register_table(entry).await.expect("table should register"); metadata_backend - .put_json( - "warehouse", - current_location, - test_table_metadata_json(&table_uuid, "s3://warehouse/tables/table-id"), - ) + .put_json("warehouse", current_location, test_table_metadata_json(&table_uuid, &warehouse_location)) .await; let current = get_table_metadata_location_response(&store, "warehouse", &namespace, "events") .await @@ -8822,11 +11135,7 @@ async fn metadata_location_api_loads_and_updates_current_pointer() { ); let next_location = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002.metadata.json"; metadata_backend - .put_json( - "warehouse", - next_location, - test_table_metadata_json(&table_uuid, "s3://warehouse/tables/table-id"), - ) + .put_json("warehouse", next_location, test_table_metadata_json(&table_uuid, &warehouse_location)) .await; let updated = update_table_metadata_location_response( @@ -9573,7 +11882,7 @@ async fn legacy_commit_rejects_mismatched_table_uuid_before_commit() { new_metadata_location: Some(mismatched_location.to_string()), requirements: Vec::new(), updates: Vec::new(), - _identifier: None, + identifier: None, writer: Some("pyiceberg".to_string()), }, ) diff --git a/rustfs/src/admin/handlers/table_catalog/view.rs b/rustfs/src/admin/handlers/table_catalog/view.rs index 2b4ca5d31..b18b7e5ce 100644 --- a/rustfs/src/admin/handlers/table_catalog/view.rs +++ b/rustfs/src/admin/handlers/table_catalog/view.rs @@ -43,8 +43,9 @@ impl Operation for RestCreateViewHandler { let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; let table_bucket_enabled = table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let publication_backend = TableCommitObjectBackend::preauthorized(metadata_backend); let response = - create_view_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; + create_view_response(&store, &publication_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; build_json_response(StatusCode::OK, &response) } } @@ -87,17 +88,20 @@ pub struct RestReplaceViewHandler {} #[async_trait::async_trait] impl Operation for RestReplaceViewHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; let namespace = namespace_from_params(¶ms)?; let view = view_name_from_params(¶ms)?; let resource = TableCatalogResource::view(&warehouse, &namespace, &view); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + install_table_catalog_s3_request_info(&mut req, &principal)?; ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; - let request = read_json_body::(req.input).await?; + let request = read_rest_commit_view_request(std::mem::take(&mut req.input)).await?; let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = replace_view_response(&store, &metadata_backend, &warehouse, &namespace, &view, request).await?; + let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); + let result = replace_view_response(&store, &commit_backend, &warehouse, &namespace, &view, request).await; + let response = commit_backend.finish(result).await?; build_json_response(StatusCode::OK, &response) } } diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 806035b70..2d62f16c9 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -1478,7 +1478,7 @@ async fn retain_table_data_plane_publication_guard( .map_err(|err| s3_error!(InternalError, "failed to acquire table publication guard: {}", err))?; let mut state = retained.state.lock(); state.keys.insert(key); - state.guards.push(guard); + state.guards.push(Box::new(guard)); drop(state); req.extensions.insert(retained); Ok(()) diff --git a/rustfs/src/table_catalog/iceberg/manifest.rs b/rustfs/src/table_catalog/iceberg/manifest.rs index cdc0a2635..82960d255 100644 --- a/rustfs/src/table_catalog/iceberg/manifest.rs +++ b/rustfs/src/table_catalog/iceberg/manifest.rs @@ -16,6 +16,8 @@ use std::io::Read; use super::super::*; +const AVRO_ZSTANDARD_MAX_WINDOW_LOG: u32 = 27; + #[derive(Debug, Clone, PartialEq)] pub(crate) struct ManifestDataFileReference { pub location: String, @@ -66,6 +68,7 @@ pub(crate) struct DecodedManifestList { pub(crate) struct DecodedManifest { pub references: Vec, pub decoded_size: usize, + pub partition_spec_id: Option, } pub(crate) fn manifest_paths_from_manifest_list_avro(data: &[u8]) -> TableCatalogStoreResult> { @@ -92,6 +95,25 @@ pub(crate) fn decode_manifest_list_avro(data: &[u8]) -> TableCatalogStoreResult< .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to read manifest list Avro: {err}")))?; let format_version = avro_record_format_version(reader.writer_schema(), &["sequence_number", "min_sequence_number"], "manifest list")?; + if format_version == 2 { + let apache_avro::Schema::Record(record) = reader.writer_schema() else { + return Err(TableCatalogStoreError::Invalid("manifest list Avro schema must be a record".to_string())); + }; + for field in [ + "added_files_count", + "existing_files_count", + "deleted_files_count", + "added_rows_count", + "existing_rows_count", + "deleted_rows_count", + ] { + if !record.lookup.contains_key(field) { + return Err(TableCatalogStoreError::Invalid(format!( + "Iceberg v2 manifest list Avro schema is missing {field}" + ))); + } + } + } let mut manifest_paths = Vec::new(); for value in reader { if manifest_paths.len() >= TABLE_MANIFEST_AVRO_MAX_RECORDS { @@ -115,24 +137,12 @@ pub(crate) fn decode_manifest_list_avro(data: &[u8]) -> TableCatalogStoreResult< sequence_number: avro_record_field(&value, "sequence_number").and_then(avro_i64_value), min_sequence_number: avro_record_field(&value, "min_sequence_number").and_then(avro_i64_value), added_snapshot_id: avro_record_field(&value, "added_snapshot_id").and_then(avro_i64_value), - added_files_count: avro_record_field(&value, "added_files_count") - .and_then(avro_i32_value) - .and_then(|value| u64::try_from(value).ok()), - existing_files_count: avro_record_field(&value, "existing_files_count") - .and_then(avro_i32_value) - .and_then(|value| u64::try_from(value).ok()), - deleted_files_count: avro_record_field(&value, "deleted_files_count") - .and_then(avro_i32_value) - .and_then(|value| u64::try_from(value).ok()), - added_rows_count: avro_record_field(&value, "added_rows_count") - .and_then(avro_i64_value) - .and_then(|value| u64::try_from(value).ok()), - existing_rows_count: avro_record_field(&value, "existing_rows_count") - .and_then(avro_i64_value) - .and_then(|value| u64::try_from(value).ok()), - deleted_rows_count: avro_record_field(&value, "deleted_rows_count") - .and_then(avro_i64_value) - .and_then(|value| u64::try_from(value).ok()), + added_files_count: avro_nullable_non_negative_i32(&value, "added_files_count", "manifest list")?, + existing_files_count: avro_nullable_non_negative_i32(&value, "existing_files_count", "manifest list")?, + deleted_files_count: avro_nullable_non_negative_i32(&value, "deleted_files_count", "manifest list")?, + added_rows_count: avro_nullable_non_negative_i64(&value, "added_rows_count", "manifest list")?, + existing_rows_count: avro_nullable_non_negative_i64(&value, "existing_rows_count", "manifest list")?, + deleted_rows_count: avro_nullable_non_negative_i64(&value, "deleted_rows_count", "manifest list")?, }); } Ok(DecodedManifestList { @@ -171,6 +181,17 @@ pub(crate) fn decode_manifest_avro(data: &[u8]) -> TableCatalogStoreResult().ok()) + .filter(|value| *value >= 0) + .ok_or_else(|| TableCatalogStoreError::Invalid("manifest partition-spec-id metadata is invalid".to_string())) + }) + .transpose()?; let mut files = Vec::new(); for value in reader { if files.len() >= TABLE_MANIFEST_AVRO_MAX_RECORDS { @@ -202,6 +223,9 @@ pub(crate) fn decode_manifest_avro(data: &[u8]) -> TableCatalogStoreResult TableCatalogStoreResult) -> TableCatalogSto enum AvroContainerCodec { Null, Deflate, + Snappy, + Zstandard, } fn validate_avro_container(data: &[u8]) -> TableCatalogStoreResult { @@ -300,6 +325,8 @@ fn validate_avro_container(data: &[u8]) -> TableCatalogStoreResult { let codec = match codec.unwrap_or(b"null") { b"null" => AvroContainerCodec::Null, b"deflate" => AvroContainerCodec::Deflate, + b"snappy" => AvroContainerCodec::Snappy, + b"zstandard" => AvroContainerCodec::Zstandard, codec => { return Err(TableCatalogStoreError::Unsupported(format!( "Avro codec {} is not supported for table commit validation", @@ -369,6 +396,34 @@ fn avro_block_decoded_size(codec: AvroContainerCodec, block: &[u8], remaining_si usize::try_from(decoded_size) .map_err(|_| TableCatalogStoreError::Invalid("Avro decoded data size is invalid".to_string())) } + AvroContainerCodec::Snappy => { + let data_end = block + .len() + .checked_sub(4) + .ok_or_else(|| TableCatalogStoreError::Invalid("Avro snappy block is missing its checksum".to_string()))?; + let decoded_size = snap::raw::decompress_len(&block[..data_end]) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to inspect Avro snappy block: {err}")))?; + if decoded_size > remaining_size { + return Err(TableCatalogStoreError::Invalid("Avro decoded data exceeds the commit limit".to_string())); + } + Ok(decoded_size) + } + AvroContainerCodec::Zstandard => { + let limit = remaining_size + .checked_add(1) + .ok_or_else(|| TableCatalogStoreError::Invalid("Avro decoded data size limit overflowed".to_string()))?; + let limit = u64::try_from(limit) + .map_err(|_| TableCatalogStoreError::Invalid("Avro decoded data size limit is invalid".to_string()))?; + let mut decoder = zstd::stream::read::Decoder::new(block) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to decompress Avro zstandard block: {err}")))?; + decoder + .window_log_max(AVRO_ZSTANDARD_MAX_WINDOW_LOG) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to bound Avro zstandard window: {err}")))?; + let decoded_size = std::io::copy(&mut decoder.take(limit), &mut std::io::sink()) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to decompress Avro zstandard block: {err}")))?; + usize::try_from(decoded_size) + .map_err(|_| TableCatalogStoreError::Invalid("Avro decoded data size is invalid".to_string())) + } } } @@ -445,6 +500,40 @@ fn avro_record_value_fields(value: &apache_avro::types::Value) -> Option TableCatalogStoreResult> { + let Some(value) = avro_record_field(value, field) else { + return Ok(None); + }; + match avro_non_union_value(value) { + apache_avro::types::Value::Null => Ok(None), + apache_avro::types::Value::Int(value) => u64::try_from(*value) + .map(Some) + .map_err(|_| TableCatalogStoreError::Invalid(format!("{label} field {field} must be a non-negative int"))), + _ => Err(TableCatalogStoreError::Invalid(format!("{label} field {field} must be a nullable int"))), + } +} + +fn avro_nullable_non_negative_i64( + value: &apache_avro::types::Value, + field: &str, + label: &str, +) -> TableCatalogStoreResult> { + let Some(value) = avro_record_field(value, field) else { + return Ok(None); + }; + match avro_non_union_value(value) { + apache_avro::types::Value::Null => Ok(None), + apache_avro::types::Value::Long(value) => u64::try_from(*value) + .map(Some) + .map_err(|_| TableCatalogStoreError::Invalid(format!("{label} field {field} must be a non-negative long"))), + _ => Err(TableCatalogStoreError::Invalid(format!("{label} field {field} must be a nullable long"))), + } +} + pub(crate) fn avro_non_union_value(value: &apache_avro::types::Value) -> &apache_avro::types::Value { match value { apache_avro::types::Value::Union(_, inner) => avro_non_union_value(inner), @@ -472,3 +561,127 @@ fn avro_i64_value(value: &apache_avro::types::Value) -> Option { _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_v2_manifest_lists_without_required_count_fields() { + let schema = apache_avro::Schema::parse_str( + r#"{ + "type": "record", + "name": "manifest_file", + "fields": [ + {"name": "manifest_path", "type": "string"}, + {"name": "manifest_length", "type": "long"}, + {"name": "partition_spec_id", "type": "int"}, + {"name": "content", "type": "int"}, + {"name": "sequence_number", "type": "long"}, + {"name": "min_sequence_number", "type": "long"}, + {"name": "added_snapshot_id", "type": "long"} + ] + }"#, + ) + .expect("incomplete manifest-list schema should parse"); + let data = apache_avro::Writer::new(&schema, Vec::new()) + .expect("manifest-list writer should initialize") + .into_inner() + .expect("manifest-list bytes should flush"); + + let error = match decode_manifest_list_avro(&data) { + Ok(_) => panic!("v2 count fields must be declared in the writer schema"), + Err(error) => error, + }; + assert_eq!( + error, + TableCatalogStoreError::Invalid("Iceberg v2 manifest list Avro schema is missing added_files_count".to_string()) + ); + } + + #[test] + fn rejects_negative_nullable_manifest_list_counts() { + let value = apache_avro::types::Value::Record(vec![ + ("added_files_count".to_string(), apache_avro::types::Value::Int(-1)), + ("added_rows_count".to_string(), apache_avro::types::Value::Long(-1)), + ]); + + assert_eq!( + avro_nullable_non_negative_i32(&value, "added_files_count", "manifest list") + .expect_err("negative file counts must be rejected"), + TableCatalogStoreError::Invalid("manifest list field added_files_count must be a non-negative int".to_string()) + ); + assert_eq!( + avro_nullable_non_negative_i64(&value, "added_rows_count", "manifest list") + .expect_err("negative row counts must be rejected"), + TableCatalogStoreError::Invalid("manifest list field added_rows_count must be a non-negative long".to_string()) + ); + } + + #[test] + fn rejects_manifest_partition_with_non_record_schema() { + let schema = apache_avro::Schema::parse_str( + r#"{ + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int"}, + {"name": "snapshot_id", "type": "long"}, + { + "name": "data_file", + "type": { + "type": "record", + "name": "data_file", + "fields": [ + {"name": "file_path", "type": "string"}, + {"name": "record_count", "type": "long"}, + {"name": "file_size_in_bytes", "type": "long"}, + {"name": "partition", "type": "string"} + ] + } + } + ] + }"#, + ) + .expect("manifest schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest writer should initialize"); + writer + .append_value(apache_avro::types::Value::Record(vec![ + ("status".to_string(), apache_avro::types::Value::Int(1)), + ("snapshot_id".to_string(), apache_avro::types::Value::Long(1)), + ( + "data_file".to_string(), + apache_avro::types::Value::Record(vec![ + ( + "file_path".to_string(), + apache_avro::types::Value::String("s3://warehouse/tables/table-id/data/file.parquet".to_string()), + ), + ("record_count".to_string(), apache_avro::types::Value::Long(1)), + ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), + ("partition".to_string(), apache_avro::types::Value::String("not-a-record".to_string())), + ]), + ), + ])) + .expect("manifest record should append"); + let data = writer.into_inner().expect("manifest bytes should flush"); + + let error = match decode_manifest_avro(&data) { + Ok(_) => panic!("manifest partitions must preserve their record shape"), + Err(error) => error, + }; + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest data file partition must be a record".to_string()) + ); + } + + #[test] + fn rejects_oversized_zstandard_windows() { + // Non-single-segment frame with a 2^28-byte window and one empty final block. + let compressed = [0x28, 0xb5, 0x2f, 0xfd, 0x00, 0x90, 0x01, 0x00, 0x00]; + + let error = avro_block_decoded_size(AvroContainerCodec::Zstandard, &compressed, TABLE_MANIFEST_AVRO_MAX_DECODED_SIZE) + .expect_err("zstandard windows larger than the manifest decode budget must be rejected"); + assert!(matches!(error, TableCatalogStoreError::Invalid(_))); + } +} diff --git a/rustfs/src/table_catalog/iceberg/validation.rs b/rustfs/src/table_catalog/iceberg/validation.rs index 0b96f4078..c10c30237 100644 --- a/rustfs/src/table_catalog/iceberg/validation.rs +++ b/rustfs/src/table_catalog/iceberg/validation.rs @@ -18,6 +18,8 @@ use futures::{StreamExt, TryStreamExt, stream}; use super::super::*; +const ICEBERG_MAX_USER_FIELD_ID: i32 = i32::MAX - 200; + fn normalize_warehouse_object_prefix(object_prefix: &str, max_prefix_depth: Option) -> TableCatalogStoreResult { let object_prefix = object_prefix.strip_suffix('/').unwrap_or(object_prefix); if object_prefix.is_empty() { @@ -109,18 +111,33 @@ pub(crate) fn table_object_s3_location(table_bucket: &str, object_key: &str) -> format!("s3://{table_bucket}/{object_key}") } -fn metadata_warehouse_location( +pub(crate) struct TableMetadataCommitState { + pub(crate) warehouse_location: Option, + pub(crate) format_version: Option, +} + +pub(crate) fn table_metadata_commit_state( table_bucket: &str, metadata_location: &str, metadata_object: &TableCatalogObject, - validate_location: fn(&str, &str) -> TableCatalogStoreResult<()>, -) -> TableCatalogStoreResult> { +) -> TableCatalogStoreResult { let metadata = decode_table_metadata_json(metadata_location, &metadata_object.data)?; - let Some(location) = metadata.get("location").and_then(serde_json::Value::as_str) else { - return Ok(None); - }; - validate_location(table_bucket, location)?; - Ok(Some(location.to_string())) + let warehouse_location = metadata + .get("location") + .and_then(serde_json::Value::as_str) + .map(|location| { + validate_table_warehouse_location(table_bucket, location)?; + Ok(location.to_string()) + }) + .transpose()?; + let format_version = metadata + .get("format-version") + .map(|_| table_metadata_format_version(&metadata)) + .transpose()?; + Ok(TableMetadataCommitState { + warehouse_location, + format_version, + }) } pub(crate) fn decode_table_metadata_json(metadata_location: &str, data: &[u8]) -> TableCatalogStoreResult { @@ -167,14 +184,6 @@ fn table_metadata_location_is_gzip(metadata_location: &str) -> bool { metadata_location.ends_with(".gz.metadata.json") || metadata_location.ends_with(".metadata.json.gz") } -pub(crate) fn table_metadata_warehouse_location( - table_bucket: &str, - metadata_location: &str, - metadata_object: &TableCatalogObject, -) -> TableCatalogStoreResult> { - metadata_warehouse_location(table_bucket, metadata_location, metadata_object, validate_table_warehouse_location) -} - pub(crate) fn canonical_json_sha256(metadata: &serde_json::Value) -> TableCatalogStoreResult { let canonical = serde_json::to_vec(metadata) .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode metadata digest input: {err}")))?; @@ -220,7 +229,12 @@ pub(crate) fn view_metadata_warehouse_location( metadata_location: &str, metadata_object: &TableCatalogObject, ) -> TableCatalogStoreResult> { - metadata_warehouse_location(table_bucket, metadata_location, metadata_object, validate_view_warehouse_location) + let metadata = decode_table_metadata_json(metadata_location, &metadata_object.data)?; + let Some(location) = metadata.get("location").and_then(serde_json::Value::as_str) else { + return Ok(None); + }; + validate_view_warehouse_location(table_bucket, location)?; + Ok(Some(location.to_string())) } pub(crate) fn warehouse_index_candidate_prefixes(object: &str) -> Vec<&str> { @@ -334,9 +348,99 @@ pub(crate) fn table_metadata_format_version(metadata: &serde_json::Value) -> Tab Ok(version) } +fn normalize_v1_table_metadata_update_fields(metadata: &mut serde_json::Value) -> TableCatalogStoreResult<()> { + if metadata.get("schemas").is_none() { + let mut schema = metadata + .get("schema") + .cloned() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing schema".to_string()))?; + schema + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 schema must be an object".to_string()))? + .entry("schema-id".to_string()) + .or_insert_with(|| serde_json::Value::from(0)); + let schema_id = schema + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 schema-id must be an integer".to_string()))?; + let object = metadata_object_mut(metadata)?; + object.insert("schemas".to_string(), serde_json::json!([schema])); + object.insert("current-schema-id".to_string(), serde_json::Value::from(schema_id)); + } else if metadata.get("current-schema-id").is_none() { + let schema_id = metadata + .get("schema") + .and_then(|schema| schema.get("schema-id")) + .and_then(serde_json::Value::as_i64) + .unwrap_or(0); + if !require_metadata_array(metadata, "schemas")? + .iter() + .any(|schema| schema.get("schema-id").and_then(serde_json::Value::as_i64) == Some(schema_id)) + { + return Err(TableCatalogStoreError::Invalid("Iceberg v1 current schema does not exist".to_string())); + } + metadata_object_mut(metadata)?.insert("current-schema-id".to_string(), serde_json::Value::from(schema_id)); + } + + if metadata.get("partition-specs").is_none() { + let mut fields = metadata + .get("partition-spec") + .cloned() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing partition-spec".to_string()))?; + let fields = fields + .as_array_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 partition-spec must be an array".to_string()))?; + for (index, field) in fields.iter_mut().enumerate() { + let field_id = i32::try_from(index) + .ok() + .and_then(|index| 1000_i32.checked_add(index)) + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 partition spec has too many fields".to_string()))?; + field + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 partition spec fields must be objects".to_string()))? + .entry("field-id".to_string()) + .or_insert_with(|| serde_json::Value::from(field_id)); + } + let object = metadata_object_mut(metadata)?; + object.insert("partition-specs".to_string(), serde_json::json!([{"spec-id": 0, "fields": fields}])); + object.insert("default-spec-id".to_string(), serde_json::Value::from(0)); + } else if metadata.get("default-spec-id").is_none() { + let default_spec_id = require_metadata_array(metadata, "partition-specs")? + .last() + .and_then(|spec| spec.get("spec-id")) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 default partition spec does not exist".to_string()))?; + metadata_object_mut(metadata)?.insert("default-spec-id".to_string(), serde_json::Value::from(default_spec_id)); + } + + if metadata.get("sort-orders").is_none() { + let object = metadata_object_mut(metadata)?; + object.insert("sort-orders".to_string(), serde_json::json!([{"order-id": 0, "fields": []}])); + object.insert("default-sort-order-id".to_string(), serde_json::Value::from(0)); + } else if metadata.get("default-sort-order-id").is_none() { + let default_sort_order_id = require_metadata_array(metadata, "sort-orders")? + .last() + .and_then(|order| order.get("order-id")) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 default sort order does not exist".to_string()))?; + metadata_object_mut(metadata)? + .insert("default-sort-order-id".to_string(), serde_json::Value::from(default_sort_order_id)); + } + + if metadata.get("last-partition-id").is_none() { + let last_partition_id = require_metadata_array(metadata, "partition-specs")? + .iter() + .map(max_partition_field_id) + .max() + .unwrap_or(999); + metadata_object_mut(metadata)?.insert("last-partition-id".to_string(), serde_json::Value::from(last_partition_id)); + } + Ok(()) +} + pub(crate) fn synchronize_table_metadata_version_fields(metadata: &mut serde_json::Value) -> TableCatalogStoreResult<()> { match table_metadata_format_version(metadata)? { 1 => { + normalize_v1_table_metadata_update_fields(metadata)?; if let Some(schemas) = metadata.get("schemas").and_then(serde_json::Value::as_array) && !schemas.is_empty() { @@ -404,9 +508,27 @@ pub(crate) fn synchronize_table_metadata_version_fields(metadata: &mut serde_jso metadata_object_mut(metadata)?.insert("current-schema-id".to_string(), serde_json::Value::from(schema_id)); } if metadata.get("partition-specs").is_none() { - let fields = metadata.get("partition-spec").cloned().ok_or_else(|| { + let mut fields = metadata.get("partition-spec").cloned().ok_or_else(|| { TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing partition-spec".to_string()) })?; + let fields = fields + .as_array_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 partition-spec must be an array".to_string()))?; + for (index, field) in fields.iter_mut().enumerate() { + let field_id = i32::try_from(index) + .ok() + .and_then(|index| 1000_i32.checked_add(index)) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("Iceberg v1 partition spec has too many fields".to_string()) + })?; + field + .as_object_mut() + .ok_or_else(|| { + TableCatalogStoreError::Invalid("Iceberg v1 partition spec fields must be objects".to_string()) + })? + .entry("field-id".to_string()) + .or_insert_with(|| serde_json::Value::from(field_id)); + } metadata_object_mut(metadata)? .insert("partition-specs".to_string(), serde_json::json!([{"spec-id": 0, "fields": fields}])); metadata_object_mut(metadata)?.insert("default-spec-id".to_string(), serde_json::Value::from(0)); @@ -452,11 +574,456 @@ pub(crate) fn synchronize_table_metadata_version_fields(metadata: &mut serde_jso Ok(()) } +fn validate_table_history_logs(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { + for field in ["snapshot-log", "metadata-log"] { + let Some(entries) = metadata.get(field) else { + continue; + }; + let entries = entries + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{field} must be an array")))?; + for entry in entries { + let entry = entry + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{field} entries must be JSON objects")))?; + if entry.get("timestamp-ms").and_then(serde_json::Value::as_i64).is_none() { + return Err(TableCatalogStoreError::Invalid(format!("{field} entries require integer timestamp-ms"))); + } + match field { + "snapshot-log" if entry.get("snapshot-id").and_then(serde_json::Value::as_i64).is_none() => { + return Err(TableCatalogStoreError::Invalid( + "snapshot-log entries require integer snapshot-id".to_string(), + )); + } + "metadata-log" + if !entry + .get("metadata-file") + .and_then(serde_json::Value::as_str) + .is_some_and(|location| !location.is_empty()) => + { + return Err(TableCatalogStoreError::Invalid( + "metadata-log entries require non-empty metadata-file".to_string(), + )); + } + _ => {} + } + } + } + Ok(()) +} + pub(crate) fn validate_supported_table_metadata(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { validate_supported_table_metadata_fields(metadata)?; + validate_table_history_logs(metadata)?; validate_table_metadata_references(metadata) } +pub(crate) fn validate_table_metadata_transition( + current_metadata: &serde_json::Value, + target_metadata: &serde_json::Value, +) -> TableCatalogStoreResult<()> { + let current_last_column_id = require_metadata_i32(current_metadata, "last-column-id")?; + let target_last_column_id = require_metadata_i32(target_metadata, "last-column-id")?; + if target_last_column_id < current_last_column_id { + return Err(TableCatalogStoreError::Invalid( + "last-column-id must not decrease across table metadata commits".to_string(), + )); + } + + let current_format_version = table_metadata_format_version(current_metadata)?; + let target_format_version = table_metadata_format_version(target_metadata)?; + let current_last_partition_id = table_metadata_last_partition_id(current_metadata, current_format_version)?; + let target_last_partition_id = table_metadata_last_partition_id(target_metadata, target_format_version)?; + if target_last_partition_id < current_last_partition_id { + return Err(TableCatalogStoreError::Invalid( + "last-partition-id must not decrease across table metadata commits".to_string(), + )); + } + let current_last_sequence_number = table_metadata_last_sequence_number(current_metadata, current_format_version)?; + let target_last_sequence_number = table_metadata_last_sequence_number(target_metadata, target_format_version)?; + if target_last_sequence_number < current_last_sequence_number { + return Err(TableCatalogStoreError::Invalid( + "last-sequence-number must not decrease across table metadata commits".to_string(), + )); + } + + validate_existing_partition_specs_unchanged(current_metadata, target_metadata)?; + validate_existing_metadata_entries_unchanged( + &normalized_sort_order_definitions(current_metadata, current_format_version)?, + &normalized_sort_order_definitions(target_metadata, target_format_version)?, + "sort order", + )?; + validate_existing_snapshots_unchanged(current_metadata, target_metadata, current_format_version, target_format_version)?; + + let current_schemas = table_metadata_schemas_by_id(current_metadata, current_format_version)?; + let target_schemas = table_metadata_schemas_by_id(target_metadata, target_format_version)?; + for (schema_id, current_schema) in ¤t_schemas { + if let Some(target_schema) = target_schemas.get(schema_id) + && normalized_schema_definition(current_schema, *schema_id)? + != normalized_schema_definition(target_schema, *schema_id)? + { + return Err(TableCatalogStoreError::Invalid(format!( + "existing schema {schema_id} must not be modified" + ))); + } + } + + let current_schema_id = table_metadata_current_schema_id(current_metadata, current_format_version)?; + let current_schema = current_schemas + .get(¤t_schema_id) + .ok_or_else(|| TableCatalogStoreError::Invalid("current table metadata schema does not exist".to_string()))?; + let current_fields = validate_iceberg_schema_fields(current_schema, "current schema")?; + for (target_schema_id, target_schema) in &target_schemas { + if current_schemas.contains_key(target_schema_id) { + continue; + } + let target_fields = validate_iceberg_schema_fields(target_schema, "target schema")?; + for (field_id, target_field) in &target_fields.descriptors { + match current_fields.descriptors.get(field_id) { + Some(current_field) => validate_schema_field_evolution(*field_id, current_field, target_field)?, + None if *field_id <= current_last_column_id => { + return Err(TableCatalogStoreError::Invalid(format!( + "schema field {field_id} cannot reuse a previously assigned field id" + ))); + } + None => {} + } + } + } + Ok(()) +} + +fn table_metadata_last_sequence_number(metadata: &serde_json::Value, format_version: u16) -> TableCatalogStoreResult { + if format_version == 1 { + return Ok(0); + } + require_metadata_i64(metadata, "last-sequence-number") +} + +fn normalized_sort_order_definitions( + metadata: &serde_json::Value, + format_version: u16, +) -> TableCatalogStoreResult> { + let Some(sort_orders) = metadata.get("sort-orders") else { + return if format_version == 1 { + Ok(BTreeMap::from([(0, serde_json::json!({"order-id": 0, "fields": []}))])) + } else { + Err(TableCatalogStoreError::Invalid("sort-orders must be an array".to_string())) + }; + }; + metadata_entries_by_id(sort_orders, "order-id", "sort order") +} + +fn validate_existing_snapshots_unchanged( + current_metadata: &serde_json::Value, + target_metadata: &serde_json::Value, + current_format_version: u16, + target_format_version: u16, +) -> TableCatalogStoreResult<()> { + let current = current_metadata + .get("snapshots") + .map(|snapshots| metadata_entries_by_id(snapshots, "snapshot-id", "snapshot")) + .transpose()? + .unwrap_or_default(); + let target = target_metadata + .get("snapshots") + .map(|snapshots| metadata_entries_by_id(snapshots, "snapshot-id", "snapshot")) + .transpose()? + .unwrap_or_default(); + for (snapshot_id, current_snapshot) in current { + let Some(target_snapshot) = target.get(&snapshot_id) else { + continue; + }; + let mut current_snapshot = current_snapshot; + let mut target_snapshot = target_snapshot.clone(); + if current_format_version == 1 && target_format_version == 2 { + for snapshot in [&mut current_snapshot, &mut target_snapshot] { + if snapshot.get("sequence-number").and_then(serde_json::Value::as_i64) == Some(0) + && let Some(object) = snapshot.as_object_mut() + { + object.remove("sequence-number"); + } + } + } + if current_snapshot != target_snapshot { + return Err(TableCatalogStoreError::Invalid(format!( + "existing snapshot {snapshot_id} must not be modified" + ))); + } + } + Ok(()) +} + +fn validate_existing_metadata_entries_unchanged( + current: &BTreeMap, + target: &BTreeMap, + label: &str, +) -> TableCatalogStoreResult<()> { + for (id, current_value) in current { + if let Some(target_value) = target.get(id) + && target_value != current_value + { + return Err(TableCatalogStoreError::Invalid(format!("existing {label} {id} must not be modified"))); + } + } + Ok(()) +} + +fn validate_existing_partition_specs_unchanged( + current_metadata: &serde_json::Value, + target_metadata: &serde_json::Value, +) -> TableCatalogStoreResult<()> { + let current = normalized_partition_spec_definitions(current_metadata)?; + let target = normalized_partition_spec_definitions(target_metadata)?; + for (spec_id, current_fields) in current { + if let Some(target_fields) = target.get(&spec_id) + && target_fields != ¤t_fields + { + return Err(TableCatalogStoreError::Invalid(format!( + "existing partition spec {spec_id} must not be modified" + ))); + } + } + Ok(()) +} + +fn metadata_entries_by_id( + value: &serde_json::Value, + id_field: &str, + label: &str, +) -> TableCatalogStoreResult> { + if value.is_null() { + return Ok(BTreeMap::new()); + } + let values = value + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}s must be an array")))?; + let mut entries = BTreeMap::new(); + for value in values { + let id = value + .get(id_field) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} is missing {id_field}")))?; + if entries.insert(id, value.clone()).is_some() { + return Err(TableCatalogStoreError::Invalid(format!("duplicate {label} id {id}"))); + } + } + Ok(entries) +} + +fn normalized_schema_definition(schema: &serde_json::Value, schema_id: i64) -> TableCatalogStoreResult { + let mut schema = schema.clone(); + schema + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("schema must be a JSON object".to_string()))? + .entry("schema-id".to_string()) + .or_insert_with(|| serde_json::Value::from(schema_id)); + Ok(schema) +} + +fn table_metadata_last_partition_id(metadata: &serde_json::Value, format_version: u16) -> TableCatalogStoreResult { + if format_version != 1 { + return require_metadata_i32(metadata, "last-partition-id"); + } + let field_count = require_metadata_array(metadata, "partition-spec")?.len(); + i32::try_from(field_count) + .ok() + .and_then(|field_count| 999_i32.checked_add(field_count)) + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 partition spec has too many fields".to_string())) +} + +fn table_metadata_current_schema_id(metadata: &serde_json::Value, format_version: u16) -> TableCatalogStoreResult { + if format_version == 1 { + return Ok(metadata + .get("schema") + .and_then(|schema| schema.get("schema-id")) + .and_then(serde_json::Value::as_i64) + .unwrap_or(0)); + } + metadata + .get("current-schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("current-schema-id must be an integer".to_string())) +} + +fn table_metadata_schemas_by_id( + metadata: &serde_json::Value, + format_version: u16, +) -> TableCatalogStoreResult> { + let mut schemas = BTreeMap::new(); + if let Some(values) = metadata.get("schemas") { + for schema in values + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid("schemas must be an array".to_string()))? + { + let schema_id = schema + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("schema-id must be an integer".to_string()))?; + if schemas.insert(schema_id, schema).is_some() { + return Err(TableCatalogStoreError::Invalid(format!("duplicate schema id {schema_id}"))); + } + } + } + if format_version == 1 { + let schema = metadata + .get("schema") + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing schema".to_string()))?; + let schema_id = schema.get("schema-id").and_then(serde_json::Value::as_i64).unwrap_or(0); + if let Some(known_schema) = schemas.get(&schema_id) { + let mut normalized_schema = schema.clone(); + normalized_schema + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 schema must be an object".to_string()))? + .entry("schema-id".to_string()) + .or_insert_with(|| serde_json::Value::from(schema_id)); + if *known_schema != &normalized_schema { + return Err(TableCatalogStoreError::Invalid(format!( + "Iceberg v1 current schema {schema_id} does not match schemas" + ))); + } + } else { + schemas.insert(schema_id, schema); + } + } + Ok(schemas) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum IcebergStatisticsFileKind { + Table, + Partition, +} + +pub(crate) fn validate_iceberg_statistics_file( + value: &serde_json::Value, + label: &str, + kind: IcebergStatisticsFileKind, +) -> TableCatalogStoreResult { + let object = value + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} must be a JSON object")))?; + let snapshot_id = object + .get("snapshot-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.snapshot-id must be an integer")))?; + if !object + .get("statistics-path") + .and_then(serde_json::Value::as_str) + .is_some_and(|path| !path.is_empty()) + { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.statistics-path must be a non-empty string" + ))); + } + let file_size = statistics_non_negative_i64(object, "file-size-in-bytes", label)?; + if matches!(kind, IcebergStatisticsFileKind::Table) { + let footer_size = statistics_non_negative_i64(object, "file-footer-size-in-bytes", label)?; + if footer_size > file_size { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.file-footer-size-in-bytes must not exceed file-size-in-bytes" + ))); + } + let blobs = object + .get("blob-metadata") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.blob-metadata must be an array")))?; + for blob in blobs { + validate_statistics_blob_metadata(blob, label)?; + } + } + Ok(snapshot_id) +} + +fn validate_statistics_blob_metadata(value: &serde_json::Value, label: &str) -> TableCatalogStoreResult<()> { + let object = value + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.blob-metadata entries must be JSON objects")))?; + if !object + .get("type") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| !value.is_empty()) + { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.blob-metadata type must be a non-empty string" + ))); + } + for field in ["snapshot-id", "sequence-number"] { + if object.get(field).and_then(serde_json::Value::as_i64).is_none() { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.blob-metadata {field} must be an integer" + ))); + } + } + let fields = object + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.blob-metadata fields must be an array")))?; + if fields.iter().any(|field| field.as_i64().is_none()) { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.blob-metadata fields must contain integers" + ))); + } + if let Some(properties) = object.get("properties") { + let properties = properties + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.blob-metadata properties must be a JSON object")))?; + if properties.values().any(|value| !value.is_string()) { + return Err(TableCatalogStoreError::Invalid(format!( + "{label}.blob-metadata property values must be strings" + ))); + } + } + Ok(()) +} + +fn statistics_non_negative_i64( + object: &serde_json::Map, + field: &str, + label: &str, +) -> TableCatalogStoreResult { + let value = object + .get(field) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label}.{field} must be an integer")))?; + if value < 0 { + return Err(TableCatalogStoreError::Invalid(format!("{label}.{field} must not be negative"))); + } + Ok(value) +} + +fn validate_table_statistics_references( + metadata: &serde_json::Value, + snapshot_ids: &BTreeSet, +) -> TableCatalogStoreResult<()> { + for (field, kind) in [ + ("statistics", IcebergStatisticsFileKind::Table), + ("partition-statistics", IcebergStatisticsFileKind::Partition), + ] { + let Some(values) = metadata.get(field) else { + continue; + }; + let values = values + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("table metadata field {field} must be an array")))?; + let mut snapshot_ids_with_statistics = BTreeSet::new(); + for value in values { + let snapshot_id = validate_iceberg_statistics_file(value, field, kind)?; + if !snapshot_ids.contains(&snapshot_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "{field} references missing snapshot {snapshot_id}" + ))); + } + if !snapshot_ids_with_statistics.insert(snapshot_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "{field} contains duplicate entries for snapshot {snapshot_id}" + ))); + } + } + } + Ok(()) +} + fn validate_supported_table_metadata_fields(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { table_metadata_uuid(metadata)?; table_metadata_location(metadata)?; @@ -508,12 +1075,16 @@ fn validate_supported_table_metadata_fields(metadata: &serde_json::Value) -> Tab if let Some(snapshots) = metadata.get("snapshots").and_then(serde_json::Value::as_array) { for snapshot in snapshots { validate_table_snapshot_fields(snapshot, 2)?; - let sequence_number = snapshot - .get("sequence-number") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| { + let sequence_number = match snapshot.get("sequence-number") { + Some(sequence_number) => sequence_number.as_i64().ok_or_else(|| { TableCatalogStoreError::Invalid("Iceberg v2 snapshot sequence-number must be an integer".to_string()) - })?; + })?, + None => { + return Err(TableCatalogStoreError::Invalid( + "Iceberg v2 snapshot sequence-number is required".to_string(), + )); + } + }; if sequence_number < 0 || sequence_number > last_sequence_number { return Err(TableCatalogStoreError::Invalid( "Iceberg v2 snapshot sequence-number must be between zero and last-sequence-number".to_string(), @@ -533,6 +1104,19 @@ fn validate_supported_table_metadata_fields(metadata: &serde_json::Value) -> Tab pub(crate) fn validate_table_metadata_references(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { let format_version = table_metadata_format_version(metadata)?; + let schema_fields = validate_table_schemas(metadata, format_version)?; + let current_schema_fields = current_table_schema_fields(metadata, format_version)?; + let last_column_id = require_metadata_i32(metadata, "last-column-id")?; + if last_column_id < 0 + || schema_fields + .field_ids + .last() + .is_some_and(|field_id| *field_id > last_column_id) + { + return Err(TableCatalogStoreError::Invalid( + "last-column-id must be non-negative and cover every assigned schema field id".to_string(), + )); + } let mut schema_ids = metadata_array_i32_ids(metadata, "schemas", "schema-id", "schema")?; if format_version == 1 && schema_ids.is_empty() { let schema = metadata @@ -545,6 +1129,9 @@ pub(crate) fn validate_table_metadata_references(metadata: &serde_json::Value) - .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 schema-id must be an integer".to_string()))?, None => 0, }; + if schema_id < 0 { + return Err(TableCatalogStoreError::Invalid(format!("schema id {schema_id} must not be negative"))); + } if i32::try_from(schema_id).is_err() { return Err(TableCatalogStoreError::Invalid(format!( "schema id {schema_id} exceeds the signed 32-bit range" @@ -553,11 +1140,14 @@ pub(crate) fn validate_table_metadata_references(metadata: &serde_json::Value) - schema_ids.insert(schema_id); } validate_metadata_id_reference(metadata, "current-schema-id", &schema_ids, "schema")?; + validate_partition_specs(metadata, format_version, &schema_fields, ¤t_schema_fields)?; let spec_ids = metadata_array_i32_ids(metadata, "partition-specs", "spec-id", "partition spec")?; validate_metadata_id_reference(metadata, "default-spec-id", &spec_ids, "partition spec")?; + validate_sort_orders(metadata, &schema_fields, ¤t_schema_fields)?; let sort_order_ids = metadata_array_i32_ids(metadata, "sort-orders", "order-id", "sort order")?; validate_metadata_id_reference(metadata, "default-sort-order-id", &sort_order_ids, "sort order")?; let snapshot_ids = metadata_array_ids(metadata, "snapshots", "snapshot-id", "snapshot")?; + validate_table_statistics_references(metadata, &snapshot_ids)?; let current_snapshot_id = match metadata.get("current-snapshot-id").filter(|value| !value.is_null()) { Some(current_snapshot_id) => { @@ -623,6 +1213,714 @@ pub(crate) fn validate_table_metadata_references(metadata: &serde_json::Value) - Ok(()) } +fn validate_table_schemas(metadata: &serde_json::Value, format_version: u16) -> TableCatalogStoreResult { + let mut schemas = Vec::new(); + if format_version == 1 { + let schema = metadata + .get("schema") + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing schema".to_string()))?; + let schema_id = schema.get("schema-id").and_then(serde_json::Value::as_i64).unwrap_or(0); + schemas.push((schema_id, schema)); + } + if let Some(metadata_schemas) = metadata.get("schemas") { + let metadata_schemas = metadata_schemas + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid("schemas must be an array".to_string()))?; + for schema in metadata_schemas { + let schema_id = schema + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("schema-id must be an integer".to_string()))?; + schemas.push((schema_id, schema)); + } + } + schemas.sort_by_key(|(schema_id, _)| *schema_id); + + let mut historical = BTreeMap::new(); + let mut active_fields = BTreeSet::new(); + let mut retired_fields = BTreeSet::new(); + let mut all_fields = IcebergSchemaFields::default(); + for (_, schema) in schemas { + let schema_fields = validate_iceberg_schema_fields(schema, "schema")?; + if let Some(field_id) = schema_fields + .field_ids + .iter() + .find(|field_id| retired_fields.contains(*field_id)) + { + return Err(TableCatalogStoreError::Invalid(format!( + "schema field {field_id} cannot be reused after removal" + ))); + } + for (field_id, descriptor) in &schema_fields.descriptors { + if let Some(previous) = historical.get(field_id) { + validate_schema_field_evolution(*field_id, previous, descriptor)?; + } + historical.insert(*field_id, descriptor.clone()); + } + retired_fields.extend(active_fields.difference(&schema_fields.field_ids).copied()); + active_fields = schema_fields.field_ids.clone(); + all_fields.field_ids.extend(schema_fields.field_ids); + all_fields.identifier_eligible.extend(schema_fields.identifier_eligible); + all_fields.descriptors.extend(schema_fields.descriptors); + } + Ok(all_fields) +} + +fn current_table_schema_fields( + metadata: &serde_json::Value, + format_version: u16, +) -> TableCatalogStoreResult { + if metadata.get("schemas").is_some() { + let current_schema_id = require_metadata_i32(metadata, "current-schema-id")?; + let schema = require_metadata_array(metadata, "schemas")? + .iter() + .find(|schema| schema.get("schema-id").and_then(serde_json::Value::as_i64) == Some(i64::from(current_schema_id))) + .ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "current-schema-id targets schema {current_schema_id}, which does not exist" + )) + })?; + return validate_iceberg_schema_fields(schema, "current schema"); + } + if format_version == 1 { + let schema = metadata + .get("schema") + .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v1 table metadata is missing schema".to_string()))?; + return validate_iceberg_schema_fields(schema, "schema"); + } + Err(TableCatalogStoreError::Invalid("schemas must be an array".to_string())) +} + +pub(crate) fn validate_partition_spec_sources_against_current_schema( + metadata: &serde_json::Value, + spec: &serde_json::Value, +) -> TableCatalogStoreResult<()> { + let current_schema_fields = current_table_schema_fields(metadata, table_metadata_format_version(metadata)?)?; + let fields = spec + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition spec fields must be an array".to_string()))?; + for field in fields { + let field = field + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("partition spec fields must be JSON objects".to_string()))?; + let source_id = required_positive_i32_value(field, "source-id", "partition source-id")?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition field transform must be a non-empty string".to_string()))?; + if transform == "void" { + continue; + } + let source_type = current_schema_fields.descriptors.get(&source_id).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("partition source-id {source_id} does not reference the current schema")) + })?; + if source_type.inside_collection { + return Err(TableCatalogStoreError::Invalid(format!( + "partition source-id {source_id} must not be nested in a list or map" + ))); + } + validate_transform_for_source(transform, source_type, "partition field")?; + } + Ok(()) +} + +pub(crate) fn validate_sort_order_sources_against_current_schema( + metadata: &serde_json::Value, + sort_order: &serde_json::Value, +) -> TableCatalogStoreResult<()> { + let current_schema_fields = current_table_schema_fields(metadata, table_metadata_format_version(metadata)?)?; + let fields = sort_order + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("sort order fields must be an array".to_string()))?; + for field in fields { + let field = field + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("sort order fields must be JSON objects".to_string()))?; + let source_id = required_positive_i32_value(field, "source-id", "sort field source-id")?; + let source_type = current_schema_fields.descriptors.get(&source_id).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("sort field source-id {source_id} does not reference the current schema")) + })?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Invalid("sort field transform must be a non-empty string".to_string()))?; + validate_transform_for_source(transform, source_type, "sort field")?; + } + Ok(()) +} + +fn validate_iceberg_schema(schema: &serde_json::Value, label: &str) -> TableCatalogStoreResult> { + Ok(validate_iceberg_schema_fields(schema, label)?.field_ids) +} + +fn validate_iceberg_schema_fields(schema: &serde_json::Value, label: &str) -> TableCatalogStoreResult { + let schema = schema + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} must be a JSON object")))?; + if schema.get("type").and_then(serde_json::Value::as_str) != Some("struct") { + return Err(TableCatalogStoreError::Invalid(format!("{label} type must be struct"))); + } + let fields = schema + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} fields must be an array")))?; + let mut schema_fields = IcebergSchemaFields::default(); + validate_struct_fields(fields, label, true, false, &mut schema_fields)?; + if let Some(identifier_field_ids) = schema.get("identifier-field-ids") { + let identifier_field_ids = identifier_field_ids + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} identifier-field-ids must be an array")))?; + let mut seen = BTreeSet::new(); + for field_id in identifier_field_ids { + let field_id = required_positive_i32(field_id, &format!("{label} identifier field id"))?; + if !seen.insert(field_id) || schema_fields.identifier_eligible.get(&field_id) != Some(&true) { + return Err(TableCatalogStoreError::Invalid(format!( + "{label} identifier field id {field_id} must uniquely reference a required non-floating primitive outside lists, maps, and optional structs" + ))); + } + } + } + Ok(schema_fields) +} + +#[derive(Default)] +struct IcebergSchemaFields { + field_ids: BTreeSet, + identifier_eligible: BTreeMap, + descriptors: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct IcebergSchemaFieldDescriptor { + field_type: IcebergFieldType, + required: bool, + inside_collection: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum IcebergFieldType { + Primitive(String), + Struct, + List, + Map, +} + +fn iceberg_field_type(value: &serde_json::Value, label: &str) -> TableCatalogStoreResult { + if let Some(primitive) = value.as_str() { + validate_iceberg_primitive_type(primitive, label)?; + return Ok(IcebergFieldType::Primitive(primitive.to_string())); + } + match value.get("type").and_then(serde_json::Value::as_str) { + Some("struct") => Ok(IcebergFieldType::Struct), + Some("list") => Ok(IcebergFieldType::List), + Some("map") => Ok(IcebergFieldType::Map), + _ => Err(TableCatalogStoreError::Invalid(format!("{label} contains an unsupported field type"))), + } +} + +fn insert_schema_field( + schema_fields: &mut IcebergSchemaFields, + field_id: i32, + field_type: &serde_json::Value, + required: bool, + inside_collection: bool, + label: &str, +) -> TableCatalogStoreResult<()> { + if !schema_fields.field_ids.insert(field_id) { + return Err(TableCatalogStoreError::Invalid(format!("duplicate {label} field id {field_id}"))); + } + schema_fields.descriptors.insert( + field_id, + IcebergSchemaFieldDescriptor { + field_type: iceberg_field_type(field_type, label)?, + required, + inside_collection, + }, + ); + Ok(()) +} + +fn validate_schema_field_evolution( + field_id: i32, + previous: &IcebergSchemaFieldDescriptor, + next: &IcebergSchemaFieldDescriptor, +) -> TableCatalogStoreResult<()> { + if !previous.required && next.required { + return Err(TableCatalogStoreError::Invalid(format!( + "schema field {field_id} cannot evolve from optional to required" + ))); + } + if previous.inside_collection != next.inside_collection { + return Err(TableCatalogStoreError::Invalid(format!( + "schema field {field_id} cannot move into or out of a list or map" + ))); + } + if previous.field_type == next.field_type { + return Ok(()); + } + let compatible = match (&previous.field_type, &next.field_type) { + (IcebergFieldType::Primitive(previous), IcebergFieldType::Primitive(next)) => { + primitive_type_promotion_is_valid(previous, next) + } + _ => false, + }; + if !compatible { + return Err(TableCatalogStoreError::Invalid(format!( + "schema field {field_id} has an incompatible type evolution" + ))); + } + Ok(()) +} + +fn primitive_type_promotion_is_valid(previous: &str, next: &str) -> bool { + if matches!((previous, next), ("int", "long") | ("float", "double")) { + return true; + } + let decimal = |value: &str| { + value + .strip_prefix("decimal(") + .and_then(|value| value.strip_suffix(')')) + .and_then(|parameters| parameters.split_once(',')) + .and_then(|(precision, scale)| Some((precision.trim().parse::().ok()?, scale.trim().parse::().ok()?))) + }; + matches!((decimal(previous), decimal(next)), (Some((previous_precision, previous_scale)), Some((next_precision, next_scale))) if previous_scale == next_scale && next_precision >= previous_precision) +} + +fn validate_struct_fields( + fields: &[serde_json::Value], + label: &str, + required_ancestors: bool, + inside_collection: bool, + schema_fields: &mut IcebergSchemaFields, +) -> TableCatalogStoreResult<()> { + for field in fields { + let field = field + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} fields must be JSON objects")))?; + let field_id = required_schema_field_id_value(field, "id", &format!("{label} field id"))?; + if !field + .get("name") + .and_then(serde_json::Value::as_str) + .is_some_and(|name| !name.is_empty()) + { + return Err(TableCatalogStoreError::Invalid(format!("{label} field name must be a non-empty string"))); + } + let required = field + .get("required") + .and_then(serde_json::Value::as_bool) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} field required must be a boolean")))?; + let field_type = field + .get("type") + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} field type is required")))?; + insert_schema_field(schema_fields, field_id, field_type, required, inside_collection, label)?; + let identifier_eligible = required_ancestors + && required + && !inside_collection + && field_type + .as_str() + .is_some_and(|primitive| !matches!(primitive, "float" | "double")); + schema_fields.identifier_eligible.insert(field_id, identifier_eligible); + validate_iceberg_type(field_type, label, required_ancestors && required, inside_collection, schema_fields)?; + } + Ok(()) +} + +fn validate_iceberg_type( + field_type: &serde_json::Value, + label: &str, + required_ancestors: bool, + inside_collection: bool, + schema_fields: &mut IcebergSchemaFields, +) -> TableCatalogStoreResult<()> { + if let Some(primitive) = field_type.as_str() { + return validate_iceberg_primitive_type(primitive, label); + } + let field_type = field_type + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} field type must be a string or JSON object")))?; + match field_type.get("type").and_then(serde_json::Value::as_str) { + Some("struct") => { + let fields = field_type + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} struct fields must be an array")))?; + validate_struct_fields(fields, label, required_ancestors, inside_collection, schema_fields) + } + Some("list") => { + let element_id = required_schema_field_id_value(field_type, "element-id", &format!("{label} list element-id"))?; + let element_required = field_type + .get("element-required") + .and_then(serde_json::Value::as_bool) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} list element-required must be a boolean")))?; + let element = field_type + .get("element") + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} list element is required")))?; + insert_schema_field(schema_fields, element_id, element, element_required, true, label)?; + validate_iceberg_type(element, label, false, true, schema_fields) + } + Some("map") => { + let value_required = field_type + .get("value-required") + .and_then(serde_json::Value::as_bool) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} map value-required must be a boolean")))?; + for (id_field, value_field, required) in [("key-id", "key", true), ("value-id", "value", value_required)] { + let field_id = required_schema_field_id_value(field_type, id_field, &format!("{label} map {id_field}"))?; + let value = field_type + .get(value_field) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} map {value_field} is required")))?; + insert_schema_field(schema_fields, field_id, value, required, true, label)?; + validate_iceberg_type(value, label, false, true, schema_fields)?; + } + Ok(()) + } + _ => Err(TableCatalogStoreError::Invalid(format!("{label} contains an unsupported field type"))), + } +} + +fn validate_iceberg_primitive_type(primitive: &str, label: &str) -> TableCatalogStoreResult<()> { + if matches!( + primitive, + "boolean" + | "int" + | "long" + | "float" + | "double" + | "date" + | "time" + | "timestamp" + | "timestamptz" + | "string" + | "uuid" + | "binary" + ) { + return Ok(()); + } + if let Some(length) = primitive.strip_prefix("fixed[").and_then(|value| value.strip_suffix(']')) + && length.trim().parse::().is_ok_and(|length| length > 0) + { + return Ok(()); + } + if let Some(parameters) = primitive.strip_prefix("decimal(").and_then(|value| value.strip_suffix(')')) + && let Some((precision, scale)) = parameters.split_once(',') + && !scale.contains(',') + && let (Ok(precision), Ok(scale)) = (precision.trim().parse::(), scale.trim().parse::()) + && (1..=38).contains(&precision) + && scale <= precision + { + return Ok(()); + } + Err(TableCatalogStoreError::Invalid(format!( + "{label} contains unsupported primitive type {primitive}" + ))) +} + +fn required_i32_value( + object: &serde_json::Map, + field: &str, + label: &str, +) -> TableCatalogStoreResult { + let value = object + .get(field) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} is required")))?; + required_i32(value, label) +} + +fn required_positive_i32_value( + object: &serde_json::Map, + field: &str, + label: &str, +) -> TableCatalogStoreResult { + let value = object + .get(field) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} is required")))?; + required_positive_i32(value, label) +} + +fn required_schema_field_id_value( + object: &serde_json::Map, + field: &str, + label: &str, +) -> TableCatalogStoreResult { + let field_id = required_positive_i32_value(object, field, label)?; + if field_id > ICEBERG_MAX_USER_FIELD_ID { + return Err(TableCatalogStoreError::Invalid(format!( + "{label} must not use the reserved Iceberg field ID range" + ))); + } + Ok(field_id) +} + +fn required_i32(value: &serde_json::Value, label: &str) -> TableCatalogStoreResult { + let value = value + .as_i64() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} must be an integer")))?; + i32::try_from(value).map_err(|_| TableCatalogStoreError::Invalid(format!("{label} exceeds the signed 32-bit range"))) +} + +fn required_positive_i32(value: &serde_json::Value, label: &str) -> TableCatalogStoreResult { + let value = required_i32(value, label)?; + if value <= 0 { + return Err(TableCatalogStoreError::Invalid(format!("{label} must be positive"))); + } + Ok(value) +} + +fn validate_partition_specs( + metadata: &serde_json::Value, + format_version: u16, + schema_fields: &IcebergSchemaFields, + current_schema_fields: &IcebergSchemaFields, +) -> TableCatalogStoreResult<()> { + let spec_fields = if format_version == 1 { + vec![(0, require_metadata_array(metadata, "partition-spec")?)] + } else { + metadata + .get("partition-specs") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition-specs must be an array".to_string()))? + .iter() + .map(|spec| { + let spec_id = required_i32_value( + spec.as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("partition specs must be JSON objects".to_string()))?, + "spec-id", + "partition spec-id", + )?; + let fields = spec + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition spec fields must be an array".to_string()))?; + Ok((spec_id, fields)) + }) + .collect::>>()? + }; + let default_spec_id = if format_version == 1 { + Some(0) + } else { + Some(require_metadata_i32(metadata, "default-spec-id")?) + }; + let last_partition_id = (format_version != 1) + .then(|| require_metadata_i32(metadata, "last-partition-id")) + .transpose()?; + if last_partition_id.is_some_and(|last_partition_id| last_partition_id < 0) { + return Err(TableCatalogStoreError::Invalid("last-partition-id must not be negative".to_string())); + } + let mut assigned_fields = BTreeMap::new(); + for (spec_id, fields) in spec_fields { + let mut field_ids = BTreeSet::new(); + for (field_index, field) in fields.iter().enumerate() { + let field = field + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("partition spec fields must be JSON objects".to_string()))?; + let source_id = required_positive_i32_value(field, "source-id", "partition source-id")?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("partition field transform must be a non-empty string".to_string()) + })?; + let source_fields = if default_spec_id == Some(spec_id) { + current_schema_fields + } else { + schema_fields + }; + let source_type = if transform == "void" { + None + } else { + let source_type = source_fields.descriptors.get(&source_id).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("partition source-id {source_id} does not reference a schema field")) + })?; + if source_type.inside_collection { + return Err(TableCatalogStoreError::Invalid(format!( + "partition source-id {source_id} must not be nested in a list or map" + ))); + } + Some(source_type) + }; + let field_id = if format_version == 1 { + let expected_field_id = i32::try_from(field_index) + .ok() + .and_then(|field_index| 1000_i32.checked_add(field_index)) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("Iceberg v1 partition spec has too many fields".to_string()) + })?; + match field.get("field-id") { + Some(field_id) => { + let field_id = required_positive_i32(field_id, "partition field-id")?; + if field_id != expected_field_id { + return Err(TableCatalogStoreError::Invalid( + "Iceberg v1 partition field-id must be sequential from 1000".to_string(), + )); + } + field_id + } + None => expected_field_id, + } + } else { + required_positive_i32_value(field, "field-id", "partition field-id")? + }; + if last_partition_id.is_some_and(|last_partition_id| field_id > last_partition_id) || !field_ids.insert(field_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "partition field-id {field_id} must be unique and not exceed last-partition-id" + ))); + } + if !field + .get("name") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| !value.is_empty()) + { + return Err(TableCatalogStoreError::Invalid( + "partition field name must be a non-empty string".to_string(), + )); + } + if let Some(source_type) = source_type { + validate_transform_for_source(transform, source_type, "partition field")?; + } + let identity = (source_id, transform); + if format_version != 1 + && let Some(previous) = assigned_fields.insert(field_id, identity) + && previous != identity + { + return Err(TableCatalogStoreError::Invalid(format!( + "partition field-id {field_id} is assigned to multiple partition fields" + ))); + } + } + } + Ok(()) +} + +fn validate_sort_orders( + metadata: &serde_json::Value, + schema_fields: &IcebergSchemaFields, + current_schema_fields: &IcebergSchemaFields, +) -> TableCatalogStoreResult<()> { + let Some(sort_orders) = metadata.get("sort-orders") else { + return Ok(()); + }; + let sort_orders = sort_orders + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid("sort-orders must be an array".to_string()))?; + let default_sort_order_id = metadata.get("default-sort-order-id").and_then(serde_json::Value::as_i64); + for sort_order in sort_orders { + let sort_order = sort_order + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("sort orders must be JSON objects".to_string()))?; + let order_id = required_i32_value(sort_order, "order-id", "sort order-id")?; + if order_id < 0 { + return Err(TableCatalogStoreError::Invalid("sort order-id must not be negative".to_string())); + } + let fields = sort_order + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("sort order fields must be an array".to_string()))?; + if order_id == 0 && !fields.is_empty() { + return Err(TableCatalogStoreError::Invalid( + "sort order 0 is reserved for the unsorted order".to_string(), + )); + } + if order_id > 0 && fields.is_empty() { + return Err(TableCatalogStoreError::Invalid( + "empty sort orders must use the reserved unsorted order-id 0".to_string(), + )); + } + for field in fields { + let field = field + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("sort order fields must be JSON objects".to_string()))?; + let source_id = required_positive_i32_value(field, "source-id", "sort field source-id")?; + let source_fields = if default_sort_order_id == Some(i64::from(order_id)) { + current_schema_fields + } else { + schema_fields + }; + let source_type = source_fields.descriptors.get(&source_id).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("sort field source-id {source_id} does not reference a schema field")) + })?; + let transform = field + .get("transform") + .and_then(serde_json::Value::as_str) + .filter(|transform| !transform.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Invalid("sort field transform must be a non-empty string".to_string()))?; + validate_transform_for_source(transform, source_type, "sort field")?; + if !field + .get("direction") + .and_then(serde_json::Value::as_str) + .is_some_and(|direction| matches!(direction, "asc" | "desc")) + { + return Err(TableCatalogStoreError::Invalid("sort field direction must be asc or desc".to_string())); + } + if !field + .get("null-order") + .and_then(serde_json::Value::as_str) + .is_some_and(|null_order| matches!(null_order, "nulls-first" | "nulls-last")) + { + return Err(TableCatalogStoreError::Invalid( + "sort field null-order must be nulls-first or nulls-last".to_string(), + )); + } + } + } + Ok(()) +} + +fn validate_transform_for_source( + transform: &str, + source: &IcebergSchemaFieldDescriptor, + label: &str, +) -> TableCatalogStoreResult<()> { + if transform == "void" { + return Ok(()); + } + let IcebergFieldType::Primitive(source_type) = &source.field_type else { + return Err(TableCatalogStoreError::Invalid(format!( + "{label} transform {transform} requires a primitive source type" + ))); + }; + let valid = match transform { + "identity" => true, + "year" | "month" | "day" => matches!(source_type.as_str(), "date" | "timestamp" | "timestamptz"), + "hour" => matches!(source_type.as_str(), "timestamp" | "timestamptz"), + _ => match transform_parameter(transform, "bucket") { + Some(width) => { + width > 0 + && (matches!( + source_type.as_str(), + "int" | "long" | "date" | "time" | "timestamp" | "timestamptz" | "string" | "uuid" | "binary" + ) || source_type.starts_with("decimal(") + || source_type.starts_with("fixed[")) + } + None => match transform_parameter(transform, "truncate") { + Some(width) => { + width > 0 + && (matches!(source_type.as_str(), "int" | "long" | "string" | "binary") + || source_type.starts_with("decimal(")) + } + None => false, + }, + }, + }; + if !valid { + return Err(TableCatalogStoreError::Invalid(format!( + "{label} transform {transform} is invalid for source type {source_type}" + ))); + } + Ok(()) +} + +fn transform_parameter(transform: &str, name: &str) -> Option { + transform + .strip_prefix(name) + .and_then(|value| value.strip_prefix('[')) + .and_then(|value| value.strip_suffix(']')) + .and_then(|value| value.parse::().ok()) +} + fn validate_table_snapshot_fields(snapshot: &serde_json::Value, format_version: u16) -> TableCatalogStoreResult<()> { let snapshot = snapshot .as_object() @@ -633,6 +1931,14 @@ fn validate_table_snapshot_fields(snapshot: &serde_json::Value, format_version: if snapshot.get("timestamp-ms").and_then(serde_json::Value::as_i64).is_none() { return Err(TableCatalogStoreError::Invalid("snapshot timestamp-ms must be an integer".to_string())); } + if snapshot + .get("parent-snapshot-id") + .is_some_and(|parent_snapshot_id| parent_snapshot_id.as_i64().is_none()) + { + return Err(TableCatalogStoreError::Invalid( + "snapshot parent-snapshot-id must be an integer".to_string(), + )); + } let manifest_list = snapshot .get("manifest-list") .and_then(serde_json::Value::as_str) @@ -641,6 +1947,10 @@ fn validate_table_snapshot_fields(snapshot: &serde_json::Value, format_version: if manifests.is_some_and(|manifests| !manifests.is_array()) { return Err(TableCatalogStoreError::Invalid("snapshot manifests must be an array".to_string())); } + let summary = snapshot.get("summary"); + if let Some(summary) = summary { + validate_string_map(summary, "snapshot summary")?; + } match format_version { 1 => { @@ -666,8 +1976,7 @@ fn validate_table_snapshot_fields(snapshot: &serde_json::Value, format_version: "Iceberg v2 snapshot requires manifest-list or v1-compatible manifests".to_string(), )); } - let summary = snapshot - .get("summary") + let summary = summary .and_then(serde_json::Value::as_object) .ok_or_else(|| TableCatalogStoreError::Invalid("Iceberg v2 snapshot requires summary".to_string()))?; if !summary @@ -731,24 +2040,162 @@ pub(crate) fn table_metadata_partition_spec_ids(metadata: &serde_json::Value) -> Err(TableCatalogStoreError::Invalid("table metadata has no partition specs".to_string())) } -pub(crate) fn validate_view_metadata_references(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { +pub(crate) fn validate_supported_view_metadata(metadata: &serde_json::Value) -> TableCatalogStoreResult<()> { + let object = metadata + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("view metadata must be a JSON object".to_string()))?; + let format_version = object + .get("format-version") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("view metadata is missing integer field format-version".to_string()))?; + if format_version != 1 { + return Err(TableCatalogStoreError::Unsupported(format!( + "Iceberg view format-version {format_version}" + ))); + } + for field in ["view-uuid", "location"] { + if !object + .get(field) + .and_then(serde_json::Value::as_str) + .is_some_and(|value| !value.is_empty()) + { + return Err(TableCatalogStoreError::Invalid(format!( + "view metadata is missing non-empty string field {field}" + ))); + } + } + let schemas = view_metadata_array(metadata, "schemas")?; + if schemas.is_empty() { + return Err(TableCatalogStoreError::Invalid("view metadata schemas must not be empty".to_string())); + } + for schema in schemas { + validate_iceberg_schema(schema, "view schema")?; + } let schema_ids = metadata_array_i32_ids(metadata, "schemas", "schema-id", "schema")?; validate_metadata_id_reference(metadata, "current-schema-id", &schema_ids, "schema")?; + let versions = view_metadata_array(metadata, "versions")?; + if versions.is_empty() { + return Err(TableCatalogStoreError::Invalid("view metadata versions must not be empty".to_string())); + } + if object.get("current-version-id").and_then(serde_json::Value::as_i64).is_none() { + return Err(TableCatalogStoreError::Invalid( + "view metadata is missing integer field current-version-id".to_string(), + )); + } let version_ids = metadata_array_i32_ids(metadata, "versions", "version-id", "view version")?; validate_metadata_id_reference(metadata, "current-version-id", &version_ids, "view version")?; - if let Some(versions) = metadata.get("versions").and_then(serde_json::Value::as_array) { - for version in versions { - let schema_id = version - .get("schema-id") - .and_then(serde_json::Value::as_i64) - .ok_or_else(|| TableCatalogStoreError::Invalid("view version is missing schema-id".to_string()))?; - if !schema_ids.contains(&schema_id) { - return Err(TableCatalogStoreError::Invalid(format!( - "view version schema-id targets schema {schema_id}, which does not exist" - ))); - } + for version in versions { + validate_view_version(version)?; + let schema_id = version + .get("schema-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("view version is missing schema-id".to_string()))?; + if !schema_ids.contains(&schema_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "view version schema-id targets schema {schema_id}, which does not exist" + ))); } } + let version_log = view_metadata_array(metadata, "version-log")?; + for entry in version_log { + let version_id = entry + .get("version-id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("view version-log entry is missing version-id".to_string()))?; + if entry.get("timestamp-ms").and_then(serde_json::Value::as_i64).is_none() { + return Err(TableCatalogStoreError::Invalid( + "view version-log entries require integer version-id and timestamp-ms".to_string(), + )); + } + if !version_ids.contains(&version_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "view version-log targets view version {version_id}, which does not exist" + ))); + } + } + if let Some(properties) = object.get("properties") { + validate_string_map(properties, "view metadata properties")?; + } + Ok(()) +} + +fn view_metadata_array<'a>(metadata: &'a serde_json::Value, field: &str) -> TableCatalogStoreResult<&'a Vec> { + metadata + .get(field) + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("view metadata is missing array field {field}"))) +} + +fn validate_view_version(version: &serde_json::Value) -> TableCatalogStoreResult<()> { + let version = version + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("view version must be a JSON object".to_string()))?; + for field in ["version-id", "timestamp-ms", "schema-id"] { + if version.get(field).and_then(serde_json::Value::as_i64).is_none() { + return Err(TableCatalogStoreError::Invalid(format!("view version is missing integer field {field}"))); + } + } + let summary = version + .get("summary") + .ok_or_else(|| TableCatalogStoreError::Invalid("view version is missing summary".to_string()))?; + validate_string_map(summary, "view version summary")?; + let default_namespace = version + .get("default-namespace") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("view version is missing default-namespace".to_string()))?; + if default_namespace.iter().any(|segment| !segment.is_string()) { + return Err(TableCatalogStoreError::Invalid( + "view version default-namespace must contain strings".to_string(), + )); + } + if version.get("default-catalog").is_some_and(|value| !value.is_string()) { + return Err(TableCatalogStoreError::Invalid( + "view version default-catalog must be a string".to_string(), + )); + } + let representations = version + .get("representations") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("view version is missing representations".to_string()))?; + let mut dialects = BTreeSet::new(); + for representation in representations { + let representation = representation + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("view representation must be a JSON object".to_string()))?; + let dialect = representation + .get("dialect") + .and_then(serde_json::Value::as_str) + .filter(|dialect| !dialect.is_empty()); + if representation.get("type").and_then(serde_json::Value::as_str) != Some("sql") + || representation.get("sql").and_then(serde_json::Value::as_str).is_none() + || dialect.is_none() + { + return Err(TableCatalogStoreError::Invalid( + "view representation requires type sql, sql, and dialect strings".to_string(), + )); + } + if !dialects.insert( + dialect + .ok_or_else(|| { + TableCatalogStoreError::Invalid("view representation dialect must be a non-empty string".to_string()) + })? + .to_lowercase(), + ) { + return Err(TableCatalogStoreError::Invalid( + "view version contains duplicate SQL dialect representations".to_string(), + )); + } + } + Ok(()) +} + +fn validate_string_map(value: &serde_json::Value, label: &str) -> TableCatalogStoreResult<()> { + let values = value + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} must be a JSON object")))?; + if values.values().any(|value| !value.is_string()) { + return Err(TableCatalogStoreError::Invalid(format!("{label} values must be strings"))); + } Ok(()) } @@ -792,6 +2239,9 @@ fn metadata_array_i32_ids( label: &str, ) -> TableCatalogStoreResult> { let ids = metadata_array_ids(metadata, array_field, id_field, label)?; + if let Some(id) = ids.iter().find(|id| **id < 0) { + return Err(TableCatalogStoreError::Invalid(format!("{label} id {id} must not be negative"))); + } if let Some(id) = ids.iter().find(|id| i32::try_from(**id).is_err()) { return Err(TableCatalogStoreError::Invalid(format!( "{label} id {id} exceeds the signed 32-bit range" @@ -875,14 +2325,22 @@ impl<'a, B> TableSnapshotGraphValidationContext<'a, B> { #[derive(Default)] struct SnapshotGraphReadBudget { manifest_count: usize, + manifest_traversal_count: usize, avro_bytes: usize, decoded_avro_bytes: usize, file_reference_count: usize, - manifest_lists: BTreeMap>, - manifests: BTreeMap>, + manifest_lists: BTreeMap>>, + manifests: BTreeMap, validated_live_objects: BTreeSet, } +#[derive(Clone)] +struct CachedSnapshotGraphManifest { + object_size: usize, + partition_spec_id: Option, + references: Arc>, +} + impl SnapshotGraphReadBudget { fn charge_manifests(&mut self, count: usize) -> TableCatalogStoreResult<()> { self.manifest_count = self @@ -897,6 +2355,18 @@ impl SnapshotGraphReadBudget { Ok(()) } + fn charge_manifest_traversals(&mut self, count: usize) -> TableCatalogStoreResult<()> { + self.manifest_traversal_count = self.manifest_traversal_count.checked_add(count).ok_or_else(|| { + TableCatalogStoreError::Invalid("snapshot manifest traversal count exceeds the commit limit".to_string()) + })?; + if self.manifest_traversal_count > TABLE_COMMIT_MAX_MANIFEST_TRAVERSALS { + return Err(TableCatalogStoreError::Invalid( + "snapshot manifest traversal count exceeds the commit limit".to_string(), + )); + } + Ok(()) + } + fn charge_avro_bytes(&mut self, count: usize) -> TableCatalogStoreResult<()> { self.avro_bytes = self .avro_bytes @@ -962,14 +2432,25 @@ where B: TableCatalogObjectBackend, { validate_supported_table_metadata_fields(metadata)?; + let snapshot_ids = metadata_array_ids(metadata, "snapshots", "snapshot-id", "snapshot")?; + validate_table_statistics_references(metadata, &snapshot_ids)?; + let mut budget = SnapshotGraphReadBudget::default(); + validate_table_statistics_objects(context, metadata).await?; let format_version = table_metadata_format_version(metadata)?; let snapshots = snapshots_requiring_graph_validation(current_metadata, metadata)?; - let mut budget = SnapshotGraphReadBudget::default(); for snapshot in snapshots { snapshot .get("snapshot-id") .and_then(serde_json::Value::as_i64) .ok_or_else(|| TableCatalogStoreError::Invalid("snapshot-id must be an integer".to_string()))?; + if format_version == 2 + && snapshot.get("manifests").is_some() + && !snapshot_is_retained_v1_history(current_metadata, snapshot) + { + return Err(TableCatalogStoreError::Invalid( + "new Iceberg v2 snapshots require manifest-list".to_string(), + )); + } let snapshot_sequence_number = snapshot .get("sequence-number") .and_then(serde_json::Value::as_i64) @@ -984,9 +2465,9 @@ where ) .await?; let mut seen_files = BTreeSet::new(); - for references in manifests { - for reference in references { - if !seen_files.insert(reference.location) { + for references in &manifests { + for reference in references.iter() { + if !seen_files.insert(reference.location.as_str()) { return Err(TableCatalogStoreError::Invalid( "snapshot contains a duplicate file reference".to_string(), )); @@ -997,6 +2478,163 @@ where Ok(()) } +fn snapshot_is_retained_v1_history(current_metadata: Option<&serde_json::Value>, target_snapshot: &serde_json::Value) -> bool { + let Some(current_metadata) = current_metadata else { + return false; + }; + if table_metadata_format_version(current_metadata).ok() != Some(1) { + return false; + } + let Some(snapshot_id) = target_snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) else { + return false; + }; + let Some(current_snapshot) = current_metadata + .get("snapshots") + .and_then(serde_json::Value::as_array) + .and_then(|snapshots| { + snapshots + .iter() + .find(|snapshot| snapshot.get("snapshot-id").and_then(serde_json::Value::as_i64) == Some(snapshot_id)) + }) + else { + return false; + }; + let mut current_snapshot = current_snapshot.clone(); + let mut target_snapshot = target_snapshot.clone(); + for snapshot in [&mut current_snapshot, &mut target_snapshot] { + if snapshot.get("sequence-number").and_then(serde_json::Value::as_i64) == Some(0) + && let Some(object) = snapshot.as_object_mut() + { + object.remove("sequence-number"); + } + } + current_snapshot == target_snapshot +} + +async fn validate_table_statistics_objects( + context: &TableSnapshotGraphValidationContext<'_, B>, + metadata: &serde_json::Value, +) -> TableCatalogStoreResult<()> +where + B: TableCatalogObjectBackend, +{ + if context.entry.table_bucket != context.table_bucket { + return Err(TableCatalogStoreError::Invalid( + "statistics object is outside the table bucket".to_string(), + )); + } + let warehouse_object_prefix = table_warehouse_object_prefix(context.entry)?; + let mut objects = BTreeMap::new(); + let mut total_size = 0usize; + for (field, kind) in [ + ("statistics", IcebergStatisticsFileKind::Table), + ("partition-statistics", IcebergStatisticsFileKind::Partition), + ] { + let Some(values) = metadata.get(field).and_then(serde_json::Value::as_array) else { + continue; + }; + for value in values { + let location = value + .get("statistics-path") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{field}.statistics-path must be a string")))?; + let object_key = table_catalog_object_key_from_location(context.table_bucket, location) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{field} object location is invalid")))?; + if !object_key.starts_with(&warehouse_object_prefix) { + return Err(TableCatalogStoreError::Invalid(format!("{field} object is outside the table warehouse"))); + } + let file_size = value + .get("file-size-in-bytes") + .and_then(serde_json::Value::as_u64) + .and_then(|size| usize::try_from(size).ok()) + .filter(|size| *size <= TABLE_STATISTICS_FILE_MAX_SIZE) + .ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("{field} file-size-in-bytes exceeds the validation limit")) + })?; + if let Some(previous) = objects.get(&object_key) { + if *previous != (file_size, kind) { + return Err(TableCatalogStoreError::Invalid( + "statistics object is declared with inconsistent metadata".to_string(), + )); + } + continue; + } + total_size = total_size + .checked_add(file_size) + .filter(|size| *size <= TABLE_COMMIT_MAX_STATISTICS_BYTES) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("statistics bytes exceed the commit validation limit".to_string()) + })?; + objects.insert(object_key, (file_size, kind)); + } + } + + if objects.len() > TABLE_COMMIT_MAX_STATISTICS_OBJECTS { + return Err(TableCatalogStoreError::Invalid( + "statistics object count exceeds the commit limit".to_string(), + )); + } + let backend = context.backend.clone(); + let bucket = context.table_bucket.to_string(); + stream::iter(objects) + .map(move |(object_key, (expected_size, kind))| { + let backend = backend.clone(); + let bucket = bucket.clone(); + async move { + let object = backend + .read_object_limited(&bucket, &object_key, expected_size) + .await? + .ok_or_else(|| TableCatalogStoreError::Invalid("statistics object is missing".to_string()))?; + if object.data.len() != expected_size { + return Err(TableCatalogStoreError::Invalid( + "statistics file-size-in-bytes does not match the object".to_string(), + )); + } + let valid_magic = match kind { + IcebergStatisticsFileKind::Table => object.data.starts_with(b"PFA1") && object.data.ends_with(b"PFA1"), + IcebergStatisticsFileKind::Partition => object.data.starts_with(b"PAR1") && object.data.ends_with(b"PAR1"), + }; + if !valid_magic { + return Err(TableCatalogStoreError::Invalid(match kind { + IcebergStatisticsFileKind::Table => "table statistics object is not a Puffin file".to_string(), + IcebergStatisticsFileKind::Partition => "partition statistics object is not a Parquet file".to_string(), + })); + } + Ok(()) + } + }) + .buffered(TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY) + .try_for_each(|()| async { Ok(()) }) + .await +} + +async fn validate_object_keys_exist( + backend: &B, + bucket: &str, + object_keys: impl IntoIterator, + missing_message: &'static str, +) -> TableCatalogStoreResult<()> +where + B: TableCatalogObjectBackend, +{ + let backend = backend.clone(); + let bucket = bucket.to_string(); + stream::iter(object_keys) + .map(move |object_key| { + let backend = backend.clone(); + let bucket = bucket.clone(); + async move { + if !backend.object_exists(&bucket, &object_key).await? { + return Err(TableCatalogStoreError::Invalid(missing_message.to_string())); + } + Ok(()) + } + }) + .buffered(TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY) + .try_for_each(|()| async { Ok(()) }) + .await +} + fn snapshots_requiring_graph_validation<'a>( current_metadata: Option<&serde_json::Value>, metadata: &'a serde_json::Value, @@ -1009,6 +2647,9 @@ fn snapshots_requiring_graph_validation<'a>( .ok_or_else(|| TableCatalogStoreError::Invalid("snapshots must be an array".to_string()))?; if let Some(current_metadata) = current_metadata { + if !partition_specs_preserve_existing_definitions(current_metadata, metadata)? { + return Ok(snapshots.iter().collect()); + } let current_snapshots = current_metadata .get("snapshots") .and_then(serde_json::Value::as_array) @@ -1030,29 +2671,81 @@ fn snapshots_requiring_graph_validation<'a>( .collect()); } - let mut active_snapshot_ids = BTreeSet::new(); - if let Some(snapshot_id) = metadata - .get("current-snapshot-id") - .and_then(serde_json::Value::as_i64) - .filter(|snapshot_id| *snapshot_id != -1) - { - active_snapshot_ids.insert(snapshot_id); - } - if let Some(refs) = metadata.get("refs").and_then(serde_json::Value::as_object) { - active_snapshot_ids.extend( - refs.values() - .filter_map(|reference| reference.get("snapshot-id").and_then(serde_json::Value::as_i64)), - ); - } - Ok(snapshots + Ok(snapshots.iter().collect()) +} + +fn partition_specs_preserve_existing_definitions( + current_metadata: &serde_json::Value, + metadata: &serde_json::Value, +) -> TableCatalogStoreResult { + let current_specs = normalized_partition_spec_definitions(current_metadata)?; + let target_specs = normalized_partition_spec_definitions(metadata)?; + Ok(current_specs .iter() - .filter(|snapshot| { - snapshot - .get("snapshot-id") - .and_then(serde_json::Value::as_i64) - .is_some_and(|snapshot_id| active_snapshot_ids.contains(&snapshot_id)) + .all(|(spec_id, fields)| target_specs.get(spec_id) == Some(fields))) +} + +type NormalizedPartitionFieldDefinition = (i32, i32, String, String); +type NormalizedPartitionSpecDefinitions = BTreeMap>; + +fn normalized_partition_spec_definitions( + metadata: &serde_json::Value, +) -> TableCatalogStoreResult { + let format_version = table_metadata_format_version(metadata)?; + let specs = if format_version == 1 { + vec![(0, require_metadata_array(metadata, "partition-spec")?)] + } else { + require_metadata_array(metadata, "partition-specs")? + .iter() + .map(|spec| { + let spec = spec + .as_object() + .ok_or_else(|| TableCatalogStoreError::Invalid("partition specs must be JSON objects".to_string()))?; + let spec_id = required_i32_value(spec, "spec-id", "partition spec-id")?; + let fields = spec + .get("fields") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition spec fields must be an array".to_string()))?; + Ok((spec_id, fields)) + }) + .collect::>>()? + }; + specs + .into_iter() + .map(|(spec_id, fields)| { + let fields = fields + .iter() + .enumerate() + .map(|(index, field)| { + let field = field.as_object().ok_or_else(|| { + TableCatalogStoreError::Invalid("partition spec fields must be JSON objects".to_string()) + })?; + let source_id = required_positive_i32_value(field, "source-id", "partition source-id")?; + let field_id = match field.get("field-id") { + Some(field_id) => required_positive_i32(field_id, "partition field-id")?, + None if format_version == 1 => i32::try_from(index) + .ok() + .and_then(|index| 1000_i32.checked_add(index)) + .ok_or_else(|| { + TableCatalogStoreError::Invalid("Iceberg v1 partition spec has too many fields".to_string()) + })?, + None => { + return Err(TableCatalogStoreError::Invalid("partition field-id is required".to_string())); + } + }; + let transform = field.get("transform").and_then(serde_json::Value::as_str).ok_or_else(|| { + TableCatalogStoreError::Invalid("partition field transform must be a string".to_string()) + })?; + let name = field + .get("name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| TableCatalogStoreError::Invalid("partition field name must be a string".to_string()))?; + Ok((source_id, field_id, name.to_string(), transform.to_string())) + }) + .collect::>>()?; + Ok((spec_id, fields)) }) - .collect()) + .collect() } async fn snapshot_graph_manifest_references( @@ -1062,7 +2755,7 @@ async fn snapshot_graph_manifest_references( format_version: u16, snapshot_sequence_number: i64, budget: &mut SnapshotGraphReadBudget, -) -> TableCatalogStoreResult>> +) -> TableCatalogStoreResult>>> where B: TableCatalogObjectBackend, { @@ -1071,8 +2764,9 @@ where let partition_spec_ids = table_metadata_partition_spec_ids(metadata)?; let mut manifests = Vec::with_capacity(manifest_locations.len()); let mut seen_manifest_paths = BTreeSet::new(); - for manifest_location in manifest_locations { - validate_snapshot_graph_manifest_location(&manifest_location, format_version, snapshot_sequence_number)?; + for manifest_location in manifest_locations.iter() { + budget.charge_manifest_traversals(1)?; + validate_snapshot_graph_manifest_location(manifest_location, format_version, snapshot_sequence_number)?; match manifest_location.partition_spec_id { Some(partition_spec_id) if !partition_spec_ids.contains(&partition_spec_id) => { return Err(TableCatalogStoreError::Invalid(format!( @@ -1086,7 +2780,7 @@ where } _ => {} } - if !seen_manifest_paths.insert(manifest_location.manifest_path.clone()) { + if !seen_manifest_paths.insert(manifest_location.manifest_path.as_str()) { return Err(TableCatalogStoreError::Invalid( "snapshot contains a duplicate manifest reference".to_string(), )); @@ -1096,8 +2790,8 @@ where &manifest_location.manifest_path, TableMetadataMaintenanceObjectKind::ManifestFile, )?; - let references = if let Some(references) = budget.manifests.get(&manifest_key).cloned() { - references + let cached_manifest = if let Some(manifest) = budget.manifests.get(&manifest_key) { + manifest.clone() } else { budget.charge_manifests(1)?; let manifest_object = context @@ -1109,13 +2803,37 @@ where budget.charge_avro_bytes(manifest_size)?; let decoded_manifest = decode_manifest_avro_async(manifest_object.data).await?; budget.charge_decoded_avro_bytes(decoded_manifest.decoded_size)?; - budget.charge_file_references(decoded_manifest.references.len())?; - budget.manifests.insert(manifest_key, decoded_manifest.references.clone()); - decoded_manifest.references + let manifest = CachedSnapshotGraphManifest { + object_size: manifest_size, + partition_spec_id: decoded_manifest.partition_spec_id, + references: Arc::new(decoded_manifest.references), + }; + budget.manifests.insert(manifest_key, manifest.clone()); + manifest }; + if manifest_location + .manifest_length + .is_some_and(|declared| u64::try_from(cached_manifest.object_size).ok() != Some(declared)) + { + return Err(TableCatalogStoreError::Invalid( + "manifest-list manifest_length does not match the manifest object".to_string(), + )); + } + if manifest_location.from_manifest_list + && cached_manifest + .partition_spec_id + .is_some_and(|manifest_spec_id| Some(manifest_spec_id) != manifest_location.partition_spec_id) + { + return Err(TableCatalogStoreError::Invalid( + "manifest partition-spec-id does not match its manifest-list entry".to_string(), + )); + } + let references = cached_manifest.references; + validate_snapshot_graph_manifest_content(manifest_location, references.as_ref())?; + budget.charge_file_references(references.len())?; validate_snapshot_graph_data_files( context, - &references, + references.as_ref(), budget, format_version, if manifest_location.from_manifest_list && manifest_location.format_version == 1 { @@ -1130,21 +2848,43 @@ where Ok(manifests) } +fn validate_snapshot_graph_manifest_content( + manifest: &SnapshotGraphManifestLocation, + references: &[ManifestDataFileReference], +) -> TableCatalogStoreResult<()> { + let content_matches = match manifest.content { + Some(0) => references + .iter() + .all(|reference| reference.content == ManifestDataFileContent::Data), + Some(1) => references + .iter() + .all(|reference| reference.content != ManifestDataFileContent::Data), + None => true, + Some(_) => false, + }; + if !content_matches { + return Err(TableCatalogStoreError::Invalid( + "manifest-list content does not match manifest file content".to_string(), + )); + } + Ok(()) +} + async fn snapshot_graph_manifest_locations( context: &TableSnapshotGraphValidationContext<'_, B>, snapshot: &serde_json::Value, format_version: u16, snapshot_sequence_number: i64, budget: &mut SnapshotGraphReadBudget, -) -> TableCatalogStoreResult> +) -> TableCatalogStoreResult>> where B: TableCatalogObjectBackend, { if let Some(manifest_list_location) = snapshot.get("manifest-list").and_then(serde_json::Value::as_str) { let manifest_list_key = snapshot_graph_object_key(context, manifest_list_location, TableMetadataMaintenanceObjectKind::ManifestList)?; - let references = if let Some(references) = budget.manifest_lists.get(&manifest_list_key).cloned() { - references + let references = if let Some(references) = budget.manifest_lists.get(&manifest_list_key) { + return Ok(Arc::clone(references)); } else { let manifest_list_object = context .backend @@ -1154,62 +2894,65 @@ where budget.charge_avro_bytes(manifest_list_object.data.len())?; let decoded_manifest_list = decode_manifest_list_avro_async(manifest_list_object.data).await?; budget.charge_decoded_avro_bytes(decoded_manifest_list.decoded_size)?; - budget - .manifest_lists - .insert(manifest_list_key, decoded_manifest_list.references.clone()); decoded_manifest_list.references }; - return Ok(references - .into_iter() - .map(|reference| SnapshotGraphManifestLocation { - manifest_path: reference.manifest_path, - format_version: reference.format_version, - manifest_length: reference.manifest_length, - partition_spec_id: reference.partition_spec_id, - content: reference.content, - sequence_number: reference.sequence_number, - min_sequence_number: reference.min_sequence_number, - added_snapshot_id: reference.added_snapshot_id, - added_files_count: reference.added_files_count, - existing_files_count: reference.existing_files_count, - deleted_files_count: reference.deleted_files_count, - added_rows_count: reference.added_rows_count, - existing_rows_count: reference.existing_rows_count, - deleted_rows_count: reference.deleted_rows_count, - from_manifest_list: true, - }) - .collect()); + let references = Arc::new( + references + .into_iter() + .map(|reference| SnapshotGraphManifestLocation { + manifest_path: reference.manifest_path, + format_version: reference.format_version, + manifest_length: reference.manifest_length, + partition_spec_id: reference.partition_spec_id, + content: reference.content, + sequence_number: reference.sequence_number, + min_sequence_number: reference.min_sequence_number, + added_snapshot_id: reference.added_snapshot_id, + added_files_count: reference.added_files_count, + existing_files_count: reference.existing_files_count, + deleted_files_count: reference.deleted_files_count, + added_rows_count: reference.added_rows_count, + existing_rows_count: reference.existing_rows_count, + deleted_rows_count: reference.deleted_rows_count, + from_manifest_list: true, + }) + .collect(), + ); + budget.manifest_lists.insert(manifest_list_key, Arc::clone(&references)); + return Ok(references); } let Some(manifests) = snapshot.get("manifests").and_then(serde_json::Value::as_array) else { return Err(TableCatalogStoreError::Invalid("snapshot manifest-list is required".to_string())); }; - manifests - .iter() - .map(|manifest| { - manifest - .as_str() - .filter(|manifest| !manifest.is_empty()) - .map(|manifest| SnapshotGraphManifestLocation { - manifest_path: manifest.to_string(), - format_version, - manifest_length: None, - partition_spec_id: None, - content: None, - sequence_number: Some(snapshot_sequence_number), - min_sequence_number: None, - added_snapshot_id: None, - added_files_count: None, - existing_files_count: None, - deleted_files_count: None, - added_rows_count: None, - existing_rows_count: None, - deleted_rows_count: None, - from_manifest_list: false, - }) - .ok_or_else(|| TableCatalogStoreError::Invalid("snapshot manifest location must be a string".to_string())) - }) - .collect() + Ok(Arc::new( + manifests + .iter() + .map(|manifest| { + manifest + .as_str() + .filter(|manifest| !manifest.is_empty()) + .map(|manifest| SnapshotGraphManifestLocation { + manifest_path: manifest.to_string(), + format_version, + manifest_length: None, + partition_spec_id: None, + content: None, + sequence_number: Some(snapshot_sequence_number), + min_sequence_number: None, + added_snapshot_id: None, + added_files_count: None, + existing_files_count: None, + deleted_files_count: None, + added_rows_count: None, + existing_rows_count: None, + deleted_rows_count: None, + from_manifest_list: false, + }) + .ok_or_else(|| TableCatalogStoreError::Invalid("snapshot manifest location must be a string".to_string())) + }) + .collect::>>()?, + )) } fn validate_snapshot_graph_manifest_location( @@ -1265,21 +3008,6 @@ fn validate_snapshot_graph_manifest_location( "Iceberg v2 manifest-list sequence numbers are inconsistent with the snapshot".to_string(), )); } - if [ - manifest.added_files_count, - manifest.existing_files_count, - manifest.deleted_files_count, - manifest.added_rows_count, - manifest.existing_rows_count, - manifest.deleted_rows_count, - ] - .into_iter() - .any(|count| count.is_none()) - { - return Err(TableCatalogStoreError::Invalid( - "Iceberg v2 manifest-list entry is missing required file or row counts".to_string(), - )); - } } _ => { return Err(TableCatalogStoreError::Internal(format!( @@ -1398,25 +3126,13 @@ where } } - for object_keys in live_object_keys.chunks(TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY) { - let backend = context.backend.clone(); - let bucket = context.table_bucket.to_string(); - stream::iter(object_keys.iter().cloned()) - .map(move |object_key| { - let backend = backend.clone(); - let bucket = bucket.clone(); - async move { - if !backend.object_exists(&bucket, &object_key).await? { - return Err(TableCatalogStoreError::Invalid("manifest referenced data file is missing".to_string())); - } - Ok(()) - } - }) - .buffered(TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY) - .try_for_each(|()| async { Ok(()) }) - .await?; - } - Ok(()) + validate_object_keys_exist( + context.backend, + context.table_bucket, + live_object_keys, + "manifest referenced data file is missing", + ) + .await } fn snapshot_graph_object_key( diff --git a/rustfs/src/table_catalog/mod.rs b/rustfs/src/table_catalog/mod.rs index af560459d..4011639ce 100644 --- a/rustfs/src/table_catalog/mod.rs +++ b/rustfs/src/table_catalog/mod.rs @@ -111,8 +111,12 @@ const TABLE_MANIFEST_AVRO_MAX_DECODED_SIZE: usize = 128 * 1024 * 1024; const TABLE_MANIFEST_AVRO_MAX_RECORDS: usize = 1_000_000; const TABLE_MANIFEST_AVRO_MAX_HEADER_ENTRIES: usize = 1_024; const TABLE_COMMIT_MAX_MANIFESTS: usize = 10_000; +const TABLE_COMMIT_MAX_MANIFEST_TRAVERSALS: usize = 20_000; const TABLE_COMMIT_MAX_AVRO_BYTES: usize = 512 * 1024 * 1024; const TABLE_COMMIT_MAX_FILE_REFERENCES: usize = 1_000_000; +const TABLE_COMMIT_MAX_STATISTICS_OBJECTS: usize = 1_024; +const TABLE_COMMIT_MAX_STATISTICS_BYTES: usize = 512 * 1024 * 1024; +const TABLE_STATISTICS_FILE_MAX_SIZE: usize = 128 * 1024 * 1024; pub(crate) const TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY: usize = 16; pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX; const WAREHOUSE_ROOT: &str = "warehouses"; diff --git a/rustfs/src/table_catalog/store/migration.rs b/rustfs/src/table_catalog/store/migration.rs index d8224e471..398285f88 100644 --- a/rustfs/src/table_catalog/store/migration.rs +++ b/rustfs/src/table_catalog/store/migration.rs @@ -216,7 +216,7 @@ where Ok(fence) } - pub(super) async fn acquire_table_bucket_registry_write_permit(&self) -> TableCatalogStoreResult> { + pub(super) async fn acquire_table_bucket_registry_write_permit(&self) -> TableCatalogStoreResult { let fence_path = self.paths.backing_migration_global_fence_path(); let lock_path = self.paths.backing_migration_global_fence_lock_path(); let guard = self.backend.acquire_read_lock(self.catalog_bucket(), &lock_path).await?; @@ -235,7 +235,7 @@ where pub(super) async fn acquire_object_backed_catalog_write_permit( &self, table_bucket: &str, - ) -> TableCatalogStoreResult> { + ) -> TableCatalogStoreResult { let lock_path = self.paths.backing_migration_fence_lock_path(table_bucket); let guard = self.backend.acquire_read_lock(self.catalog_bucket(), &lock_path).await?; if self.read_backing_migration_fence(table_bucket).await?.is_some() { @@ -290,7 +290,7 @@ where async fn collect_bucket_snapshot_with_locks( &self, table_bucket: &str, - guards: &mut Vec>, + guards: &mut Vec, ) -> TableCatalogStoreResult { let bucket_path = self.paths.table_bucket_entry_path(table_bucket); guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?); diff --git a/rustfs/src/table_catalog/store/mod.rs b/rustfs/src/table_catalog/store/mod.rs index 4e358651b..9e8d57129 100644 --- a/rustfs/src/table_catalog/store/mod.rs +++ b/rustfs/src/table_catalog/store/mod.rs @@ -262,6 +262,29 @@ pub(crate) trait TableCatalogStore: Send + Sync { async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()>; + async fn create_view_with_publication( + &self, + entry: ViewEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view creation requires a table-bucket publication fence".to_string(), + )); + } + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.view) + .await?; + if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.view) { + return Err(TableCatalogStoreError::Internal( + "view creation requires a view publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + self.create_view(entry).await + } + async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult>; async fn list_views_page( @@ -283,6 +306,32 @@ pub(crate) trait TableCatalogStore: Send + Sync { async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult; + async fn replace_view_with_publication( + &self, + request: ViewCommitRequest, + table_bucket_fence_required: bool, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { + if table_bucket_fence_required { + publication.begin_table_bucket(&request.table_bucket).await?; + if !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table-bucket publication fence".to_string(), + )); + } + } + publication + .prepare(&request.table_bucket, &request.namespace, &request.view) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.view) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a view publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + self.replace_view(request).await + } + async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()>; async fn get_commit_by_id( @@ -338,7 +387,7 @@ struct TableCommitLockPublication<'a, B> { struct TableCommitLockPublicationState { table_bucket: Option, table: Option<(String, String, String)>, - guards: Vec>, + guards: Vec, } impl<'a, B> TableCommitLockPublication<'a, B> { @@ -409,15 +458,17 @@ where } fn holds_table_bucket(&self, table_bucket: &str) -> bool { - self.state.lock().table_bucket.as_deref() == Some(table_bucket) + let state = self.state.lock(); + state.table_bucket.as_deref() == Some(table_bucket) && state.guards.iter().all(|guard| !guard.is_lock_lost()) } fn holds_table(&self, table_bucket: &str, namespace: &str, table: &str) -> bool { - self.state - .lock() + let state = self.state.lock(); + state .table .as_ref() .is_some_and(|held| held.0 == table_bucket && held.1 == namespace && held.2 == table) + && state.guards.iter().all(|guard| !guard.is_lock_lost()) } fn complete(&self) { @@ -438,6 +489,32 @@ pub(crate) struct TableCatalogObjectMetadata { pub mod_time: Option, } +pub(crate) struct TableCatalogLockGuard { + _guard: Box, + lock_lost: Option>, +} + +impl TableCatalogLockGuard { + pub(crate) fn stable(guard: impl Send + 'static) -> Self { + Self { + _guard: Box::new(guard), + lock_lost: None, + } + } + + fn namespace(guard: rustfs_lock::NamespaceLockGuard) -> Self { + let lock_lost = guard.lock_lost_signal(); + Self { + _guard: Box::new(guard), + lock_lost, + } + } + + pub(crate) fn is_lock_lost(&self) -> bool { + self.lock_lost.as_ref().is_some_and(|signal| signal.is_lost()) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct TableCatalogObjectListPage { pub objects: Vec, @@ -588,11 +665,11 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { Ok(TableCatalogObjectListPage { objects, is_truncated }) } - async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { self.acquire_write_lock(bucket, object).await } - async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult>; + async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult; async fn begin_table_bucket_commit_publication(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> { Ok(()) @@ -1169,6 +1246,17 @@ where } } + async fn create_view_with_publication( + &self, + entry: ViewEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.create_view_with_publication(entry, publication).await, + Self::DurableStrong(store) => store.create_view_with_publication(entry, publication).await, + } + } + async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { match self { Self::ObjectBacked(store) => store.list_views(table_bucket, namespace).await, @@ -1203,6 +1291,26 @@ where } } + async fn replace_view_with_publication( + &self, + request: ViewCommitRequest, + table_bucket_fence_required: bool, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => { + store + .replace_view_with_publication(request, table_bucket_fence_required, publication) + .await + } + Self::DurableStrong(store) => { + store + .replace_view_with_publication(request, table_bucket_fence_required, publication) + .await + } + } + } + async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { match self { Self::ObjectBacked(store) => store.drop_view(table_bucket, namespace, view).await, @@ -1686,7 +1794,7 @@ where }) } - async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { let lock = self .store .new_ns_lock(bucket, object) @@ -1696,10 +1804,10 @@ where .get_write_lock(get_lock_acquire_timeout()) .await .map_err(|err| TableCatalogStoreError::Internal(format!("failed to acquire catalog table lock: {err}")))?; - Ok(Box::new(guard)) + Ok(TableCatalogLockGuard::namespace(guard)) } - async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { let lock = self .store .new_ns_lock(bucket, object) @@ -1709,7 +1817,7 @@ where .get_read_lock(get_lock_acquire_timeout()) .await .map_err(|err| TableCatalogStoreError::Internal(format!("failed to acquire catalog migration lock: {err}")))?; - Ok(Box::new(guard)) + Ok(TableCatalogLockGuard::namespace(guard)) } } diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index b538ae85f..3cd08f321 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -1036,6 +1036,20 @@ where .await } + async fn restore_table_warehouse_index_after_failed_drop(&self, entry: &TableEntry, reason: &'static str) { + if let Err(err) = self.reserve_table_warehouse_index(entry).await { + tracing::warn!( + table_bucket = %entry.table_bucket, + namespace = %entry.namespace, + table = %entry.table, + table_id = %entry.table_id, + reason, + error = %err, + "failed to restore table warehouse index after table drop stopped" + ); + } + } + async fn delete_table_warehouse_index_if_changed(&self, current: &TableEntry, next: &TableEntry) { let Ok(current_index) = table_warehouse_index_entry(current) else { return; @@ -1316,6 +1330,15 @@ where } self.ensure_table_warehouse_prefix_available(&entry).await?; let reservation = self.reserve_table_warehouse_index(&entry).await?; + if !publication.holds_table_bucket(&entry.table_bucket) + || !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) + { + self.delete_created_table_warehouse_index(&entry, reservation, "table publication fence lost") + .await; + return Err(TableCatalogStoreError::Internal( + "table registration publication fence was lost before catalog update".to_string(), + )); + } let result = self .write_entry_unlocked(self.catalog_bucket(), &table_path, &entry, precondition) .await; @@ -1327,7 +1350,25 @@ where } async fn write_view_entry(&self, entry: ViewEntry, precondition: TableCatalogPutPrecondition) -> TableCatalogStoreResult<()> { + let publication = TableCommitLockPublication::new(&self.backend); + self.write_view_entry_with_publication(entry, precondition, &publication) + .await + } + + async fn write_view_entry_with_publication( + &self, + entry: ViewEntry, + precondition: TableCatalogPutPrecondition, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { validate_view_entry_version_and_id(&entry)?; + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view creation requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); self.require_table_bucket(&entry.table_bucket).await?; let namespace = parse_namespace_for_store(&entry.namespace)?; let view = parse_table_for_store(&entry.view)?; @@ -1353,6 +1394,17 @@ where entry.table_bucket, entry.namespace, entry.view ))); } + // Preserve catalog -> publication -> object lock order across rolling upgrades. + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.view) + .await?; + if !publication.holds_table_bucket(&entry.table_bucket) + || !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.view) + { + return Err(TableCatalogStoreError::Internal( + "view creation publication fence was lost before catalog update".to_string(), + )); + } self.write_entry_unlocked(self.catalog_bucket(), &view_path, &entry, precondition) .await } @@ -4493,16 +4545,16 @@ where validate_commit_metadata_digest(&request, &new_metadata_object)?; let table_bucket = request.table_bucket.clone(); let metadata_location = request.new_metadata_location.clone(); - let next_warehouse_location = tokio::task::spawn_blocking(move || { - table_metadata_warehouse_location(&table_bucket, &metadata_location, &new_metadata_object) + let next_metadata_state = tokio::task::spawn_blocking(move || { + table_metadata_commit_state(&table_bucket, &metadata_location, &new_metadata_object) }) .await .map_err(|err| TableCatalogStoreError::Internal(format!("table metadata parser task failed: {err}")))??; - if next_warehouse_location + let warehouse_relocation = next_metadata_state + .warehouse_location .as_ref() - .is_some_and(|warehouse_location| warehouse_location != ¤t.warehouse_location) - && !publication.holds_table_bucket(&request.table_bucket) - { + .is_some_and(|warehouse_location| warehouse_location != ¤t.warehouse_location); + if warehouse_relocation && !publication.holds_table_bucket(&request.table_bucket) { return table_commit_result( &request.table_bucket, &request.namespace, @@ -4537,9 +4589,12 @@ where let mut next = current.clone(); next.metadata_location = staged_commit_log.new_metadata_location.clone(); - if let Some(warehouse_location) = next_warehouse_location { + if let Some(warehouse_location) = next_metadata_state.warehouse_location { next.warehouse_location = warehouse_location; } + if let Some(format_version) = next_metadata_state.format_version { + next.format_version = format_version; + } next.version_token = staged_commit_log.new_version_token.clone(); next.generation = current.generation.saturating_add(1); if next.warehouse_location != current.warehouse_location { @@ -4585,6 +4640,24 @@ where ); } + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) + || (warehouse_relocation && !publication.holds_table_bucket(&request.table_bucket)) + { + self.delete_created_table_warehouse_index(&next, reservation, "table publication fence lost") + .await; + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Internal( + "table commit publication fence was lost before pointer update".to_string(), + )), + ); + } + let cas_started = Instant::now(); let cas_result = self .write_entry_unlocked( @@ -4662,20 +4735,21 @@ where ))); }; self.delete_owned_table_warehouse_index_for_drop(&entry).await?; + if !publication.holds_table_bucket(table_bucket) + || !publication.holds_table(table_bucket, &namespace.public_name(), table.as_str()) + { + self.restore_table_warehouse_index_after_failed_drop(&entry, "table publication fence lost") + .await; + return Err(TableCatalogStoreError::Internal( + "table drop publication fence was lost before catalog update".to_string(), + )); + } if let Err(err) = self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await { match self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await { Ok(None) => return Ok(()), Ok(Some((current, _))) if current == entry => { - if let Err(restore_err) = self.reserve_table_warehouse_index(&entry).await { - tracing::warn!( - table_bucket = %entry.table_bucket, - namespace = %entry.namespace, - table = %entry.table, - table_id = %entry.table_id, - error = %restore_err, - "failed to restore table warehouse index after table entry delete failure" - ); - } + self.restore_table_warehouse_index_after_failed_drop(&entry, "table entry delete failed") + .await; } Ok(Some(_)) => { return Err(TableCatalogStoreError::Internal(format!( @@ -4703,6 +4777,15 @@ where self.write_view_entry(entry, TableCatalogPutPrecondition::IfAbsent).await } + async fn create_view_with_publication( + &self, + entry: ViewEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + self.write_view_entry_with_publication(entry, TableCatalogPutPrecondition::IfAbsent, publication) + .await + } + async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { let namespace = parse_namespace_for_store(namespace)?; let mut entries = Vec::new(); @@ -4757,8 +4840,26 @@ where } async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { + let publication = TableCommitLockPublication::new(&self.backend); + self.replace_view_with_publication(request, true, &publication).await + } + + async fn replace_view_with_publication( + &self, + request: ViewCommitRequest, + table_bucket_fence_required: bool, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(&request.namespace)?; let view = parse_table_for_store(&request.view)?; + if table_bucket_fence_required { + publication.begin_table_bucket(&request.table_bucket).await?; + if !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table-bucket publication fence".to_string(), + )); + } + } let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; let namespace_path = self.paths.namespace_entry_path(&request.table_bucket, &namespace); let _namespace_guard = self @@ -4767,6 +4868,16 @@ where .await?; let view_path = self.paths.view_entry_path(&request.table_bucket, &namespace, &view); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &view_path).await?; + // Preserve catalog -> publication -> object lock order across rolling upgrades. + publication + .prepare(&request.table_bucket, &request.namespace, &request.view) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.view) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); let Some((current, current_etag)) = self .read_view_with_etag_unlocked(&request.table_bucket, &namespace, &view) .await? @@ -4814,6 +4925,14 @@ where }) .await .map_err(|err| TableCatalogStoreError::Internal(format!("view metadata parser task failed: {err}")))??; + let warehouse_relocation = next_warehouse_location + .as_deref() + .is_some_and(|location| location != current.warehouse_location); + if warehouse_relocation && !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view warehouse relocation requires a table-bucket publication fence".to_string(), + )); + } let mut next = current; next.metadata_location = request.new_metadata_location; @@ -4822,13 +4941,40 @@ where } next.version_token = format!("token-{}", Uuid::new_v4()); next.generation = next.generation.saturating_add(1); - self.write_entry_unlocked( - self.catalog_bucket(), - &view_path, - &next, - TableCatalogPutPrecondition::IfMatch(current_etag), - ) - .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.view) + || ((table_bucket_fence_required || warehouse_relocation) && !publication.holds_table_bucket(&request.table_bucket)) + { + return Err(TableCatalogStoreError::Internal( + "view replacement publication fence was lost before catalog update".to_string(), + )); + } + let write_result = self + .write_entry_unlocked( + self.catalog_bucket(), + &view_path, + &next, + TableCatalogPutPrecondition::IfMatch(current_etag), + ) + .await; + if let Err(err) = write_result { + match self + .read_view_with_etag_unlocked(&request.table_bucket, &namespace, &view) + .await + { + Ok(Some((persisted, _))) if persisted == next => {} + Ok(_) => return Err(err), + Err(read_err) => { + tracing::warn!( + table_bucket = %request.table_bucket, + namespace = %request.namespace, + view = %request.view, + error = %read_err, + "failed to verify view state after an ambiguous catalog update" + ); + return Err(err); + } + } + } Ok(ViewCommitResult { view: next }) } diff --git a/rustfs/src/table_catalog/store/strong.rs b/rustfs/src/table_catalog/store/strong.rs index e1d313129..2ac5e94d6 100644 --- a/rustfs/src/table_catalog/store/strong.rs +++ b/rustfs/src/table_catalog/store/strong.rs @@ -554,7 +554,7 @@ where // Ordinary mutations hold the global migration read lock before the local write lock; migration takes the // write side before invoking its dedicated snapshot mutation methods. - async fn acquire_snapshot_write_permit(&self) -> TableCatalogStoreResult> { + async fn acquire_snapshot_write_permit(&self) -> TableCatalogStoreResult { let lock_path = TableCatalogObjectPaths::default().backing_migration_global_fence_lock_path(); self.object_backend.acquire_read_lock(RUSTFS_META_BUCKET, &lock_path).await } @@ -1775,7 +1775,7 @@ where request: &TableCommitRequest, namespace: &Namespace, table: &IdentifierSegment, - next_warehouse_location: Option, + next_metadata_state: TableMetadataCommitState, ) -> TableCatalogStoreResult { let key = Self::table_key(&request.table_bucket, namespace, table); let current = Self::validate_new_table_commit_locked(state, &key, request)?; @@ -1802,9 +1802,12 @@ where let mut next = current; next.metadata_location = commit_log.new_metadata_location.clone(); - if let Some(warehouse_location) = next_warehouse_location { + if let Some(warehouse_location) = next_metadata_state.warehouse_location { next.warehouse_location = warehouse_location; } + if let Some(format_version) = next_metadata_state.format_version { + next.format_version = format_version; + } Self::ensure_table_warehouse_prefix_available_locked(state, &next, &key)?; next.version_token = commit_log.new_version_token.clone(); next.generation = next.generation.saturating_add(1); @@ -2170,6 +2173,7 @@ where let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let key = Self::table_key(&entry.table_bucket, &namespace, &table); + let publication_identity = (entry.table_bucket.clone(), entry.namespace.clone(), entry.table.clone()); let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; @@ -2198,6 +2202,13 @@ where StrongSnapshotWritePostcondition::TablePresent(entry), ) }; + if !publication.holds_table_bucket(&publication_identity.0) + || !publication.holds_table(&publication_identity.0, &publication_identity.1, &publication_identity.2) + { + return Err(TableCatalogStoreError::Internal( + "table registration publication fence was lost before snapshot update".to_string(), + )); + } self.finalize_snapshot_write(snapshot, precondition, postcondition).await } @@ -2479,9 +2490,15 @@ where let result = match prepared_result { Ok((result, Some((snapshot, precondition)))) => { let postcondition = Self::commit_write_postcondition(&request.table_bucket, &result.commit_log); - self.finalize_snapshot_write(snapshot, precondition, postcondition) - .await - .map(|_| result) + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) { + Err(TableCatalogStoreError::Internal( + "table commit publication fence was lost before snapshot update".to_string(), + )) + } else { + self.finalize_snapshot_write(snapshot, precondition, postcondition) + .await + .map(|_| result) + } } Ok((result, None)) => Ok(result), Err(err) => Err(err), @@ -2519,8 +2536,8 @@ where validate_commit_metadata_digest(&request, &new_metadata_object)?; let table_bucket = request.table_bucket.clone(); let metadata_location = request.new_metadata_location.clone(); - let next_warehouse_location = tokio::task::spawn_blocking(move || { - table_metadata_warehouse_location(&table_bucket, &metadata_location, &new_metadata_object) + let next_metadata_state = tokio::task::spawn_blocking(move || { + table_metadata_commit_state(&table_bucket, &metadata_location, &new_metadata_object) }) .await .map_err(|err| TableCatalogStoreError::Internal(format!("table metadata parser task failed: {err}")))??; @@ -2539,11 +2556,11 @@ where )) })? }; - if next_warehouse_location + let warehouse_relocation = next_metadata_state + .warehouse_location .as_ref() - .is_some_and(|warehouse_location| warehouse_location != ¤t_warehouse_location) - && !publication.holds_table_bucket(&request.table_bucket) - { + .is_some_and(|warehouse_location| warehouse_location != ¤t_warehouse_location); + if warehouse_relocation && !publication.holds_table_bucket(&request.table_bucket) { return table_commit_result( &request.table_bucket, &request.namespace, @@ -2561,7 +2578,7 @@ where let prepared_result = { let state = self.state.lock().await; let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - match Self::apply_commit_locked(&mut draft_state, &request, &namespace, &table, next_warehouse_location) { + match Self::apply_commit_locked(&mut draft_state, &request, &namespace, &table, next_metadata_state) { Ok(result) => Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version) .map(|snapshot| (result, snapshot, precondition)), Err(err) => Err(err), @@ -2570,7 +2587,16 @@ where let result = match prepared_result { Ok((result, snapshot, precondition)) => { let postcondition = Self::commit_write_postcondition(&request.table_bucket, &result.commit_log); - match self.finalize_snapshot_write(snapshot, precondition, postcondition).await { + let snapshot_result = if publication.holds_table(&request.table_bucket, &request.namespace, &request.table) + && (!warehouse_relocation || publication.holds_table_bucket(&request.table_bucket)) + { + self.finalize_snapshot_write(snapshot, precondition, postcondition).await + } else { + Err(TableCatalogStoreError::Internal( + "table commit publication fence was lost before snapshot update".to_string(), + )) + }; + match snapshot_result { Ok(()) => Ok(result), Err(err) => { let replay = { @@ -2647,13 +2673,26 @@ where }, ) }; + if !publication.holds_table_bucket(table_bucket) + || !publication.holds_table(table_bucket, &namespace.public_name(), table.as_str()) + { + return Err(TableCatalogStoreError::Internal( + "table drop publication fence was lost before snapshot update".to_string(), + )); + } self.finalize_snapshot_write(snapshot, precondition, postcondition).await } async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()> { - let _migration_guard = self.acquire_snapshot_write_permit().await?; - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; + let publication = TableCommitLockPublication::new(&self.object_backend); + self.create_view_with_publication(entry, &publication).await + } + + async fn create_view_with_publication( + &self, + entry: ViewEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { validate_view_entry_version_and_id(&entry)?; validate_view_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; let namespace = parse_namespace_for_store(&entry.namespace)?; @@ -2663,7 +2702,26 @@ where "view metadata location must be inside the view metadata directory".to_string(), )); } + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view creation requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + let _migration_guard = self.acquire_snapshot_write_permit().await?; + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.view) + .await?; + if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.view) { + return Err(TableCatalogStoreError::Internal( + "view creation requires a table publication fence".to_string(), + )); + } + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; let key = Self::table_key(&entry.table_bucket, &namespace, &view); + let publication_identity = (entry.table_bucket.clone(), entry.namespace.clone(), entry.view.clone()); let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; @@ -2682,6 +2740,13 @@ where StrongSnapshotWritePostcondition::ViewPresent(entry), ) }; + if !publication.holds_table_bucket(&publication_identity.0) + || !publication.holds_table(&publication_identity.0, &publication_identity.1, &publication_identity.2) + { + return Err(TableCatalogStoreError::Internal( + "view creation publication fence was lost before snapshot update".to_string(), + )); + } self.finalize_snapshot_write(snapshot, precondition, postcondition).await } @@ -2751,11 +2816,38 @@ where } async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { + let publication = TableCommitLockPublication::new(&self.object_backend); + self.replace_view_with_publication(request, true, &publication).await + } + + async fn replace_view_with_publication( + &self, + request: ViewCommitRequest, + table_bucket_fence_required: bool, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { + if table_bucket_fence_required { + publication.begin_table_bucket(&request.table_bucket).await?; + if !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table-bucket publication fence".to_string(), + )); + } + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); let _migration_guard = self.acquire_snapshot_write_permit().await?; - let write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; let namespace = parse_namespace_for_store(&request.namespace)?; let view = parse_table_for_store(&request.view)?; + publication + .prepare(&request.table_bucket, &request.namespace, &request.view) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.view) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table publication fence".to_string(), + )); + } + let write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; let key = Self::table_key(&request.table_bucket, &namespace, &view); let expected_view_id = { let state = self.state.lock().await; @@ -2798,7 +2890,7 @@ where let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; - let (snapshot, precondition, next, postcondition) = { + let (snapshot, precondition, next, postcondition, warehouse_relocation) = { let state = self.state.lock().await; Self::ensure_identifier_is_unambiguous_locked(&state, &key)?; let Some(current) = state.views.get(&key).cloned() else { @@ -2828,6 +2920,14 @@ where "current view metadata location does not match expected location".to_string(), )); } + let warehouse_relocation = next_warehouse_location + .as_deref() + .is_some_and(|location| location != current.warehouse_location); + if warehouse_relocation && !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view warehouse relocation requires a table-bucket publication fence".to_string(), + )); + } let mut next = current; next.metadata_location = request.new_metadata_location; @@ -2843,8 +2943,16 @@ where precondition, next.clone(), StrongSnapshotWritePostcondition::ViewPresent(next), + warehouse_relocation, ) }; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.view) + || ((table_bucket_fence_required || warehouse_relocation) && !publication.holds_table_bucket(&request.table_bucket)) + { + return Err(TableCatalogStoreError::Internal( + "view replacement publication fence was lost before snapshot update".to_string(), + )); + } self.finalize_snapshot_write(snapshot, precondition, postcondition).await?; Ok(ViewCommitResult { view: next }) } diff --git a/rustfs/src/table_catalog/test_support.rs b/rustfs/src/table_catalog/test_support.rs index 3dac79309..9e49a39a2 100644 --- a/rustfs/src/table_catalog/test_support.rs +++ b/rustfs/src/table_catalog/test_support.rs @@ -55,23 +55,35 @@ pub(crate) fn table_metadata_json(table_uuid: &str, location: &str) -> serde_jso }) } -pub(crate) fn manifest_list_avro_bytes(manifest_paths: &[&str], sequence_number: i64, snapshot_id: i64) -> Vec { - let manifests = manifest_paths - .iter() - .map(|manifest_path| (*manifest_path, 0, sequence_number, snapshot_id)) - .collect::>(); - manifest_list_avro_entries_with_partition_specs(&manifests) -} - -pub(crate) fn manifest_list_avro_entries(manifests: &[(&str, i64, i64)]) -> Vec { +pub(crate) fn manifest_list_avro_bytes(manifests: &[(&str, usize)], sequence_number: i64, snapshot_id: i64) -> Vec { let manifests = manifests .iter() - .map(|(manifest_path, sequence_number, snapshot_id)| (*manifest_path, 0, *sequence_number, *snapshot_id)) + .map(|(manifest_path, manifest_length)| (*manifest_path, *manifest_length, 0, sequence_number, snapshot_id)) .collect::>(); manifest_list_avro_entries_with_partition_specs(&manifests) } -pub(crate) fn manifest_list_avro_entries_with_partition_specs(manifests: &[(&str, i32, i64, i64)]) -> Vec { +pub(crate) fn manifest_list_avro_entries(manifests: &[(&str, usize, i64, i64)]) -> Vec { + let manifests = manifests + .iter() + .map(|(manifest_path, manifest_length, sequence_number, snapshot_id)| { + (*manifest_path, *manifest_length, 0, *sequence_number, *snapshot_id) + }) + .collect::>(); + manifest_list_avro_entries_with_partition_specs(&manifests) +} + +pub(crate) fn manifest_list_avro_entries_with_partition_specs(manifests: &[(&str, usize, i32, i64, i64)]) -> Vec { + let manifests = manifests + .iter() + .map(|(path, length, spec_id, sequence_number, snapshot_id)| { + (*path, *length, *spec_id, 0, *sequence_number, *snapshot_id) + }) + .collect::>(); + manifest_list_avro_entries_with_content(&manifests) +} + +pub(crate) fn manifest_list_avro_entries_with_content(manifests: &[(&str, usize, i32, i32, i64, i64)]) -> Vec { let schema = apache_avro::Schema::parse_str( r#" { @@ -97,16 +109,19 @@ pub(crate) fn manifest_list_avro_entries_with_partition_specs(manifests: &[(&str ) .expect("manifest list avro schema should parse"); let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest list writer should initialize"); - for (manifest_path, partition_spec_id, sequence_number, snapshot_id) in manifests { + for (manifest_path, manifest_length, partition_spec_id, content, sequence_number, snapshot_id) in manifests { writer .append_value(apache_avro::types::Value::Record(vec![ ( "manifest_path".to_string(), apache_avro::types::Value::String((*manifest_path).to_string()), ), - ("manifest_length".to_string(), apache_avro::types::Value::Long(1)), + ( + "manifest_length".to_string(), + apache_avro::types::Value::Long(i64::try_from(*manifest_length).expect("test manifest length should fit")), + ), ("partition_spec_id".to_string(), apache_avro::types::Value::Int(*partition_spec_id)), - ("content".to_string(), apache_avro::types::Value::Int(0)), + ("content".to_string(), apache_avro::types::Value::Int(*content)), ("sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), ("min_sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)), @@ -122,7 +137,86 @@ pub(crate) fn manifest_list_avro_entries_with_partition_specs(manifests: &[(&str writer.into_inner().expect("manifest list avro bytes should flush") } +pub(crate) fn manifest_list_avro_entries_with_nullable_counts(manifests: &[(&str, usize, i32, i64, i64)]) -> Vec { + let schema = apache_avro::Schema::parse_str( + r#" + { + "type": "record", + "name": "manifest_file", + "fields": [ + {"name": "manifest_path", "type": "string"}, + {"name": "manifest_length", "type": "long"}, + {"name": "partition_spec_id", "type": "int"}, + {"name": "content", "type": "int"}, + {"name": "sequence_number", "type": "long"}, + {"name": "min_sequence_number", "type": "long"}, + {"name": "added_snapshot_id", "type": "long"}, + {"name": "added_files_count", "type": ["null", "int"], "default": null}, + {"name": "existing_files_count", "type": ["null", "int"], "default": null}, + {"name": "deleted_files_count", "type": ["null", "int"], "default": null}, + {"name": "added_rows_count", "type": ["null", "long"], "default": null}, + {"name": "existing_rows_count", "type": ["null", "long"], "default": null}, + {"name": "deleted_rows_count", "type": ["null", "long"], "default": null} + ] + } + "#, + ) + .expect("manifest list avro schema should parse"); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("manifest list writer should initialize"); + for (manifest_path, manifest_length, partition_spec_id, sequence_number, snapshot_id) in manifests { + writer + .append_value(apache_avro::types::Value::Record(vec![ + ( + "manifest_path".to_string(), + apache_avro::types::Value::String((*manifest_path).to_string()), + ), + ( + "manifest_length".to_string(), + apache_avro::types::Value::Long(i64::try_from(*manifest_length).expect("test manifest length should fit")), + ), + ("partition_spec_id".to_string(), apache_avro::types::Value::Int(*partition_spec_id)), + ("content".to_string(), apache_avro::types::Value::Int(0)), + ("sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), + ("min_sequence_number".to_string(), apache_avro::types::Value::Long(*sequence_number)), + ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(*snapshot_id)), + ( + "added_files_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "existing_files_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "deleted_files_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "added_rows_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "existing_rows_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ( + "deleted_rows_count".to_string(), + apache_avro::types::Value::Union(0, Box::new(apache_avro::types::Value::Null)), + ), + ])) + .expect("manifest list record should append"); + } + writer.into_inner().expect("manifest list avro bytes should flush") +} + pub(crate) fn manifest_avro_bytes(files: &[(&str, i32, i32, i64, i64)]) -> Vec { + manifest_avro_bytes_with_partition_spec(files, None) +} + +pub(crate) fn manifest_avro_bytes_with_partition_spec( + files: &[(&str, i32, i32, i64, i64)], + partition_spec_id: Option, +) -> Vec { let schema = apache_avro::Schema::parse_str( r#" { @@ -141,6 +235,7 @@ pub(crate) fn manifest_avro_bytes(files: &[(&str, i32, i32, i64, i64)]) -> Vec Vec Vec, - guard: Arc>>>, + guard: Arc>>, } impl BlockingObjectPublication { @@ -812,7 +915,7 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { .collect()) } - async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { self.lock_attempts.lock().await.push((bucket.to_string(), object.to_string())); { let mut state = self.state.lock().await; @@ -828,10 +931,10 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { .or_insert_with(|| std::sync::Arc::new(tokio::sync::RwLock::new(()))) .clone() }; - Ok(Box::new(lock.write_owned().await)) + Ok(TableCatalogLockGuard::stable(lock.write_owned().await)) } - async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { // The admin fake implemented only acquire_write_lock, so the trait's // default read->write delegation made read acquisitions observable in // lock_attempts as well; keep that (backlog#1837 PR2). @@ -850,7 +953,7 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { .or_insert_with(|| std::sync::Arc::new(tokio::sync::RwLock::new(()))) .clone() }; - Ok(Box::new(lock.read_owned().await)) + Ok(TableCatalogLockGuard::stable(lock.read_owned().await)) } } @@ -1165,6 +1268,8 @@ pub(crate) struct TestTableCatalogStore { pub(crate) fail_put_table_bucket: tokio::sync::Mutex, pub(crate) register_table_pause: Option, pub(crate) commit_table_pause: Option, + pub(crate) create_view_pause: Option, + pub(crate) replace_view_pause: Option, } #[async_trait::async_trait] @@ -1473,6 +1578,10 @@ impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { entry.table_bucket, entry.namespace ))); } + if let Some(pause) = &self.create_view_pause { + pause.started.notify_one(); + pause.release.notified().await; + } self.views.lock().await.push(entry); Ok(()) } @@ -1531,6 +1640,10 @@ impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { "current view metadata location does not match expected location".to_string(), )); } + if let Some(pause) = &self.replace_view_pause { + pause.started.notify_one(); + pause.release.notified().await; + } let mut next = current; next.metadata_location = request.new_metadata_location; next.version_token = "token-view-committed".to_string(); diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs index 1572c3b16..cfc050343 100644 --- a/rustfs/src/table_catalog/tests.rs +++ b/rustfs/src/table_catalog/tests.rs @@ -338,6 +338,107 @@ async fn catalog_backings_fence_direct_commits_with_publication_lock() { assert_direct_commit_uses_publication_lock(&strong_store, &strong_backend).await; } +#[derive(Default)] +struct LosingTestPublication { + table_checks: std::sync::atomic::AtomicUsize, +} + +#[async_trait::async_trait] +impl TableCommitPublication for LosingTestPublication { + async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn prepare(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + fn holds_table_bucket(&self, _table_bucket: &str) -> bool { + true + } + + fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool { + self.table_checks.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 + } + + fn complete(&self) {} +} + +async fn assert_view_replacement_rechecks_publication_fence(store: &S, backend: &TestCatalogObjectBackend) +where + S: TableCatalogStore, +{ + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let current_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let next_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_view(test_view_entry(bucket, &namespace, &view, current_metadata.clone())) + .await + .expect("view should be created"); + backend + .seed_object( + bucket, + &next_metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": format!("s3://{bucket}/views/view-id") + })) + .expect("view metadata should encode"), + ) + .await; + + let error = store + .replace_view_with_publication( + ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata.clone(), + new_metadata_location: next_metadata, + }, + true, + &LosingTestPublication::default(), + ) + .await + .expect_err("a lost publication fence must stop the view replacement"); + assert_matches!( + error, + TableCatalogStoreError::Internal(message) if message.contains("publication fence was lost") + ); + + let loaded = store + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect("view lookup should succeed") + .expect("view should remain present"); + assert_eq!(loaded.metadata_location, current_metadata); + assert_eq!(loaded.version_token, "token-v1"); + assert_eq!(loaded.generation, 1); +} + +#[tokio::test] +async fn catalog_backings_stop_view_replacement_after_publication_fence_loss() { + let object_backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(object_backend.clone()); + assert_view_replacement_rechecks_publication_fence(&object_store, &object_backend).await; + + let strong_backend = TestCatalogObjectBackend::default(); + let strong_store = StrongTableCatalogStore::new(strong_backend.clone()); + assert_view_replacement_rechecks_publication_fence(&strong_store, &strong_backend).await; +} + #[tokio::test] async fn strong_table_registration_and_drop_acquire_publication_before_migration_read_lock() { let backend = TestCatalogObjectBackend::default(); @@ -795,17 +896,23 @@ async fn strong_catalog_view_replace_rejects_identity_recreation_during_metadata let replace_current_metadata = current_metadata.clone(); let replace = tokio::spawn(async move { replace_store - .replace_view(ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: replace_namespace.public_name(), - view: replace_view.as_str().to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: replace_current_metadata, - new_metadata_location: new_metadata, - }) + .replace_view_with_publication( + ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: replace_namespace.public_name(), + view: replace_view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: replace_current_metadata, + new_metadata_location: new_metadata, + }, + false, + &UnserializedTestPublication, + ) .await }); - metadata_read.wait_started().await; + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, metadata_read.wait_started()) + .await + .expect("the replacement should reach the paused metadata read"); store .drop_view(bucket, &namespace.public_name(), view.as_str()) @@ -927,20 +1034,38 @@ fn object_cleanup_report<'a>( .expect("metadata maintenance object cleanup report should exist") } -fn manifest_list_avro_bytes(manifest_paths: &[&str]) -> Vec { - manifest_list_avro_bytes_with_spec(manifest_paths, 0) +fn manifest_list_avro_bytes(manifests: &[(&str, usize)]) -> Vec { + manifest_list_avro_bytes_with_spec(manifests, 0) } -fn manifest_list_avro_bytes_with_spec(manifest_paths: &[&str], partition_spec_id: i32) -> Vec { - // Historical fixed values of this file's fixtures: sequence 7, snapshot 20. - let manifests = manifest_paths +fn manifest_list_avro_bytes_with_spec(manifests: &[(&str, usize)], partition_spec_id: i32) -> Vec { + manifest_list_avro_bytes_with_spec_and_content(manifests, partition_spec_id, 0) +} + +fn manifest_list_avro_bytes_with_spec_and_content(manifests: &[(&str, usize)], partition_spec_id: i32, content: i32) -> Vec { + let manifests = manifests .iter() - .map(|path| (*path, partition_spec_id, 7_i64, 20_i64)) + .map(|(path, length)| (*path, *length, partition_spec_id, content, 7_i64, 20_i64)) .collect::>(); - crate::table_catalog::test_support::manifest_list_avro_entries_with_partition_specs(&manifests) + crate::table_catalog::test_support::manifest_list_avro_entries_with_content(&manifests) } -fn v1_manifest_list_avro_bytes(manifest_path: &str) -> Vec { +fn manifest_list_avro_bytes_with_spec_and_null_counts( + manifests: &[(&str, usize)], + partition_spec_id: i32, + null_counts: bool, +) -> Vec { + if !null_counts { + return manifest_list_avro_bytes_with_spec(manifests, partition_spec_id); + } + let manifests = manifests + .iter() + .map(|(path, length)| (*path, *length, partition_spec_id, 7_i64, 20_i64)) + .collect::>(); + crate::table_catalog::test_support::manifest_list_avro_entries_with_nullable_counts(&manifests) +} + +fn v1_manifest_list_avro_bytes(manifest_path: &str, manifest_length: usize) -> Vec { let schema = apache_avro::Schema::parse_str( r#" { @@ -960,7 +1085,10 @@ fn v1_manifest_list_avro_bytes(manifest_path: &str) -> Vec { writer .append_value(apache_avro::types::Value::Record(vec![ ("manifest_path".to_string(), apache_avro::types::Value::String(manifest_path.to_string())), - ("manifest_length".to_string(), apache_avro::types::Value::Long(1)), + ( + "manifest_length".to_string(), + apache_avro::types::Value::Long(i64::try_from(manifest_length).expect("test manifest length should fit")), + ), ("partition_spec_id".to_string(), apache_avro::types::Value::Int(0)), ("added_snapshot_id".to_string(), apache_avro::types::Value::Long(10)), ])) @@ -997,6 +1125,9 @@ fn v1_manifest_avro_bytes(data_file_path: &str) -> Vec { ) .expect("v1 manifest schema should parse"); let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("v1 manifest writer should initialize"); + writer + .add_user_metadata("partition-spec-id".to_string(), "0") + .expect("v1 manifest partition spec metadata should write"); writer .append_value(apache_avro::types::Value::Record(vec![ ("status".to_string(), apache_avro::types::Value::Int(1)), @@ -1059,13 +1190,31 @@ fn iceberg_metadata_validation_accepts_complete_v1_and_v2_shapes() { "schema-id": 0, "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] }, - "partition-spec": [], + "partition-spec": [{"source-id": 1, "name": "id", "transform": "identity"}], "properties": {}, "snapshots": [], "snapshot-log": [], "metadata-log": [] }); validate_supported_table_metadata(&v1).expect("complete Iceberg v1 metadata should validate"); + + let mut v1_retired_partition_source = v1.clone(); + v1_retired_partition_source["schemas"] = serde_json::json!([ + { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + {"type": "struct", "schema-id": 1, "fields": []} + ]); + v1_retired_partition_source["current-schema-id"] = serde_json::Value::from(1); + v1_retired_partition_source["schema"] = serde_json::json!({"type": "struct", "schema-id": 1, "fields": []}); + validate_supported_table_metadata(&v1_retired_partition_source) + .expect_err("the v1 partition spec must bind to the current schema rather than a historical schema"); + + let mut negative_v1_schema_id = v1; + negative_v1_schema_id["schema"]["schema-id"] = serde_json::Value::from(-1); + validate_supported_table_metadata(&negative_v1_schema_id).expect_err("Iceberg v1 schema IDs must not be negative"); } #[test] @@ -1099,6 +1248,529 @@ fn iceberg_metadata_validation_rejects_incomplete_v2_and_dangling_references() { assert!(matches!(error, TableCatalogStoreError::Invalid(_))); } +#[test] +fn iceberg_metadata_validation_rejects_invalid_primitive_types_and_field_ids() { + let metadata = table_metadata_json_for_validation(); + for invalid_type in [ + "banana", + "decimal(0,0)", + "decimal(10,11)", + "fixed[0]", + "fixed[2147483648]", + "decimal(10)", + ] { + let mut invalid = metadata.clone(); + invalid["schemas"][0]["fields"][0]["type"] = serde_json::Value::from(invalid_type); + validate_supported_table_metadata(&invalid).expect_err("invalid Iceberg primitive types must be rejected"); + } + for valid_type in [ + "decimal(38,38)", + "decimal(9, 2)", + "decimal( 9 , 2 )", + "fixed[16]", + "fixed[ 16 ]", + "timestamptz", + "uuid", + ] { + let mut valid = metadata.clone(); + valid["schemas"][0]["fields"][0]["type"] = serde_json::Value::from(valid_type); + validate_supported_table_metadata(&valid).expect("standard Iceberg primitive types should validate"); + } + for invalid_id in [0, -1] { + let mut invalid = metadata.clone(); + invalid["schemas"][0]["fields"][0]["id"] = serde_json::Value::from(invalid_id); + validate_supported_table_metadata(&invalid).expect_err("Iceberg field IDs must be positive"); + } + let mut reserved_id = metadata; + reserved_id["schemas"][0]["fields"][0]["id"] = serde_json::Value::from(2_147_483_448_i64); + validate_supported_table_metadata(&reserved_id).expect_err("reserved Iceberg metadata field IDs must be rejected"); + + let mut negative_schema_id = table_metadata_json_for_validation(); + negative_schema_id["schemas"][0]["schema-id"] = serde_json::Value::from(-1); + negative_schema_id["current-schema-id"] = serde_json::Value::from(-1); + validate_supported_table_metadata(&negative_schema_id).expect_err("Iceberg schema IDs must not be negative"); + + let mut negative_last_partition_id = table_metadata_json_for_validation(); + negative_last_partition_id["last-partition-id"] = serde_json::Value::from(-1); + validate_supported_table_metadata(&negative_last_partition_id).expect_err("Iceberg last-partition-id must not be negative"); +} + +#[test] +fn iceberg_metadata_validation_enforces_schema_evolution() { + let mut promoted = table_metadata_json_for_validation(); + promoted["schemas"][0]["fields"][0]["type"] = serde_json::Value::from("int"); + promoted["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + })); + promoted["current-schema-id"] = serde_json::Value::from(1); + validate_supported_table_metadata(&promoted).expect("int fields may promote to long"); + + let mut decimal_promotion = table_metadata_json_for_validation(); + decimal_promotion["schemas"][0]["fields"][0]["type"] = serde_json::Value::from("decimal(9, 2)"); + decimal_promotion["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [{"id": 1, "name": "id", "required": true, "type": "decimal( 10 , 2 )"}] + })); + decimal_promotion["current-schema-id"] = serde_json::Value::from(1); + validate_supported_table_metadata(&decimal_promotion) + .expect("decimal precision promotion must accept optional parameter whitespace"); + + let mut incompatible = promoted; + incompatible["schemas"][1]["fields"][0]["type"] = serde_json::Value::from("string"); + let error = validate_supported_table_metadata(&incompatible).expect_err("field IDs must retain compatible types"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("schema field 1 has an incompatible type evolution".to_string()) + ); + + let mut moved_into_collection = table_metadata_json_for_validation(); + moved_into_collection["last-column-id"] = serde_json::Value::from(2); + moved_into_collection["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [{ + "id": 2, + "name": "items", + "required": true, + "type": { + "type": "list", + "element-id": 1, + "element-required": true, + "element": "long" + } + }] + })); + moved_into_collection["current-schema-id"] = serde_json::Value::from(1); + validate_supported_table_metadata(&moved_into_collection).expect_err("a schema field ID must not move into a list or map"); + + let mut reused = table_metadata_json_for_validation(); + reused["schemas"] = serde_json::json!([ + { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + {"type": "struct", "schema-id": 1, "fields": []}, + { + "type": "struct", + "schema-id": 2, + "fields": [{"id": 1, "name": "replacement", "required": true, "type": "long"}] + } + ]); + reused["current-schema-id"] = serde_json::Value::from(2); + let error = validate_supported_table_metadata(&reused).expect_err("removed Iceberg field IDs must never be reused"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("schema field 1 cannot be reused after removal".to_string()) + ); + + let mut stale_last_column_id = table_metadata_json_for_validation(); + stale_last_column_id["last-column-id"] = serde_json::Value::from(0); + let error = validate_supported_table_metadata(&stale_last_column_id) + .expect_err("last-column-id must cover nested and top-level assigned field IDs"); + assert_eq!( + error, + TableCatalogStoreError::Invalid( + "last-column-id must be non-negative and cover every assigned schema field id".to_string() + ) + ); +} + +#[test] +fn iceberg_metadata_transition_preserves_assignment_watermarks_and_schema_history() { + let current = table_metadata_json_for_validation(); + + let mut lower_column_watermark = current.clone(); + lower_column_watermark["last-column-id"] = serde_json::Value::from(0); + let error = validate_table_metadata_transition(¤t, &lower_column_watermark) + .expect_err("last-column-id must not decrease across commits"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("last-column-id must not decrease across table metadata commits".to_string()) + ); + + let mut lower_partition_watermark = current.clone(); + lower_partition_watermark["last-partition-id"] = serde_json::Value::from(998); + let error = validate_table_metadata_transition(¤t, &lower_partition_watermark) + .expect_err("last-partition-id must not decrease across commits"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("last-partition-id must not decrease across table metadata commits".to_string()) + ); + + let mut current_sequence = current.clone(); + current_sequence["last-sequence-number"] = serde_json::Value::from(2); + let mut lower_sequence_watermark = current_sequence.clone(); + lower_sequence_watermark["last-sequence-number"] = serde_json::Value::from(1); + let error = validate_table_metadata_transition(¤t_sequence, &lower_sequence_watermark) + .expect_err("last-sequence-number must not decrease across commits"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("last-sequence-number must not decrease across table metadata commits".to_string()) + ); + + let mut current_partitioned = current.clone(); + current_partitioned["partition-specs"] = serde_json::json!([{ + "spec-id": 0, + "fields": [{ + "source-id": 1, + "field-id": 1000, + "name": "id", + "transform": "identity" + }] + }]); + current_partitioned["last-partition-id"] = serde_json::Value::from(1000); + let mut modified_partition = current_partitioned.clone(); + modified_partition["partition-specs"][0]["fields"][0]["name"] = serde_json::Value::from("renamed_id"); + let error = validate_table_metadata_transition(¤t_partitioned, &modified_partition) + .expect_err("published partition specs must be immutable"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("existing partition spec 0 must not be modified".to_string()) + ); + + let mut current_sorted = current.clone(); + current_sorted["sort-orders"] = serde_json::json!([ + {"order-id": 0, "fields": []}, + { + "order-id": 1, + "fields": [{ + "source-id": 1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + } + ]); + current_sorted["default-sort-order-id"] = serde_json::Value::from(1); + let mut modified_sort = current_sorted.clone(); + modified_sort["sort-orders"][1]["fields"][0]["direction"] = serde_json::Value::from("desc"); + let error = + validate_table_metadata_transition(¤t_sorted, &modified_sort).expect_err("published sort orders must be immutable"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("existing sort order 1 must not be modified".to_string()) + ); + + let mut current_with_snapshot = current.clone(); + current_with_snapshot["last-sequence-number"] = serde_json::Value::from(1); + current_with_snapshot["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + let mut modified_snapshot = current_with_snapshot.clone(); + modified_snapshot["snapshots"][0]["timestamp-ms"] = serde_json::Value::from(2); + let error = validate_table_metadata_transition(¤t_with_snapshot, &modified_snapshot) + .expect_err("published snapshots must be immutable"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("existing snapshot 10 must not be modified".to_string()) + ); + + let mut modified_history = current.clone(); + modified_history["schemas"][0]["fields"][0]["type"] = serde_json::Value::from("string"); + let error = validate_table_metadata_transition(¤t, &modified_history) + .expect_err("published schema definitions must be immutable"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("existing schema 0 must not be modified".to_string()) + ); + + let mut current_with_retired_id = current.clone(); + current_with_retired_id["schemas"] = serde_json::json!([ + { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + {"type": "struct", "schema-id": 1, "fields": []} + ]); + current_with_retired_id["current-schema-id"] = serde_json::Value::from(1); + let mut reused_id = current_with_retired_id.clone(); + reused_id["schemas"] = serde_json::json!([ + {"type": "struct", "schema-id": 1, "fields": []}, + { + "type": "struct", + "schema-id": 2, + "fields": [{"id": 1, "name": "replacement", "required": true, "type": "long"}] + } + ]); + reused_id["current-schema-id"] = serde_json::Value::from(2); + let error = validate_table_metadata_transition(¤t_with_retired_id, &reused_id) + .expect_err("new schemas must not reuse a previously assigned field ID"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("schema field 1 cannot reuse a previously assigned field id".to_string()) + ); + + let mut valid = current.clone(); + valid["last-column-id"] = serde_json::Value::from(2); + valid["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [ + {"id": 1, "name": "renamed_id", "required": true, "type": "long"}, + {"id": 2, "name": "value", "required": false, "type": "string"} + ] + })); + valid["current-schema-id"] = serde_json::Value::from(1); + validate_table_metadata_transition(¤t, &valid) + .expect("renaming an existing field and allocating a new field ID must remain valid"); +} + +#[test] +fn iceberg_metadata_validation_enforces_identifier_field_contracts() { + let mut metadata = table_metadata_json_for_validation(); + metadata["last-column-id"] = serde_json::Value::from(10); + metadata["schemas"][0]["fields"] = serde_json::json!([ + {"id": 1, "name": "id", "required": true, "type": "long"}, + {"id": 2, "name": "optional_id", "required": false, "type": "string"}, + {"id": 3, "name": "float_id", "required": true, "type": "float"}, + { + "id": 4, + "name": "required_parent", + "required": true, + "type": { + "type": "struct", + "fields": [{"id": 5, "name": "nested_id", "required": true, "type": "string"}] + } + }, + { + "id": 6, + "name": "optional_parent", + "required": false, + "type": { + "type": "struct", + "fields": [{"id": 7, "name": "nested_id", "required": true, "type": "long"}] + } + }, + { + "id": 8, + "name": "ids", + "required": true, + "type": {"type": "list", "element-id": 9, "element-required": true, "element": "long"} + } + ]); + metadata["schemas"][0]["identifier-field-ids"] = serde_json::json!([1, 5]); + validate_supported_table_metadata(&metadata).expect("required primitive fields in required structs may identify rows"); + + for invalid_id in [2, 3, 4, 7, 9] { + let mut invalid = metadata.clone(); + invalid["schemas"][0]["identifier-field-ids"] = serde_json::json!([invalid_id]); + validate_supported_table_metadata(&invalid) + .expect_err("optional, floating, complex, collection, and optional-parent fields must not identify rows"); + } +} + +#[test] +fn iceberg_metadata_validation_binds_partition_fields_to_schema_and_field_identity() { + let mut missing_source = table_metadata_json_for_validation(); + missing_source["partition-specs"] = serde_json::json!([{ + "spec-id": 0, + "fields": [{"source-id": 99, "field-id": 1000, "name": "missing", "transform": "identity"}] + }]); + missing_source["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&missing_source).expect_err("partition source IDs must reference schema fields"); + + let mut reassigned = table_metadata_json_for_validation(); + reassigned["last-column-id"] = serde_json::Value::from(2); + reassigned["schemas"][0]["fields"] = serde_json::json!([ + {"id": 1, "name": "id", "required": true, "type": "long"}, + {"id": 2, "name": "category", "required": false, "type": "string"} + ]); + reassigned["partition-specs"] = serde_json::json!([ + { + "spec-id": 0, + "fields": [{"source-id": 1, "field-id": 1000, "name": "id", "transform": "identity"}] + }, + { + "spec-id": 1, + "fields": [{"source-id": 2, "field-id": 1000, "name": "category", "transform": "identity"}] + } + ]); + reassigned["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&reassigned) + .expect_err("a partition field ID must not be reassigned to a different source or transform"); + + reassigned["partition-specs"][1]["fields"][0] = + serde_json::json!({"source-id": 1, "field-id": 1000, "name": "renamed_id", "transform": "identity"}); + validate_supported_table_metadata(&reassigned) + .expect("a historical partition field may retain its ID when only its name changes"); + + let mut v1_missing_source = serde_json::json!({ + "format-version": 1, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-updated-ms": 1, + "last-column-id": 1, + "schema": { + "type": "struct", + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + }, + "partition-spec": [{"source-id": 2, "name": "missing", "transform": "identity"}] + }); + validate_supported_table_metadata(&v1_missing_source) + .expect_err("Iceberg v1 partition source IDs must reference schema fields"); + v1_missing_source["partition-spec"][0]["source-id"] = serde_json::Value::from(1); + v1_missing_source["partition-spec"][0]["field-id"] = serde_json::Value::from(1001); + validate_supported_table_metadata(&v1_missing_source) + .expect_err("explicit Iceberg v1 partition field IDs must retain sequential compatibility IDs"); + + let mut nested_source = table_metadata_json_for_validation(); + nested_source["last-column-id"] = serde_json::Value::from(4); + nested_source["schemas"][0]["fields"] = serde_json::json!([ + { + "id": 1, + "name": "payload", + "required": true, + "type": { + "type": "struct", + "fields": [{"id": 2, "name": "event_date", "required": true, "type": "date"}] + } + }, + { + "id": 3, + "name": "dates", + "required": true, + "type": {"type": "list", "element-id": 4, "element-required": true, "element": "date"} + } + ]); + nested_source["partition-specs"] = serde_json::json!([{ + "spec-id": 0, + "fields": [{"source-id": 2, "field-id": 1000, "name": "event_day", "transform": "day"}] + }]); + nested_source["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&nested_source).expect("a primitive nested in a struct may be a partition source"); + + nested_source["partition-specs"][0]["fields"][0]["source-id"] = serde_json::Value::from(4); + validate_supported_table_metadata(&nested_source).expect_err("a primitive nested in a list must not be a partition source"); +} + +#[test] +fn iceberg_metadata_validation_binds_defaults_to_current_schema() { + let mut partitioned = table_metadata_json_for_validation(); + partitioned["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({"type": "struct", "schema-id": 1, "fields": []})); + partitioned["current-schema-id"] = serde_json::Value::from(1); + partitioned["partition-specs"] = serde_json::json!([ + {"spec-id": 0, "fields": []}, + { + "spec-id": 1, + "fields": [{"source-id": 1, "field-id": 1000, "name": "id", "transform": "identity"}] + } + ]); + partitioned["default-spec-id"] = serde_json::Value::from(1); + partitioned["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&partitioned).expect_err("the default partition spec must bind to the current schema"); + partitioned["partition-specs"][1]["fields"][0]["transform"] = serde_json::Value::from("void"); + validate_supported_table_metadata(&partitioned) + .expect("a void partition field may retain a source removed from the current schema"); + partitioned["partition-specs"][1]["fields"][0]["transform"] = serde_json::Value::from("identity"); + partitioned["default-spec-id"] = serde_json::Value::from(0); + validate_supported_table_metadata(&partitioned) + .expect("a non-default historical partition spec may retain a source removed from the current schema"); + + let mut sorted = table_metadata_json_for_validation(); + sorted["schemas"] + .as_array_mut() + .expect("schemas should be an array") + .push(serde_json::json!({"type": "struct", "schema-id": 1, "fields": []})); + sorted["current-schema-id"] = serde_json::Value::from(1); + sorted["sort-orders"] = serde_json::json!([ + {"order-id": 0, "fields": []}, + { + "order-id": 1, + "fields": [{ + "source-id": 1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + } + ]); + sorted["default-sort-order-id"] = serde_json::Value::from(1); + validate_supported_table_metadata(&sorted).expect_err("the default sort order must bind to the current schema"); + sorted["default-sort-order-id"] = serde_json::Value::from(0); + validate_supported_table_metadata(&sorted) + .expect("a non-default historical sort order may retain a source removed from the current schema"); +} + +#[test] +fn iceberg_metadata_validation_binds_transforms_to_source_types() { + let mut valid = table_metadata_json_for_validation(); + valid["schemas"][0]["fields"][0]["type"] = serde_json::Value::from("date"); + valid["partition-specs"] = serde_json::json!([{ + "spec-id": 0, + "fields": [{"source-id": 1, "field-id": 1000, "name": "day", "transform": "day"}] + }]); + valid["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&valid).expect("day transforms may bind to date fields"); + + let mut invalid_source = valid; + invalid_source["schemas"][0]["fields"][0]["type"] = serde_json::Value::from("string"); + validate_supported_table_metadata(&invalid_source).expect_err("day transforms must reject string fields"); + + let mut invalid_width = table_metadata_json_for_validation(); + invalid_width["partition-specs"] = serde_json::json!([{ + "spec-id": 0, + "fields": [{"source-id": 1, "field-id": 1000, "name": "bucket", "transform": "bucket[0]"}] + }]); + invalid_width["last-partition-id"] = serde_json::Value::from(1000); + validate_supported_table_metadata(&invalid_width).expect_err("bucket widths must be positive"); +} + +#[test] +fn iceberg_metadata_validation_enforces_sort_order_fields() { + let mut metadata = table_metadata_json_for_validation(); + metadata["sort-orders"] = serde_json::json!([{ + "order-id": 1, + "fields": [{ + "source-id": 1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + }]); + metadata["default-sort-order-id"] = serde_json::Value::from(1); + validate_supported_table_metadata(&metadata).expect("complete sort order fields should validate"); + + for (field, invalid_value) in [ + ("source-id", serde_json::Value::from(99)), + ("transform", serde_json::Value::from("")), + ("direction", serde_json::Value::from("ascending")), + ("null-order", serde_json::Value::from("first")), + ] { + let mut invalid = metadata.clone(); + invalid["sort-orders"][0]["fields"][0][field] = invalid_value; + validate_supported_table_metadata(&invalid).expect_err("invalid Iceberg sort fields must be rejected"); + } + + let mut reserved_unsorted = metadata; + reserved_unsorted["sort-orders"][0]["order-id"] = serde_json::Value::from(0); + reserved_unsorted["default-sort-order-id"] = serde_json::Value::from(0); + validate_supported_table_metadata(&reserved_unsorted).expect_err("sort order 0 must remain unsorted"); +} + #[test] fn iceberg_metadata_validation_enforces_snapshot_and_ref_contracts() { let mut metadata = table_metadata_json_for_validation(); @@ -1122,6 +1794,10 @@ fn iceberg_metadata_validation_enforces_snapshot_and_ref_contracts() { empty_operation["snapshots"][0]["summary"]["operation"] = serde_json::Value::from(""); validate_supported_table_metadata(&empty_operation).expect_err("snapshot operation must not be empty"); + let mut non_string_summary = metadata.clone(); + non_string_summary["snapshots"][0]["summary"]["added-records"] = serde_json::Value::from(1); + validate_supported_table_metadata(&non_string_summary).expect_err("snapshot summary values must be strings"); + let mut missing_timestamp = metadata.clone(); missing_timestamp["snapshots"][0] .as_object_mut() @@ -1129,6 +1805,14 @@ fn iceberg_metadata_validation_enforces_snapshot_and_ref_contracts() { .remove("timestamp-ms"); validate_supported_table_metadata(&missing_timestamp).expect_err("snapshot timestamp must be required"); + let mut malformed_snapshot_log = metadata.clone(); + malformed_snapshot_log["snapshot-log"] = serde_json::json!([{"timestamp-ms": 1, "snapshot-id": "10"}]); + validate_supported_table_metadata(&malformed_snapshot_log).expect_err("snapshot log entries must use integer snapshot IDs"); + + let mut malformed_metadata_log = metadata.clone(); + malformed_metadata_log["metadata-log"] = serde_json::json!([{"timestamp-ms": 1, "metadata-file": ""}]); + validate_supported_table_metadata(&malformed_metadata_log).expect_err("metadata log entries must identify a metadata file"); + let mut mismatched_main = metadata.clone(); mismatched_main["snapshots"] .as_array_mut() @@ -1152,6 +1836,309 @@ fn iceberg_metadata_validation_enforces_snapshot_and_ref_contracts() { validate_supported_table_metadata(&invalid_tag).expect_err("tags must reject branch-only retention fields"); } +#[test] +fn iceberg_metadata_validation_rejects_duplicate_snapshot_statistics() { + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(1); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + let statistics = serde_json::json!({ + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/stats.puffin", + "file-size-in-bytes": 1, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + }); + metadata["statistics"] = serde_json::json!([statistics.clone(), statistics]); + + let error = validate_supported_table_metadata(&metadata).expect_err("a snapshot must not have duplicate statistics entries"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("statistics contains duplicate entries for snapshot 10".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_bounds_changed_statistics_object_fanout() { + let backend = TestCatalogObjectBackend::default(); + let mut metadata = table_metadata_json_for_validation(); + let object_count = TABLE_COMMIT_MAX_STATISTICS_OBJECTS + 1; + metadata["last-sequence-number"] = serde_json::Value::from(object_count); + metadata["snapshots"] = serde_json::Value::Array( + (1..=object_count) + .map(|snapshot_id| { + serde_json::json!({ + "snapshot-id": snapshot_id, + "sequence-number": snapshot_id, + "timestamp-ms": snapshot_id, + "manifest-list": format!("s3://warehouse/tables/table-id/metadata/snap-{snapshot_id}.avro"), + "summary": {"operation": "append"} + }) + }) + .collect(), + ); + metadata["partition-statistics"] = serde_json::Value::Array( + (1..=object_count) + .map(|snapshot_id| { + serde_json::json!({ + "snapshot-id": snapshot_id, + "statistics-path": format!( + "s3://warehouse/tables/table-id/metadata/partition-stats-{snapshot_id}.parquet" + ), + "file-size-in-bytes": 1 + }) + }) + .collect(), + ); + let entry = TableEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: "analytics".to_string(), + table: "events".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://warehouse/tables/table-id".to_string(), + metadata_location: "metadata/00001.metadata.json".to_string(), + version_token: "token-v1".to_string(), + generation: 1, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect_err("statistics object fanout must be bounded before storage lookups"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("statistics object count exceeds the commit limit".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_bounds_changed_statistics_object_bytes() { + let backend = TestCatalogObjectBackend::default(); + let mut metadata = table_metadata_json_for_validation(); + let object_count = TABLE_COMMIT_MAX_STATISTICS_BYTES / TABLE_STATISTICS_FILE_MAX_SIZE + 1; + metadata["last-sequence-number"] = serde_json::Value::from(object_count); + metadata["snapshots"] = serde_json::Value::Array( + (1..=object_count) + .map(|snapshot_id| { + serde_json::json!({ + "snapshot-id": snapshot_id, + "sequence-number": snapshot_id, + "timestamp-ms": snapshot_id, + "manifest-list": format!("s3://warehouse/tables/table-id/metadata/snap-{snapshot_id}.avro"), + "summary": {"operation": "append"} + }) + }) + .collect(), + ); + metadata["partition-statistics"] = serde_json::Value::Array( + (1..=object_count) + .map(|snapshot_id| { + serde_json::json!({ + "snapshot-id": snapshot_id, + "statistics-path": format!( + "s3://warehouse/tables/table-id/metadata/partition-stats-{snapshot_id}.parquet" + ), + "file-size-in-bytes": TABLE_STATISTICS_FILE_MAX_SIZE + }) + }) + .collect(), + ); + let entry = TableEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: "analytics".to_string(), + table: "events".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://warehouse/tables/table-id".to_string(), + metadata_location: "metadata/00001.metadata.json".to_string(), + version_token: "token-v1".to_string(), + generation: 1, + state: TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect_err("statistics bytes must be bounded before storage lookups"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("statistics bytes exceed the commit validation limit".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_rechecks_retained_statistics_locations() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let mut current = table_metadata_json_for_validation(); + current["last-sequence-number"] = serde_json::Value::from(1); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/missing-history.avro", + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + current["statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/another-table/metadata/stats.puffin", + "file-size-in-bytes": 1, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + }]); + let target = current.clone(); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect_err("retained statistics must remain inside the table warehouse"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("statistics object is outside the table warehouse".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_rejects_non_puffin_table_statistics() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let mut current = table_metadata_json_for_validation(); + current["last-sequence-number"] = serde_json::Value::from(1); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let mut target = current.clone(); + let statistics = b"notpuffin".to_vec(); + target["statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/stats.puffin", + "file-size-in-bytes": statistics.len(), + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + }]); + backend + .seed_object("warehouse", "tables/table-id/metadata/stats.puffin", statistics) + .await; + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect_err("table statistics must be a Puffin file"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("table statistics object is not a Puffin file".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_rejects_non_parquet_partition_statistics() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let mut current = table_metadata_json_for_validation(); + current["last-sequence-number"] = serde_json::Value::from(1); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let mut target = current.clone(); + let statistics = b"notparquet".to_vec(); + target["partition-statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/partition-stats.parquet", + "file-size-in-bytes": statistics.len() + }]); + backend + .seed_object("warehouse", "tables/table-id/metadata/partition-stats.parquet", statistics) + .await; + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect_err("partition statistics must be a Parquet file"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("partition statistics object is not a Parquet file".to_string()) + ); +} + +#[tokio::test] +async fn snapshot_validation_rejects_statistics_size_mismatch() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let mut current = table_metadata_json_for_validation(); + current["last-sequence-number"] = serde_json::Value::from(1); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/snap-10.avro", + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let mut target = current.clone(); + let statistics = b"PFA1PFA1".to_vec(); + target["statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": "s3://warehouse/tables/table-id/metadata/stats.puffin", + "file-size-in-bytes": statistics.len() + 1, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + }]); + backend + .seed_object("warehouse", "tables/table-id/metadata/stats.puffin", statistics) + .await; + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect_err("statistics lengths must match the published object"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("statistics file-size-in-bytes does not match the object".to_string()) + ); +} + #[test] fn iceberg_metadata_validation_bounds_snapshot_sequence_numbers() { let mut v2 = table_metadata_json_for_validation(); @@ -1170,6 +2157,16 @@ fn iceberg_metadata_validation_bounds_snapshot_sequence_numbers() { "Iceberg v2 snapshot sequence-number must be between zero and last-sequence-number".to_string() ) ); + v2["last-sequence-number"] = serde_json::Value::from(0); + v2["snapshots"][0] + .as_object_mut() + .expect("snapshot should be an object") + .remove("sequence-number"); + let error = validate_supported_table_metadata(&v2).expect_err("Iceberg v2 snapshots must include sequence-number"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("Iceberg v2 snapshot sequence-number is required".to_string()) + ); let mut v1 = serde_json::json!({ "format-version": 1, @@ -1193,6 +2190,11 @@ fn iceberg_metadata_validation_bounds_snapshot_sequence_numbers() { ); v1["snapshots"][0]["sequence-number"] = serde_json::Value::from(0); validate_supported_table_metadata(&v1).expect("a zero v1 compatibility sequence should validate"); + v1["snapshots"][0] + .as_object_mut() + .expect("snapshot should be an object") + .remove("sequence-number"); + validate_supported_table_metadata(&v1).expect("Iceberg v1 snapshots may omit sequence-number"); } #[test] @@ -1261,7 +2263,7 @@ fn iceberg_metadata_version_synchronization_builds_complete_v2_shape() { } #[test] -fn iceberg_manifest_validation_accepts_deflate_and_rejects_unknown_content() { +fn iceberg_manifest_validation_accepts_standard_codecs_and_rejects_unknown_content() { let schema = apache_avro::Schema::parse_str( r#" { @@ -1275,22 +2277,29 @@ fn iceberg_manifest_validation_accepts_deflate_and_rejects_unknown_content() { "#, ) .expect("manifest list schema should parse"); - let mut writer = apache_avro::Writer::with_codec(&schema, Vec::new(), apache_avro::Codec::Deflate(Default::default())) - .expect("compressed manifest list writer should initialize"); - writer - .append_value(apache_avro::types::Value::Record(vec![ - ( - "manifest_path".to_string(), - apache_avro::types::Value::String("s3://warehouse/tables/table-id/metadata/manifest.avro".to_string()), - ), - ("partition_spec_id".to_string(), apache_avro::types::Value::Int(0)), - ])) - .expect("compressed manifest list record should append"); - let compressed = writer.into_inner().expect("compressed manifest list should flush"); - let references = manifest_list_references_from_manifest_list_avro(&compressed) - .expect("deflate-compressed manifest lists should be supported with bounded decoding"); - assert_eq!(references.len(), 1); - assert_eq!(references[0].manifest_path, "s3://warehouse/tables/table-id/metadata/manifest.avro"); + for (label, codec) in [ + ("null", apache_avro::Codec::Null), + ("deflate", apache_avro::Codec::Deflate(Default::default())), + ("snappy", apache_avro::Codec::Snappy), + ("zstandard", apache_avro::Codec::Zstandard(Default::default())), + ] { + let mut writer = apache_avro::Writer::with_codec(&schema, Vec::new(), codec) + .expect("compressed manifest list writer should initialize"); + writer + .append_value(apache_avro::types::Value::Record(vec![ + ( + "manifest_path".to_string(), + apache_avro::types::Value::String("s3://warehouse/tables/table-id/metadata/manifest.avro".to_string()), + ), + ("partition_spec_id".to_string(), apache_avro::types::Value::Int(0)), + ])) + .expect("compressed manifest list record should append"); + let compressed = writer.into_inner().expect("compressed manifest list should flush"); + let references = manifest_list_references_from_manifest_list_avro(&compressed) + .unwrap_or_else(|error| panic!("{label}-compressed manifest lists should be supported: {error}")); + assert_eq!(references.len(), 1); + assert_eq!(references[0].manifest_path, "s3://warehouse/tables/table-id/metadata/manifest.avro"); + } let unknown_content = manifest_avro_bytes_with_status(&[("s3://warehouse/tables/table-id/data/part.parquet", 3, 1)]); let error = data_file_references_from_manifest_avro(&unknown_content) @@ -1366,7 +2375,7 @@ async fn iceberg_snapshot_graph_rejects_unknown_partition_specs() { .seed_object( "warehouse", "tables/table-id/metadata/snap-10.avro", - manifest_list_avro_bytes_with_spec(&[manifest_location], 7), + manifest_list_avro_bytes_with_spec(&[(manifest_location, 1)], 7), ) .await; let mut metadata = table_metadata_json_for_validation(); @@ -1391,6 +2400,118 @@ async fn iceberg_snapshot_graph_rejects_unknown_partition_specs() { ); } +#[tokio::test] +async fn iceberg_snapshot_graph_accepts_null_manifest_list_counts() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list_location = "s3://warehouse/tables/table-id/metadata/snap-10.avro"; + let manifest_location = "s3://warehouse/tables/table-id/metadata/manifest-10.avro"; + let manifest = manifest_avro_bytes(&[]); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/snap-10.avro", + manifest_list_avro_bytes_with_spec_and_null_counts(&[(manifest_location, manifest.len())], 0, true), + ) + .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/manifest-10.avro", manifest) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list_location, + "summary": {"operation": "append"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect("nullable v2 manifest-list counts must remain compatible"); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_revalidates_unchanged_snapshots_after_spec_removal() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list_location = "s3://warehouse/tables/table-id/metadata/snap-10.avro"; + let manifest_location = "s3://warehouse/tables/table-id/metadata/manifest-10.avro"; + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/snap-10.avro", + manifest_list_avro_bytes_with_spec(&[(manifest_location, 1)], 7), + ) + .await; + let mut current = table_metadata_json_for_validation(); + current["partition-specs"] + .as_array_mut() + .expect("partition specs should be an array") + .push(serde_json::json!({"spec-id": 7, "fields": []})); + current["last-sequence-number"] = serde_json::Value::from(7); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list_location, + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let mut target = current.clone(); + target["partition-specs"] + .as_array_mut() + .expect("partition specs should be an array") + .retain(|spec| spec.get("spec-id").and_then(serde_json::Value::as_i64) != Some(7)); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect_err("removing a spec referenced by an unchanged snapshot must fail"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("snapshot manifest references missing partition spec 7".to_string()) + ); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_skips_unchanged_history_after_spec_addition() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let mut current = table_metadata_json_for_validation(); + current["last-sequence-number"] = serde_json::Value::from(1); + current["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/missing-history.avro", + "summary": {"operation": "append"} + }]); + current["current-snapshot-id"] = serde_json::Value::from(10); + current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let mut target = current.clone(); + target["partition-specs"] + .as_array_mut() + .expect("partition specs should be an array") + .push(serde_json::json!({"spec-id": 7, "fields": []})); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect("adding a partition spec must not reread unchanged snapshot history"); +} + #[tokio::test] async fn iceberg_snapshot_graph_allows_missing_deleted_files() { let backend = TestCatalogObjectBackend::default(); @@ -1400,19 +2521,16 @@ async fn iceberg_snapshot_graph_allows_missing_deleted_files() { let manifest_list_location = "s3://warehouse/tables/table-id/metadata/snap-10.avro"; let manifest_location = "s3://warehouse/tables/table-id/metadata/manifest-10.avro"; let deleted_data_location = "s3://warehouse/tables/table-id/data/deleted.parquet"; + let manifest_bytes = manifest_avro_bytes_with_status(&[(deleted_data_location, 0, 2)]); backend .seed_object( "warehouse", "tables/table-id/metadata/snap-10.avro", - manifest_list_avro_bytes(&[manifest_location]), + manifest_list_avro_bytes(&[(manifest_location, manifest_bytes.len())]), ) .await; backend - .seed_object( - "warehouse", - "tables/table-id/metadata/manifest-10.avro", - manifest_avro_bytes_with_status(&[(deleted_data_location, 0, 2)]), - ) + .seed_object("warehouse", "tables/table-id/metadata/manifest-10.avro", manifest_bytes) .await; let mut metadata = table_metadata_json_for_validation(); metadata["last-sequence-number"] = serde_json::Value::from(7); @@ -1469,19 +2587,16 @@ async fn iceberg_v2_snapshot_graph_accepts_reused_v1_manifests() { let manifest_list_location = "s3://warehouse/tables/table-id/metadata/snap-v1.avro"; let manifest_location = "s3://warehouse/tables/table-id/metadata/manifest-v1.avro"; let data_location = "s3://warehouse/tables/table-id/data/v1.parquet"; + let manifest_bytes = v1_manifest_avro_bytes(data_location); backend .seed_object( "warehouse", "tables/table-id/metadata/snap-v1.avro", - v1_manifest_list_avro_bytes(manifest_location), + v1_manifest_list_avro_bytes(manifest_location, manifest_bytes.len()), ) .await; backend - .seed_object( - "warehouse", - "tables/table-id/metadata/manifest-v1.avro", - v1_manifest_avro_bytes(data_location), - ) + .seed_object("warehouse", "tables/table-id/metadata/manifest-v1.avro", manifest_bytes) .await; backend .seed_object("warehouse", "tables/table-id/data/v1.parquet", vec![1]) @@ -1549,7 +2664,7 @@ async fn iceberg_snapshot_change_validation_skips_unchanged_history() { } #[tokio::test] -async fn iceberg_snapshot_registration_validates_only_active_snapshot_heads() { +async fn iceberg_snapshot_registration_validates_all_retained_snapshots() { let backend = TestCatalogObjectBackend::default(); let namespace = Namespace::parse("analytics").expect("namespace should parse"); let table = IdentifierSegment::parse("events").expect("table should parse"); @@ -1579,9 +2694,13 @@ async fn iceberg_snapshot_registration_validates_only_active_snapshot_heads() { metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 11}}); let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); - validate_table_snapshot_changes(&context, None, &metadata) + let error = validate_table_snapshot_changes(&context, None, &metadata) .await - .expect("registration should validate active snapshot heads without traversing all history"); + .expect_err("registration must validate every retained snapshot"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("snapshot manifest-list object is missing".to_string()) + ); } #[tokio::test] @@ -1593,19 +2712,16 @@ async fn iceberg_snapshot_graph_counts_shared_manifests_once() { let manifest_list = "s3://warehouse/tables/table-id/metadata/shared-list.avro"; let manifest = "s3://warehouse/tables/table-id/metadata/shared-manifest.avro"; let data_file = "s3://warehouse/tables/table-id/data/shared.parquet"; + let manifest_bytes = manifest_avro_bytes(&[(data_file, 0)]); backend .seed_object( "warehouse", "tables/table-id/metadata/shared-list.avro", - manifest_list_avro_bytes(&[manifest]), + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len())]), ) .await; backend - .seed_object( - "warehouse", - "tables/table-id/metadata/shared-manifest.avro", - manifest_avro_bytes(&[(data_file, 0)]), - ) + .seed_object("warehouse", "tables/table-id/metadata/shared-manifest.avro", manifest_bytes) .await; backend .seed_object("warehouse", "tables/table-id/data/shared.parquet", vec![1]) @@ -1637,6 +2753,142 @@ async fn iceberg_snapshot_graph_counts_shared_manifests_once() { .expect("shared manifest objects must consume the commit budget only once"); } +#[tokio::test] +async fn iceberg_snapshot_graph_revalidates_cached_manifest_declarations() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest = "s3://warehouse/tables/table-id/metadata/shared-manifest.avro"; + let data_file = "s3://warehouse/tables/table-id/data/shared.parquet"; + let manifest_bytes = manifest_avro_bytes_with_status_and_partition_spec(&[(data_file, 0, 1)], 0); + let manifest_length = manifest_bytes.len(); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/list-10.avro", + manifest_list_avro_bytes(&[(manifest, manifest_length)]), + ) + .await; + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/list-11.avro", + manifest_list_avro_bytes(&[(manifest, manifest_length + 1)]), + ) + .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/shared-manifest.avro", manifest_bytes) + .await; + backend + .seed_object("warehouse", "tables/table-id/data/shared.parquet", vec![1]) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([ + { + "snapshot-id": 10, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": "s3://warehouse/tables/table-id/metadata/list-10.avro", + "summary": {"operation": "append"} + }, + { + "snapshot-id": 11, + "sequence-number": 7, + "timestamp-ms": 2, + "manifest-list": "s3://warehouse/tables/table-id/metadata/list-11.avro", + "summary": {"operation": "append"} + } + ]); + metadata["current-snapshot-id"] = serde_json::Value::from(11); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 11}}); + let current_metadata = table_metadata_json_for_validation(); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t_metadata), &metadata) + .await + .expect_err("every manifest-list declaration must match the cached manifest object"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest-list manifest_length does not match the manifest object".to_string()) + ); + + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/list-11.avro", + manifest_list_avro_bytes_with_spec(&[(manifest, manifest_length)], 1), + ) + .await; + metadata["partition-specs"] + .as_array_mut() + .expect("partition specs should be an array") + .push(serde_json::json!({"spec-id": 1, "fields": []})); + + let error = validate_table_snapshot_changes(&context, Some(¤t_metadata), &metadata) + .await + .expect_err("every cached manifest must match each manifest-list partition spec declaration"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest partition-spec-id does not match its manifest-list entry".to_string()) + ); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_bounds_shared_manifest_traversals() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list = "s3://warehouse/tables/table-id/metadata/shared-list.avro"; + let manifest = "s3://warehouse/tables/table-id/metadata/shared-manifest.avro"; + let data_file = "s3://warehouse/tables/table-id/data/shared.parquet"; + let manifest_bytes = manifest_avro_bytes(&[(data_file, 0)]); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/shared-list.avro", + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len())]), + ) + .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/shared-manifest.avro", manifest_bytes) + .await; + backend + .seed_object("warehouse", "tables/table-id/data/shared.parquet", vec![1]) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::Value::Array( + (0..=TABLE_COMMIT_MAX_MANIFEST_TRAVERSALS) + .map(|index| { + let snapshot_id = i64::try_from(index + 1).expect("snapshot id should fit in i64"); + serde_json::json!({ + "snapshot-id": snapshot_id, + "sequence-number": 7, + "timestamp-ms": snapshot_id, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + }) + }) + .collect(), + ); + let current_snapshot_id = i64::try_from(TABLE_COMMIT_MAX_MANIFEST_TRAVERSALS + 1).expect("snapshot id should fit in i64"); + metadata["current-snapshot-id"] = serde_json::Value::from(current_snapshot_id); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": current_snapshot_id}}); + let current_metadata = table_metadata_json_for_validation(); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, Some(¤t_metadata), &metadata) + .await + .expect_err("logical manifest traversals must remain bounded across shared snapshots"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("snapshot manifest traversal count exceeds the commit limit".to_string()) + ); +} + #[tokio::test] async fn iceberg_snapshot_graph_accepts_more_than_ten_thousand_live_files() { let backend = TestCatalogObjectBackend::default(); @@ -1645,14 +2897,6 @@ async fn iceberg_snapshot_graph_accepts_more_than_ten_thousand_live_files() { let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); let manifest_list = "s3://warehouse/tables/table-id/metadata/boundary-list.avro"; let manifest = "s3://warehouse/tables/table-id/metadata/boundary-manifest.avro"; - backend - .seed_object( - "warehouse", - "tables/table-id/metadata/boundary-list.avro", - manifest_list_avro_bytes(&[manifest]), - ) - .await; - let data_file_count = 10_001; let data_files = (0..data_file_count) .map(|index| format!("s3://warehouse/tables/table-id/data/part-{index:05}.parquet")) @@ -1669,13 +2913,17 @@ async fn iceberg_snapshot_graph_accepts_more_than_ten_thousand_live_files() { .await; } let references = data_files.iter().map(|data_file| (data_file.as_str(), 0)).collect::>(); + let manifest_bytes = manifest_avro_bytes(&references); backend .seed_object( "warehouse", - "tables/table-id/metadata/boundary-manifest.avro", - manifest_avro_bytes(&references), + "tables/table-id/metadata/boundary-list.avro", + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len())]), ) .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/boundary-manifest.avro", manifest_bytes) + .await; let mut metadata = table_metadata_json_for_validation(); metadata["last-sequence-number"] = serde_json::Value::from(7); @@ -1696,7 +2944,7 @@ async fn iceberg_snapshot_graph_accepts_more_than_ten_thousand_live_files() { } #[tokio::test] -async fn iceberg_v2_snapshot_graph_accepts_embedded_v2_manifests() { +async fn iceberg_v2_snapshot_graph_rejects_embedded_v2_manifests() { let backend = TestCatalogObjectBackend::default(); let namespace = Namespace::parse("analytics").expect("namespace should parse"); let table = IdentifierSegment::parse("events").expect("table should parse"); @@ -1726,13 +2974,17 @@ async fn iceberg_v2_snapshot_graph_accepts_embedded_v2_manifests() { metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); - validate_table_snapshot_changes(&context, None, &metadata) + let error = validate_table_snapshot_changes(&context, None, &metadata) .await - .expect("embedded v2 manifests should use the enclosing snapshot sequence bound"); + .expect_err("new v2 snapshots must use a manifest list"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("new Iceberg v2 snapshots require manifest-list".to_string()) + ); } #[tokio::test] -async fn iceberg_snapshot_graph_accepts_delete_files_in_data_directory() { +async fn iceberg_snapshot_graph_rejects_delete_files_in_data_manifest() { let backend = TestCatalogObjectBackend::default(); let namespace = Namespace::parse("analytics").expect("namespace should parse"); let table = IdentifierSegment::parse("events").expect("table should parse"); @@ -1740,20 +2992,62 @@ async fn iceberg_snapshot_graph_accepts_delete_files_in_data_directory() { let manifest_list = "s3://warehouse/tables/table-id/metadata/list-10.avro"; let manifest = "s3://warehouse/tables/table-id/metadata/snap-delete-manifest.avro"; let delete_file = "s3://warehouse/tables/table-id/data/position-deletes.parquet"; + let manifest_bytes = manifest_avro_bytes(&[(delete_file, 1)]); backend .seed_object( "warehouse", "tables/table-id/metadata/list-10.avro", - manifest_list_avro_bytes(&[manifest]), + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len())]), ) .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/snap-delete-manifest.avro", manifest_bytes) + .await; + backend + .seed_object("warehouse", "tables/table-id/data/position-deletes.parquet", vec![1]) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list, + "summary": {"operation": "delete"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect_err("a data manifest must not contain delete files"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest-list content does not match manifest file content".to_string()) + ); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_accepts_delete_files_in_delete_manifest() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list = "s3://warehouse/tables/table-id/metadata/list-10.avro"; + let manifest = "s3://warehouse/tables/table-id/metadata/snap-delete-manifest.avro"; + let delete_file = "s3://warehouse/tables/table-id/data/position-deletes.parquet"; + let manifest_bytes = manifest_avro_bytes(&[(delete_file, 1)]); backend .seed_object( "warehouse", - "tables/table-id/metadata/snap-delete-manifest.avro", - manifest_avro_bytes(&[(delete_file, 1)]), + "tables/table-id/metadata/list-10.avro", + manifest_list_avro_bytes_with_spec_and_content(&[(manifest, manifest_bytes.len())], 0, 1), ) .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/snap-delete-manifest.avro", manifest_bytes) + .await; backend .seed_object("warehouse", "tables/table-id/data/position-deletes.parquet", vec![1]) .await; @@ -1772,7 +3066,97 @@ async fn iceberg_snapshot_graph_accepts_delete_files_in_data_directory() { validate_table_snapshot_changes(&context, None, &metadata) .await - .expect("manifest content should identify delete files stored under the data directory"); + .expect("a delete manifest may contain delete files regardless of their object directory"); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_rejects_manifest_length_mismatch() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list = "s3://warehouse/tables/table-id/metadata/list-20.avro"; + let manifest = "s3://warehouse/tables/table-id/metadata/manifest-20.avro"; + let data_file = "s3://warehouse/tables/table-id/data/part-20.parquet"; + let manifest_bytes = manifest_avro_bytes(&[(data_file, 0)]); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/list-20.avro", + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len() + 1)]), + ) + .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/manifest-20.avro", manifest_bytes) + .await; + backend + .seed_object("warehouse", "tables/table-id/data/part-20.parquet", vec![1]) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(20); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 20}}); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect_err("manifest-list lengths must match the published manifest object"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest-list manifest_length does not match the manifest object".to_string()) + ); +} + +#[tokio::test] +async fn iceberg_snapshot_graph_rejects_manifest_partition_spec_mismatch() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list = "s3://warehouse/tables/table-id/metadata/list-20.avro"; + let manifest = "s3://warehouse/tables/table-id/metadata/manifest-20.avro"; + let data_file = "s3://warehouse/tables/table-id/data/part-20.parquet"; + let manifest_bytes = manifest_avro_bytes_with_status_and_partition_spec(&[(data_file, 0, 1)], 7); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/list-20.avro", + manifest_list_avro_bytes(&[(manifest, manifest_bytes.len())]), + ) + .await; + backend + .seed_object("warehouse", "tables/table-id/metadata/manifest-20.avro", manifest_bytes) + .await; + backend + .seed_object("warehouse", "tables/table-id/data/part-20.parquet", vec![1]) + .await; + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(20); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 20}}); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + + let error = validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect_err("manifest headers must agree with their manifest-list entry"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("manifest partition-spec-id does not match its manifest-list entry".to_string()) + ); } #[tokio::test] @@ -1797,8 +3181,6 @@ fn manifest_avro_bytes(files: &[(&str, i32)]) -> Vec { } fn manifest_avro_bytes_with_status(files: &[(&str, i32, i32)]) -> Vec { - // Historical fixed values of this file's fixtures: snapshot 20, sequence 7 - // (the shared constructor takes snapshot_id fourth, sequence fifth). let files = files .iter() .map(|(path, content, status)| (*path, *content, *status, 20_i64, 7_i64)) @@ -1806,6 +3188,14 @@ fn manifest_avro_bytes_with_status(files: &[(&str, i32, i32)]) -> Vec { crate::table_catalog::test_support::manifest_avro_bytes(&files) } +fn manifest_avro_bytes_with_status_and_partition_spec(files: &[(&str, i32, i32)], partition_spec_id: i32) -> Vec { + let files = files + .iter() + .map(|(path, content, status)| (*path, *content, *status, 20_i64, 7_i64)) + .collect::>(); + crate::table_catalog::test_support::manifest_avro_bytes_with_partition_spec(&files, Some(partition_spec_id)) +} + fn manifest_avro_bytes_with_dt_partition(files: &[(&str, i32, &str)]) -> Vec { let schema = apache_avro::Schema::parse_str( r#" @@ -1839,6 +3229,9 @@ fn manifest_avro_bytes_with_dt_partition(files: &[(&str, i32, &str)]) -> Vec ) .expect("partitioned manifest avro schema should parse"); let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("partitioned manifest writer should initialize"); + writer + .add_user_metadata("partition-spec-id".to_string(), "0") + .expect("manifest partition spec metadata should write"); for (file_path, content, partition_value) in files { writer .append_value(apache_avro::types::Value::Record(vec![ @@ -1887,6 +3280,7 @@ fn manifest_avro_bytes_with_sort_order(files: &[(&str, i32, i32)]) -> Vec { "fields": [ {"name": "content", "type": "int"}, {"name": "file_path", "type": "string"}, + {"name": "partition", "type": {"type": "record", "name": "partition", "fields": []}}, {"name": "record_count", "type": "long"}, {"name": "file_size_in_bytes", "type": "long"}, {"name": "sort_order_id", "type": ["null", "int"], "default": null} @@ -1899,6 +3293,9 @@ fn manifest_avro_bytes_with_sort_order(files: &[(&str, i32, i32)]) -> Vec { ) .expect("sort-order manifest avro schema should parse"); let mut writer = apache_avro::Writer::new(&schema, Vec::new()).expect("sorted manifest writer should initialize"); + writer + .add_user_metadata("partition-spec-id".to_string(), "0") + .expect("manifest partition spec metadata should write"); for (file_path, content, sort_order_id) in files { writer .append_value(apache_avro::types::Value::Record(vec![ @@ -1911,6 +3308,7 @@ fn manifest_avro_bytes_with_sort_order(files: &[(&str, i32, i32)]) -> Vec { apache_avro::types::Value::Record(vec![ ("content".to_string(), apache_avro::types::Value::Int(*content)), ("file_path".to_string(), apache_avro::types::Value::String((*file_path).to_string())), + ("partition".to_string(), apache_avro::types::Value::Record(Vec::new())), ("record_count".to_string(), apache_avro::types::Value::Long(1)), ("file_size_in_bytes".to_string(), apache_avro::types::Value::Long(1)), ( @@ -2299,6 +3697,61 @@ async fn object_table_catalog_store_persists_view_entries_and_blocks_non_empty_n store.drop_namespace(bucket, &namespace.public_name()).await.unwrap(); } +#[tokio::test] +async fn object_catalog_view_replacement_recovers_after_committed_write_response_loss() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let current_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let next_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_view(test_view_entry(bucket, &namespace, &view, current_metadata.clone())) + .await + .unwrap(); + backend + .seed_object( + bucket, + &next_metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": format!("s3://{bucket}/views/view-id") + })) + .expect("view metadata should encode"), + ) + .await; + let view_path = store.paths.view_entry_path(bucket, &namespace, &view); + backend.fail_after_next_put(RUSTFS_META_BUCKET, &view_path).await; + + let replaced = store + .replace_view(ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata.clone(), + }) + .await + .expect("an exact persisted replacement must prove the ambiguous write succeeded"); + + assert_eq!(replaced.view.metadata_location, next_metadata); + assert_eq!(replaced.view.generation, 2); + let loaded = store + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect("view lookup should succeed") + .expect("view should remain present"); + assert_eq!(loaded, replaced.view); +} + #[tokio::test] async fn maintenance_dry_run_keeps_current_metadata() { let backend = TestCatalogObjectBackend::default(); @@ -5206,14 +6659,13 @@ async fn maintenance_worker_preserves_queued_dry_run_after_delete_is_enabled() { let data_file = format!("{table_root}data/part-00001.parquet"); let orphan_data = format!("{table_root}data/orphan.parquet"); let now = OffsetDateTime::UNIX_EPOCH + Duration::seconds(100); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, b"data".to_vec()).await; backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; backend @@ -6140,14 +7592,13 @@ async fn maintenance_reachability_expands_manifest_avro_references() { let manifest = format!("{metadata_dir}/manifest-10.avro"); let data_file = format!("{table_root}data/part-00001.parquet"); let delete_file = format!("{table_root}delete/pos-00001.parquet"); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0), (&delete_file, 1)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0), (&delete_file, 1)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, b"data".to_vec()).await; backend.seed_object(bucket, &delete_file, b"delete".to_vec()).await; backend @@ -6281,14 +7732,13 @@ async fn maintenance_reachability_uses_table_warehouse_object_paths() { let manifest = "tables/table-id/metadata/manifest-10.avro".to_string(); let data_file = "tables/table-id/data/part-00001.parquet".to_string(); let orphan_data = "tables/table-id/data/orphan.parquet".to_string(); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, b"data".to_vec()).await; backend.seed_object(bucket, &orphan_data, b"orphan".to_vec()).await; backend @@ -6407,14 +7857,13 @@ async fn maintenance_dry_run_reports_unreachable_manifest_data_and_delete_candid let orphan_manifest = format!("{metadata_dir}/manifest-orphan.avro"); let orphan_data = format!("{table_root}data/orphan.parquet"); let orphan_delete = format!("{table_root}delete/orphan-delete.parquet"); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, b"data".to_vec()).await; backend.seed_object(bucket, &orphan_manifest, manifest_avro_bytes(&[])).await; backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; @@ -6485,14 +7934,13 @@ async fn maintenance_delete_removes_only_planned_unreachable_table_objects() { let data_file = format!("{table_root}data/part-00001.parquet"); let orphan_manifest = format!("{metadata_dir}/manifest-orphan.avro"); let orphan_data = format!("{table_root}data/orphan.parquet"); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&data_file, 0)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, b"data".to_vec()).await; backend.seed_object(bucket, &orphan_manifest, manifest_avro_bytes(&[])).await; backend.seed_object(bucket, &orphan_data, b"orphan-data".to_vec()).await; @@ -6947,18 +8395,13 @@ async fn compaction_plan_reports_row_level_delete_files_without_rewrite_candidat let data_file = format!("{data_dir}/part-left.parquet"); let position_delete_file = format!("{delete_dir}/pos-left.parquet"); let equality_delete_file = format!("{delete_dir}/eq-left.parquet"); + let manifest_bytes = manifest_avro_bytes(&[(&data_file, 0), (&position_delete_file, 1), (&equality_delete_file, 2)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes(&[(&data_file, 0), (&position_delete_file, 1), (&equality_delete_file, 2)]), - ) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &data_file, parquet_i32_bytes(&[1, 2])).await; backend .seed_object(bucket, &position_delete_file, b"position-delete".to_vec()) @@ -7130,18 +8573,13 @@ async fn compaction_commit_rewrites_small_data_files_and_advances_pointer() { let retained_values = (10..20_000).collect::>(); let retained_parquet = parquet_i32_bytes(&retained_values); let small_file_threshold_bytes = u64::try_from(left_parquet.len().max(right_parquet.len())).unwrap(); + let manifest_bytes = manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0), (&retained_data, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0), (&retained_data, 0)]), - ) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &left_data, left_parquet).await; backend.seed_object(bucket, &right_data, right_parquet).await; backend.seed_object(bucket, &retained_data, retained_parquet).await; @@ -7309,22 +8747,17 @@ async fn compaction_commit_keeps_partition_rewrite_groups_isolated() { let other_partition_parquet = parquet_i32_bytes(&[5, 6]); let small_file_threshold_bytes = u64::try_from(left_parquet.len().max(right_parquet.len()).max(other_partition_parquet.len())).unwrap(); + let manifest_bytes = manifest_avro_bytes_with_dt_partition(&[ + (&left_data, 0, "2026-06-24"), + (&right_data, 0, "2026-06-24"), + (&other_partition_data, 0, "2026-06-25"), + ]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes_with_dt_partition(&[ - (&left_data, 0, "2026-06-24"), - (&right_data, 0, "2026-06-24"), - (&other_partition_data, 0, "2026-06-25"), - ]), - ) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &left_data, left_parquet).await; backend.seed_object(bucket, &right_data, right_parquet).await; backend @@ -7495,18 +8928,14 @@ async fn compaction_commit_preserves_sort_order_and_keeps_groups_isolated() { let other_sort_parquet = parquet_i32_bytes(&[5, 6]); let small_file_threshold_bytes = u64::try_from(left_parquet.len().max(right_parquet.len()).max(other_sort_parquet.len())).unwrap(); + let manifest_bytes = + manifest_avro_bytes_with_sort_order(&[(&left_data, 0, 7), (&right_data, 0, 7), (&other_sort_data, 0, 8)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes_with_sort_order(&[(&left_data, 0, 7), (&right_data, 0, 7), (&other_sort_data, 0, 8)]), - ) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &left_data, left_parquet).await; backend.seed_object(bucket, &right_data, right_parquet).await; backend.seed_object(bucket, &other_sort_data, other_sort_parquet).await; @@ -7632,14 +9061,13 @@ async fn compaction_commit_rejects_schema_mismatch_without_advancing_pointer() { let manifest = format!("{metadata_dir}/manifest-20.avro"); let left_data = format!("{data_dir}/part-left.parquet"); let right_data = format!("{data_dir}/part-right.parquet"); + let manifest_bytes = manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object(bucket, &manifest, manifest_avro_bytes(&[(&left_data, 0), (&right_data, 0)])) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &left_data, parquet_i32_bytes(&[1, 2])).await; backend.seed_object(bucket, &right_data, parquet_i64_bytes(&[3, 4])).await; backend @@ -7708,18 +9136,13 @@ async fn compaction_commit_rejects_deleted_manifest_entries_without_advancing_po let manifest = format!("{metadata_dir}/manifest-20.avro"); let left_data = format!("{data_dir}/part-left.parquet"); let deleted_data = format!("{data_dir}/part-deleted.parquet"); + let manifest_bytes = manifest_avro_bytes_with_status(&[(&left_data, 0, 1), (&deleted_data, 0, 2)]); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; backend - .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[&manifest])) - .await; - backend - .seed_object( - bucket, - &manifest, - manifest_avro_bytes_with_status(&[(&left_data, 0, 1), (&deleted_data, 0, 2)]), - ) + .seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())])) .await; + backend.seed_object(bucket, &manifest, manifest_bytes).await; backend.seed_object(bucket, &left_data, parquet_i32_bytes(&[1, 2])).await; backend.seed_object(bucket, &deleted_data, parquet_i32_bytes(&[3, 4])).await; backend @@ -11643,31 +13066,45 @@ async fn strong_catalog_does_not_guess_view_history_from_a_concurrent_replacemen let first_expected_metadata = initial_metadata.clone(); let first_replace = tokio::spawn(async move { first_replace_store - .replace_view(ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: first_namespace_name, - view: first_view_name, - expected_version_token: "token-v1".to_string(), - expected_metadata_location: first_expected_metadata, - new_metadata_location: first_metadata, - }) + .replace_view_with_publication( + ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: first_namespace_name, + view: first_view_name, + expected_version_token: "token-v1".to_string(), + expected_metadata_location: first_expected_metadata, + new_metadata_location: first_metadata, + }, + false, + &UnserializedTestPublication, + ) .await }); - recovery_read.wait_started().await; - second - .replace_view(ViewCommitRequest { - table_bucket: bucket.to_string(), - namespace: namespace.public_name(), - view: view.as_str().to_string(), - expected_version_token: "token-v1".to_string(), - expected_metadata_location: initial_metadata, - new_metadata_location: second_metadata.clone(), - }) + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, recovery_read.wait_started()) .await - .expect("second writer should publish a different replacement"); + .expect("the first replacement should reach its paused recovery read"); + tokio::time::timeout( + TABLE_CATALOG_TEST_TIMEOUT, + second.replace_view_with_publication( + ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: initial_metadata, + new_metadata_location: second_metadata.clone(), + }, + false, + &UnserializedTestPublication, + ), + ) + .await + .expect("the independent replacement should not wait for publication serialization") + .expect("second writer should publish a different replacement"); recovery_read.release(); - let error = first_replace + let error = tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, first_replace) .await + .expect("the first replacement should finish after its recovery read is released") .expect("first replacement task should join") .expect_err("a different generation-two view must not prove the first replacement succeeded"); assert_matches!(error, TableCatalogStoreError::Internal(message) if message.contains("injected put failure")); @@ -12407,6 +13844,76 @@ async fn catalog_backings_reject_table_view_identifier_collisions() { } } +#[tokio::test] +async fn catalog_backings_persist_table_format_upgrade_and_replay() { + for mode in [TableCatalogBackingMode::ObjectBacked, TableCatalogBackingMode::DurableStrong] { + let backend = TestCatalogObjectBackend::default(); + let store = ConfiguredTableCatalogStore::new_for_test(backend.clone(), mode); + let bucket = format!("format-upgrade-{mode:?}").to_ascii_lowercase(); + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(&bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(&bucket, &namespace)) + .await + .expect("namespace should be created"); + let mut entry = test_table_entry(&bucket, &namespace, &table, current_metadata.clone()); + entry.format_version = 1; + store.create_table(entry).await.expect("v1 table should be created"); + backend + .seed_object( + &bucket, + &next_metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": format!("s3://{bucket}/tables/table-id") + })) + .expect("target metadata should encode"), + ) + .await; + let request = TableCommitRequest { + table_bucket: bucket.clone(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "format-upgrade-commit".to_string(), + idempotency_key: Some("format-upgrade-request".to_string()), + operation: "upgrade-format-version".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata, + requirements: Vec::new(), + writer: Some("iceberg-rest/test".to_string()), + }; + + let committed = store + .commit_table(request.clone()) + .await + .expect("format upgrade should commit"); + assert_eq!(committed.table.format_version, 2); + let replay = store + .commit_table(request) + .await + .expect("exact format upgrade replay should succeed"); + assert_eq!(replay, committed); + + let restarted = ConfiguredTableCatalogStore::new_for_test(backend, mode); + let loaded = restarted + .load_table(&bucket, &namespace.public_name(), table.as_str()) + .await + .expect("restarted catalog should load") + .expect("upgraded table should persist"); + assert_eq!(loaded.format_version, 2); + assert_eq!(loaded.metadata_location, committed.table.metadata_location); + } +} + #[tokio::test] async fn catalog_backings_hide_and_reject_mutation_of_inactive_resources() { for mode in [TableCatalogBackingMode::ObjectBacked, TableCatalogBackingMode::DurableStrong] { diff --git a/scripts/table-catalog/failure_coverage.py b/scripts/table-catalog/failure_coverage.py index aae4f1e93..db1614294 100644 --- a/scripts/table-catalog/failure_coverage.py +++ b/scripts/table-catalog/failure_coverage.py @@ -132,18 +132,8 @@ def failure_probe_plan(warehouse: str, namespace: str, table: str, rest_path: st "expected-version-token": "stale-token-from-previous-load", "expected-metadata-location": "current-metadata-location-from-load-table", "new-metadata-location": f"s3://{warehouse}/tables/table-id/metadata/conflict_probe.metadata.json", - "requirements": [ - { - "type": "assert-current-snapshot-id", - "snapshot-id": 0, - } - ], - "updates": [ - { - "action": "set-current-schema", - "schema-id": 0, - } - ], + "requirements": [], + "updates": [], }, ), probe_step( @@ -157,6 +147,8 @@ def failure_probe_plan(warehouse: str, namespace: str, table: str, rest_path: st "expected-version-token": "current-version-token-from-load-table", "expected-metadata-location": "current-metadata-location-from-load-table", "new-metadata-location": f"s3://{warehouse}/tables/table-id/metadata/does_not_exist.metadata.json", + "requirements": [], + "updates": [], }, ), probe_step( diff --git a/scripts/table-catalog/test_failure_coverage.py b/scripts/table-catalog/test_failure_coverage.py index 164e8f492..60773fa71 100644 --- a/scripts/table-catalog/test_failure_coverage.py +++ b/scripts/table-catalog/test_failure_coverage.py @@ -48,6 +48,8 @@ class FailureCoverageTest(unittest.TestCase): self.assertIn("expected-version-token", by_name["stale-token-commit-conflict"]["body"]) self.assertIn("expected-metadata-location", by_name["stale-token-commit-conflict"]["body"]) self.assertIn("new-metadata-location", by_name["stale-token-commit-conflict"]["body"]) + self.assertEqual(by_name["stale-token-commit-conflict"]["body"]["requirements"], []) + self.assertEqual(by_name["stale-token-commit-conflict"]["body"]["updates"], []) self.assertNotIn("base", by_name["stale-token-commit-conflict"]["body"]) self.assertEqual( by_name["diagnostics-after-finalization-gap"]["path"], @@ -56,6 +58,8 @@ class FailureCoverageTest(unittest.TestCase): self.assertEqual(by_name["diagnostics-after-finalization-gap"]["method"], "GET") self.assertEqual(by_name["recovery-repairs-idempotency-index"]["method"], "POST") self.assertIn("does_not_exist.metadata.json", json.dumps(by_name["missing-metadata-object-rejected"])) + self.assertEqual(by_name["missing-metadata-object-rejected"]["body"]["requirements"], []) + self.assertEqual(by_name["missing-metadata-object-rejected"]["body"]["updates"], []) self.assertNotIn("base", by_name["missing-metadata-object-rejected"]["body"]) def test_cli_prints_failure_matrix_and_probe_plan(self) -> None: From cfa9276fad318c742662010b1e59fac9af131e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 16 Aug 2026 05:27:23 +0800 Subject: [PATCH 27/71] fix(admin): serialize replication metrics in minio-go wire shapes (#6127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(admin): pin minio-go Metrics/MetricsV2 wire contract for replication metrics Red-light evidence for backlog#1675 P1-11: ?replication-metrics[=2] serializes the internal snake_case BucketStats family straight onto the wire, while minio-go's replication.Metrics/MetricsV2 expect camelCase tags (currStats/queueStats/replicaCount/queued/...). Go's decoder is case-insensitive but does not ignore underscores, so 'mc replicate status' shows all zeros without any error. The rewritten snapshot tests assert the minio-go tags (plus a synthesized queueStats node — the aggregation path leaves queue_stats.nodes empty today) and fail against the current pass-through serialization. * fix(admin): serialize replication metrics in minio-go wire shapes ?replication-metrics[=2] and the admin replicationmetrics endpoint serialized the internal snake_case BucketStats family straight onto the wire, so 'mc replicate status' decoded all zeros without any error (backlog#1675 P1-11). The internal structs cannot be renamed: they are the intra-cluster peer-RPC wire format (rmp_serde to_vec_named in node_service.rs), pinned by a new regression test. - New admin/replication_metrics_wire.rs: Serialize-only projections onto minio-go replication.Metrics (v1 body, currStats) and MetricsV2 (uptime/currStats/queueStats/downtimeInfo) with the exact json tags; per-target failed becomes the TimedErrStats envelope fed from the FailStats rolling window; the queue peak is dual-emitted as max (MinIO server tag) and peak (minio-go tag). - queueStats synthesizes one node from the bucket queue snapshot — the aggregation path leaves queue_stats.nodes empty, and mc treats an empty node list as 'no data' — and carries transfer summaries (Large/Small/Total) derived from the per-target xfer rates. - Both endpoints share the DTOs; source-health extension keys (provider_available/cluster_complete/...) ride along and are ignored by Go decoders. - Widen the ecstore replication_stats_boundary re-exports (BucketReplicationStat/InQueueMetric/XferStats) so the admin facade chain can name the projected types. * fix(replication): carry failure rolling windows through cluster aggregation Review: both metrics endpoints aggregate first, and FailStats::merge dropped the process-local samples (which also never cross the peer-RPC wire — serde-skipped), so lastMinute/lastHour serialized as zero right after a failure while totals was nonzero. - FailStats gains serializable last_minute/last_hour window snapshots (serde default: old nodes read zeros, new fields are ignored by old decoders), recomputed on every add_size and re-stamped at the per-node collection point (get_latest_replication_stats), and summed by merge. - The wire DTO takes the component-wise max of the live samples and the snapshot, so both the single-node and the aggregated path report the window. - Regression test drives a stat through rmp round trip + merge before serialization, as requested. Also restore the #[allow(dead_code)] attribute to route_policy — the new module declaration had been inserted between the attribute and its item, which broke the -D warnings CI lanes. * fix(replication): bin transfer summaries at 128 MiB and keep window refresh off the hot path Second review round: - update_xfer_rate split at 1 MiB while the minio-go transferSummary labels (and RustFS's own worker-pool split) mean >= 128 MiB for Large, so a 2 MiB replication reported under Large with Small stuck at zero. The producer now bins on MIN_LARGE_OBJ_SIZE; a MetricsV2 assertion covers 2 MiB / 127 MiB / exactly 128 MiB. - add_size no longer recomputes the rolling windows: two full one-hour-deque scans per failure under the bucket-stats write lock made failure bursts quadratic (30k events ~2.1s). The windows are stamped only at the collection point (get_latest_replication_stats, which serves both the local leg and the peer RPC); the aggregation regression now drives that path explicitly before the RPC round trip and merge. * fix(replication): average transfer summaries --------- Co-authored-by: overtrue --- crates/ecstore/src/api/mod.rs | 23 +- crates/ecstore/src/bucket/replication/mod.rs | 2 +- .../bucket/replication/replication_state.rs | 6 + .../replication/replication_stats_boundary.rs | 8 +- crates/replication/src/stats.rs | 33 +- rustfs/src/admin/handlers/replication.rs | 9 +- rustfs/src/admin/mod.rs | 1 + rustfs/src/admin/replication_metrics_wire.rs | 673 ++++++++++++++++++ rustfs/src/admin/router.rs | 80 ++- rustfs/src/admin/storage_api.rs | 4 + 10 files changed, 808 insertions(+), 31 deletions(-) create mode 100644 rustfs/src/admin/replication_metrics_wire.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index d93b717bb..d201c4ff5 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -184,17 +184,18 @@ pub mod bucket { mrf_backlog_observability_snapshot, }; pub use crate::bucket::replication::{ - BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot, - DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry, - MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, - REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, - REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, - ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt, - ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, - ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, - ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, - ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, - TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent, + BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats, + DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric, + MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, + REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, + REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, + ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig, + ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput, + ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO, + ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge, + ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError, + ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus, + VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool, get_global_replication_stats, init_background_replication, invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, diff --git a/crates/ecstore/src/bucket/replication/mod.rs b/crates/ecstore/src/bucket/replication/mod.rs index c1ae1345c..1c4ac52d4 100644 --- a/crates/ecstore/src/bucket/replication/mod.rs +++ b/crates/ecstore/src/bucket/replication/mod.rs @@ -81,6 +81,6 @@ pub use replication_queue_boundary::{ pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus}; pub use replication_scanner_bridge::ReplicationScannerBridge; pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog}; -pub use replication_stats_boundary::{BucketReplicationStats, BucketStats}; +pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats}; pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage}; pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge; diff --git a/crates/ecstore/src/bucket/replication/replication_state.rs b/crates/ecstore/src/bucket/replication/replication_state.rs index d16a507e5..cd0e303f3 100644 --- a/crates/ecstore/src/bucket/replication/replication_state.rs +++ b/crates/ecstore/src/bucket/replication/replication_state.rs @@ -704,6 +704,12 @@ impl ReplicationStats { } else { BucketReplicationStats::new() }; + // Stamp the serializable failure windows from the live samples: the + // samples themselves do not cross the peer-RPC wire, so this snapshot + // is what cluster aggregation and the metrics endpoints see. + for stat in replication_stats.stats.values_mut() { + stat.fail_stats.refresh_windows(); + } let uptime = if cache.contains_key(bucket) { SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) diff --git a/crates/ecstore/src/bucket/replication/replication_stats_boundary.rs b/crates/ecstore/src/bucket/replication/replication_stats_boundary.rs index ad3ef737f..f1917bb2a 100644 --- a/crates/ecstore/src/bucket/replication/replication_stats_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_stats_boundary.rs @@ -15,7 +15,9 @@ #[cfg(test)] pub(crate) use rustfs_replication::FailStats; pub(crate) use rustfs_replication::{ - ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope, - SRMetricsSummary, XferStats, + ActiveWorkerStat, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope, SRMetricsSummary, }; -pub use rustfs_replication::{BucketReplicationStats, BucketStats}; +// Public so the admin wire DTOs (rustfs/src/admin/replication_metrics_wire.rs) +// can project the internal stats onto the minio-go response shapes through +// the storage_api facade chain. +pub use rustfs_replication::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats}; diff --git a/crates/replication/src/stats.rs b/crates/replication/src/stats.rs index a6c000d43..eab14930d 100644 --- a/crates/replication/src/stats.rs +++ b/crates/replication/src/stats.rs @@ -520,6 +520,14 @@ struct FailureSample { pub struct FailStats { pub count: i64, pub size: i64, + /// Rolling-window snapshots refreshed at collection time + /// ([`Self::refresh_windows`]). The raw samples (`recent`) are process + /// local (serde-skipped), so these fields are what survives the peer-RPC + /// wire and [`Self::merge`]-based cluster aggregation. + #[serde(default)] + pub last_minute: FailedMetric, + #[serde(default)] + pub last_hour: FailedMetric, #[serde(skip)] recent: VecDeque, } @@ -537,6 +545,17 @@ impl FailStats { self.prune(observed_at); } + /// Recompute the serializable rolling-window snapshots from the local + /// samples. Called at the collection point (per-node stats snapshot), + /// never on the failure hot path — the two deque scans are O(window) and + /// `add_size` runs under the bucket-stats write lock. Only meaningful on + /// the live per-node struct: a deserialized or merged struct has no + /// samples, and refreshing it would wipe the aggregated windows. + pub fn refresh_windows(&mut self) { + self.last_minute = self.recent_since(Duration::from_secs(60)); + self.last_hour = self.recent_since(Duration::from_secs(3600)); + } + fn prune(&mut self, observed_at: Instant) { while self .recent @@ -565,6 +584,16 @@ impl FailStats { Self { count: self.count.saturating_add(other.count), size: self.size.saturating_add(other.size), + // The window snapshots sum across nodes; the raw samples do not + // travel and stay empty on aggregated structs. + last_minute: FailedMetric { + count: self.last_minute.count.saturating_add(other.last_minute.count), + size: self.last_minute.size.saturating_add(other.last_minute.size), + }, + last_hour: FailedMetric { + count: self.last_hour.count.saturating_add(other.last_hour.count), + size: self.last_hour.size.saturating_add(other.last_hour.size), + }, recent: VecDeque::new(), } } @@ -636,7 +665,9 @@ impl BucketReplicationStat { } pub fn update_xfer_rate(&mut self, size: i64, duration: Duration) { - if size > 1024 * 1024 { + // Same boundary as the worker-pool split and minio-go's + // Large/Small transfer-summary labels: >= 128 MiB is "large". + if size >= crate::runtime::MIN_LARGE_OBJ_SIZE { self.xfer_rate_lrg.add_size(size, duration); } else { self.xfer_rate_sml.add_size(size, duration); diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 0d0c1e8ce..a2a6054c4 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -535,8 +535,13 @@ impl Operation for GetReplicationMetricsHandler { let bucket_stats = cluster_replication_stats(bucket, app_context_from_req(&req)).await; - let data = serde_json::to_vec(&bucket_stats.replication_stats) - .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "serialize failed"))?; + // Same minio-go `replication.Metrics` wire shape as + // `?replication-metrics` — the internal snake_case stats are the peer + // RPC wire format and must not leak here. + let data = serde_json::to_vec(&crate::admin::replication_metrics_wire::MetricsWire::from( + &bucket_stats.replication_stats, + )) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "serialize failed"))?; let mut headers = HeaderMap::new(); headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index be2ea82d9..d61abf932 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -17,6 +17,7 @@ mod auth; pub mod console; pub mod handlers; mod plugin_contract; +pub(crate) mod replication_metrics_wire; // Contract inventory is validated by tests before later runtime integration. #[allow(dead_code)] pub(crate) mod route_policy; diff --git a/rustfs/src/admin/replication_metrics_wire.rs b/rustfs/src/admin/replication_metrics_wire.rs new file mode 100644 index 000000000..15c9cad81 --- /dev/null +++ b/rustfs/src/admin/replication_metrics_wire.rs @@ -0,0 +1,673 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Serialize-only wire projections of the internal replication statistics +//! onto the minio-go `replication.Metrics` / `replication.MetricsV2` json +//! shapes consumed by `mc replicate status` (`?replication-metrics[=2]` and +//! the admin `replicationmetrics` endpoint). +//! +//! Red line: the internal `BucketStats` family in +//! `crates/replication/src/stats.rs` is ALSO the intra-cluster peer-RPC wire +//! format — `node_service.rs` encodes it with `rmp_serde::to_vec_named`, so +//! its Rust field names travel between nodes as msgpack map keys. Renaming +//! those serde names would break mixed-version clusters mid rolling upgrade. +//! All madmin/minio-go interop therefore happens in these DTOs; never add +//! `#[serde(rename)]` to the internal structs instead. +//! +//! Field names below are the exact json tags of minio-go +//! `pkg/replication/replication.go` (v7.0.91). Keys minio-go does not know +//! are RustFS extensions; Go decoders ignore unknown keys. `max`/`peak` are +//! both emitted for the queue peak because the MinIO server writes `max` +//! while minio-go reads `peak` (an upstream drift); emitting both keeps every +//! decoder working. + +use serde::Serialize; +use std::collections::HashMap; +use std::time::Duration; + +use crate::admin::storage_api::replication::{ + BucketReplicationStat as InternalReplicationStat, BucketReplicationStats as InternalReplicationStats, BucketStats, + InQueueMetric as InternalInQueueMetric, XferStats as InternalXferStats, +}; + +/// minio-go `replication.RStat`. +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub(crate) struct RStatWire { + #[serde(rename = "count")] + pub count: f64, + #[serde(rename = "bytes")] + pub bytes: i64, +} + +/// minio-go `replication.TimedErrStats`. +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub(crate) struct TimedErrStatsWire { + #[serde(rename = "lastMinute")] + pub last_minute: RStatWire, + #[serde(rename = "lastHour")] + pub last_hour: RStatWire, + #[serde(rename = "totals")] + pub totals: RStatWire, +} + +impl TimedErrStatsWire { + fn add(self, other: TimedErrStatsWire) -> TimedErrStatsWire { + fn add(a: RStatWire, b: RStatWire) -> RStatWire { + RStatWire { + count: a.count + b.count, + bytes: a.bytes.saturating_add(b.bytes), + } + } + TimedErrStatsWire { + last_minute: add(self.last_minute, other.last_minute), + last_hour: add(self.last_hour, other.last_hour), + totals: add(self.totals, other.totals), + } + } +} + +/// minio-go `replication.QStat`. +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub(crate) struct QStatWire { + #[serde(rename = "count")] + pub count: f64, + #[serde(rename = "bytes")] + pub bytes: f64, +} + +/// minio-go `replication.InQueueMetric`, with the queue peak emitted under +/// both `peak` (minio-go tag) and `max` (MinIO server tag). +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub(crate) struct InQueueMetricWire { + #[serde(rename = "curr")] + pub curr: QStatWire, + #[serde(rename = "avg")] + pub avg: QStatWire, + #[serde(rename = "max")] + pub max: QStatWire, + #[serde(rename = "peak")] + pub peak: QStatWire, +} + +impl From<&InternalInQueueMetric> for InQueueMetricWire { + fn from(metric: &InternalInQueueMetric) -> Self { + fn qstat(bytes: i64, count: i64) -> QStatWire { + QStatWire { + count: count as f64, + bytes: bytes as f64, + } + } + let peak = qstat(metric.max.bytes, metric.max.count); + InQueueMetricWire { + curr: qstat(metric.curr.bytes, metric.curr.count), + avg: qstat(metric.avg.bytes, metric.avg.count), + max: peak, + peak, + } + } +} + +/// minio-go `replication.XferStats`. +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub(crate) struct XferStatsWire { + #[serde(rename = "avgRate")] + pub avg_rate: f64, + #[serde(rename = "peakRate")] + pub peak_rate: f64, + #[serde(rename = "currRate")] + pub curr_rate: f64, +} + +#[derive(Default)] +struct XferStatsAverage { + sum: XferStatsWire, + active: u32, +} + +impl XferStatsAverage { + fn add_active(&mut self, stats: XferStatsWire) { + if stats.peak_rate <= 0.0 { + return; + } + self.add_raw(stats); + self.active += 1; + } + + fn add_raw(&mut self, stats: XferStatsWire) { + self.sum.avg_rate += stats.avg_rate; + self.sum.curr_rate += stats.curr_rate; + self.sum.peak_rate = self.sum.peak_rate.max(stats.peak_rate); + } + + fn finish(self) -> XferStatsWire { + let active = self.active; + self.finish_with_divisor(active) + } + + fn finish_with_divisor(self, divisor: u32) -> XferStatsWire { + if divisor == 0 { + return self.sum; + } + XferStatsWire { + avg_rate: self.sum.avg_rate / f64::from(divisor), + peak_rate: self.sum.peak_rate, + curr_rate: self.sum.curr_rate / f64::from(divisor), + } + } +} + +impl From<&InternalXferStats> for XferStatsWire { + fn from(stats: &InternalXferStats) -> Self { + XferStatsWire { + avg_rate: stats.avg, + peak_rate: stats.peak, + curr_rate: stats.curr, + } + } +} + +/// minio-go `replication.WorkerStat`. RustFS does not track per-bucket worker +/// occupancy yet, so this always reports zeros. +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub(crate) struct WorkerStatWire { + #[serde(rename = "curr")] + pub curr: i32, + #[serde(rename = "avg")] + pub avg: f32, + #[serde(rename = "max")] + pub max: i32, +} + +/// minio-go `replication.ReplMRFStats`. RustFS does not track the 5-minute / +/// dropped MRF windows, so this always reports zeros; the durable backlog is +/// enumerable via `/v3/replication/mrf` instead. +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub(crate) struct ReplMrfStatsWire { + #[serde(rename = "failedCount_last5min")] + pub last_failed_count: u64, + #[serde(rename = "droppedCount_since_uptime")] + pub total_dropped_count: u64, + #[serde(rename = "droppedBytes_since_uptime")] + pub total_dropped_bytes: u64, +} + +/// minio-go `replication.CounterSummary`. +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub(crate) struct CounterSummaryWire { + #[serde(rename = "last1hr")] + pub last1hr: u64, + #[serde(rename = "last1m")] + pub last1m: u64, + #[serde(rename = "total")] + pub total: u64, +} + +/// minio-go `replication.TargetMetrics` (one remote target / ARN). +#[derive(Debug, Default, Serialize)] +pub(crate) struct TargetMetricsWire { + #[serde(rename = "replicationCount")] + pub replicated_count: i64, + #[serde(rename = "completedReplicationSize")] + pub replicated_size: i64, + /// Bandwidth limit for this target. The tag says "bits" but both MinIO + /// and minio-go treat the value as bytes/sec; keep bytes/sec. + #[serde(rename = "limitInBits")] + pub bandwidth_limit_bytes_per_sec: i64, + #[serde(rename = "currentBandwidth")] + pub current_bandwidth_bytes_per_sec: f64, + #[serde(rename = "failed")] + pub failed: TimedErrStatsWire, + #[serde(rename = "failedReplicationSize")] + pub failed_size: i64, + #[serde(rename = "failedReplicationCount")] + pub failed_count: i64, +} + +fn target_timed_err_stats(stat: &InternalReplicationStat) -> TimedErrStatsWire { + // Cluster aggregation merges FailStats without the process-local samples, + // so the serializable window snapshots (refreshed at each node's + // collection point, summed by merge) are authoritative here; the live + // samples only ever agree with or lag them, so take the larger. + let sampled_minute = stat.fail_stats.recent_since(Duration::from_secs(60)); + let sampled_hour = stat.fail_stats.recent_since(Duration::from_secs(3600)); + let window = |sampled_count: i64, sampled_size: i64, snapshot_count: i64, snapshot_size: i64| RStatWire { + count: sampled_count.max(snapshot_count) as f64, + bytes: sampled_size.max(snapshot_size), + }; + TimedErrStatsWire { + last_minute: window( + sampled_minute.count, + sampled_minute.size, + stat.fail_stats.last_minute.count, + stat.fail_stats.last_minute.size, + ), + last_hour: window( + sampled_hour.count, + sampled_hour.size, + stat.fail_stats.last_hour.count, + stat.fail_stats.last_hour.size, + ), + totals: RStatWire { + count: stat.failed.count as f64, + bytes: stat.failed.size, + }, + } +} + +impl From<&InternalReplicationStat> for TargetMetricsWire { + fn from(stat: &InternalReplicationStat) -> Self { + TargetMetricsWire { + replicated_count: stat.replicated_count, + replicated_size: stat.replicated_size, + bandwidth_limit_bytes_per_sec: stat.bandwidth_limit_bytes_per_sec, + current_bandwidth_bytes_per_sec: stat.current_bandwidth_bytes_per_sec, + failed: target_timed_err_stats(stat), + failed_size: stat.failed.size, + failed_count: stat.failed.count, + } + } +} + +/// minio-go `replication.Metrics` — the `currStats` member of `MetricsV2` and +/// the whole v1 response body. The trailing snake_case fields are RustFS +/// source-health extension keys (ignored by Go decoders) carried over from +/// the previous response shape. +#[derive(Debug, Default, Serialize)] +pub(crate) struct MetricsWire { + #[serde(rename = "Stats")] + pub stats: HashMap, + #[serde(rename = "completedReplicationSize")] + pub replicated_size: i64, + #[serde(rename = "replicaSize")] + pub replica_size: i64, + #[serde(rename = "replicaCount")] + pub replica_count: i64, + #[serde(rename = "replicationCount")] + pub replicated_count: i64, + #[serde(rename = "failed")] + pub failed: TimedErrStatsWire, + #[serde(rename = "queued")] + pub queued: InQueueMetricWire, + // RustFS extension keys (source health of the aggregation). + pub provider_available: bool, + pub cluster_complete: bool, + pub observed_node_count: u32, + pub expected_node_count: u32, +} + +impl From<&InternalReplicationStats> for MetricsWire { + fn from(stats: &InternalReplicationStats) -> Self { + let mut failed = TimedErrStatsWire::default(); + let mut targets = HashMap::with_capacity(stats.stats.len()); + for (arn, stat) in &stats.stats { + let target = TargetMetricsWire::from(stat); + failed = failed.add(target.failed); + targets.insert(arn.clone(), target); + } + MetricsWire { + stats: targets, + replicated_size: stats.replicated_size, + replica_size: stats.replica_size, + replica_count: stats.replica_count, + replicated_count: stats.replicated_count, + failed, + queued: InQueueMetricWire::from(&stats.q_stat), + provider_available: stats.provider_available, + cluster_complete: stats.cluster_complete, + observed_node_count: stats.observed_node_count, + expected_node_count: stats.expected_node_count, + } + } +} + +/// minio-go `replication.ReplQNodeStats`. +#[derive(Debug, Default, Serialize)] +pub(crate) struct ReplQNodeStatsWire { + #[serde(rename = "nodeName")] + pub node_name: String, + #[serde(rename = "uptime")] + pub uptime: i64, + #[serde(rename = "activeWorkers")] + pub workers: WorkerStatWire, + #[serde(rename = "transferSummary")] + pub xfer_stats: XferSummaryWire, + #[serde(rename = "tgtTransferStats")] + pub tgt_xfer_stats: TargetXferSummaryWire, + #[serde(rename = "queueStats")] + pub q_stats: InQueueMetricWire, + #[serde(rename = "mrfStats")] + pub mrf_stats: ReplMrfStatsWire, + #[serde(rename = "retries")] + pub retries: CounterSummaryWire, + #[serde(rename = "errors")] + pub errors: CounterSummaryWire, +} + +/// minio-go `replication.ReplQueueStats`. +#[derive(Debug, Default, Serialize)] +pub(crate) struct ReplQueueStatsWire { + #[serde(rename = "nodes")] + pub nodes: Vec, +} + +/// minio-go `replication.MetricsV2` — the `?replication-metrics=2` body. +#[derive(Debug, Default, Serialize)] +pub(crate) struct MetricsV2Wire { + #[serde(rename = "uptime")] + pub uptime: i64, + #[serde(rename = "currStats")] + pub current_stats: MetricsWire, + #[serde(rename = "queueStats")] + pub queue_stats: ReplQueueStatsWire, + #[serde(rename = "downtimeInfo")] + pub downtime_info: HashMap, +} + +/// `transferSummary` map keyed by minio-go `MetricName` (Large/Small/Total). +type XferSummaryWire = HashMap<&'static str, XferStatsWire>; +/// `tgtTransferStats` map keyed by target ARN. +type TargetXferSummaryWire = HashMap; + +fn transfer_summaries(stats: &InternalReplicationStats) -> (XferSummaryWire, TargetXferSummaryWire) { + let mut per_target: TargetXferSummaryWire = HashMap::new(); + let mut large_summary = XferStatsAverage::default(); + let mut small_summary = XferStatsAverage::default(); + let mut total_summary = XferStatsAverage::default(); + let mut active_targets = 0; + for (arn, stat) in &stats.stats { + let large = XferStatsWire::from(&stat.xfer_rate_lrg); + let small = XferStatsWire::from(&stat.xfer_rate_sml); + let mut target_total = XferStatsAverage::default(); + target_total.add_active(large); + target_total.add_active(small); + let total = target_total.finish(); + per_target.insert(arn.clone(), HashMap::from([("Large", large), ("Small", small), ("Total", total)])); + if large.peak_rate > 0.0 || small.peak_rate > 0.0 { + active_targets += 1; + large_summary.add_raw(large); + small_summary.add_raw(small); + total_summary.add_raw(large); + total_summary.add_raw(small); + } + } + let summary = HashMap::from([ + ("Large", large_summary.finish_with_divisor(active_targets)), + ("Small", small_summary.finish_with_divisor(active_targets)), + ("Total", total_summary.finish_with_divisor(active_targets)), + ]); + (summary, per_target) +} + +impl MetricsV2Wire { + /// Project the aggregated internal stats onto the `MetricsV2` shape. + /// + /// The aggregation path leaves `queue_stats.nodes` empty today, so a + /// single node entry is synthesized from the bucket queue snapshot — + /// `mc replicate status` derives its queue/worker panels from + /// `queueStats.nodes` and treats an empty list as "no data". + pub(crate) fn from_stats(bucket_stats: &BucketStats, node_name: &str) -> Self { + let (xfer_stats, tgt_xfer_stats) = transfer_summaries(&bucket_stats.replication_stats); + let mut nodes: Vec = bucket_stats + .queue_stats + .nodes + .iter() + .map(|node| ReplQNodeStatsWire { + node_name: node_name.to_string(), + uptime: bucket_stats.uptime, + q_stats: InQueueMetricWire::from(&node.q_stats), + ..Default::default() + }) + .collect(); + if nodes.is_empty() { + nodes.push(ReplQNodeStatsWire { + node_name: node_name.to_string(), + uptime: bucket_stats.uptime, + q_stats: InQueueMetricWire::from(&bucket_stats.replication_stats.q_stat), + xfer_stats: xfer_stats.clone(), + tgt_xfer_stats: tgt_xfer_stats.clone(), + ..Default::default() + }); + } else { + // Attach the transfer summaries to the first node; the internal + // snapshot does not attribute transfer rates per node. + if let Some(first) = nodes.first_mut() { + first.xfer_stats = xfer_stats.clone(); + first.tgt_xfer_stats = tgt_xfer_stats.clone(); + } + } + + MetricsV2Wire { + uptime: bucket_stats.uptime, + current_stats: MetricsWire::from(&bucket_stats.replication_stats), + queue_stats: ReplQueueStatsWire { nodes }, + downtime_info: HashMap::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_bucket_stats() -> BucketStats { + let mut stats = BucketStats { + uptime: 42, + ..Default::default() + }; + stats.replication_stats.replica_count = 2; + stats.replication_stats.replica_size = 128; + stats.replication_stats.replicated_count = 9; + stats.replication_stats.replicated_size = 4096; + let target = stats + .replication_stats + .stats + .entry("arn:minio:replication::t:b".to_string()) + .or_default(); + target.replicated_count = 9; + target.replicated_size = 4096; + target.failed.count = 3; + target.failed.size = 900; + target.bandwidth_limit_bytes_per_sec = 1024; + target.current_bandwidth_bytes_per_sec = 512.5; + stats + .replication_stats + .q_stat + .curr + .now_count + .store(4, std::sync::atomic::Ordering::Relaxed); + stats + .replication_stats + .q_stat + .curr + .now_bytes + .store(1200, std::sync::atomic::Ordering::Relaxed); + stats.replication_stats.q_stat = stats.replication_stats.q_stat.snapshot(); + stats + } + + #[test] + fn metrics_wire_matches_minio_go_tags() { + let stats = sample_bucket_stats(); + let json = serde_json::to_value(MetricsWire::from(&stats.replication_stats)).expect("v1 wire should serialize"); + + assert_eq!(json["replicaCount"], 2); + assert_eq!(json["replicaSize"], 128); + assert_eq!(json["replicationCount"], 9); + assert_eq!(json["completedReplicationSize"], 4096); + assert_eq!(json["queued"]["curr"]["count"], 4.0); + assert_eq!(json["queued"]["curr"]["bytes"], 1200.0); + let target = &json["Stats"]["arn:minio:replication::t:b"]; + assert_eq!(target["replicationCount"], 9); + assert_eq!(target["completedReplicationSize"], 4096); + assert_eq!(target["limitInBits"], 1024); + assert_eq!(target["currentBandwidth"], 512.5); + // failed is the madmin TimedErrStats envelope, not the internal + // {count,size} pair. + assert_eq!(target["failed"]["totals"]["count"], 3.0); + assert_eq!(target["failed"]["totals"]["bytes"], 900); + assert!(target["failed"].get("count").is_none()); + // Aggregate failed mirrors the per-target totals. + assert_eq!(json["failed"]["totals"]["count"], 3.0); + } + + #[test] + fn metrics_v2_wire_synthesizes_queue_node() { + let stats = sample_bucket_stats(); + let json = serde_json::to_value(MetricsV2Wire::from_stats(&stats, "node-1:9000")).expect("v2 wire should serialize"); + + assert_eq!(json["uptime"], 42); + assert_eq!(json["currStats"]["replicaCount"], 2); + let node = &json["queueStats"]["nodes"][0]; + assert_eq!(node["nodeName"], "node-1:9000"); + assert_eq!(node["uptime"], 42); + assert_eq!(node["queueStats"]["curr"]["count"], 4.0); + // The queue peak is emitted under both the minio-go tag (`peak`) and + // the MinIO server tag (`max`). + assert_eq!(node["queueStats"]["peak"], node["queueStats"]["max"]); + assert!(node["activeWorkers"].get("curr").is_some()); + assert!(node["transferSummary"].get("Total").is_some()); + assert_eq!(json["downtimeInfo"], serde_json::json!({})); + } + + /// minio-go's transferSummary labels mean >= 128 MiB for Large; the + /// producer must bin on the same boundary (MIN_LARGE_OBJ_SIZE, shared + /// with the worker-pool split), or a 2 MiB replication shows under Large + /// while Small stays zero. + #[test] + fn transfer_summary_bins_on_the_128_mib_boundary() { + const MIB: i64 = 1024 * 1024; + let mut stats = BucketStats::default(); + let stat = stats + .replication_stats + .stats + .entry("arn:minio:replication::t:b".to_string()) + .or_default(); + stat.update_xfer_rate(2 * MIB, std::time::Duration::from_secs(1)); + stat.update_xfer_rate(127 * MIB, std::time::Duration::from_secs(1)); + stat.update_xfer_rate(128 * MIB, std::time::Duration::from_secs(1)); + + let json = serde_json::to_value(MetricsV2Wire::from_stats(&stats, "node-1")).expect("v2 wire should serialize"); + let summary = &json["queueStats"]["nodes"][0]["tgtTransferStats"]["arn:minio:replication::t:b"]; + let small_peak = summary["Small"]["peakRate"].as_f64().expect("Small peakRate"); + let large_peak = summary["Large"]["peakRate"].as_f64().expect("Large peakRate"); + assert!( + (small_peak - (127 * MIB) as f64).abs() < 1.0, + "2 MiB and 127 MiB transfers must bin as Small (peak {small_peak})" + ); + assert!( + (large_peak - (128 * MIB) as f64).abs() < 1.0, + "exactly 128 MiB must bin as Large (peak {large_peak})" + ); + } + + #[test] + fn transfer_summaries_average_active_bins_and_targets() { + let mut stats = BucketStats::default(); + let first = stats.replication_stats.stats.entry("target-a".to_string()).or_default(); + first.xfer_rate_sml.avg = 50.0; + first.xfer_rate_sml.curr = 40.0; + first.xfer_rate_sml.peak = 60.0; + first.xfer_rate_lrg.avg = 100.0; + first.xfer_rate_lrg.curr = 80.0; + first.xfer_rate_lrg.peak = 120.0; + + let second = stats.replication_stats.stats.entry("target-b".to_string()).or_default(); + second.xfer_rate_sml.avg = 30.0; + second.xfer_rate_sml.curr = 20.0; + second.xfer_rate_sml.peak = 40.0; + + let json = serde_json::to_value(MetricsV2Wire::from_stats(&stats, "node-1")).expect("v2 wire should serialize"); + let node = &json["queueStats"]["nodes"][0]; + let target_a = &node["tgtTransferStats"]["target-a"]["Total"]; + assert_eq!(target_a["avgRate"], 75.0); + assert_eq!(target_a["currRate"], 60.0); + assert_eq!(target_a["peakRate"], 120.0); + + let summary = &node["transferSummary"]; + assert_eq!(summary["Small"]["avgRate"], 40.0); + assert_eq!(summary["Small"]["currRate"], 30.0); + assert_eq!(summary["Large"]["avgRate"], 50.0); + assert_eq!(summary["Total"]["avgRate"], 90.0); + assert_eq!(summary["Total"]["currRate"], 70.0); + assert_eq!(summary["Total"]["peakRate"], 120.0); + } + + /// Review regression: both metrics endpoints aggregate first, and the + /// FailStats merge drops the process-local samples — the rolling windows + /// must survive a peer-RPC round trip plus aggregation and still reach + /// the wire body. + #[test] + fn failure_windows_survive_aggregation_before_serialization() { + // Node A: live failure; the windows are stamped at the collection + // point (get_latest_replication_stats calls refresh_windows before + // the stats cross the wire), never on the failure hot path. + let mut node_a = crate::admin::storage_api::replication::BucketReplicationStat::default(); + node_a.fail_stats.add_size(512, None::<&std::io::Error>); + node_a.fail_stats.refresh_windows(); + node_a.failed = node_a.fail_stats.to_metric(); + + // Node A's stats cross the peer RPC wire: the samples are dropped, + // the window snapshots travel. + let encoded = rmp_serde::to_vec_named(&node_a).expect("stat should encode"); + let remote: crate::admin::storage_api::replication::BucketReplicationStat = + rmp_serde::from_slice(&encoded).expect("stat should decode"); + + // Aggregation merges the remote stat with an empty local one. + let merged_fail = remote.fail_stats.merge(&Default::default()); + let aggregated = crate::admin::storage_api::replication::BucketReplicationStat { + failed: merged_fail.to_metric(), + fail_stats: merged_fail, + ..Default::default() + }; + + let mut stats = BucketStats::default(); + stats + .replication_stats + .stats + .insert("arn:minio:replication::t:b".to_string(), aggregated); + + let json = serde_json::to_value(MetricsWire::from(&stats.replication_stats)).expect("wire should serialize"); + let failed = &json["Stats"]["arn:minio:replication::t:b"]["failed"]; + assert_eq!(failed["totals"]["count"], 1.0); + assert_eq!( + failed["lastMinute"]["count"], 1.0, + "the rolling minute window must survive RPC + aggregation" + ); + assert_eq!(failed["lastMinute"]["bytes"], 512); + assert_eq!(failed["lastHour"]["count"], 1.0); + } + + /// Pin the intra-cluster peer-RPC wire format of the internal stats: it + /// is msgpack with the Rust field names as map keys + /// (`rmp_serde::to_vec_named` in node_service.rs). If someone "fixes" + /// the interop bug by renaming the internal serde fields instead of using + /// these DTOs, this test fails and points them here. + #[test] + fn internal_bucket_stats_rpc_wire_stays_snake_case() { + let stats = sample_bucket_stats(); + let encoded = rmp_serde::to_vec_named(&stats).expect("internal stats should encode"); + let value: serde_json::Value = rmp_serde::from_slice(&encoded).expect("named msgpack should decode generically"); + + assert!( + value.get("replication_stats").is_some(), + "peer RPC key replication_stats must not be renamed" + ); + assert!(value["replication_stats"].get("q_stat").is_some()); + assert!(value.get("queue_stats").is_some()); + assert!(value.get("proxy_stats").is_some()); + + let decoded: BucketStats = rmp_serde::from_slice(&encoded).expect("round-trip through the peer RPC wire"); + assert_eq!(decoded.replication_stats.replica_count, 2); + } +} diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 1f3ac8856..9eb893792 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -1548,7 +1548,8 @@ async fn build_replication_metrics_response( let bucket_stats = apply_replication_metrics_bandwidth_report(bucket_stats, collect_replication_metrics_bandwidth(bucket)); let bucket_stats = apply_replication_metrics_runtime_fields(bucket_stats, route, replication_metrics_uptime_seconds()); - let body = serialize_replication_metrics_body(&bucket_stats, route)?; + let node_name = crate::runtime_sources::current_local_node_name().await.unwrap_or_default(); + let body = serialize_replication_metrics_body(&bucket_stats, route, &node_name)?; let mut resp = S3Response::with_status(Body::from(body), StatusCode::OK); resp.headers @@ -1608,12 +1609,24 @@ fn apply_replication_metrics_runtime_fields( bucket_stats } -fn serialize_replication_metrics_body(bucket_stats: &BucketStats, route: ReplicationExtRoute) -> S3Result> { +/// Serialize the metrics body in the minio-go wire shapes +/// (`replication.Metrics` for v1, `replication.MetricsV2` for v2). The +/// internal `BucketStats` serde names are the intra-cluster peer-RPC wire +/// format and must never appear here — see +/// `crate::admin::replication_metrics_wire`. +fn serialize_replication_metrics_body( + bucket_stats: &BucketStats, + route: ReplicationExtRoute, + node_name: &str, +) -> S3Result> { + use crate::admin::replication_metrics_wire::{MetricsV2Wire, MetricsWire}; match route { ReplicationExtRoute::MetricsV1 => { - serde_json::to_vec(&bucket_stats.replication_stats).map_err(|e| s3_error!(InternalError, "{e}")) + serde_json::to_vec(&MetricsWire::from(&bucket_stats.replication_stats)).map_err(|e| s3_error!(InternalError, "{e}")) + } + ReplicationExtRoute::MetricsV2 => { + serde_json::to_vec(&MetricsV2Wire::from_stats(bucket_stats, node_name)).map_err(|e| s3_error!(InternalError, "{e}")) } - ReplicationExtRoute::MetricsV2 => serde_json::to_vec(bucket_stats).map_err(|e| s3_error!(InternalError, "{e}")), ReplicationExtRoute::Check | ReplicationExtRoute::ResetStart | ReplicationExtRoute::ResetStatus => { Err(s3_error!(InternalError, "invalid route for metrics response")) } @@ -4147,22 +4160,37 @@ mod tests { assert!(err.message().unwrap_or_default().contains("rule-stale")); } + /// The v1 body must decode into minio-go `replication.Metrics` (exact + /// json tags); Go's decoder matches case-insensitively but does not + /// ignore underscores, so the internal snake_case names read as all-zero. #[test] - fn serialize_replication_metrics_body_v1_returns_replication_stats_only() { + fn serialize_replication_metrics_body_v1_returns_minio_go_metrics_shape() { let mut stats = BucketStats { uptime: 99, ..Default::default() }; stats.replication_stats.replica_count = 7; + stats.replication_stats.replicated_size = 2048; + stats + .replication_stats + .stats + .entry("arn:minio:replication::t:b".to_string()) + .or_default() + .replicated_count = 5; stats.proxy_stats.put_total = 3; - let body = - serialize_replication_metrics_body(&stats, ReplicationExtRoute::MetricsV1).expect("metrics v1 body should serialize"); + let body = serialize_replication_metrics_body(&stats, ReplicationExtRoute::MetricsV1, "node-1:9000") + .expect("metrics v1 body should serialize"); let payload: serde_json::Value = serde_json::from_slice(&body).expect("body should be json"); - assert_eq!(payload["replica_count"], 7); + assert_eq!(payload["replicaCount"], 7); + assert_eq!(payload["completedReplicationSize"], 2048); + assert_eq!(payload["Stats"]["arn:minio:replication::t:b"]["replicationCount"], 5); assert!(payload.get("uptime").is_none()); assert!(payload.get("proxy_stats").is_none()); + // The internal snake_case names must not leak into the wire body. + assert!(payload.get("replica_count").is_none()); + assert!(payload.get("q_stat").is_none()); } #[test] @@ -4248,22 +4276,48 @@ mod tests { assert_eq!(target.current_bandwidth_bytes_per_sec, 3000.0); } + /// The v2 body must decode into minio-go `replication.MetricsV2` + /// (`uptime`/`currStats`/`queueStats`); `mc replicate status` reads + /// `currStats` and `queueStats.nodes` and silently shows zeros when the + /// keys do not match. #[test] - fn serialize_replication_metrics_body_v2_returns_full_bucket_stats() { + fn serialize_replication_metrics_body_v2_returns_minio_go_metrics_v2_shape() { let mut stats = BucketStats { uptime: 99, ..Default::default() }; stats.replication_stats.replica_count = 7; + stats + .replication_stats + .q_stat + .curr + .now_count + .store(4, std::sync::atomic::Ordering::Relaxed); + stats + .replication_stats + .q_stat + .curr + .now_bytes + .store(1200, std::sync::atomic::Ordering::Relaxed); + stats.replication_stats.q_stat = stats.replication_stats.q_stat.snapshot(); stats.proxy_stats.put_total = 3; - let body = - serialize_replication_metrics_body(&stats, ReplicationExtRoute::MetricsV2).expect("metrics v2 body should serialize"); + let body = serialize_replication_metrics_body(&stats, ReplicationExtRoute::MetricsV2, "node-1:9000") + .expect("metrics v2 body should serialize"); let payload: serde_json::Value = serde_json::from_slice(&body).expect("body should be json"); assert_eq!(payload["uptime"], 99); - assert_eq!(payload["replication_stats"]["replica_count"], 7); - assert_eq!(payload["proxy_stats"]["put_total"], 3); + assert_eq!(payload["currStats"]["replicaCount"], 7); + assert_eq!(payload["currStats"]["queued"]["curr"]["count"], 4.0); + // The queue snapshot must surface at least one node: mc derives the + // worker/queue panels from queueStats.nodes and treats an empty list + // as "no data". + assert_eq!(payload["queueStats"]["nodes"][0]["queueStats"]["curr"]["count"], 4.0); + assert_eq!(payload["queueStats"]["nodes"][0]["uptime"], 99); + // The internal snake_case names must not leak into the wire body. + assert!(payload.get("replication_stats").is_none()); + assert!(payload.get("queue_stats").is_none()); + assert!(payload.get("proxy_stats").is_none()); } #[test] diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 17841d1de..d630f557f 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -417,6 +417,10 @@ pub(crate) mod replication { }; pub(crate) type BucketReplicationResyncStatus = super::ecstore_bucket::replication::BucketReplicationResyncStatus; pub(crate) type BucketStats = super::ecstore_bucket::replication::BucketStats; + pub(crate) type BucketReplicationStats = super::ecstore_bucket::replication::BucketReplicationStats; + pub(crate) type BucketReplicationStat = super::ecstore_bucket::replication::BucketReplicationStat; + pub(crate) type InQueueMetric = super::ecstore_bucket::replication::InQueueMetric; + pub(crate) type XferStats = super::ecstore_bucket::replication::XferStats; pub(crate) type ReplicationStatusType = super::ecstore_bucket::replication::ReplicationStatusType; pub(crate) type ResyncOpts = super::ecstore_bucket::replication::ResyncOpts; pub(crate) type ResyncStatusType = super::ecstore_bucket::replication::ResyncStatusType; From c1f66969d7da8375bdd803e3d69072e57de5fc04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 16 Aug 2026 05:34:17 +0800 Subject: [PATCH 28/71] fix(site-replication): merge incoming ILM expiry documents instead of overwriting (#6130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(site-replication): pin ILM expiry merge contract for incoming lc-config Red-light evidence for backlog#1675 P1-1: the lc-config receiver overwrites the whole local lifecycle config with whatever the peer sends (and deletes it wholesale on peer delete), so an expiry-only document erases the receiver's local tier/transition rules, and peer transition rules get installed across sites. The new tests pin the MinIO mergeWithCurrentLCConfig semantics plus RustFS hardening: - incoming expiry documents merge with (never replace) local rules - local transition sides are authoritative for same-id rules - incoming transition fields are discarded at the trust boundary - dropped expiry rules strip the expiry side but keep transitions; pure-expiry rules are removed - delete merges with the empty set instead of dropping the config - disabled rules survive; abort-mpu-only rules stay site-local - deterministic order (idempotent re-delivery) and expiry_updated_at stamping for the staleness axis All fail against the current overwrite implementation (identity extraction of merge_incoming_lifecycle_config). * fix(site-replication): merge incoming ILM expiry documents instead of overwriting The lc-config receiver replaced the whole local lifecycle config with the peer's document (and deleted it wholesale on peer delete), so an expiry-only update erased the receiver's local tier/transition rules, and a peer's transition rules were installed across sites (backlog#1675 P1-1). Receiver (apply_bucket_meta_item): - lc-config now merges via merge_incoming_lifecycle_config, mirroring MinIO's mergeWithCurrentLCConfig with a trust-boundary hardening: incoming transition fields are discarded outright; the local transition side of a same-id rule is authoritative. A peer delete merges with the empty set — pure-expiry rules go away, transition rules survive with their expiry side cleared, and only an empty result deletes the config file. - Staleness moves to the expiry axis (config.expiry_updated_at): lifecycle_config_updated_at also moves on local transition-only edits, which shadowed newer peer expiry updates. - Receiver-side replicateILMExpiry gate, symmetric with the sender hook (previously any peer could install expiry rules while the option was off). - Rule order is deterministic (local order, incoming-new appended), so re-delivering the same document is byte-stable and does not rewrite bucket metadata per broadcast. Sender: - Both admin choke points — the bucket-meta hook and the SRInfo bucket entry feeding bootstrap/repair and consistency views — now emit only the expiry subset (transition fields stripped, non-expiry rules dropped). MinIO receivers install incoming rules verbatim, so transition rules must never leave the site. An unparseable local config is forwarded unfiltered rather than degraded to a delete. Not covered here (follow-up): a two-site e2e with a real tier backend to exercise transition-rule preservation end to end; receiver-side validate_transition_tier for merged configs. * fix(site-replication): close ILM merge review findings Adversarial review of the lc-config merge surfaced four real defects, all fixed here: - Deletion tombstone regression: with the staleness axis moved to the in-config expiry_updated_at, a deleted lifecycle config fell back to UNIX_EPOCH and any delayed stale broadcast could resurrect deleted expiry rules. The axis now falls back to the whole-config write time (which survives deletion in bucket metadata as the deletion's lower bound), also covering legacy configs that predate the axis field. - MinIO zero-rule documents: MinIO's delete tombstone / transition-only state marshals a lifecycle document with no , which the strict s3s deserializer rejects — the receiver now recognizes it as the 'no expiry rules here' statement (delete semantics) instead of erroring on every MinIO heal pass. - Inflated expiry axis at the sender: PutBucketLifecycle stamped expiry_updated_at unconditionally, so a transition-only edit advanced the axis and let this site's stale expiry subset shadow and roll back newer peer expiry edits fleet-wide. The stamp is now conditional (expiry subset present before or after the edit, MinIO parity), the hook item travels with the config's expiry axis (UNIX_EPOCH when the site has none), and the SRInfo bucket entry feeds bootstrap/repair the same axis instead of the whole-config write time. - Del-marker parity: MinIO's CloneNonTransition never emits del-marker or abort-mpu fields, so treating del_marker_expiration as traveling expiry let a MinIO broadcast delete this site's del-marker-only rules. Both fields are now site-local on every edge: stripped from outbound subsets and inbound rules, restored from the local side on same-id merges, and never a deletion criterion. Receiver-side validation of merged configs (object-lock / tier constraints, MinIO runs finalLcCfg.Validate) remains a follow-up. * fix(site-replication): close the second ILM review round - Missed-delete repair: a deleted expiry state now travels through bootstrap/repair as an explicit timestamped lc-config delete item (lifecycle_expiry_statement distinguishes deletion — whole-config write time advanced past the created backfill — from never-configured buckets and from transition-only configs without an expiry axis, which say nothing). A peer that missed the live delete converges on repair; the receiver's staleness guard protects newer peer state. - Strict tombstone recognition: only a well-delimited zero-rule document maps to delete semantics; truncated or foreign payloads that fail the strict deserializer are rejected instead of being treated as a delete that erases local expiry rules. - Staleness fallback axis narrowed: the whole-config write time is used only for deleted or legacy-with-expiry state. A present transition-only config without an expiry axis compares at epoch — its whole-config time moves on transition edits and must not shadow or block independent peer expiry updates and same-timestamp repairs. * fix(site-replication): validate tombstone children structurally Second review round: a well-delimited root could still smuggle malformed content — e.g. passed the no- --- crates/ecstore/src/api/mod.rs | 3 +- crates/ecstore/src/bucket/metadata_sys.rs | 18 + rustfs/src/admin/handlers/site_replication.rs | 936 +++++++++++++++++- rustfs/src/admin/storage_api.rs | 28 + rustfs/src/app/bucket_usecase.rs | 41 +- 5 files changed, 1001 insertions(+), 25 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index d201c4ff5..f18da095a 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -135,7 +135,8 @@ pub mod bucket { pub use crate::bucket::metadata_sys::ConfigWriteLockProbe; pub use crate::bucket::metadata_sys::{ BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, - capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy, + acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete, + delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config, get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config, diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 40aa0bd77..94709e4ed 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -656,6 +656,16 @@ pub async fn update_under_transaction_lock( update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await } +/// Clear one config file while the caller holds this bucket's transaction lock. +pub async fn delete_under_transaction_lock( + guard: &BucketMetadataMutationGuard, + bucket: &str, + config_file: &str, +) -> Result { + guard.ensure_valid(bucket)?; + delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await +} + pub async fn update_quota_if_incarnation( bucket: &str, data: Vec, @@ -795,6 +805,14 @@ pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result Result { + acquire_config_write_guard_for_incarnation(get_bucket_metadata_sys()?, bucket, Some(expected_incarnation_id)).await +} + pub(crate) async fn acquire_bucket_metadata_transaction_lock_in( ctx: &crate::runtime::instance::InstanceContext, bucket: &str, diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 471a624d4..495db4284 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -2323,18 +2323,31 @@ fn append_bootstrap_bucket_items( }, )?; if replicate_ilm_expiry { - append_bootstrap_bucket_item( - &mut plan.bucket_items, - bucket, - "lc-config", - bucket.expiry_lc_config.clone(), - bucket.expiry_lc_config_updated_at, - |item, value| { - item.expiry_lc_config = Some(value); - item.expiry_updated_at = item.updated_at; - Ok(()) - }, - )?; + if bucket.expiry_lc_config.is_some() { + append_bootstrap_bucket_item( + &mut plan.bucket_items, + bucket, + "lc-config", + bucket.expiry_lc_config.clone(), + bucket.expiry_lc_config_updated_at, + |item, value| { + item.expiry_lc_config = Some(value); + // `updated_at` here is the entry's expiry axis (see the + // SRBucketInfo construction), not the wall clock. + item.expiry_updated_at = item.updated_at; + Ok(()) + }, + )?; + } else if bucket.expiry_lc_config_updated_at.is_some() { + // Expiry rules were removed at this axis (lifecycle_expiry_statement): + // an explicit timestamped delete item, so a peer that missed the + // live delete converges on bootstrap/repair instead of keeping + // stale expiry rules. The receiver's staleness guard protects a + // peer whose expiry state is newer. + let mut item = bootstrap_bucket_meta_item(bucket, "lc-config", bucket.expiry_lc_config_updated_at); + item.expiry_updated_at = item.updated_at; + plan.bucket_items.push(item); + } } append_bootstrap_bucket_item( &mut plan.bucket_items, @@ -4220,13 +4233,23 @@ pub async fn site_replication_delete_bucket_hook(bucket: &str, force_delete: boo broadcast_site_replication_json(&path, &serde_json::json!({})).await } -pub async fn site_replication_bucket_meta_hook(item: SRBucketMeta) -> S3Result<()> { +pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Result<()> { let Some(runtime) = runtime_site_replication_targets().await? else { return Ok(()); }; if item.r#type == "lc-config" && !site_replication_state_replicates_ilm_expiry(&runtime.state) { return Ok(()); } + if item.r#type == "lc-config" { + // Only the expiry subset travels (MinIO peers install incoming rules + // verbatim, so transition rules must never leave this site). An empty + // subset becomes a delete, which the receiver merges with the empty + // set — local transition rules there survive. + item.expiry_lc_config = item + .expiry_lc_config + .and_then(|raw| lifecycle_expiry_subset_xml(raw.as_bytes())) + .map(|data| String::from_utf8_lossy(&data).into_owned()); + } broadcast_site_replication_json_with_runtime( &runtime, "/rustfs/admin/v3/site-replication/peer/bucket-meta", @@ -4319,7 +4342,14 @@ async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml); entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml); entry.quota_config = raw_config_to_base64(&metadata.quota_config_json); - entry.expiry_lc_config = raw_config_to_base64(&metadata.lifecycle_config_xml); + // Expiry subset only: this entry feeds both the bootstrap/repair + // plan (peers must not receive transition rules) and cross-site + // consistency views (transition rules are site-local and would + // read as false mismatches). A deleted expiry state is a `None` + // value with the deletion's axis so repair can converge peers + // that missed the live delete. + let expiry_statement = lifecycle_expiry_statement(&metadata); + entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone()); entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml); entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at); entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at); @@ -4328,7 +4358,11 @@ async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at); entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at); entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at); - entry.expiry_lc_config_updated_at = maybe_time(metadata.lifecycle_config_updated_at); + // The expiry axis, not the whole-config write time: local + // transition-only edits inflate the latter, and a repair item + // stamped with it could out-rank a newer real expiry edit on a + // third site. + entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis); entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at); entry.replication_targets_online = Some(site_replication_targets_online(&bucket.name, &metadata.replication_config_xml).await); @@ -6847,6 +6881,300 @@ fn merge_incoming_replication_config( Some(ReplicationConfiguration { role, rules }) } +/// Merge a peer's ILM expiry document into the local lifecycle config. +/// +/// Mirrors MinIO's `mergeWithCurrentLCConfig` with one hardening: incoming +/// site-local fields (transitions, abort-multipart, del-marker expiration — +/// exactly what MinIO's `CloneNonTransition` sender never emits) are +/// discarded outright at the trust boundary, whatever the peer sends. Local +/// site-local fields always survive; a delete (`incoming == None`) therefore +/// merges with the empty set instead of dropping the whole config. +fn merge_incoming_lifecycle_config( + incoming: Option, + local: Option, + updated_at: Option, +) -> Option { + // Incoming rules reduced to their traveling expiry side. Rules with no + // expiry semantics after the strip are not installed. + let mut incoming_by_id: HashMap = HashMap::new(); + let mut incoming_order: Vec = Vec::new(); + for mut rule in incoming.into_iter().flat_map(|config| config.rules) { + strip_site_local_lifecycle_fields(&mut rule); + if !lifecycle_rule_has_expiry(&rule) { + continue; + } + let id = rule.id.clone().unwrap_or_default(); + if incoming_by_id.insert(id.clone(), rule).is_none() { + incoming_order.push(id); + } + } + + // Local order first, incoming-new appended: repeated delivery of the same + // document is byte-stable, so bucket metadata is written once, not on + // every broadcast. + let local_expiry_updated_at = local.as_ref().and_then(|config| config.expiry_updated_at.clone()); + let mut rules: Vec = Vec::new(); + for mut rule in local.into_iter().flat_map(|config| config.rules) { + let id = rule.id.clone().unwrap_or_default(); + if let Some(mut incoming_rule) = incoming_by_id.remove(&id) { + incoming_order.retain(|pending| pending != &id); + // The incoming expiry side wins; the local site-local side is + // authoritative (MinIO CloneNonTransition + restore). + incoming_rule.transitions = rule.transitions.take(); + incoming_rule.noncurrent_version_transitions = rule.noncurrent_version_transitions.take(); + incoming_rule.abort_incomplete_multipart_upload = rule.abort_incomplete_multipart_upload.take(); + incoming_rule.del_marker_expiration = rule.del_marker_expiration.take(); + rules.push(incoming_rule); + } else if lifecycle_rule_has_expiry(&rule) { + // Expiry rule dropped upstream: strip only the traveling expiry + // side; the rule survives while any site-local action remains. + rule.expiration = None; + rule.noncurrent_version_expiration = None; + if lifecycle_rule_has_transition(&rule) + || rule.abort_incomplete_multipart_upload.is_some() + || rule.del_marker_expiration.is_some() + { + rules.push(rule); + } + } else { + // No traveling expiry semantics (transition-only / abort-mpu-only + // / del-marker-only): not managed by expiry replication, keep + // untouched. + rules.push(rule); + } + } + for id in incoming_order { + if let Some(rule) = incoming_by_id.remove(&id) { + rules.push(rule); + } + } + + if rules.is_empty() { + return None; + } + + Some(s3s::dto::BucketLifecycleConfiguration { + rules, + // Record the expiry axis the staleness guard compares on. (The PUT + // path stamps `expiry_updated_at` only when the expiry subset + // changes, so this axis is not inflated by transition-only edits.) + expiry_updated_at: updated_at.map(s3s::dto::Timestamp::from).or(local_expiry_updated_at), + }) +} + +/// True when the rule carries the expiry semantics that `replicateILMExpiry` +/// propagates. Del-marker expiration and abort-multipart are deliberately +/// excluded: MinIO's sender never emits them (`CloneNonTransition` drops +/// both), so treating them as traveling state would let a MinIO peer's +/// broadcast delete this site's del-marker-only rules. +fn lifecycle_rule_has_expiry(rule: &s3s::dto::LifecycleRule) -> bool { + rule.expiration.is_some() || rule.noncurrent_version_expiration.is_some() +} + +fn lifecycle_rule_has_transition(rule: &s3s::dto::LifecycleRule) -> bool { + rule.transitions.as_ref().is_some_and(|transitions| !transitions.is_empty()) + || rule + .noncurrent_version_transitions + .as_ref() + .is_some_and(|transitions| !transitions.is_empty()) +} + +/// Remove the fields that never travel between sites (MinIO +/// `CloneNonTransition` parity). +fn strip_site_local_lifecycle_fields(rule: &mut s3s::dto::LifecycleRule) { + rule.transitions = None; + rule.noncurrent_version_transitions = None; + rule.abort_incomplete_multipart_upload = None; + rule.del_marker_expiration = None; +} + +/// Reduce a lifecycle XML document to the expiry subset that is allowed to +/// travel between sites (what MinIO's sender emits): transition fields are +/// stripped and rules left with no expiry semantics are dropped. Returns +/// `None` when nothing remains — the receiver then merges with the empty set, +/// which is exactly the "no expiry rules here" statement. A document that +/// fails to parse is forwarded unfiltered (`Some(original)`): the receiver +/// merge strips it anyway, and turning a local parse error into a `None` +/// would delete the peers' replicated expiry rules. +fn lifecycle_expiry_subset_xml(raw: &[u8]) -> Option> { + if raw.is_empty() { + return None; + } + let config: s3s::dto::BucketLifecycleConfiguration = match deserialize(raw) { + Ok(config) => config, + Err(err) => { + warn!("failed to parse local lifecycle config for expiry replication; forwarding unfiltered: {err}"); + return Some(raw.to_vec()); + } + }; + let expiry_updated_at = config.expiry_updated_at.clone(); + let rules: Vec = config + .rules + .into_iter() + .filter_map(|mut rule| { + strip_site_local_lifecycle_fields(&mut rule); + lifecycle_rule_has_expiry(&rule).then_some(rule) + }) + .collect(); + if rules.is_empty() { + return None; + } + let subset = s3s::dto::BucketLifecycleConfiguration { + rules, + expiry_updated_at, + }; + match serialize(&subset) { + Ok(data) => Some(data), + Err(err) => { + warn!("failed to serialize lifecycle expiry subset; forwarding unfiltered: {err}"); + Some(raw.to_vec()) + } + } +} + +/// The expiry replication axis persisted in a lifecycle XML document, if any. +/// Used for the SRInfo bucket entry so bootstrap/repair items carry the +/// expiry axis instead of the whole-config write time (which local +/// transition-only edits inflate). +fn lifecycle_expiry_updated_at(raw: &[u8]) -> Option { + if raw.is_empty() { + return None; + } + deserialize::(raw) + .ok() + .and_then(|config| config.expiry_updated_at) + .map(OffsetDateTime::from) +} + +/// The timestamp an incoming lc-config item must beat to be applied. +/// +/// - Present config with the expiry axis: the axis itself. +/// - Present legacy config that has expiry rules but predates the axis +/// field: the whole-config write time bounds its last expiry edit. +/// - Present transition-only config without the axis: `UNIX_EPOCH` — there +/// is no local expiry state to protect, and the whole-config time moves on +/// transition edits, which must not shadow independent peer expiry updates. +/// - Absent config: the whole-config write time — it survives deletion in +/// bucket metadata as the deletion's lower bound, so a delayed stale +/// broadcast cannot resurrect deleted expiry rules. +fn local_lifecycle_staleness_axis( + local: Option<&s3s::dto::BucketLifecycleConfiguration>, + whole_config_axis: OffsetDateTime, +) -> OffsetDateTime { + match local { + Some(config) => match config.expiry_updated_at.clone() { + Some(axis) => OffsetDateTime::from(axis), + None if config.rules.iter().any(lifecycle_rule_has_expiry) => whole_config_axis, + None => OffsetDateTime::UNIX_EPOCH, + }, + None => whole_config_axis, + } +} + +/// Recognize MinIO's zero-rule lifecycle tombstone (its delete / +/// transition-only state marshals `` with no `` +/// child, which the strict s3s deserializer rejects). Only a well-delimited +/// document qualifies as the "no expiry rules here" statement; truncated or +/// otherwise malformed payloads are rejected rather than treated as a delete +/// that would erase local expiry rules. +fn is_zero_rule_lifecycle_tombstone(raw: &[u8]) -> bool { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Tombstone { + #[serde(rename = "@xmlns")] + _xmlns: Option, + #[serde(rename = "ExpiryUpdatedAt")] + _expiry_updated_at: Option, + } + + let mut reader = quick_xml::Reader::from_reader(raw); + let mut depth = 0usize; + let mut seen_root = false; + let mut closed_root = false; + let mut seen_declaration = false; + let well_formed_document = loop { + match reader.read_event() { + Ok(quick_xml::events::Event::Start(element)) => { + if depth == 0 { + if seen_root || closed_root || element.name().as_ref() != b"LifecycleConfiguration" { + break false; + } + seen_root = true; + } + depth += 1; + } + Ok(quick_xml::events::Event::Empty(element)) => { + if depth == 0 { + if seen_root || closed_root || element.name().as_ref() != b"LifecycleConfiguration" { + break false; + } + seen_root = true; + closed_root = true; + } + } + Ok(quick_xml::events::Event::End(_)) => { + if depth == 0 { + break false; + } + depth -= 1; + if depth == 0 { + closed_root = true; + } + } + Ok(quick_xml::events::Event::Decl(_)) => { + if seen_declaration || seen_root || depth != 0 { + break false; + } + seen_declaration = true; + } + Ok(quick_xml::events::Event::DocType(_)) => break false, + Ok(quick_xml::events::Event::Text(text)) if depth == 0 && !text.iter().all(u8::is_ascii_whitespace) => { + break false; + } + Ok(quick_xml::events::Event::Text(_)) => {} + Ok(quick_xml::events::Event::CData(_)) if depth == 0 => break false, + Ok(quick_xml::events::Event::Comment(_) | quick_xml::events::Event::PI(_)) => {} + Ok(quick_xml::events::Event::Eof) => break seen_root && closed_root && depth == 0, + Ok(_) if depth == 0 => break false, + Ok(_) => {} + Err(_) => break false, + } + }; + + well_formed_document && quick_xml::de::from_reader::<_, Tombstone>(raw).is_ok() +} + +/// The ILM expiry statement this site contributes to its SRInfo bucket entry +/// (feeding bootstrap/repair and consistency views), if any. +/// `Some((subset_b64, axis))` — a `None` subset means "expiry rules were +/// removed at `axis`" and travels as an explicit timestamped delete item, so +/// a peer that missed the live delete still converges on repair. +fn lifecycle_expiry_statement( + metadata: &crate::admin::storage_api::bucket::metadata::BucketMetadata, +) -> Option<(Option, OffsetDateTime)> { + if metadata.lifecycle_config_xml.is_empty() { + // Deleted vs never configured: the whole-config write time survives + // deletion in bucket metadata and strictly exceeds the created-time + // backfill only after a real write. + return (metadata.lifecycle_config_updated_at > metadata.created).then_some((None, metadata.lifecycle_config_updated_at)); + } + let axis = lifecycle_expiry_updated_at(&metadata.lifecycle_config_xml); + match lifecycle_expiry_subset_xml(&metadata.lifecycle_config_xml) { + Some(subset) => { + // Legacy documents predate the axis field; their whole-config + // write time bounds the last expiry edit. + let axis = axis.unwrap_or(metadata.lifecycle_config_updated_at); + Some((raw_config_to_base64(&subset), axis)) + } + // Transition-only config: with an expiry axis the site once had + // expiry rules and properly removed them — the delete travels at + // that axis. Without one there is nothing to say (a delete stamped + // off the whole-config time would let a local transition edit erase + // newer peer expiry state). + None => axis.map(|axis| (None, axis)), + } +} + fn replication_rule_deployment_id(rule: &ReplicationRule) -> Option { if let Some(rule_id) = rule.id.as_deref() { if let Some(deployment_id) = rule_id.strip_prefix("site-repl-") @@ -7928,7 +8256,12 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { } else { None }; - if let Ok(bucket_meta) = metadata_sys::get(&item.bucket).await { + // lc-config staleness is judged on the expiry axis inside its merge block + // below: `lifecycle_config_updated_at` moves on local transition-only + // edits too, which would shadow newer peer expiry updates. + if item.r#type != "lc-config" + && let Ok(bucket_meta) = metadata_sys::get(&item.bucket).await + { let local_updated_at = bucket_meta_local_updated_at(&bucket_meta, config_file); if is_stale_update(local_updated_at, incoming_updated_at) { return Ok(()); @@ -7970,6 +8303,72 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { None }; + let (merged_lifecycle_config, lifecycle_guard) = if item.r#type == "lc-config" { + // Receiver-side gate, symmetric with the sender hook: a peer must not + // install expiry rules here while `replicateILMExpiry` is off. When + // the state cannot be read, fall through and apply (pre-gate + // behavior) rather than silently dropping a legitimate update. Note + // the gate acks with 200 — the sender treats the item as delivered + // and will not retry; items skipped inside the enable-flag + // propagation window are healed by repair, not by retry. + if let Ok(state) = load_site_replication_state().await + && !site_replication_state_replicates_ilm_expiry(&state) + { + return Ok(()); + } + + let incoming = match item.expiry_lc_config.as_ref() { + Some(raw) => { + let data = decode_bucket_meta_wire_value(raw); + match deserialize::(&data) { + Ok(config) => Some(config), + // MinIO's delete tombstone / transition-only state is a + // zero-rule document the strict deserializer rejects; it + // means "no expiry rules here" (delete semantics). Any + // other malformed payload is rejected — treating it as a + // delete would let a bad payload erase local expiry rules. + Err(_) if is_zero_rule_lifecycle_tombstone(&data) => None, + Err(e) => return Err(s3_error!(InvalidRequest, "invalid lifecycle config: {e}")), + } + } + None => None, + }; + let lifecycle_guard = + metadata_sys::acquire_bucket_metadata_transaction_lock_for_incarnation(&item.bucket, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + let local_metadata = metadata_sys::get_config_from_disk(&item.bucket) + .await + .map_err(ApiError::from)?; + let local = if local_metadata.lifecycle_config_xml.is_empty() { + None + } else { + Some( + deserialize::(&local_metadata.lifecycle_config_xml).map_err(|e| { + S3Error::with_message(S3ErrorCode::InternalError, format!("invalid local lifecycle config: {e}")) + })?, + ) + }; + let whole_config_axis = local_metadata.lifecycle_config_updated_at; + if is_stale_update(local_lifecycle_staleness_axis(local.as_ref(), whole_config_axis), incoming_updated_at) { + return Ok(()); + } + let local_absent = local.is_none(); + let merged = match merge_incoming_lifecycle_config(incoming, local, incoming_updated_at) { + Some(config) => Some( + serialize(&config) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize lifecycle failed: {e}")))?, + ), + None => { + skip_config_write = local_absent; + None + } + }; + (merged, Some(lifecycle_guard)) + } else { + (None, None) + }; + let data = match item.r#type.as_str() { "policy" => item .policy @@ -7986,7 +8385,7 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { "object-lock-config" => decode_bucket_meta_wire_option(item.object_lock_config), "sse-config" => decode_bucket_meta_wire_option(item.sse_config), "replication-config" => merged_replication_config, - "lc-config" => decode_bucket_meta_wire_option(item.expiry_lc_config), + "lc-config" => merged_lifecycle_config, "cors-config" => decode_bucket_meta_wire_option(item.cors), _ => unreachable!(), }; @@ -8018,16 +8417,29 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { .map_err(ApiError::from)?; } } else { - metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) + if let Some(guard) = lifecycle_guard.as_ref() { + metadata_sys::update_under_transaction_lock(guard, &item.bucket, config_file, data) + .await + .map_err(ApiError::from)?; + } else { + metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + } + } + } else { + if let Some(guard) = lifecycle_guard.as_ref() { + metadata_sys::delete_under_transaction_lock(guard, &item.bucket, config_file) + .await + .map_err(ApiError::from)?; + } else { + metadata_sys::delete_if_incarnation(&item.bucket, config_file, expected_incarnation_id) .await .map_err(ApiError::from)?; } - } else { - metadata_sys::delete_if_incarnation(&item.bucket, config_file, expected_incarnation_id) - .await - .map_err(ApiError::from)?; } } + drop(lifecycle_guard); drop(targets_guard); if item.r#type == "replication-config" { @@ -12645,6 +13057,87 @@ mod tests { assert!(!plan.bucket_items.iter().any(|item| item.r#type == "lc-config")); } + /// A deleted expiry state (entry value None, axis set) must travel as an + /// explicit timestamped delete item — a peer that missed the live delete + /// otherwise keeps stale expiry rules through every repair (review + /// finding). + #[test] + fn test_site_replication_bootstrap_plan_emits_timestamped_lifecycle_delete() { + let deleted_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let mut info = SRInfo::default(); + info.state.peers.insert( + "remote-dep".to_string(), + PeerInfo { + replicate_ilm_expiry: true, + ..peer("remote", "https://remote.example.com") + }, + ); + info.buckets.insert( + "photos".to_string(), + SRBucketInfo { + bucket: "photos".to_string(), + expiry_lc_config: None, + expiry_lc_config_updated_at: Some(deleted_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + }, + ); + + let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build"); + + let item = plan + .bucket_items + .iter() + .find(|item| item.r#type == "lc-config") + .expect("a deleted expiry state must produce an lc-config delete item"); + assert!(item.expiry_lc_config.is_none(), "delete items carry no config body"); + assert_eq!(item.expiry_updated_at, Some(deleted_at)); + assert_eq!(item.updated_at, Some(deleted_at)); + } + + /// What each local lifecycle state contributes to the SRInfo entry: + /// deletions are timestamped statements, never-configured buckets and + /// transition-only configs without an expiry axis say nothing. + #[test] + fn test_lifecycle_expiry_statement_matrix() { + let created = OffsetDateTime::from_unix_timestamp(1_600_000_000).expect("timestamp"); + let mut meta = crate::admin::storage_api::bucket::metadata::BucketMetadata::new("photos"); + meta.created = created; + // Never configured: load backfills the write time to `created`. + meta.lifecycle_config_updated_at = created; + assert!(lifecycle_expiry_statement(&meta).is_none()); + + // Deleted: the write time survives deletion and exceeds creation. + let deleted_at = created + time::Duration::seconds(100); + meta.lifecycle_config_updated_at = deleted_at; + let (subset, axis) = lifecycle_expiry_statement(&meta).expect("deletion is a statement"); + assert!(subset.is_none()); + assert_eq!(axis, deleted_at); + + // Present with expiry rules and the axis: subset + axis travel. + let expiry_axis = created + time::Duration::seconds(50); + let mut config = lc_config(vec![lc_rule("e1", Some(7), None)]); + config.expiry_updated_at = Some(s3s::dto::Timestamp::from(expiry_axis)); + meta.lifecycle_config_xml = serialize(&config).expect("serialize config"); + let (subset, axis) = lifecycle_expiry_statement(&meta).expect("expiry config is a statement"); + assert!(subset.is_some()); + assert_eq!(axis.unix_timestamp(), expiry_axis.unix_timestamp()); + + // Transition-only without an axis: nothing to say (a delete stamped + // off the whole-config time would erase newer peer expiry state). + meta.lifecycle_config_xml = serialize(&lc_config(vec![lc_rule("t1", None, Some(30))])).expect("serialize config"); + assert!(lifecycle_expiry_statement(&meta).is_none()); + + // Transition-only WITH an axis: expiry rules were properly removed — + // the delete travels at that axis. + let mut transition_only = lc_config(vec![lc_rule("t1", None, Some(30))]); + transition_only.expiry_updated_at = Some(s3s::dto::Timestamp::from(expiry_axis)); + meta.lifecycle_config_xml = serialize(&transition_only).expect("serialize config"); + let (subset, axis) = lifecycle_expiry_statement(&meta).expect("removed expiry state is a statement"); + assert!(subset.is_none()); + assert_eq!(axis.unix_timestamp(), expiry_axis.unix_timestamp()); + } + #[test] fn test_site_replication_repair_request_is_strict_and_requires_explicit_mode() { assert!(serde_json::from_str::(r#"{"mode":"dry-run"}"#).is_ok()); @@ -14299,6 +14792,405 @@ mod tests { assert!(merge_incoming_replication_config(Some(site_repl_config("home")), None).is_none()); } + fn lc_rule(id: &str, expiry_days: Option, transition_days: Option) -> s3s::dto::LifecycleRule { + s3s::dto::LifecycleRule { + id: Some(id.to_string()), + status: s3s::dto::ExpirationStatus::from_static(s3s::dto::ExpirationStatus::ENABLED), + prefix: Some(String::new()), + expiration: expiry_days.map(|days| s3s::dto::LifecycleExpiration { + days: Some(days), + ..Default::default() + }), + transitions: transition_days.map(|days| { + vec![s3s::dto::Transition { + days: Some(days), + storage_class: Some(s3s::dto::TransitionStorageClass::from_static(s3s::dto::TransitionStorageClass::GLACIER)), + date: None, + }] + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + } + } + + fn lc_config(rules: Vec) -> s3s::dto::BucketLifecycleConfiguration { + s3s::dto::BucketLifecycleConfiguration { + rules, + expiry_updated_at: None, + } + } + + fn rule_ids(config: &s3s::dto::BucketLifecycleConfiguration) -> Vec<&str> { + config.rules.iter().filter_map(|rule| rule.id.as_deref()).collect() + } + + /// P1-1 red-light: an incoming expiry-only document must not erase the + /// receiver's local transition/tiering rules (today the receiver + /// overwrites the whole lifecycle config). + #[test] + fn test_merge_incoming_lifecycle_preserves_local_transition_rule() { + let merged = merge_incoming_lifecycle_config( + Some(lc_config(vec![lc_rule("e1", Some(7), None)])), + Some(lc_config(vec![lc_rule("t1", None, Some(30))])), + None, + ) + .expect("merge should keep rules"); + + let mut ids = rule_ids(&merged); + ids.sort_unstable(); + assert_eq!(ids, vec!["e1", "t1"]); + let t1 = merged.rules.iter().find(|rule| rule.id.as_deref() == Some("t1")).unwrap(); + assert!(t1.transitions.as_ref().is_some_and(|t| !t.is_empty()), "local transition must survive"); + } + + /// Same-id incoming rule updates the expiry side but the local transition + /// side is authoritative (MinIO `CloneNonTransition` + restore). + #[test] + fn test_merge_incoming_lifecycle_same_id_keeps_local_transition() { + let merged = merge_incoming_lifecycle_config( + Some(lc_config(vec![lc_rule("r1", Some(7), None)])), + Some(lc_config(vec![lc_rule("r1", Some(1), Some(30))])), + None, + ) + .expect("merge should keep rules"); + + assert_eq!(merged.rules.len(), 1); + let r1 = &merged.rules[0]; + assert_eq!(r1.expiration.as_ref().and_then(|e| e.days), Some(7), "incoming expiry wins"); + assert!( + r1.transitions.as_ref().is_some_and(|t| !t.is_empty()), + "local transition is authoritative" + ); + } + + /// Trust boundary: whatever the peer sends, its transition fields never + /// land here — a new incoming rule is stripped to its expiry parts. + #[test] + fn test_merge_incoming_lifecycle_strips_incoming_transitions() { + let merged = merge_incoming_lifecycle_config( + Some(lc_config(vec![lc_rule("r1", Some(7), Some(1))])), + Some(lc_config(vec![lc_rule("t1", None, Some(30))])), + None, + ) + .expect("merge should keep rules"); + + let r1 = merged.rules.iter().find(|rule| rule.id.as_deref() == Some("r1")).unwrap(); + assert!( + r1.transitions.as_ref().is_none_or(|t| t.is_empty()), + "incoming transition fields must be discarded" + ); + } + + /// A local rule whose expiry part was dropped upstream loses only the + /// expiry fields; a pure-expiry rule disappears entirely. + #[test] + fn test_merge_incoming_lifecycle_dropped_rule_strips_expiry_keeps_transition() { + let merged = merge_incoming_lifecycle_config( + Some(lc_config(vec![lc_rule("other", Some(3), None)])), + Some(lc_config(vec![ + lc_rule("mixed", Some(1), Some(30)), + lc_rule("pure-expiry", Some(2), None), + ])), + None, + ) + .expect("merge should keep rules"); + + let mut ids = rule_ids(&merged); + ids.sort_unstable(); + assert_eq!(ids, vec!["mixed", "other"], "pure-expiry rule not in the incoming set is removed"); + let mixed = merged.rules.iter().find(|rule| rule.id.as_deref() == Some("mixed")).unwrap(); + assert!(mixed.expiration.is_none(), "expiry side cleared"); + assert!(mixed.transitions.as_ref().is_some_and(|t| !t.is_empty()), "transition side kept"); + } + + /// Peer lifecycle delete merges with the empty set: local transition rules + /// survive with their expiry parts cleared; only when nothing remains does + /// the whole config disappear. + #[test] + fn test_merge_incoming_lifecycle_delete_merges_with_empty() { + let merged = merge_incoming_lifecycle_config( + None, + Some(lc_config(vec![ + lc_rule("mixed", Some(1), Some(30)), + lc_rule("pure-expiry", Some(2), None), + ])), + None, + ) + .expect("transition rules must survive a peer lifecycle delete"); + assert_eq!(rule_ids(&merged), vec!["mixed"]); + assert!(merged.rules[0].expiration.is_none()); + + assert!( + merge_incoming_lifecycle_config(None, Some(lc_config(vec![lc_rule("pure-expiry", Some(2), None)])), None).is_none(), + "an all-expiry config deletes cleanly" + ); + } + + /// Disabled rules must survive the merge like enabled ones — the merge + /// must not reuse ENABLED-filtered helpers. + #[test] + fn test_merge_incoming_lifecycle_keeps_disabled_transition_rule() { + let mut disabled = lc_rule("t-disabled", None, Some(30)); + disabled.status = s3s::dto::ExpirationStatus::from_static(s3s::dto::ExpirationStatus::DISABLED); + + let merged = merge_incoming_lifecycle_config( + Some(lc_config(vec![lc_rule("e1", Some(7), None)])), + Some(lc_config(vec![disabled])), + None, + ) + .expect("merge should keep rules"); + + let mut ids = rule_ids(&merged); + ids.sort_unstable(); + assert_eq!(ids, vec!["e1", "t-disabled"]); + } + + /// Abort-multipart-only rules carry no expiry semantics: local ones stay + /// untouched, incoming ones are not installed (they are site-local, like + /// MinIO's sender-side filter). + #[test] + fn test_merge_incoming_lifecycle_abort_mpu_rules_stay_local() { + let abort_only = |id: &str| s3s::dto::LifecycleRule { + id: Some(id.to_string()), + status: s3s::dto::ExpirationStatus::from_static(s3s::dto::ExpirationStatus::ENABLED), + prefix: Some(String::new()), + abort_incomplete_multipart_upload: Some(s3s::dto::AbortIncompleteMultipartUpload { + days_after_initiation: Some(3), + }), + del_marker_expiration: None, + expiration: None, + filter: None, + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + transitions: None, + }; + + let merged = merge_incoming_lifecycle_config( + Some(lc_config(vec![abort_only("incoming-abort"), lc_rule("e1", Some(7), None)])), + Some(lc_config(vec![abort_only("local-abort")])), + None, + ) + .expect("merge should keep rules"); + + let mut ids = rule_ids(&merged); + ids.sort_unstable(); + assert_eq!( + ids, + vec!["e1", "local-abort"], + "incoming abort-mpu rule is not installed; local one survives" + ); + } + + /// Repeated delivery of the same document must be byte-stable (rule order + /// deterministic), or every broadcast rewrites bucket metadata. + #[test] + fn test_merge_incoming_lifecycle_is_idempotent() { + let incoming = || Some(lc_config(vec![lc_rule("e1", Some(7), None), lc_rule("e2", Some(9), None)])); + let local = Some(lc_config(vec![lc_rule("t1", None, Some(30))])); + + let once = merge_incoming_lifecycle_config(incoming(), local, None).expect("first merge"); + let twice = merge_incoming_lifecycle_config(incoming(), Some(once.clone()), None).expect("second merge"); + + assert_eq!( + serialize(&once).expect("serialize once"), + serialize(&twice).expect("serialize twice"), + "merge must be idempotent for identical input" + ); + } + + /// The merged config records the expiry axis timestamp so the staleness + /// guard compares expiry updates against expiry updates (a local + /// transition-only edit must not shadow newer peer expiry updates). + #[test] + fn test_merge_incoming_lifecycle_stamps_expiry_updated_at() { + let updated_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let merged = merge_incoming_lifecycle_config(Some(lc_config(vec![lc_rule("e1", Some(7), None)])), None, Some(updated_at)) + .expect("merge should keep rules"); + + let stamped = merged.expiry_updated_at.expect("expiry_updated_at must be stamped"); + assert_eq!(OffsetDateTime::from(stamped).unix_timestamp(), updated_at.unix_timestamp()); + } + + /// MinIO's sender never emits del-marker-expiration rules + /// (CloneNonTransition drops them), so a MinIO expiry broadcast must not + /// delete this site's del-marker-only rules, and the local del-marker + /// side of a same-id rule is authoritative. + #[test] + fn test_merge_incoming_lifecycle_del_marker_rules_stay_local() { + let del_marker_only = |id: &str| { + let mut rule = lc_rule(id, None, None); + rule.del_marker_expiration = Some(s3s::dto::DelMarkerExpiration { days: Some(3) }); + rule + }; + + // A local del-marker-only rule survives an incoming expiry document + // that does not mention it. + let merged = merge_incoming_lifecycle_config( + Some(lc_config(vec![lc_rule("e1", Some(7), None)])), + Some(lc_config(vec![del_marker_only("dm-local")])), + None, + ) + .expect("merge should keep rules"); + let mut ids = rule_ids(&merged); + ids.sort_unstable(); + assert_eq!(ids, vec!["dm-local", "e1"]); + + // Same-id: the incoming expiry side wins, the local del-marker / + // abort-mpu side is authoritative and an incoming del-marker field is + // discarded at the trust boundary. + let mut local_mixed = lc_rule("r1", Some(1), None); + local_mixed.del_marker_expiration = Some(s3s::dto::DelMarkerExpiration { days: Some(3) }); + local_mixed.abort_incomplete_multipart_upload = Some(s3s::dto::AbortIncompleteMultipartUpload { + days_after_initiation: Some(5), + }); + let mut incoming_mixed = lc_rule("r1", Some(7), None); + incoming_mixed.del_marker_expiration = Some(s3s::dto::DelMarkerExpiration { days: Some(9) }); + + let merged = + merge_incoming_lifecycle_config(Some(lc_config(vec![incoming_mixed])), Some(lc_config(vec![local_mixed])), None) + .expect("merge should keep rules"); + let r1 = &merged.rules[0]; + assert_eq!(r1.expiration.as_ref().and_then(|e| e.days), Some(7)); + assert_eq!(r1.del_marker_expiration.as_ref().and_then(|d| d.days), Some(3), "local del-marker wins"); + assert_eq!( + r1.abort_incomplete_multipart_upload + .as_ref() + .and_then(|a| a.days_after_initiation), + Some(5), + "local abort-mpu wins" + ); + } + + /// Only a well-delimited zero-rule `` document is + /// the delete statement; truncated or foreign payloads must be rejected, + /// not treated as a delete that erases local expiry rules. + #[test] + fn test_zero_rule_lifecycle_tombstone_recognition() { + assert!(is_zero_rule_lifecycle_tombstone( + b"2026-01-01T00:00:00Z" + )); + assert!(is_zero_rule_lifecycle_tombstone( + b"\n" + )); + assert!(is_zero_rule_lifecycle_tombstone(b"")); + + // Documents with rules are not tombstones (they must parse strictly). + assert!(!is_zero_rule_lifecycle_tombstone( + b"x" + )); + // Truncated / malformed / foreign payloads are rejected. + assert!(!is_zero_rule_lifecycle_tombstone(b"")); + assert!(!is_zero_rule_lifecycle_tombstone(b"")); + assert!(!is_zero_rule_lifecycle_tombstone(b"garbage")); + assert!(!is_zero_rule_lifecycle_tombstone(b"")); + assert!(!is_zero_rule_lifecycle_tombstone(b"")); + // Malformed children inside a well-delimited root are still rejected + // (second review round): a dangling open tag, stray text, an + // unclosed child, or nested markup is not a tombstone. + assert!(!is_zero_rule_lifecycle_tombstone( + b"" + )); + assert!(!is_zero_rule_lifecycle_tombstone( + b"stray text" + )); + assert!(!is_zero_rule_lifecycle_tombstone( + b"" + )); + assert!(!is_zero_rule_lifecycle_tombstone( + b"" + )); + assert!(!is_zero_rule_lifecycle_tombstone( + b"&bogus;" + )); + assert!(!is_zero_rule_lifecycle_tombstone(b"")); + assert!(!is_zero_rule_lifecycle_tombstone( + b"2026-01-01T00:00:00Z" + )); + assert!(!is_zero_rule_lifecycle_tombstone(b"&bogus;")); + } + + #[test] + fn test_lifecycle_merge_holds_metadata_transaction_across_read_and_write() { + let source = include_str!("site_replication.rs"); + let apply = source + .split("async fn apply_bucket_meta_item") + .nth(1) + .and_then(|rest| rest.split("fn group_info_requires_upsert").next()) + .expect("apply_bucket_meta_item source"); + let acquire = apply + .find("acquire_bucket_metadata_transaction_lock_for_incarnation") + .expect("lifecycle merge transaction acquisition"); + let read = apply.find("get_config_from_disk").expect("fresh lifecycle config read"); + let write = apply + .find("update_under_transaction_lock") + .expect("lifecycle config write under transaction"); + + assert!( + acquire < read && read < write, + "the transaction must span the lifecycle read, merge, and write" + ); + } + + /// The staleness axis an incoming lc-config item must beat: the expiry + /// axis when present; the whole-config write time only for deleted or + /// legacy-with-expiry state; epoch for a transition-only config (its + /// whole-config time moves on transition edits and must not shadow + /// independent peer expiry updates — review finding). + #[test] + fn test_local_lifecycle_staleness_axis_selection() { + let whole = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let axis_ts = OffsetDateTime::from_unix_timestamp(1_600_000_000).expect("timestamp"); + + let mut with_axis = lc_config(vec![lc_rule("e1", Some(7), None)]); + with_axis.expiry_updated_at = Some(s3s::dto::Timestamp::from(axis_ts)); + assert_eq!(local_lifecycle_staleness_axis(Some(&with_axis), whole), axis_ts); + + let legacy_with_expiry = lc_config(vec![lc_rule("e1", Some(7), None)]); + assert_eq!(local_lifecycle_staleness_axis(Some(&legacy_with_expiry), whole), whole); + + let transition_only = lc_config(vec![lc_rule("t1", None, Some(30))]); + assert_eq!( + local_lifecycle_staleness_axis(Some(&transition_only), whole), + OffsetDateTime::UNIX_EPOCH, + "a transition-only config has no expiry state to protect" + ); + + assert_eq!(local_lifecycle_staleness_axis(None, whole), whole, "deletion lower bound"); + } + + /// Sender-side filter: only the expiry subset leaves this site. MinIO + /// peers install incoming rules verbatim, so a full document would plant + /// this site's transition rules there. + #[test] + fn test_lifecycle_expiry_subset_xml_strips_transitions() { + let full = serialize(&lc_config(vec![lc_rule("mixed", Some(1), Some(30)), lc_rule("t-only", None, Some(7))])) + .expect("serialize full config"); + + let subset = lifecycle_expiry_subset_xml(&full).expect("expiry subset should remain"); + let parsed: s3s::dto::BucketLifecycleConfiguration = deserialize(&subset).expect("subset should parse"); + assert_eq!(rule_ids(&parsed), vec!["mixed"]); + assert!(parsed.rules[0].transitions.is_none(), "transition side must not travel"); + + let transition_only = + serialize(&lc_config(vec![lc_rule("t-only", None, Some(7))])).expect("serialize transition-only config"); + assert!( + lifecycle_expiry_subset_xml(&transition_only).is_none(), + "a transition-only config states 'no expiry rules' (delete semantics)" + ); + assert!(lifecycle_expiry_subset_xml(b"").is_none()); + } + + /// A local parse failure must forward the document unfiltered — mapping + /// it to `None` would delete the peers' replicated expiry rules. + #[test] + fn test_lifecycle_expiry_subset_xml_forwards_unparseable_config() { + let garbage = b""; + assert_eq!(lifecycle_expiry_subset_xml(garbage).as_deref(), Some(garbage.as_slice())); + } + // `role` is part of the bucket's S3-visible configuration. Repairing a reverse rule must // drop only a sender-owned site-replication ARN, never an operator's own role — the same // rule the merge path applies, so both paths agree on what is ours to rewrite. diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index d630f557f..ec89f5e4e 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -321,6 +321,34 @@ pub(crate) mod metadata_sys { crate::storage::storage_api::acquire_bucket_metadata_transaction_lock(bucket).await } + pub(crate) async fn acquire_bucket_metadata_transaction_lock_for_incarnation( + bucket: &str, + expected_incarnation_id: uuid::Uuid, + ) -> Result { + super::ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock_for_incarnation( + bucket, + expected_incarnation_id, + ) + .await + } + + pub(crate) async fn update_under_transaction_lock( + guard: &super::ecstore_bucket::metadata_sys::BucketMetadataMutationGuard, + bucket: &str, + config_file: &str, + data: Vec, + ) -> Result { + super::ecstore_bucket::metadata_sys::update_under_transaction_lock(guard, bucket, config_file, data).await + } + + pub(crate) async fn delete_under_transaction_lock( + guard: &super::ecstore_bucket::metadata_sys::BucketMetadataMutationGuard, + bucket: &str, + config_file: &str, + ) -> Result { + super::ecstore_bucket::metadata_sys::delete_under_transaction_lock(guard, bucket, config_file).await + } + pub(crate) async fn update_bucket_targets_under_transaction_lock( guard: &super::ecstore_bucket::metadata_sys::BucketMetadataMutationGuard, bucket: &str, diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index fdc711a17..7d55035e9 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -1158,6 +1158,19 @@ fn lifecycle_has_expiry_rules(config: &BucketLifecycleConfiguration) -> bool { }) } +/// Status-independent presence of the expiry subset that site replication +/// propagates (`replicateILMExpiry`): expiration / noncurrent-version +/// expiration only. Distinct from [`lifecycle_has_expiry_rules`], which +/// filters on ENABLED for scanner scheduling — editing a Disabled expiry rule +/// must still advance the replication axis. Del-marker expiration and +/// abort-multipart are site-local and never travel. +fn lifecycle_rules_have_expiry(config: &BucketLifecycleConfiguration) -> bool { + config + .rules + .iter() + .any(|rule| rule.expiration.is_some() || rule.noncurrent_version_expiration.is_some()) +} + fn lifecycle_has_abort_multipart_rules(config: &BucketLifecycleConfiguration) -> bool { config.rules.iter().any(|rule| { rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED) @@ -2186,7 +2199,24 @@ impl DefaultBucketUsecase { return Err(s3_error!(InvalidArgument, "{err}")); } - input_cfg.expiry_updated_at = Some(Timestamp::from(time::OffsetDateTime::now_utc())); + // Stamp the expiry axis only when the expiry subset can have changed + // (MinIO: HasExpiry() || expiryRuleRemoved). Site-replication peers + // judge lc-config staleness on this axis; a transition-only edit that + // advanced it would let this site's stale expiry subset shadow — and + // roll back — a newer peer expiry edit fleet-wide. + let previous_expiry_updated_at = match metadata_sys::get_lifecycle_config(&bucket).await { + Ok((previous, _)) => { + if lifecycle_rules_have_expiry(&input_cfg) || lifecycle_rules_have_expiry(&previous) { + Some(Timestamp::from(time::OffsetDateTime::now_utc())) + } else { + previous.expiry_updated_at + } + } + // No previous config (or unreadable): stamping is the + // conservative pre-existing behavior. + Err(_) => lifecycle_rules_have_expiry(&input_cfg).then(|| Timestamp::from(time::OffsetDateTime::now_utc())), + }; + input_cfg.expiry_updated_at = previous_expiry_updated_at; let data = serialize_config(&input_cfg)?; update_bucket_config_for_incarnation(&bucket, BUCKET_LIFECYCLE_CONFIG, data, expected_incarnation_id) .await @@ -2197,7 +2227,14 @@ impl DefaultBucketUsecase { let mut item = sr_bucket_meta_item(bucket.clone(), "lc-config"); item.expiry_lc_config = Some(serialize_config(&input_cfg).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?); - item.expiry_updated_at = item.updated_at; + // The item travels with the expiry axis, not the wall clock: a site + // whose expiry knowledge is old (or absent — UNIX_EPOCH) must not + // out-rank newer peer expiry state at the receivers. + item.expiry_updated_at = input_cfg + .expiry_updated_at + .clone() + .map(time::OffsetDateTime::from) + .or(Some(time::OffsetDateTime::UNIX_EPOCH)); if let Err(err) = site_replication_bucket_meta_hook(item).await { warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle hook failed"); } From 04b9c8fd3627486f5b0f37d451b3ae8fa0938325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 16 Aug 2026 05:55:15 +0800 Subject: [PATCH 29/71] fix(admin): stream madmin ReplicationMRF documents from /v3/replication/mrf (#6126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(admin): pin madmin ReplicationMRF stream contract for /v3/replication/mrf Red-light evidence for backlog#1675 P1-13 (mrf half): madmin's BucketReplicationMRF decodes the response one ReplicationMRF document at a time, so the current aggregate envelope decodes as a single phantom row with an empty object in 'mc replicate backlog'. The new contract tests assert the desired bare-document stream (exact madmin json tags, empty body for an empty backlog) and fail against the current render_mrf_backlog extraction, which preserves the envelope-only behavior: - mrf_stream_renders_bare_madmin_documents: envelope keys leak, no per-entry documents - mrf_stream_renders_empty_body_for_no_entries: empty backlog still renders the envelope (phantom row) - mrf_aggregate_envelope_retains_counters: PerObjectEntriesAvailable never advertises the enumerable stream * fix(admin): stream madmin ReplicationMRF documents from /v3/replication/mrf The mrf endpoint returned a single aggregate envelope, which madmin's json.Decoder loop decoded as one phantom row (empty object) in 'mc replicate backlog' (backlog#1675 P1-13, mrf half; the diff half was fixed in #5799 and this mirrors its pattern). - Default response is now a bare stream of ReplicationMRF documents (exact madmin json tags; Size/TargetARNs as ignored extension keys) built from the durable backlog ledger; an empty backlog renders an empty body, so mc shows zero rows instead of a phantom row. - The aggregate counter envelope moves behind ?aggregate=true (RustFS extension) and now advertises PerObjectEntriesAvailable whenever the durable backlog is readable. - An unreadable backlog is signalled out-of-band via x-rustfs-replication-mrf-backlog-unavailable (mirrors the diff truncation header) plus a warn event, since the bare stream cannot carry source health. - The madmin node parameter is accepted but documented as a no-op: the durable ledger is cluster-shared with no per-node attribution. - Delete-marker purge entries fall back to the marker version id so those rows keep a version identity. * fix(admin): fail the mrf stream request when the durable ledger is unreadable Review: madmin only decodes the body of a 200, so the out-of-band unavailability header was invisible to it and an unreadable ledger read as a clean zero-row backlog. Stream mode now returns 503; aggregate mode keeps the availability fields. * fix(admin): gate, bound, and null-map the mrf stream Second review round: - Authorization: the default stream enumerates object names and version ids, which a metrics-only principal must not see — it now requires admin:ReplicationDiff (MinIO parity, route policy updated); ?aggregate=true carries no object identities and keeps admin:GetReplicationMetrics. - The nil UUID is RustFS's in-memory null-version sentinel and now leaves as the S3 wire token 'null' instead of a zero UUID (a pre-versioning object scanned after versioning + existing-object replication can persist it into the ledger). - The durable ledger is not bounded by the in-memory pending cap and the body is buffered before send; the stream now stops at 10,000 documents and signals truncation via x-rustfs-replication-mrf-truncated (mirroring the diff endpoint) plus a warn event, instead of staging an unbounded body. * fix(admin): reject truncated MRF streams --------- Co-authored-by: Zhengchao An --- rustfs/src/admin/handlers/replication.rs | 363 +++++++++++++++++++++-- rustfs/src/admin/route_policy.rs | 5 +- 2 files changed, 350 insertions(+), 18 deletions(-) diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index a2a6054c4..890a11800 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -1158,7 +1158,7 @@ struct MrfResponse { fn build_mrf_response( bucket: String, bucket_stats: &BucketStats, - durable: crate::admin::storage_api::replication::DurableMrfBacklog, + durable: &crate::admin::storage_api::replication::DurableMrfBacklog, ) -> MrfResponse { let observation_scope = if bucket_stats.replication_stats.cluster_complete { "cluster_aggregated" @@ -1228,7 +1228,10 @@ fn build_mrf_response( total_failed_size, queued_count: queued.count, queued_size: queued.bytes, - per_object_entries_available: false, + // The default (non-aggregate) response mode streams the durable + // backlog per object, so the enumerable API exists whenever the + // backlog is readable. + per_object_entries_available: durable.available, runtime_stats_available: bucket_stats.replication_stats.provider_available, cluster_complete: bucket_stats.replication_stats.cluster_complete, observed_node_count: bucket_stats.replication_stats.observed_node_count, @@ -1240,23 +1243,165 @@ fn build_mrf_response( } } +/// One durable MRF backlog entry rendered for the default (madmin-compatible) +/// stream. Field names are the exact json tags of madmin-go `ReplicationMRF` +/// (replication-api.go), which `mc replicate backlog` decodes one JSON +/// document at a time. `Size` and `TargetARNs` are RustFS extension keys with +/// no madmin counterpart; Go decoders ignore unknown keys. +#[derive(Debug, Serialize)] +struct MrfEntryDocument { + /// The durable backlog is a cluster-shared ledger with no per-node + /// attribution, so the madmin `nodeName` tag is always empty. + #[serde(rename = "nodeName")] + node_name: String, + #[serde(rename = "bucket")] + bucket: String, + #[serde(rename = "object")] + object: String, + #[serde(rename = "versionId")] + version_id: String, + #[serde(rename = "retryCount")] + retry_count: i32, + #[serde(rename = "Size")] + size: i64, + #[serde(rename = "TargetARNs", skip_serializing_if = "Vec::is_empty")] + target_arns: Vec, +} + +/// Upper bound on the number of documents one stream response emits. The +/// durable ledger is not bounded by the in-memory pending cap (recovery can +/// persist far larger generations), and the body is buffered before send, so +/// an unbounded read could stage hundreds of MB per request. The handler +/// rejects a response beyond this bound instead of returning a partial 200. +const REPLICATION_MRF_MAX_STREAM_ENTRIES: usize = 10_000; + +/// Project the durable backlog into madmin `ReplicationMRF` documents, +/// scoped to `bucket` when it is non-empty (madmin allows an empty bucket to +/// mean "across all buckets"), bounded by +/// [`REPLICATION_MRF_MAX_STREAM_ENTRIES`]. Returns the documents and whether +/// the backlog was truncated. +fn mrf_entry_documents( + bucket: &str, + durable: &crate::admin::storage_api::replication::DurableMrfBacklog, +) -> (Vec, bool) { + let mut documents = Vec::new(); + let mut truncated = false; + for entry in durable + .entries + .iter() + .filter(|entry| bucket.is_empty() || entry.bucket == bucket) + { + if documents.len() >= REPLICATION_MRF_MAX_STREAM_ENTRIES { + truncated = true; + break; + } + documents.push(MrfEntryDocument { + node_name: String::new(), + bucket: entry.bucket.clone(), + object: entry.object.clone(), + // Delete-marker purge entries track the marker version separately; + // fall back to it so those rows still carry a version identity. + // The nil UUID is RustFS's in-memory null-version sentinel and + // must leave as the S3 wire token, not a zero UUID. + version_id: entry + .version_id + .or(entry.delete_marker_version_id) + .map(|v| { + if v.is_nil() { + rustfs_filemeta::NULL_VERSION_ID.to_string() + } else { + v.to_string() + } + }) + .unwrap_or_default(), + retry_count: entry.retry_count, + size: entry.size, + target_arns: entry.target_arns.clone(), + }); + } + (documents, truncated) +} + +/// Render the MRF backlog as a response body. +/// +/// Default (madmin-compatible) mode emits one `ReplicationMRF` JSON document +/// per line with no envelope — madmin's `BucketReplicationMRF` reads the body +/// with a `json.Decoder` loop, so an envelope object would decode as a single +/// entry whose `"Bucket"` key case-insensitively matches +/// `ReplicationMRF.Bucket` (a phantom row in `mc replicate backlog`), and an +/// empty backlog must render an empty body so the loop ends on io.EOF with +/// zero rows. +/// +/// `aggregate=true` (RustFS extension) keeps the enveloped counter shape; +/// backlog-source health (`RuntimeStatsAvailable`/`DurableBacklogAvailable`) +/// is only representable there — an unreadable ledger fails the stream +/// request outright in the handler (madmin only decodes the body of a 200, +/// so an empty stream would read as a healthy zero-row backlog). +fn render_mrf_backlog( + response: &MrfResponse, + durable: &crate::admin::storage_api::replication::DurableMrfBacklog, + aggregate: bool, +) -> Result<(Vec, bool), serde_json::Error> { + if aggregate { + return Ok((serde_json::to_vec(response)?, false)); + } + + let (documents, truncated) = mrf_entry_documents(&response.bucket, durable); + let mut data = Vec::new(); + for entry in documents { + serde_json::to_writer(&mut data, &entry)?; + data.push(b'\n'); + } + Ok((data, truncated)) +} + +fn ensure_complete_mrf_stream(truncated: bool) -> S3Result<()> { + if truncated { + return Err(S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "durable MRF backlog exceeds the stream limit; narrow the bucket scope or drain the backlog".to_string(), + )); + } + Ok(()) +} + /// `GET /v3/replication/mrf` /// /// Reports the failed-replication backlog (MinIO's MRF concept) for a bucket. /// -/// Compatibility note: MinIO returns a stream of individual MRF entries. RustFS -/// deliberately returns aggregate runtime and durable counters instead. -/// `PerObjectEntriesAvailable` remains false until an enumerable API exists. -/// `PerTargetDurableEntriesAvailable` is false when the durable backlog includes -/// older entries that cannot be attributed to a target. +/// The default response is a madmin-compatible stream of `ReplicationMRF` +/// documents built from the durable backlog ledger (in-memory failures that +/// have not been flushed yet — the persister runs every few seconds — are not +/// visible). `?aggregate=true` (RustFS extension) returns the enveloped +/// runtime + durable counter shape instead; `PerTargetDurableEntriesAvailable` +/// is false there when the durable backlog includes older entries that cannot +/// be attributed to a target. +/// +/// The madmin `node` parameter is accepted but has no filtering effect: the +/// durable ledger is cluster-shared with no per-node attribution, so every +/// node serves the same (complete) backlog. +/// +/// Authorization: the stream requires `admin:ReplicationDiff` (it enumerates +/// object names and version ids, MinIO parity); `?aggregate=true` carries no +/// object identities and requires only `admin:GetReplicationMetrics`. pub struct ReplicationMrfHandler {} #[async_trait::async_trait] impl Operation for ReplicationMrfHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - validate_replication_admin_request(&req, AdminAction::GetReplicationMetricsAction).await?; - let queries = extract_query_params(&req.uri); + let aggregate = queries.get("aggregate").map(String::as_str) == Some("true"); + // The default stream enumerates object names and version ids, which + // a metrics-only principal must not see; gate it on the same action + // MinIO uses for this endpoint. The aggregate counters carry no + // object identities and keep the metrics action. + let action = if aggregate { + AdminAction::GetReplicationMetricsAction + } else { + AdminAction::ReplicationDiff + }; + validate_replication_admin_request(&req, action).await?; + let Some(bucket) = queries.get("bucket").filter(|b| !b.is_empty()).cloned() else { return Err(s3_error!(InvalidRequest, "bucket is required")); }; @@ -1280,14 +1425,48 @@ impl Operation for ReplicationMrfHandler { return Err(ApiError::from(err).into()); } + if let Some(node) = queries.get("node").filter(|node| !node.is_empty() && node.as_str() != "all") { + // The durable backlog ledger is cluster-shared with no per-node + // attribution, so a node-scoped request still sees the complete + // (superset) backlog. + debug!(node = %node, "replication mrf node filter has no effect on the cluster-shared backlog"); + } + let durable = crate::admin::storage_api::replication::read_durable_mrf_backlog(store).await; let bucket_stats = cluster_replication_stats(&bucket, app_context_from_req(&req)).await; - let response = build_mrf_response(bucket, &bucket_stats, durable); + let response = build_mrf_response(bucket, &bucket_stats, &durable); - let data = serde_json::to_vec(&response) + if !durable.available && !aggregate { + // The madmin stream has no envelope to carry source health, and + // madmin only decodes the body of a 200 — an empty stream would + // read as a clean, healthy zero-row backlog. Fail loudly instead; + // aggregate mode still reports the availability fields. + tracing::warn!( + bucket = %response.bucket, + "durable MRF backlog is unreadable; failing the stream request — use aggregate=true to see source health" + ); + return Err(S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "durable MRF backlog is unreadable; retry, or use aggregate=true for source health".to_string(), + )); + } + + let (data, truncated) = render_mrf_backlog(&response, &durable, aggregate) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize failed: {e}")))?; let mut headers = HeaderMap::new(); headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + if truncated { + tracing::warn!( + event = "replication_mrf_stream_rejected", + component = "admin", + subsystem = "replication", + result = "rejected", + bucket = %response.bucket, + max_entries = REPLICATION_MRF_MAX_STREAM_ENTRIES, + "replication mrf stream exceeds the response limit" + ); + } + ensure_complete_mrf_stream(truncated)?; Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), headers)) } } @@ -1297,7 +1476,8 @@ mod tests { use super::{ REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, RemoteTargetCredentialsRequest, RemoteTargetRequest, ReplicationDiffEntry, SUPPORTED_REMOTE_TARGET_API, TargetUpdateOp, build_mrf_response, extract_query_params, - parse_remote_target_update_ops, render_replication_diff, unique_replication_peers, validate_remote_target_tls_settings, + parse_remote_target_update_ops, render_mrf_backlog, render_replication_diff, unique_replication_peers, + validate_remote_target_tls_settings, }; use crate::admin::storage_api::bucket::target::{BucketTarget, LatencyStat}; use crate::admin::storage_api::replication::{BucketStats, DurableMrfBacklog, MrfOpKind, MrfReplicateEntry}; @@ -1515,7 +1695,7 @@ mod tests { ], }; - let response = build_mrf_response("bucket-a".to_string(), &stats, durable); + let response = build_mrf_response("bucket-a".to_string(), &stats, &durable); let json = serde_json::to_value(response).expect("MRF response should serialize"); assert_eq!(json["TotalFailedCount"], 3); @@ -1527,7 +1707,9 @@ mod tests { assert_eq!(json["RuntimeStatsAvailable"], true); assert_eq!(json["ClusterComplete"], false); assert_eq!(json["Targets"][0]["ObservationScope"], "partial_cluster"); - assert_eq!(json["PerObjectEntriesAvailable"], false); + // The bare stream enumerates the durable backlog per object, so a + // readable backlog advertises the enumerable API. + assert_eq!(json["PerObjectEntriesAvailable"], true); assert_eq!(json["PerTargetDurableEntriesAvailable"], true); let targets = json["Targets"].as_array().expect("targets should serialize as an array"); @@ -1574,7 +1756,7 @@ mod tests { }], }; - let response = build_mrf_response("bucket-a".to_string(), &stats, durable); + let response = build_mrf_response("bucket-a".to_string(), &stats, &durable); let json = serde_json::to_value(response).expect("MRF response should serialize"); assert_eq!(json["DurableBacklogAvailable"], true); @@ -1592,7 +1774,7 @@ mod tests { #[test] fn mrf_response_distinguishes_unavailable_sources_from_valid_zero() { - let unavailable = build_mrf_response("bucket-a".to_string(), &BucketStats::default(), DurableMrfBacklog::default()); + let unavailable = build_mrf_response("bucket-a".to_string(), &BucketStats::default(), &DurableMrfBacklog::default()); let unavailable_json = serde_json::to_value(unavailable).expect("unavailable response should serialize"); assert_eq!(unavailable_json["RuntimeStatsAvailable"], false); assert_eq!(unavailable_json["DurableBacklogAvailable"], false); @@ -1605,7 +1787,7 @@ mod tests { let valid_empty = build_mrf_response( "bucket-a".to_string(), &valid_empty_stats, - DurableMrfBacklog { + &DurableMrfBacklog { available: true, entries: Vec::new(), }, @@ -1618,6 +1800,153 @@ mod tests { assert_eq!(valid_empty_json["PerTargetDurableEntriesAvailable"], true); } + fn sample_durable_backlog() -> DurableMrfBacklog { + DurableMrfBacklog { + available: true, + entries: vec![ + MrfReplicateEntry { + bucket: "bucket-a".to_string(), + object: "object-a".to_string(), + version_id: Some(uuid::Uuid::from_u128(7)), + retry_count: 2, + size: 250, + op: MrfOpKind::Object, + target_arns: vec!["arn-a".to_string()], + ..Default::default() + }, + MrfReplicateEntry { + bucket: "other-bucket".to_string(), + object: "object-b".to_string(), + version_id: None, + retry_count: 0, + size: 999, + op: MrfOpKind::Object, + target_arns: Vec::new(), + ..Default::default() + }, + ], + } + } + + /// madmin's `BucketReplicationMRF` decodes the body one `ReplicationMRF` + /// JSON document at a time; the default response must therefore be a bare + /// document stream with madmin's exact json tags, not an envelope. + #[test] + fn mrf_stream_renders_bare_madmin_documents() { + let durable = sample_durable_backlog(); + let response = build_mrf_response("bucket-a".to_string(), &BucketStats::default(), &durable); + + let (body, _) = render_mrf_backlog(&response, &durable, false).expect("stream body should serialize"); + let text = String::from_utf8(body).expect("body should be utf-8"); + let lines: Vec<&str> = text.lines().filter(|line| !line.trim().is_empty()).collect(); + + // Only the entry matching the requested bucket is streamed. + assert_eq!(lines.len(), 1, "expected one MRF document, got: {text}"); + let doc: serde_json::Value = serde_json::from_str(lines[0]).expect("each line should be a JSON document"); + assert_eq!(doc["bucket"], "bucket-a"); + assert_eq!(doc["object"], "object-a"); + assert_eq!(doc["versionId"], uuid::Uuid::from_u128(7).to_string()); + assert_eq!(doc["retryCount"], 2); + // madmin `ReplicationMRF` has a `nodeName` tag; the durable backlog is + // cluster-shared, so RustFS reports an empty node name. + assert_eq!(doc["nodeName"], ""); + // The envelope keys must not leak into the stream: a `"Bucket"` key + // would case-insensitively populate `ReplicationMRF.Bucket` and render + // a phantom row in `mc replicate backlog`. + assert!(doc.get("Bucket").is_none()); + assert!(doc.get("Targets").is_none()); + } + + /// An empty backlog must produce an empty body: madmin's decoder loop then + /// terminates on io.EOF with zero rows instead of one phantom row. + #[test] + fn mrf_stream_renders_empty_body_for_no_entries() { + let durable = DurableMrfBacklog { + available: true, + entries: Vec::new(), + }; + let response = build_mrf_response("bucket-a".to_string(), &BucketStats::default(), &durable); + + let (body, _) = render_mrf_backlog(&response, &durable, false).expect("stream body should serialize"); + assert!( + body.is_empty(), + "empty backlog must serialize to an empty body, got: {}", + String::from_utf8_lossy(&body) + ); + } + + /// `?aggregate=true` (RustFS extension) keeps the enveloped counter shape. + #[test] + fn mrf_aggregate_envelope_retains_counters() { + let durable = sample_durable_backlog(); + let response = build_mrf_response("bucket-a".to_string(), &BucketStats::default(), &durable); + + let (body, _) = render_mrf_backlog(&response, &durable, true).expect("aggregate body should serialize"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("aggregate body should be one JSON object"); + assert_eq!(json["Bucket"], "bucket-a"); + assert_eq!(json["DurableCount"], 1); + assert_eq!(json["DurableBacklogAvailable"], true); + // The bare stream is an enumerable per-object API, so the aggregate + // shell now truthfully advertises it whenever the backlog is readable. + assert_eq!(json["PerObjectEntriesAvailable"], true); + } + + /// The nil UUID is RustFS's in-memory null-version sentinel; the wire + /// token is `null`, never the zero UUID (second review round). + #[test] + fn mrf_stream_maps_nil_version_to_null_token() { + let durable = DurableMrfBacklog { + available: true, + entries: vec![MrfReplicateEntry { + bucket: "bucket-a".to_string(), + object: "null-version-object".to_string(), + version_id: Some(uuid::Uuid::nil()), + retry_count: 1, + size: 10, + op: MrfOpKind::Object, + ..Default::default() + }], + }; + let response = build_mrf_response("bucket-a".to_string(), &BucketStats::default(), &durable); + + let (body, truncated) = render_mrf_backlog(&response, &durable, false).expect("stream body should serialize"); + assert!(!truncated); + let doc: serde_json::Value = + serde_json::from_str(String::from_utf8(body).expect("utf-8").lines().next().expect("one line")) + .expect("line should be a JSON document"); + assert_eq!(doc["versionId"], "null"); + } + + /// The durable ledger is not bounded by the in-memory pending cap; the + /// stream must stop at the documented bound and signal truncation + /// (second review round). + #[test] + fn mrf_stream_truncates_at_the_documented_bound() { + let entries = (0..super::REPLICATION_MRF_MAX_STREAM_ENTRIES + 1) + .map(|index| MrfReplicateEntry { + bucket: "bucket-a".to_string(), + object: format!("object-{index}"), + retry_count: 1, + op: MrfOpKind::Object, + ..Default::default() + }) + .collect(); + let durable = DurableMrfBacklog { + available: true, + entries, + }; + let response = build_mrf_response("bucket-a".to_string(), &BucketStats::default(), &durable); + + let (body, truncated) = render_mrf_backlog(&response, &durable, false).expect("stream body should serialize"); + assert!(truncated, "one entry past the bound must signal truncation"); + assert_eq!( + String::from_utf8(body).expect("utf-8").lines().count(), + super::REPLICATION_MRF_MAX_STREAM_ENTRIES + ); + let error = super::ensure_complete_mrf_stream(truncated).expect_err("partial streams must not return 200"); + assert_eq!(error.code(), &s3s::S3ErrorCode::ServiceUnavailable); + } + #[test] fn test_extract_query_params_decodes_percent_encoded_values() { let uri: Uri = "/rustfs/admin/v3/list-remote-targets?bucket=foo%2Fbar&flag=a+b" diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 774ec7175..474d3ad0f 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -1459,10 +1459,13 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ REPLICATION_DIFF, RouteRiskLevel::Sensitive, ), + // The default stream enumerates object names/version ids and requires + // ReplicationDiff (MinIO parity); only ?aggregate=true relaxes to + // GetReplicationMetrics in the handler. admin( HttpMethod::Get, "/rustfs/admin/v3/replication/mrf", - GET_REPLICATION_METRICS, + REPLICATION_DIFF, RouteRiskLevel::Sensitive, ), ]; From dcf3e4b9e8c618459911cca0a2bdae5503fd6106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 16 Aug 2026 05:56:04 +0800 Subject: [PATCH 30/71] fix(replication): transport and persist LWW timestamps for tag, retention, and legal hold (#6129) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(replication): pin missing LWW timestamp header transport Red-light tests for the replication timestamp three-header contract: - put_object_headers_carry_replication_timestamp_headers pins that PutObjectOptions::header() must emit the x-{rustfs,minio}-source-replication-{tagging,retention,legalhold}-timestamp headers when the internal timestamps are set (currently missing). - test_put_opts_from_headers_gates_replication_timestamp_persistence_on_authorization and test_complete_multipart_opts_persist_replication_timestamps_when_authorized pin that an authorized replication PUT / multipart complete must persist the inbound timestamps into the internal metadata keys while unauthorized requests must not (currently never persisted). - fake_s3_target journals the three timestamp headers per request (ReplicationTimestampHeaders on RequestRecord) so sender-side e2e assertions can observe what a real target receives; self-test included. * fix(replication): transport and persist LWW timestamps for tag, retention, and legal hold Active-active conflict resolution for concurrent tag/retention/legal-hold edits needs the source's per-category modification times on both sides of the wire; the three AdvancedPutOptions timestamp fields were dead and the headers were neither sent nor parsed. - Emit x-{rustfs,minio}-source-replication-{tagging,retention,legalhold}- timestamp from PutObjectOptions::header(); names and RFC3339 values interoperate with MinIO (minio-go constants.go, object-api-options.go), pinned by a header_compat wire-name test. - Default the three AdvancedPutOptions timestamps to UNIX_EPOCH and skip epoch values in header(), so "never modified" is not sent as a modification made now. - Parse the headers only on authorized replication PUTs and multipart completes, expose them as Option on ObjectOptions, and persist them into the dual-prefix internal metadata keys so the outbound pass (replication_target_boundary) reads the source's timestamps instead of the mod_time fallback. - Record the local tagging timestamp in the PutObjectTagging and DeleteObjectTagging eval metadata, mirroring the object-lock handlers; without it the sender only ever had the mod_time fallback to offer. Receiver-side LWW comparison (keep newer stored category metadata over a stale inbound copy) is left as a TODO at the parse site. * fix(replication): load the stored tagging timestamp independently of remaining tags Review: DeleteObjectTagging persists the tagging-timestamp internal key but leaves the object tagless, and the outbound mapper only loaded the key inside the user_tags-nonempty branch — the deletion's LWW timestamp stayed at the epoch and the header was omitted, so the deletion could never win conflict resolution on the replica. The stored key is now loaded unconditionally; the mod_time fallback still applies only while tags exist (MinIO parity), and a tagless object without the key keeps the epoch default (no header). Deletion-path regression test added. * fix(storage): reserve replication transport names at metadata ingest Second review round: a client PUT of x-amz-meta-x-rustfs-source-replication-tagging-timestamp materialized the bare transport key as stored user metadata. The outbound replication header builder forwards user metadata verbatim on a server-authorized request, so the receiver would persist the attacker-chosen value as trusted internal LWW state — and for a tagless object nothing later overwrites it. The ingest namespacing guard now reserves the whole x-rustfs-source- / x-minio-source- families (the new timestamps and their siblings: source-mtime/-etag/-version-id/-replication-request), folding forged keys back under x-amz-meta-. Forged-ingress regression covers both prefixes and a sibling. * fix(replication): harden timestamp replay * fix(app): route retention helper through facade --------- Co-authored-by: overtrue --- crates/e2e_test/src/fake_s3_target/mod.rs | 92 +++++++- .../ecstore/src/bucket/bucket_target_sys.rs | 74 +++++- .../replication_target_boundary.rs | 166 ++++++++++++- crates/ecstore/src/object_api/types.rs | 6 + crates/utils/src/http/header_compat.rs | 29 +++ rustfs/src/app/multipart_usecase.rs | 8 +- rustfs/src/app/object_usecase.rs | 21 +- rustfs/src/app/storage_api.rs | 5 +- rustfs/src/storage/ecfs.rs | 12 +- rustfs/src/storage/options.rs | 223 +++++++++++++++++- rustfs/src/storage/storage_api.rs | 5 +- 11 files changed, 606 insertions(+), 35 deletions(-) diff --git a/crates/e2e_test/src/fake_s3_target/mod.rs b/crates/e2e_test/src/fake_s3_target/mod.rs index d094f8cdb..c8ddcecf3 100644 --- a/crates/e2e_test/src/fake_s3_target/mod.rs +++ b/crates/e2e_test/src/fake_s3_target/mod.rs @@ -76,6 +76,18 @@ const SOURCE_MTIME_HEADERS: [&str; 2] = ["x-rustfs-source-mtime", "x-minio-sourc const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] = ["x-rustfs-source-replication-request", "x-minio-source-replication-request"]; const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"]; +const SOURCE_TAGGING_TIMESTAMP_HEADERS: [&str; 2] = [ + "x-rustfs-source-replication-tagging-timestamp", + "x-minio-source-replication-tagging-timestamp", +]; +const SOURCE_RETENTION_TIMESTAMP_HEADERS: [&str; 2] = [ + "x-rustfs-source-replication-retention-timestamp", + "x-minio-source-replication-retention-timestamp", +]; +const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [ + "x-rustfs-source-replication-legalhold-timestamp", + "x-minio-source-replication-legalhold-timestamp", +]; const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"]; const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"]; @@ -118,6 +130,25 @@ pub enum FaultAction { WrongEtag, } +/// Replication LWW timestamp headers observed on a request, journaled so +/// sender-side tests can assert what a real target would receive. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ReplicationTimestampHeaders { + pub tagging: Option, + pub retention: Option, + pub legalhold: Option, +} + +impl ReplicationTimestampHeaders { + fn from_headers(headers: &HeaderMap) -> Self { + Self { + tagging: header_value(headers, &SOURCE_TAGGING_TIMESTAMP_HEADERS).map(bounded_journal_value), + retention: header_value(headers, &SOURCE_RETENTION_TIMESTAMP_HEADERS).map(bounded_journal_value), + legalhold: header_value(headers, &SOURCE_LEGALHOLD_TIMESTAMP_HEADERS).map(bounded_journal_value), + } + } +} + /// Credential-free request metadata retained for deterministic assertions. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RequestRecord { @@ -131,6 +162,7 @@ pub struct RequestRecord { pub part_number: Option, pub content_length: Option, pub consumed_bytes: Option, + pub replication_timestamps: ReplicationTimestampHeaders, pub fault: Option, } @@ -536,7 +568,15 @@ impl S3Access for FaultAccess { .get(CONTENT_LENGTH) .and_then(|value| value.to_str().ok()) .and_then(|value| value.parse().ok()); - let fault = record_request(&self.control, operation, context.method().clone(), parsed, content_length); + let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers()); + let fault = record_request( + &self.control, + operation, + context.method().clone(), + parsed, + content_length, + replication_timestamps, + ); if let Some(RequestFault { action: FaultAction::Status(status), .. @@ -589,6 +629,7 @@ fn record_request( method: Method, parsed: ParsedRequest, content_length: Option, + replication_timestamps: ReplicationTimestampHeaders, ) -> Option { let mut state = lock(control); let action = parsed @@ -613,6 +654,7 @@ fn record_request( part_number: parsed.part_number, content_length, consumed_bytes: None, + replication_timestamps, fault: action.clone(), }); action.map(|action| RequestFault { sequence, action }) @@ -1699,6 +1741,52 @@ mod tests { .await?) } + #[tokio::test] + async fn journals_replication_timestamp_headers() -> Result<(), BoxError> { + let target = FakeS3Target::start().await?; + target.create_bucket("target-bucket"); + let client = client(&target); + + client + .put_object() + .bucket("target-bucket") + .key("plain") + .body(ByteStream::from_static(b"plain")) + .send() + .await?; + client + .put_object() + .bucket("target-bucket") + .key("stamped") + .body(ByteStream::from_static(b"stamped")) + .customize() + .map_request(move |mut request| { + let headers = request.headers_mut(); + headers.insert("x-rustfs-source-replication-tagging-timestamp", "2026-01-02T03:04:05Z"); + headers.insert("x-minio-source-replication-retention-timestamp", "2026-01-02T03:04:06Z"); + headers.insert("x-rustfs-source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z"); + Ok::<_, std::convert::Infallible>(request) + }) + .send() + .await?; + + let requests = target.requests(); + let plain = requests + .iter() + .find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("plain")) + .expect("plain PUT must be journaled"); + assert_eq!(plain.replication_timestamps, ReplicationTimestampHeaders::default()); + + let stamped = requests + .iter() + .find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("stamped")) + .expect("stamped PUT must be journaled"); + assert_eq!(stamped.replication_timestamps.tagging.as_deref(), Some("2026-01-02T03:04:05Z")); + assert_eq!(stamped.replication_timestamps.retention.as_deref(), Some("2026-01-02T03:04:06Z")); + assert_eq!(stamped.replication_timestamps.legalhold.as_deref(), Some("2026-01-02T03:04:07Z")); + Ok(()) + } + macro_rules! assert_sdk_error { ($error:expr, $status:expr, $code:expr) => {{ let error = &$error; @@ -2985,6 +3073,7 @@ mod tests { part_number: None, }, Some(0), + ReplicationTimestampHeaders::default(), ); } let records = lock(&control).requests.clone(); @@ -3006,6 +3095,7 @@ mod tests { part_number: None, }, None, + ReplicationTimestampHeaders::default(), ); { let bounded_records = lock(&bounded_control); diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 1e1e18dc0..c6bcc5742 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -58,7 +58,9 @@ use rustfs_utils::http::{ }; use rustfs_utils::http::{ SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, - SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header, + SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST, + SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID, + insert_header, }; use rustls_pki_types::pem::PemObject; use serde::{Deserialize, Serialize}; @@ -1476,9 +1478,12 @@ impl Default for AdvancedPutOptions { replication_status: ReplicationStatusType::Pending, source_mtime: OffsetDateTime::now_utc(), replication_request: false, - retention_timestamp: OffsetDateTime::now_utc(), - tagging_timestamp: OffsetDateTime::now_utc(), - legalhold_timestamp: OffsetDateTime::now_utc(), + // UNIX_EPOCH means "never modified": header() must not emit a + // timestamp header for it, otherwise a receiver would treat an + // unset category as a modification made right now. + retention_timestamp: OffsetDateTime::UNIX_EPOCH, + tagging_timestamp: OffsetDateTime::UNIX_EPOCH, + legalhold_timestamp: OffsetDateTime::UNIX_EPOCH, replication_validity_check: false, } } @@ -1675,6 +1680,16 @@ impl PutObjectOptions { ); } + for (suffix, timestamp) in [ + (SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, self.internal.tagging_timestamp), + (SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, self.internal.retention_timestamp), + (SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, self.internal.legalhold_timestamp), + ] { + if timestamp.unix_timestamp() != 0 { + insert_header(&mut header, suffix, timestamp.format(&Rfc3339).unwrap_or_default()); + } + } + if self.internal.replication_request { insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); } @@ -2842,6 +2857,57 @@ mod tests { ); } + #[test] + fn put_object_headers_carry_replication_timestamp_headers() { + // MinIO receivers resolve concurrent tag/retention/legal-hold edits by + // last-writer-wins on these headers (object-api-options.go parses them + // as RFC3339); a replica without them loses every conflict resolution. + let mut opts = PutObjectOptions::default(); + opts.internal.replication_request = true; + let tagging = OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("valid timestamp"); + let retention = OffsetDateTime::from_unix_timestamp(1_700_000_002).expect("valid timestamp"); + let legalhold = OffsetDateTime::from_unix_timestamp(1_700_000_003).expect("valid timestamp"); + opts.internal.tagging_timestamp = tagging; + opts.internal.retention_timestamp = retention; + opts.internal.legalhold_timestamp = legalhold; + + let header = opts.header(); + for (suffix, expected) in [ + ("source-replication-tagging-timestamp", tagging), + ("source-replication-retention-timestamp", retention), + ("source-replication-legalhold-timestamp", legalhold), + ] { + assert_eq!( + rustfs_utils::http::get_header(&header, suffix).as_deref(), + Some(expected.format(&Rfc3339).expect("RFC3339 timestamp").as_str()), + "replication put requests must carry the {suffix} header" + ); + } + } + + #[test] + fn put_object_headers_omit_unset_replication_timestamps() { + // UNIX_EPOCH means "never modified on the source"; sending it would + // make the receiver treat an unset category as a fresh modification. + let mut opts = PutObjectOptions::default(); + opts.internal.replication_request = true; + opts.internal.tagging_timestamp = OffsetDateTime::UNIX_EPOCH; + opts.internal.retention_timestamp = OffsetDateTime::UNIX_EPOCH; + opts.internal.legalhold_timestamp = OffsetDateTime::UNIX_EPOCH; + + let header = opts.header(); + for suffix in [ + "source-replication-tagging-timestamp", + "source-replication-retention-timestamp", + "source-replication-legalhold-timestamp", + ] { + assert!( + rustfs_utils::http::get_header(&header, suffix).is_none(), + "unset {suffix} must not be sent to replication targets" + ); + } + } + #[tokio::test] async fn get_remote_target_client_internal_rejects_loopback_endpoint() { let sys = BucketTargetSys::default(); diff --git a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs index 156c94cd7..e82ab7e52 100644 --- a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs @@ -27,8 +27,10 @@ use rustfs_utils::http::{ AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, - SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map, - is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key, ssec_replication_transport_header, + SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, + SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, + get_str, insert_header_map, is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key, + ssec_replication_transport_header, }; use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; @@ -119,6 +121,27 @@ fn classify_replication_source_encryption(metadata: &HashMap) -> } } +fn is_legacy_source_replication_timestamp_key(key: &str) -> bool { + fn has_prefix_and_suffix(key: &str, prefix: &str, suffix: &str) -> bool { + let key = key.as_bytes(); + key.len() == prefix.len() + suffix.len() + && key[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes()) + && key[prefix.len()..].eq_ignore_ascii_case(suffix.as_bytes()) + } + + [ + SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, + SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, + SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, + ] + .iter() + .any(|suffix| { + ["x-rustfs-", "x-minio-"] + .iter() + .any(|prefix| has_prefix_and_suffix(key, prefix, suffix)) + }) +} + pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap) -> bool { rustfs_replication::is_ssec_encrypted(user_defined) } @@ -176,6 +199,11 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo) continue; } + if is_legacy_source_replication_timestamp_key(key) { + meta.insert(format!("x-amz-meta-{key}"), value.to_string()); + continue; + } + if is_internal_key(key) || is_standard_header(key) { continue; } @@ -259,15 +287,23 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo) if !tags.is_empty() { put_options.user_tags = tags; - put_options.internal.tagging_timestamp = - if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) { - OffsetDateTime::parse(×tamp, &Rfc3339) - .map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))? - } else { - object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) - }; } } + // Load the stored tagging timestamp independently of whether any tags + // remain: DeleteObjectTagging leaves the object tagless but stamps this + // key, and the deletion's LWW timestamp must still reach the replica. + // With no stored key, fall back to mod_time only while tags exist + // (MinIO parity); a tagless object without the key was never tagged and + // keeps the epoch default (no header). + put_options.internal.tagging_timestamp = if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) + { + OffsetDateTime::parse(×tamp, &Rfc3339) + .map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))? + } else if !put_options.user_tags.is_empty() { + object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) + } else { + OffsetDateTime::UNIX_EPOCH + }; let metadata = &*object_info.user_defined; @@ -283,13 +319,15 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo) put_options.cache_control = cache_control.to_string(); } - if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE) { + if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE).filter(|mode| !mode.is_empty()) { put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str())); } if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) { - put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339) - .map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?; + if !retain_until_date.is_empty() { + put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339) + .map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?; + } put_options.internal.retention_timestamp = if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) { OffsetDateTime::parse(×tamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH) @@ -694,6 +732,110 @@ mod tests { assert!(options.internal.replication_request); } + /// DeleteObjectTagging leaves the object tagless but stamps the + /// tagging-timestamp internal key; the deletion's LWW timestamp must + /// still be loaded (and therefore sent) so the replica can order the + /// deletion against concurrent tag edits. + #[test] + fn replication_put_options_carry_tagging_timestamp_after_tag_deletion() { + let mut metadata = std::collections::HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, SUFFIX_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z".to_string()); + + let object_info = ObjectInfo { + user_defined: Arc::new(metadata), + user_tags: Arc::new(String::new()), + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + version_id: Some(Uuid::nil()), + ..Default::default() + }; + + let (options, _) = replication_put_object_options("", &object_info).expect("build put options"); + + assert!(options.user_tags.is_empty()); + assert_eq!( + options.internal.tagging_timestamp, + OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp"), + "the stored tagging timestamp must load independently of remaining tags" + ); + + // A tagless object without the stored key was never tagged: the epoch + // default keeps the header unsent. + let untagged = ObjectInfo { + user_tags: Arc::new(String::new()), + mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")), + version_id: Some(Uuid::nil()), + ..Default::default() + }; + let (options, _) = replication_put_object_options("", &untagged).expect("build put options"); + assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH); + } + + #[test] + fn replication_put_options_do_not_promote_legacy_user_timestamp_metadata() { + let legacy_keys = [ + "x-rustfs-source-replication-tagging-timestamp", + "x-rustfs-source-replication-retention-timestamp", + "x-rustfs-source-replication-legalhold-timestamp", + "x-minio-source-replication-tagging-timestamp", + "x-minio-source-replication-retention-timestamp", + "x-minio-source-replication-legalhold-timestamp", + ]; + let object_info = ObjectInfo { + user_defined: Arc::new( + legacy_keys + .iter() + .map(|key| (key.to_string(), "2099-01-02T03:04:05Z".to_string())) + .collect(), + ), + ..Default::default() + }; + + let (options, _) = replication_put_object_options("", &object_info).expect("build put options"); + + for legacy_key in legacy_keys { + assert!(!options.user_metadata.contains_key(legacy_key)); + assert_eq!( + options + .user_metadata + .get(&format!("x-amz-meta-{legacy_key}")) + .map(String::as_str), + Some("2099-01-02T03:04:05Z") + ); + } + assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH); + assert_eq!(options.internal.retention_timestamp, OffsetDateTime::UNIX_EPOCH); + assert_eq!(options.internal.legalhold_timestamp, OffsetDateTime::UNIX_EPOCH); + } + + #[test] + fn replication_put_options_carry_retention_timestamp_after_clear() { + let mut metadata = HashMap::from([ + (AMZ_OBJECT_LOCK_MODE.to_string(), String::new()), + (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), String::new()), + ]); + rustfs_utils::http::insert_str(&mut metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, "2026-01-02T03:04:05Z".to_string()); + let object_info = ObjectInfo { + user_defined: Arc::new(metadata), + ..Default::default() + }; + + let (options, _) = replication_put_object_options("", &object_info).expect("retention clear must replicate"); + + assert!(options.mode.is_none()); + assert_eq!(options.retain_until_date, OffsetDateTime::UNIX_EPOCH); + assert_eq!( + options.internal.retention_timestamp, + OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp") + ); + let headers = options.header(); + assert!(!headers.contains_key(AMZ_OBJECT_LOCK_MODE)); + assert!(!headers.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE)); + assert_eq!( + rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP).as_deref(), + Some("2026-01-02T03:04:05Z") + ); + } + #[test] fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() { use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER}; diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 291b9039a..624cb7593 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -277,6 +277,12 @@ pub struct ObjectOptions { /// fence avoids recursively acquiring the read lock behind a queued writer. pub bucket_lifecycle_lock_fence: Option, pub replication_request: bool, + /// Source-cluster LWW timestamps carried by an authorized replication + /// request; None when the source never modified the category. Only the + /// replication-authorized options builders may set these. + pub replication_tagging_timestamp: Option, + pub replication_retention_timestamp: Option, + pub replication_legalhold_timestamp: Option, /// Authorized SSE-C replication passthrough: the body is already /// ciphertext, so the write path must not encrypt or compress it and /// stores the restored encryption metadata verbatim. Only the diff --git a/crates/utils/src/http/header_compat.rs b/crates/utils/src/http/header_compat.rs index 38486ca07..c36632d8e 100644 --- a/crates/utils/src/http/header_compat.rs +++ b/crates/utils/src/http/header_compat.rs @@ -50,6 +50,14 @@ pub const SUFFIX_SOURCE_DELETEMARKER: &str = "source-deletemarker"; pub const SUFFIX_SOURCE_PROXY_REQUEST: &str = "source-proxy-request"; pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request"; pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check"; +// LWW timestamps for replicated tag/retention/legal-hold modifications. MinIO +// declares these with mixed case (internal/http/headers.go: +// X-Minio-Source-Replication-Tagging-Timestamp / -Retention-Timestamp / +// -LegalHold-Timestamp); HTTP header names compare case-insensitively, so the +// lowercase suffix forms interoperate. Values are RFC3339 on the wire. +pub const SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP: &str = "source-replication-tagging-timestamp"; +pub const SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP: &str = "source-replication-retention-timestamp"; +pub const SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP: &str = "source-replication-legalhold-timestamp"; pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc"; /// Returns true if the key is object-encryption metadata understood by RustFS or MinIO. @@ -196,6 +204,27 @@ mod tests { assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42)); } + #[test] + fn replication_timestamp_headers_match_minio_wire_names() { + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z"); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, "2026-01-02T03:04:06Z"); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, "2026-01-02T03:04:07Z"); + + // The exact names MinIO's object-api-options.go reads (its Get() + // canonicalizes case, so a case-insensitive match is wire-equivalent). + for name in [ + "X-Minio-Source-Replication-Tagging-Timestamp", + "X-Minio-Source-Replication-Retention-Timestamp", + "X-Minio-Source-Replication-LegalHold-Timestamp", + "x-rustfs-source-replication-tagging-timestamp", + "x-rustfs-source-replication-retention-timestamp", + "x-rustfs-source-replication-legalhold-timestamp", + ] { + assert!(headers.contains_key(name), "replication timestamp header {name} must be written"); + } + } + #[test] fn test_get_header() { let mut headers = HeaderMap::new(); diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 1b871a0df..a20b13188 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -44,8 +44,8 @@ use super::storage_api::multipart_usecase::io::{HashReader, WriteEncryption, Wri use super::storage_api::multipart_usecase::object_utils::to_s3s_etag; use super::storage_api::multipart_usecase::options::{ copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization, - get_content_sha256_with_query, get_opts, namespace_reserved_user_metadata, parse_copy_source_range, - put_opts_with_replication_authorization, validate_archive_content_encoding, + get_content_sha256_with_query, get_opts, has_replication_retention_update, namespace_reserved_user_metadata, + parse_copy_source_range, put_opts_with_replication_authorization, validate_archive_content_encoding, }; use super::storage_api::multipart_usecase::request_context::spawn_traced_join; use super::storage_api::multipart_usecase::s3_api::multipart::{ @@ -777,7 +777,9 @@ impl DefaultMultipartUsecase { let mut metadata = create_multipart_upload_metadata(input_metadata, &req.headers, tagging, storage_class.as_ref()); - let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some(); + let has_explicit_object_lock_retention = object_lock_mode.is_some() + || object_lock_retain_until_date.is_some() + || has_replication_retention_update(&req.headers, replication_authorized); let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?; if let Some(object_lock_metadata) = build_put_like_object_lock_metadata( &bucket, diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 3bf35bc1b..913d6eb52 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -86,8 +86,8 @@ use super::storage_api::object_usecase::object_utils::to_s3s_etag; use super::storage_api::object_usecase::options::{ copy_dst_opts_with_replication_authorization, copy_src_opts, del_opts_with_versioning, extract_metadata, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_content_sha256_with_query, get_opts, - namespace_reserved_user_metadata, normalize_content_encoding_for_storage, preserve_unclassified_user_metadata, - put_opts_with_replication_authorization, validate_archive_content_encoding, + has_replication_retention_update, namespace_reserved_user_metadata, normalize_content_encoding_for_storage, + preserve_unclassified_user_metadata, put_opts_with_replication_authorization, validate_archive_content_encoding, }; use super::storage_api::object_usecase::request_context::{self, spawn_traced, spawn_traced_join}; use super::storage_api::object_usecase::s3_api::multipart::parse_list_parts_params; @@ -5799,7 +5799,9 @@ impl DefaultObjectUsecase { )?; let mut metadata = metadata.unwrap_or_default(); - let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some(); + let has_explicit_object_lock_retention = object_lock_mode.is_some() + || object_lock_retain_until_date.is_some() + || has_replication_retention_update(&req.headers, inbound_replication_put); let object_lock_config_stage_start = put_stage_metrics_enabled.then(Instant::now); let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?; rustfs_io_metrics::record_put_object_stage_duration_from("app_object_lock_config_lookup", object_lock_config_stage_start); @@ -10081,6 +10083,19 @@ mod tests { assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("COMPLIANCE")); assert!(metadata.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER)); assert_eq!(metadata.get("x-amz-meta-x-amz-object-lock-mode").map(String::as_str), Some("GOVERNANCE")); + + let mut replication_headers = HeaderMap::new(); + insert_header(&mut replication_headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + insert_header( + &mut replication_headers, + rustfs_utils::http::SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, + "2026-01-01T00:00:00Z", + ); + let mut replica_metadata = HashMap::new(); + let explicit_clear = has_replication_retention_update(&replication_headers, true); + apply_bucket_default_lock_retention("bucket", &state, &mut replica_metadata, explicit_clear).unwrap(); + assert!(!replica_metadata.contains_key(AMZ_OBJECT_LOCK_MODE_LOWER)); + assert!(!replica_metadata.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER)); } fn pax_record(key: &str, value: &[u8]) -> Vec { diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index b97b16bcb..839b8a38f 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -996,8 +996,9 @@ pub(crate) mod options { copy_dst_opts_with_replication_authorization, copy_src_opts, del_opts_with_versioning, extract_metadata, extract_metadata_from_mime, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts_with_replication_authorization, get_content_sha256_with_query, get_opts, - namespace_reserved_user_metadata, normalize_content_encoding_for_storage, parse_copy_source_range, - preserve_unclassified_user_metadata, put_opts_with_replication_authorization, validate_archive_content_encoding, + has_replication_retention_update, namespace_reserved_user_metadata, normalize_content_encoding_for_storage, + parse_copy_source_range, preserve_unclassified_user_metadata, put_opts_with_replication_authorization, + validate_archive_content_encoding, }; } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index ce4be49d5..9369d7224 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -45,7 +45,7 @@ use rustfs_targets::EventName; use rustfs_utils::http::headers::{ AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, }; -use rustfs_utils::http::{SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, insert_str}; +use rustfs_utils::http::{SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, insert_str}; use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error}; use std::collections::HashMap; use std::fmt::Debug; @@ -461,6 +461,11 @@ impl S3 for FS { let mut eval_metadata = HashMap::new(); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default()); + insert_str( + &mut eval_metadata, + SUFFIX_TAGGING_TIMESTAMP, + OffsetDateTime::now_utc().format(&Rfc3339).unwrap_or_default(), + ); opts.eval_metadata = Some(eval_metadata); } @@ -1645,6 +1650,11 @@ impl S3 for FS { let mut eval_metadata = HashMap::new(); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default()); + insert_str( + &mut eval_metadata, + SUFFIX_TAGGING_TIMESTAMP, + OffsetDateTime::now_utc().format(&Rfc3339).unwrap_or_default(), + ); opts.eval_metadata = Some(eval_metadata); } diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 609b2ea62..ac920b5c3 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -17,11 +17,13 @@ use crate::storage::storage_api::options_consumer::contract::{object::HTTPPrecon use http::header::{IF_MATCH, IF_NONE_MATCH}; use http::{HeaderMap, HeaderValue}; use rustfs_utils::http::{ - AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_FORCE_DELETE, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, - SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST, - SUFFIX_SOURCE_VERSION_ID, get_header, + AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_FORCE_DELETE, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, + SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, + SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, + SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, + SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID, SUFFIX_TAGGING_TIMESTAMP, get_header, header_compat::{MINIO_ENCRYPTION_PREFIX, RUSTFS_ENCRYPTION_PREFIX}, - insert_header_map, + insert_header_map, insert_str, metadata_compat::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX}, }; use rustfs_utils::http::{ @@ -422,6 +424,9 @@ pub fn get_complete_multipart_upload_opts_with_replication_authorization( preserve_etag, ..Default::default() }; + if replication_request { + apply_replication_timestamps_from_headers(headers, &mut opts); + } apply_replica_status_from_headers(headers, &mut opts, replication_request_authorized); fill_conditional_writes_opts_from_header(headers, &mut opts)?; @@ -458,6 +463,12 @@ pub fn put_opts_from_headers(headers: &HeaderMap, metadata: HashMap put_opts_from_headers_with_replication_authorization(headers, metadata, false) } +pub(crate) fn has_replication_retention_update(headers: &HeaderMap, replication_request_authorized: bool) -> bool { + replication_request_authorized + && get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") + && replication_timestamp_header(headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP).is_some() +} + pub fn put_opts_from_headers_with_replication_authorization( headers: &HeaderMap, metadata: HashMap, @@ -479,6 +490,7 @@ pub fn put_opts_from_headers_with_replication_authorization( if let Some(crc) = get_header(headers, SUFFIX_REPLICATION_SSEC_CRC) { insert_header_map(&mut opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC, crc.into_owned()); } + apply_replication_timestamps_from_headers(headers, &mut opts); } Ok(opts) } @@ -504,6 +516,47 @@ fn replication_source_mtime(headers: &HeaderMap) -> Option, suffix: &str) -> Option { + let value = get_header(headers, suffix)?; + let value = value.trim(); + match time::OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339) { + Ok(timestamp) => Some(timestamp), + Err(err) => { + tracing::warn!("Invalid {} value '{}' (replication request=true): {}", suffix, value, err); + None + } + } +} + +/// Callers must gate on an authorized replication request: these headers are +/// trusted source-cluster state, not client input. +fn apply_replication_timestamps_from_headers(headers: &HeaderMap, opts: &mut ObjectOptions) { + opts.replication_tagging_timestamp = replication_timestamp_header(headers, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP); + opts.replication_retention_timestamp = replication_timestamp_header(headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP); + opts.replication_legalhold_timestamp = replication_timestamp_header(headers, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP); + + // Persist into the internal metadata keys so a later outbound replication + // pass (replication_target_boundary) reads the source's modification + // times instead of falling back to mod_time. + // TODO(P1-6): receiver-side LWW is still missing — when the stored + // per-category timestamp is newer than the inbound one, the existing + // tags/retention/legal-hold should win instead of being overwritten. + for (timestamp, suffix) in [ + (opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP), + (opts.replication_retention_timestamp, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP), + (opts.replication_legalhold_timestamp, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP), + ] { + if let Some(timestamp) = timestamp + && let Ok(value) = timestamp.format(&time::format_description::well_known::Rfc3339) + { + insert_str(&mut opts.user_defined, suffix, value); + } + } +} + fn apply_replica_status_from_headers(headers: &HeaderMap, opts: &mut ObjectOptions, authorized: bool) { if !authorized { return; @@ -663,6 +716,13 @@ fn is_reserved_user_metadata_key(key: &str) -> bool { || starts_with_ignore_ascii_case(key, MINIO_INTERNAL_PREFIX) || starts_with_ignore_ascii_case(key, RUSTFS_ENCRYPTION_PREFIX) || starts_with_ignore_ascii_case(key, MINIO_ENCRYPTION_PREFIX) + // Replication transport names (source-replication timestamps, + // source-mtime/-etag/-version-id, ...). A bare stored key with one of + // these names is forwarded verbatim by the outbound replication + // header builder on a server-authorized request, so the receiver + // would persist attacker-chosen values as trusted internal LWW state. + || starts_with_ignore_ascii_case(key, "x-rustfs-source-") + || starts_with_ignore_ascii_case(key, "x-minio-source-") } fn stored_user_metadata_key(key: &str) -> String { @@ -1082,15 +1142,16 @@ mod tests { del_opts_with_versioning, detect_content_type_from_object_name, extract_metadata, extract_metadata_from_mime, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts, get_complete_multipart_upload_opts_with_replication_authorization, get_default_opts, get_opts, - namespace_reserved_user_metadata, parse_copy_source_range, put_opts, put_opts_from_headers, - put_opts_from_headers_with_replication_authorization, put_opts_with_replication_authorization, + has_replication_retention_update, namespace_reserved_user_metadata, parse_copy_source_range, put_opts, + put_opts_from_headers, put_opts_from_headers_with_replication_authorization, put_opts_with_replication_authorization, validate_archive_content_encoding, }; use http::{HeaderMap, HeaderValue}; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, - SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header, + SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, + SUFFIX_SOURCE_VERSION_ID, insert_header, }; use s3s::S3ErrorCode; use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration}; @@ -1520,6 +1581,24 @@ mod tests { assert!(opts.preserve_etag.is_none()); } + #[test] + fn test_replication_retention_update_requires_authorization() { + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, "2026-01-01T00:00:00Z"); + + assert!(!has_replication_retention_update(&headers, false)); + assert!(has_replication_retention_update(&headers, true)); + + let mut missing_request = HeaderMap::new(); + insert_header( + &mut missing_request, + SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, + "2026-01-01T00:00:00Z", + ); + assert!(!has_replication_retention_update(&missing_request, true)); + } + #[test] fn test_put_opts_from_headers_gates_ssec_passthrough_on_authorization() { use rustfs_utils::http::object_encryption_keys::{ @@ -1599,6 +1678,136 @@ mod tests { assert!(opts_invalid.mod_time.is_none()); } + /// A client PUT must not materialize the replication transport names as + /// bare stored user-metadata keys: the outbound replication header + /// builder forwards user metadata verbatim on a server-authorized + /// request, so a bare `x-rustfs-source-replication-*-timestamp` key would + /// deliver an attacker-chosen value into the replica's trusted internal + /// LWW state (for a tagless object nothing later overwrites it). + #[test] + fn test_replication_transport_names_cannot_be_forged_via_user_metadata() { + let mut headers = HeaderMap::new(); + for name in [ + "x-amz-meta-x-rustfs-source-replication-tagging-timestamp", + "x-amz-meta-x-minio-source-replication-legalhold-timestamp", + "x-rustfs-meta-x-rustfs-source-replication-retention-timestamp", + "x-amz-meta-x-rustfs-source-mtime", + ] { + headers.insert( + http::header::HeaderName::from_static(name), + HeaderValue::from_static("2026-01-02T03:04:05Z"), + ); + } + + let metadata = extract_metadata(&headers); + + for forged in [ + "x-rustfs-source-replication-tagging-timestamp", + "x-minio-source-replication-legalhold-timestamp", + "x-rustfs-source-replication-retention-timestamp", + "x-rustfs-source-mtime", + ] { + assert!( + !metadata.contains_key(forged), + "{forged} must not be storable as a bare user-metadata key" + ); + } + // The values survive, namespaced back under the user-metadata prefix. + assert_eq!( + metadata + .get("x-amz-meta-x-rustfs-source-replication-tagging-timestamp") + .map(String::as_str), + Some("2026-01-02T03:04:05Z") + ); + } + + #[test] + fn test_put_opts_from_headers_gates_replication_timestamp_persistence_on_authorization() { + // Sender-side LWW state (replication_target_boundary.rs) is read back + // from these internal metadata keys, so an authorized replication PUT + // must persist the inbound timestamp headers; an unauthorized client + // must not be able to forge them. + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + insert_header(&mut headers, "source-replication-tagging-timestamp", "2026-01-02T03:04:05Z"); + insert_header(&mut headers, "source-replication-retention-timestamp", "2026-01-02T03:04:06Z"); + insert_header(&mut headers, "source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z"); + + let untrusted = put_opts_from_headers(&headers, HashMap::new()).expect("ordinary PUT options should be created"); + for suffix in [ + "tagging-timestamp", + "objectlock-retention-timestamp", + "objectlock-legalhold-timestamp", + ] { + assert!( + rustfs_utils::http::get_str(&untrusted.user_defined, suffix).is_none(), + "unauthorized clients must not persist the {suffix} internal key" + ); + } + assert!(untrusted.replication_tagging_timestamp.is_none()); + assert!(untrusted.replication_retention_timestamp.is_none()); + assert!(untrusted.replication_legalhold_timestamp.is_none()); + + let trusted = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true) + .expect("authorized replication request should parse"); + let parse = |value: &str| { + time::OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339).expect("valid RFC3339") + }; + assert_eq!(trusted.replication_tagging_timestamp, Some(parse("2026-01-02T03:04:05Z"))); + assert_eq!(trusted.replication_retention_timestamp, Some(parse("2026-01-02T03:04:06Z"))); + assert_eq!(trusted.replication_legalhold_timestamp, Some(parse("2026-01-02T03:04:07Z"))); + for (suffix, expected) in [ + ("tagging-timestamp", "2026-01-02T03:04:05Z"), + ("objectlock-retention-timestamp", "2026-01-02T03:04:06Z"), + ("objectlock-legalhold-timestamp", "2026-01-02T03:04:07Z"), + ] { + assert_eq!( + rustfs_utils::http::get_str(&trusted.user_defined, suffix).as_deref(), + Some(expected), + "authorized replication must persist the {suffix} internal key" + ); + } + } + + #[test] + fn test_complete_multipart_opts_persist_replication_timestamps_when_authorized() { + let mut headers = HeaderMap::new(); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + insert_header(&mut headers, "replication-actual-object-size", "1"); + insert_header(&mut headers, "source-replication-tagging-timestamp", "2026-01-02T03:04:05Z"); + insert_header(&mut headers, "source-replication-retention-timestamp", "2026-01-02T03:04:06Z"); + insert_header(&mut headers, "source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z"); + + let untrusted = get_complete_multipart_upload_opts(&headers).expect("ordinary multipart options should be created"); + for suffix in [ + "tagging-timestamp", + "objectlock-retention-timestamp", + "objectlock-legalhold-timestamp", + ] { + assert!( + rustfs_utils::http::get_str(&untrusted.user_defined, suffix).is_none(), + "unauthorized multipart completes must not persist the {suffix} internal key" + ); + } + + let trusted = get_complete_multipart_upload_opts_with_replication_authorization(&headers, true) + .expect("authorized multipart complete should parse"); + for (suffix, expected) in [ + ("tagging-timestamp", "2026-01-02T03:04:05Z"), + ("objectlock-retention-timestamp", "2026-01-02T03:04:06Z"), + ("objectlock-legalhold-timestamp", "2026-01-02T03:04:07Z"), + ] { + assert_eq!( + rustfs_utils::http::get_str(&trusted.user_defined, suffix).as_deref(), + Some(expected), + "authorized multipart completes must persist the {suffix} internal key" + ); + } + assert!(trusted.replication_tagging_timestamp.is_some()); + assert!(trusted.replication_retention_timestamp.is_some()); + assert!(trusted.replication_legalhold_timestamp.is_some()); + } + #[test] fn test_put_opts_from_headers_with_replica_status() { let mut headers = HeaderMap::new(); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index f4f084a31..5d217f18e 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -185,8 +185,9 @@ pub(crate) mod options_consumer { copy_dst_opts_with_replication_authorization, copy_src_opts, del_opts_with_versioning, extract_metadata, extract_metadata_from_mime, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts_with_replication_authorization, get_content_sha256_with_query, get_opts, - namespace_reserved_user_metadata, normalize_content_encoding_for_storage, parse_copy_source_range, - preserve_unclassified_user_metadata, put_opts_with_replication_authorization, validate_archive_content_encoding, + has_replication_retention_update, namespace_reserved_user_metadata, normalize_content_encoding_for_storage, + parse_copy_source_range, preserve_unclassified_user_metadata, put_opts_with_replication_authorization, + validate_archive_content_encoding, }; pub(crate) mod contract { From 526d6f667eeb5d2fa99393442cd9834be7928083 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 16 Aug 2026 09:44:00 +0800 Subject: [PATCH 31/71] perf(ecstore): defer pending inline data shards (#6137) --- .../src/set_disk/core/io_primitives.rs | 216 +++++++++++++++++- 1 file changed, 214 insertions(+), 2 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index fff99a3cc..43057104d 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -749,6 +749,63 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( } } +fn data_read_inline_missing_shards_are_pending( + candidate: &FileInfo, + parts_metadata: &[FileInfo], + errors: &[Option], + disks: &[Option], + fanout_order: &[usize], + scheduled_fanout_len: usize, +) -> bool { + let Ok(erasure) = coding::Erasure::try_new_with_options( + candidate.erasure.data_blocks, + candidate.erasure.parity_blocks, + candidate.erasure.block_size, + candidate.uses_legacy_checksum, + ) else { + return false; + }; + let distribution = &candidate.erasure.distribution; + let mut data_shards_seen_or_pending = vec![false; erasure.data_shards]; + let mut missing_pending_data_shards = 0usize; + + for (disk_index, file_info) in parts_metadata.iter().enumerate() { + let Some(&block_index) = distribution.get(disk_index) else { + return false; + }; + if block_index == 0 || block_index > erasure.data_shards { + continue; + } + if !disks.get(disk_index).is_some_and(Option::is_some) { + return false; + } + + let data_slot = block_index - 1; + if file_info.name.is_empty() { + let scheduled_and_not_failed = fanout_order + .get(..scheduled_fanout_len) + .is_some_and(|scheduled_disks| scheduled_disks.contains(&disk_index)) + && errors.get(disk_index).is_some_and(Option::is_none); + if scheduled_and_not_failed { + data_shards_seen_or_pending[data_slot] = true; + missing_pending_data_shards = missing_pending_data_shards.saturating_add(1); + continue; + } + return false; + } + if file_info.erasure.index != block_index + || !file_info.has_valid_erasure_geometry() + || !metadata_early_stop_candidate_matches(file_info, candidate) + || file_info.data.as_ref().is_none_or(|data| data.is_empty()) + { + return false; + } + data_shards_seen_or_pending[data_slot] = true; + } + + missing_pending_data_shards > 0 && data_shards_seen_or_pending.into_iter().all(|seen_or_pending| seen_or_pending) +} + pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str { match err { DiskError::FileNotFound | DiskError::VolumeNotFound => GET_METADATA_RESPONSE_NOT_FOUND, @@ -2532,6 +2589,7 @@ impl SetDisks { } while let Some(result) = join_set.join_next().await { + let mut defer_pending_inline_data_shard = false; match result { Ok((index, res, elapsed)) => match res { Ok(file_info) => { @@ -2574,8 +2632,22 @@ impl SetDisks { { None => true, Some(reason) => { - force_full_wait = true; final_miss_reason_override = Some(reason); + if bounded_fanout + && reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD + && data_read_inline_missing_shards_are_pending( + candidate, + &ress, + &errors, + disks, + &fanout_order, + next_fanout_index, + ) + { + defer_pending_inline_data_shard = true; + } else { + force_full_wait = true; + } false } }, @@ -2621,6 +2693,7 @@ impl SetDisks { let pending_responses = join_set.len(); let should_hedge_single_pending_data_read = read_data && !force_full_wait + && !defer_pending_inline_data_shard && pending_responses == 1 && accumulator.can_still_reach_early_stop_with_pending(pending_responses); if bounded_fanout && force_full_wait { @@ -2633,6 +2706,7 @@ impl SetDisks { next_fanout_index = next_fanout_index.saturating_add(1); } } else if bounded_fanout + && !defer_pending_inline_data_shard && next_fanout_index < disks.len() && (!accumulator.can_still_reach_early_stop_with_pending(pending_responses) || should_hedge_single_pending_data_read) @@ -5790,9 +5864,20 @@ mod tests { object: &str, payload: &[u8], uses_legacy_checksum: bool, + ) -> Vec { + inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, uses_legacy_checksum, 2, 2).await + } + + async fn inline_metadata_fanout_fileinfos_with_geometry( + bucket: &str, + object: &str, + payload: &[u8], + uses_legacy_checksum: bool, + data_shards: usize, + parity_shards: usize, ) -> Vec { let distribution_key = metadata_distribution_key(bucket, object); - let mut base = FileInfo::new(&distribution_key, 2, 2); + let mut base = FileInfo::new(&distribution_key, data_shards, parity_shards); base.volume = bucket.to_string(); base.name = object.to_string(); base.size = i64::try_from(payload.len()).expect("test payload should fit i64"); @@ -5855,6 +5940,21 @@ mod tests { install_inline_metadata_fanout_files(disks, bucket, object, files).await; } + async fn install_inline_metadata_fanout_fileinfo_with_geometry( + disks: &[Option], + bucket: &str, + object: &str, + payload: &[u8], + data_shards: usize, + parity_shards: usize, + mutate: impl FnOnce(&mut [FileInfo]), + ) { + let mut files = + inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, false, data_shards, parity_shards).await; + mutate(&mut files); + install_inline_metadata_fanout_files(disks, bucket, object, files).await; + } + async fn install_inline_metadata_fanout_files(disks: &[Option], bucket: &str, object: &str, files: Vec) { let distribution = files .first() @@ -6075,6 +6175,118 @@ mod tests { drop(dirs); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn bounded_metadata_early_stop_waits_for_pending_inline_data_shard() { + const DISKS: usize = 6; + const DATA_SHARDS: usize = 4; + const PARITY_SHARDS: usize = 2; + let bucket = "bounded-inline-data-get-pending-shard-bucket"; + let object = + object_with_initial_data_shards(bucket, "bounded-inline-data-get-pending-shard-object", DATA_SHARDS, DATA_SHARDS); + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_inline_metadata_fanout_fileinfo_with_geometry( + &disks, + bucket, + &object, + b"verified inline payload", + DATA_SHARDS, + PARITY_SHARDS, + |_| {}, + ) + .await; + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let fanout_order = bounded_metadata_fanout_order(bucket, &object, DISKS, PARITY_SHARDS); + let distribution_key = metadata_distribution_key(bucket, &object); + let distribution = FileInfo::new(&distribution_key, DATA_SHARDS, PARITY_SHARDS) + .erasure + .distribution; + let paused_data_disk = *fanout_order + .iter() + .take(DATA_SHARDS) + .find(|disk_index| { + distribution + .get(**disk_index) + .is_some_and(|block_index| (1..=DATA_SHARDS).contains(block_index)) + }) + .expect("initial fanout should include a data shard to pause"); + let hedged_parity_disk = fanout_order[DATA_SHARDS]; + let unscheduled_parity_disk = fanout_order[DATA_SHARDS + 1]; + + let barrier = rename_fanout_barrier::arm(&object, paused_data_disk, rename_fanout_barrier::PHASE_READ_VERSION); + let tracker = rename_fanout_barrier::observe_tasks(&object); + let calls = disk_call_counters::observe(&object); + let disks_for_read = disks.clone(); + let object_for_read = object.clone(); + let mut read = tokio::spawn(async move { + SetDisks::read_all_fileinfo_observed( + &disks_for_read, + bucket, + bucket, + &object_for_read, + "", + true, + false, + false, + true, + PARITY_SHARDS, + ) + .await + }); + + tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused()) + .await + .expect("initial data shard should pause before returning"); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + while calls.for_disk(disk_call_counters::KIND_READ_VERSION, hedged_parity_disk) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("bounded fanout should hedge one parity disk while the data shard is pending"); + + assert!( + tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await.is_err(), + "inline data-read early-stop must wait for a scheduled missing data shard instead of forcing full wait" + ); + + barrier.release(); + let (parts_metadata, errs, diagnostics) = read + .await + .expect("metadata read task should not panic") + .expect("pending data shard should let the inline verifier finish"); + + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + 5, + "pending data-shard defer should not schedule the final parity disk" + ); + assert_eq!( + calls.for_disk(disk_call_counters::KIND_READ_VERSION, unscheduled_parity_disk), + 0, + "the remaining parity disk must stay unissued when pending data verification succeeds" + ); + assert_eq!( + tracker.running(), + 0, + "early-stop should drain spawned read_version tasks before returning" + ); + assert_eq!(diagnostics.total_responses(), 5); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 5); + assert!(errs.iter().all(Option::is_none)); + }, + ) + .await; + + drop(dirs); + } + #[tokio::test] async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() { let bucket = "legacy-inline-data-get-fanout-bucket"; From 0d86c50760105867bb31ceed0e7def4073a5f93c Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 09:50:16 +0800 Subject: [PATCH 32/71] fix(ecstore): classify remote inline early-stop misses (#6136) --- crates/ecstore/src/set_disk/core/io_primitives.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 43057104d..a251644ea 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -666,7 +666,8 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( parts_metadata: &[FileInfo], disks: &[Option], ) -> Option<&'static str> { - if !candidate.inline_data() { + // `inline_data` excludes remote objects; this diagnostic reports them separately. + if !rustfs_utils::http::contains_key_str(&candidate.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA) { return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE); } if candidate.is_compressed() @@ -6369,6 +6370,13 @@ mod tests { Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE) ); + let mut remote = candidate.clone(); + remote.transition_status = TRANSITION_COMPLETE.to_string(); + assert_eq!( + data_read_early_stop_inline_body_miss_reason(bucket, object, &remote, &parts_metadata, &disks).await, + Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE) + ); + let mut transformed = candidate.clone(); rustfs_utils::http::insert_str(&mut transformed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); assert_eq!( From d172d05e868b64745c1007bd6ea9b37cfe854472 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Sun, 16 Aug 2026 10:27:59 +0800 Subject: [PATCH 33/71] fix(ecstore): overlap metacache reader deadlines (#6098) * fix(ecstore): overlap metacache reader deadlines * fix(admin): avoid span guards across awaits --------- Co-authored-by: Henry Guo --- .../ecstore/src/cache_value/metacache_set.rs | 52 ++++++++++++++++++- rustfs/src/admin/handlers/audit.rs | 15 +++--- rustfs/src/admin/handlers/event.rs | 17 +++--- 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/crates/ecstore/src/cache_value/metacache_set.rs b/crates/ecstore/src/cache_value/metacache_set.rs index dfce7dbf3..003960bac 100644 --- a/crates/ecstore/src/cache_value/metacache_set.rs +++ b/crates/ecstore/src/cache_value/metacache_set.rs @@ -15,6 +15,7 @@ use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout}; use crate::disk::error::DiskError; use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions}; +use futures::future::join_all; use metrics::counter; use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof}; use std::{ @@ -655,6 +656,7 @@ async fn list_path_raw_inner( errs.push(None); } let mut pending_entries: Vec> = vec![None; readers.len()]; + let mut peek_outcomes: Vec> = std::iter::repeat_with(|| None).take(readers.len()).collect(); loop { let mut current = MetaCacheEntry::default(); @@ -676,6 +678,21 @@ async fn list_path_raw_inner( let mut has_err = 0; let mut agree = 0; + // Start every missing head read in the same round so one stalled + // disk cannot multiply the wait budget by the erasure-set width. + // Outcomes are still consumed below in stable disk-index order. + let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| { + if errs[i].is_some() || pending_entries[i].is_some() { + return None; + } + + let cancel = &revjob_rx; + Some(async move { (i, peek_with_timeout(cancel, reader, peek_timeout).await) }) + }); + for (i, outcome) in join_all(concurrent_peeks).await { + peek_outcomes[i] = Some(outcome); + } + for (i, r) in readers.iter_mut().enumerate() { if errs[i].is_some() { has_err += 1; @@ -685,7 +702,10 @@ async fn list_path_raw_inner( let entry = if let Some(entry) = pending_entries[i].take() { entry } else { - match peek_with_timeout(&revjob_rx, r, peek_timeout).await { + let Some(outcome) = peek_outcomes[i].take() else { + return Err(DiskError::Unexpected); + }; + match outcome { PeekOutcome::Ready(res) => { if let Some(entry) = res { // info!("read entry disk: {}, name: {}", i, entry.name); @@ -1295,6 +1315,36 @@ mod tests { assert_eq!(err, DiskError::Timeout); } + #[tokio::test(start_paused = true)] + async fn list_path_raw_bounds_multiple_stalled_readers_by_one_peek_deadline() { + let peek_timeout = Duration::from_millis(20); + let started = tokio::time::Instant::now(); + let err = list_path_raw( + CancellationToken::new(), + ListPathRawOptions { + disks: vec![None, None, None, None], + min_disks: 1, + test_reader_behaviors: vec![ + TestReaderBehavior::Stall, + TestReaderBehavior::Stall, + TestReaderBehavior::Stall, + TestReaderBehavior::Stall, + ], + peek_timeout: Some(peek_timeout), + ..Default::default() + }, + ) + .await + .expect_err("all stalled readers should fail the listing"); + + assert_eq!(err, DiskError::Timeout); + assert_eq!( + started.elapsed(), + peek_timeout, + "reader deadlines must overlap instead of accumulating once per disk" + ); + } + #[tokio::test] async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() { let entry = MetaCacheEntry { diff --git a/rustfs/src/admin/handlers/audit.rs b/rustfs/src/admin/handlers/audit.rs index dd8bfa17f..d00c7ed49 100644 --- a/rustfs/src/admin/handlers/audit.rs +++ b/rustfs/src/admin/handlers/audit.rs @@ -40,7 +40,7 @@ use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::LazyLock; -use tracing::{Span, error, info, warn}; +use tracing::{error, info, warn}; const LOG_COMPONENT_ADMIN_API: &str = "admin_api"; const LOG_SUBSYSTEM_AUDIT_TARGET: &str = "audit_target"; @@ -278,8 +278,6 @@ pub struct AuditTargetConfig {} #[async_trait::async_trait] impl Operation for AuditTargetConfig { async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let span = Span::current(); - let _enter = span.enter(); let (target_type, target_name) = extract_target_params(¶ms)?; authorize_audit_admin_request(&req, AdminAction::SetBucketTargetAction).await?; @@ -345,8 +343,6 @@ pub struct ListAuditTargets {} #[async_trait::async_trait] impl Operation for ListAuditTargets { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - let span = Span::current(); - let _enter = span.enter(); authorize_audit_admin_request(&req, AdminAction::GetBucketTargetAction).await?; let mut runtime_statuses = HashMap::new(); @@ -370,8 +366,6 @@ pub struct RemoveAuditTarget {} #[async_trait::async_trait] impl Operation for RemoveAuditTarget { async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let span = Span::current(); - let _enter = span.enter(); let (target_type, target_name) = extract_target_params(¶ms)?; authorize_audit_admin_request(&req, AdminAction::SetBucketTargetAction).await?; @@ -838,6 +832,13 @@ mod tests { extract_block_between_markers(src, "impl Operation for ListAuditTargets", "pub struct RemoveAuditTarget"); let delete_block = extract_block_between_markers(src, "impl Operation for RemoveAuditTarget", "#[cfg(test)]"); + for block in [put_block, list_block, delete_block] { + assert!( + !block.contains(".enter()"), + "async audit handlers must rely on request-future instrumentation instead of holding span guards across awaits" + ); + } + assert!( put_block.contains("authorize_audit_admin_request(&req, AdminAction::SetBucketTargetAction).await?;"), "audit target writes should require SetBucketTargetAction" diff --git a/rustfs/src/admin/handlers/event.rs b/rustfs/src/admin/handlers/event.rs index f4a1ba2d1..a820e1fc0 100644 --- a/rustfs/src/admin/handlers/event.rs +++ b/rustfs/src/admin/handlers/event.rs @@ -43,7 +43,7 @@ use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::LazyLock; -use tracing::{Span, error, info, warn}; +use tracing::{error, info, warn}; const LOG_COMPONENT_ADMIN_API: &str = "admin_api"; @@ -333,8 +333,6 @@ pub struct NotificationTarget {} #[async_trait::async_trait] impl Operation for NotificationTarget { async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let span = Span::current(); - let _enter = span.enter(); let (target_type, target_name) = extract_target_params(¶ms)?; let context = app_context_from_req(&req); @@ -401,8 +399,6 @@ pub struct ListNotificationTargets {} #[async_trait::async_trait] impl Operation for ListNotificationTargets { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - let span = Span::current(); - let _enter = span.enter(); authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?; refresh_persisted_module_switches_from_store().await.map_err(|err| { warn!( @@ -439,8 +435,6 @@ pub struct ListTargetsArns {} #[async_trait::async_trait] impl Operation for ListTargetsArns { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - let span = Span::current(); - let _enter = span.enter(); authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?; if let Some(reason) = notification_target_operation_block_reason( "querying notification target ARNs for bucket associations from the console", @@ -485,8 +479,6 @@ pub struct RemoveNotificationTarget {} #[async_trait::async_trait] impl Operation for RemoveNotificationTarget { async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - let span = Span::current(); - let _enter = span.enter(); let (target_type, target_name) = extract_target_params(¶ms)?; let context = app_context_from_req(&req); @@ -1006,6 +998,13 @@ mod tests { extract_block_between_markers(src, "impl Operation for ListTargetsArns", "pub struct RemoveNotificationTarget"); let delete_block = extract_block_between_markers(src, "impl Operation for RemoveNotificationTarget", "fn extract_param"); + for block in [put_block, list_block, arns_block, delete_block] { + assert!( + !block.contains(".enter()"), + "async notification handlers must rely on request-future instrumentation instead of holding span guards across awaits" + ); + } + assert!( put_block.contains("authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;"), "notification target writes should require SetBucketTargetAction" From 8d3511c1b3e5bf0d1aa9d9768f5c6855c89df39e Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Sun, 16 Aug 2026 10:28:11 +0800 Subject: [PATCH 34/71] fix(heal): preserve timeout budget across retries (#6101) Co-authored-by: Henry Guo --- crates/heal/src/error.rs | 9 ++++-- crates/heal/src/heal/manager.rs | 50 ++++++++++++++++++++------------- crates/heal/src/heal/task.rs | 40 +++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 23 deletions(-) diff --git a/crates/heal/src/error.rs b/crates/heal/src/error.rs index 68ca7eef4..413f02c54 100644 --- a/crates/heal/src/error.rs +++ b/crates/heal/src/error.rs @@ -82,8 +82,8 @@ impl Error { /// Whether a heal operation can be retried without changing its inputs. pub(crate) fn is_recoverable_heal(&self) -> bool { match self { - Error::TaskCancelled => false, - Error::TaskTimeout | Error::TransientSkip { .. } => true, + Error::TaskCancelled | Error::TaskTimeout => false, + Error::TransientSkip { .. } => true, Error::Storage(err) => { err.is_quorum_error() || matches!( @@ -165,4 +165,9 @@ mod tests { assert!(Error::Storage(EcstoreError::DiskNotFound).is_recoverable_heal()); assert!(Error::Storage(EcstoreError::VolumeNotFound).is_recoverable_heal()); } + + #[test] + fn task_timeout_is_terminal() { + assert!(!Error::TaskTimeout.is_recoverable_heal()); + } } diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index fa831b129..79ba6b169 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -673,6 +673,12 @@ fn retry_request_for_result(task: &HealTask, result: &Result<()>) -> Option<(Hea Some((request, delay, error)) } +async fn retry_request_for_result_with_budget(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> { + let (_, delay, error) = retry_request_for_result(task, result)?; + let request = task.retry_request_with_remaining_timeout().await.ok()?; + Some((request, delay, error)) +} + fn recoverable_heal_retry_delay(retry_attempt: u32) -> Duration { let retry_attempt = retry_attempt.clamp(1, 5); let delay = Duration::from_secs(2_u64.saturating_pow(retry_attempt)); @@ -690,7 +696,7 @@ pub struct HealConfig { pub max_concurrent_heals: usize, /// Maximum concurrent heal tasks allowed for a single erasure set pub max_concurrent_per_set: usize, - /// Task timeout + /// Aggregate task execution timeout across recoverable retries pub task_timeout: Duration, /// Queue size pub queue_size: usize, @@ -3106,7 +3112,7 @@ impl HealManager { "Heal scheduler task started" ); let result = task.execute().await; - let retry_request = retry_request_for_result(task.as_ref(), &result); + let retry_request = retry_request_for_result_with_budget(task.as_ref(), &result).await; match &result { Ok(_) => { debug!( @@ -4539,6 +4545,25 @@ mod tests { assert!(retry_error.contains("Lock acquisition timeout")); } + #[tokio::test] + async fn retry_request_for_result_preserves_remaining_timeout_budget() { + let storage: Arc = Arc::new(MockStorage); + let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None); + request.options.timeout = Some(Duration::from_secs(60)); + let task = HealTask::from_request(request, storage); + let result = task.execute().await; + + let (retry_request, _, _) = retry_request_for_result_with_budget(&task, &result) + .await + .expect("read quorum failure should retain the unused timeout budget"); + let remaining = retry_request + .options + .timeout + .expect("configured timeout should remain present"); + assert!(remaining < Duration::from_secs(60)); + assert!(remaining > Duration::from_secs(59)); + } + #[test] fn test_retry_request_for_incomplete_heal_rename() { let storage: Arc = Arc::new(MockStorage); @@ -6054,7 +6079,7 @@ mod tests { process_manager_queue_once(&manager).await; let defaulted_status = tokio::time::timeout(Duration::from_secs(1), async { loop { - if let Ok(status @ HealTaskStatus::Retrying { .. }) = manager.get_task_status(&defaulted_id).await { + if let Ok(status @ HealTaskStatus::Timeout) = manager.get_task_status(&defaulted_id).await { break status; } tokio::task::yield_now().await; @@ -6062,23 +6087,8 @@ mod tests { }) .await .expect("configured timeout should finish the task"); - assert!(matches!(defaulted_status, HealTaskStatus::Retrying { .. })); - assert_eq!( - manager - .retrying_heals - .lock() - .await - .get(&defaulted_id) - .expect("timed out task should retain its retry request") - .request - .options - .timeout, - Some(Duration::ZERO) - ); - manager - .cancel_task(&defaulted_id) - .await - .expect("retrying timeout task should be cancelled"); + assert_eq!(defaulted_status, HealTaskStatus::Timeout); + assert!(manager.retrying_heals.lock().await.get(&defaulted_id).is_none()); let mut explicit = bucket_request("explicit-timeout", HealPriority::Normal, HealRequestSource::Admin); explicit.options.timeout = Some(Duration::from_secs(60)); diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 916b51822..62123c418 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -196,7 +196,7 @@ pub struct HealOptions { /// Whether to skip namespace locking #[serde(default)] pub no_lock: bool, - /// Timeout + /// Aggregate execution timeout across recoverable manager retries pub timeout: Option, /// pool index pub pool_index: Option, @@ -442,6 +442,14 @@ impl HealTask { } } + pub(crate) async fn retry_request_with_remaining_timeout(&self) -> Result { + let mut request = self.retry_request(); + if self.options.timeout.is_some() { + request.options.timeout = self.remaining_timeout().await?; + } + Ok(request) + } + pub(crate) fn from_replacement_recovery_request( request: HealRequest, storage: Arc, @@ -2657,6 +2665,36 @@ mod tests { use super::super::storage_api::status::BucketInfo; + #[tokio::test] + async fn retry_request_carries_remaining_timeout_budget() { + let storage: Arc = Arc::new(MockStorage::default()); + let mut request = HealRequest::bucket("bucket".to_string()); + request.options.timeout = Some(Duration::from_secs(100)); + let task = HealTask::from_request(request, storage.clone()); + *task.task_start_instant.write().await = Some(Instant::now() - Duration::from_secs(40)); + + let retry = task + .retry_request_with_remaining_timeout() + .await + .expect("first retry should retain the unused timeout budget"); + let first_remaining = retry.options.timeout.expect("configured timeout should remain present"); + assert!(first_remaining <= Duration::from_secs(60)); + assert!(first_remaining > Duration::from_secs(59)); + + let retry_task = HealTask::from_request(retry, storage); + *retry_task.task_start_instant.write().await = Some(Instant::now() - Duration::from_secs(20)); + let second_retry = retry_task + .retry_request_with_remaining_timeout() + .await + .expect("second retry should retain only the unused aggregate budget"); + let second_remaining = second_retry + .options + .timeout + .expect("configured timeout should remain present"); + assert!(second_remaining <= Duration::from_secs(40)); + assert!(second_remaining > Duration::from_secs(39)); + } + #[test] fn format_result_requires_every_requested_target_to_be_ok() { let result = HealResultItem { From e26668e62c8dbb9dc710ed4bc0c03766e371d970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 16 Aug 2026 10:28:45 +0800 Subject: [PATCH 35/71] fix(ecstore): mint bucket-target ARNs in the madmin arn:minio partition (#6128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(ecstore): pin madmin-compatible ARN partition contract Red-light evidence for backlog#1675 P1-7: madmin-go's ParseARN hard-rejects any ARN that does not start with 'arn:minio:', while RustFS generates and only accepts 'arn:rustfs:'. mc/madmin tooling therefore cannot decode RustFS remote-target listings, and MinIO-era replication configs are rejected as StaleTarget when re-registered. The new tests pin the target contract (generate arn:minio:, parse both partitions, reject unknown partitions) and fail against the current single-partition gate. * fix(ecstore): mint bucket-target ARNs in the madmin arn:minio partition madmin-go's ParseARN hard-rejects any partition other than 'arn:minio:', so native mc/madmin tooling could not decode RustFS remote-target listings, and re-registering a MinIO-era replication config failed its StaleTarget check against freshly minted arn:rustfs: targets (backlog#1675 P1-7, route A). - ARN Display now emits 'arn:minio:'; FromStr accepts a {minio, rustfs} partition whitelist (the legacy partition stays readable forever for persisted bucket-targets.json / replication configs). The whitelist is the only structural gate — BucketTargetType::from_str never fails — so it deliberately rejects foreign partitions such as arn:aws:. - No data migration: every runtime match between targets, rules and stats keys is full-string equality, so existing arn:rustfs: targets keep matching their persisted rules; site replication already preserves MinIO-era ARNs on reconcile (pinned by existing tests). - Rolling upgrade note: upgrade all cluster nodes before creating new remote targets — a not-yet-upgraded node rejects remove-remote-target for a freshly minted arn:minio: ARN with BucketRemoteArnInvalid. - Out of scope: notification/SQS ARNs (crates/targets) keep the arn:rustfs:sqs: partition; they have their own compatibility story. --- crates/ecstore/src/bucket/target/arn.rs | 56 +++++++++++++++++-- rustfs/src/admin/handlers/site_replication.rs | 5 +- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/crates/ecstore/src/bucket/target/arn.rs b/crates/ecstore/src/bucket/target/arn.rs index 0bf0f175a..e680daca9 100644 --- a/crates/ecstore/src/bucket/target/arn.rs +++ b/crates/ecstore/src/bucket/target/arn.rs @@ -40,7 +40,14 @@ impl ARN { impl Display for ARN { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket) + // The `minio` partition is deliberate: madmin-go's ParseARN + // hard-rejects any other partition, so native mc/madmin tooling can + // only decode remote-target ARNs minted in this form (backlog#1675 + // P1-7). Legacy `arn:rustfs:` ARNs persisted by older releases stay + // readable via the FromStr whitelist below; runtime matching between + // targets and replication rules is by full-string equality, so mixed + // partitions coexist safely. + write!(f, "arn:minio:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket) } } @@ -48,7 +55,12 @@ impl FromStr for ARN { type Err = std::io::Error; fn from_str(s: &str) -> Result { - if !s.starts_with("arn:rustfs:") { + // Partition whitelist, not just an `arn:` check: `BucketTargetType:: + // from_str(...).unwrap_or_default()` below never fails, so this is + // the only structural gate rejecting foreign ARNs. `arn:rustfs:` is + // the legacy partition and must stay accepted forever (persisted + // bucket-targets.json / replication configs from older releases). + if !s.starts_with("arn:minio:") && !s.starts_with("arn:rustfs:") { return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format")); } @@ -101,14 +113,50 @@ mod tests { } /// RustFS commonly generates ARNs with an empty region: - /// `arn:rustfs:replication:::`. + /// `arn:minio:replication:::`. #[test] fn from_str_handles_empty_region_segment() { - let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse"); + let parsed = ARN::from_str("arn:minio:replication::depl-123:bucket-a").expect("valid ARN must parse"); assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService); assert_eq!(parsed.region, "", "region segment is empty in this form"); assert_eq!(parsed.id, "depl-123"); assert_eq!(parsed.bucket, "bucket-a"); } + + /// madmin-go's `ParseARN` hard-rejects anything that does not start with + /// `arn:minio:`, so generated ARNs must use the `minio` partition or the + /// native mc/madmin tooling cannot decode remote-target listings. + #[test] + fn display_emits_minio_partition() { + let arn = ARN::new( + BucketTargetType::ReplicationService, + "depl-123".to_string(), + String::new(), + "bucket-a".to_string(), + ); + + assert_eq!(arn.to_string(), "arn:minio:replication::depl-123:bucket-a"); + } + + /// Persisted bucket-targets.json files from older RustFS releases carry + /// `arn:rustfs:` ARNs; the legacy partition must stay parseable forever. + #[test] + fn from_str_accepts_legacy_rustfs_partition() { + let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("legacy ARN must parse"); + + assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService); + assert_eq!(parsed.region, "us-east-1"); + assert_eq!(parsed.id, "depl-123"); + assert_eq!(parsed.bucket, "bucket-a"); + } + + /// The partition whitelist is the only structural gate: `BucketTargetType:: + /// from_str(...).unwrap_or_default()` never fails, so any 6-segment string + /// would otherwise parse as `type=None`. + #[test] + fn from_str_rejects_unknown_partition() { + assert!(ARN::from_str("arn:aws:replication::depl-123:bucket-a").is_err()); + assert!(ARN::from_str("not-an-arn").is_err()); + } } diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 495db4284..d9604fe6f 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -15305,7 +15305,10 @@ mod tests { assert!(!target.secure); assert_eq!(target.target_bucket, "photos"); assert_eq!(target.deployment_id, "remote"); - assert_eq!(target.arn, "arn:rustfs:replication::remote:photos"); + // Freshly minted ARNs use the `minio` partition so madmin-go tooling + // can parse them; legacy `arn:rustfs:` targets are preserved as-is + // (see the MinIO-era preservation test below). + assert_eq!(target.arn, "arn:minio:replication::remote:photos"); assert_eq!(target.region, "us-east-1"); let credentials = target .credentials From 81d7b7d07a81e0b9deded47aeab9cefff8494415 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 11:20:52 +0800 Subject: [PATCH 36/71] chore(ecstore): drop the client dead_code blanket (#6138) --- .../ecstore/src/client/api_error_response.rs | 21 ------- crates/ecstore/src/client/api_get_object.rs | 28 ++++++++++ crates/ecstore/src/client/api_put_object.rs | 2 +- .../src/client/api_put_object_common.rs | 4 -- .../src/client/api_put_object_streaming.rs | 12 ---- crates/ecstore/src/client/api_s3_datatypes.rs | 56 ++++++------------- crates/ecstore/src/client/checksum.rs | 4 ++ crates/ecstore/src/client/constants.rs | 3 - crates/ecstore/src/client/credentials.rs | 31 ++++------ crates/ecstore/src/client/mod.rs | 1 - crates/ecstore/src/client/object_api_utils.rs | 33 ----------- .../ecstore/src/client/provider_versions.rs | 2 + crates/ecstore/src/client/transition_api.rs | 41 ++++++++++++++ crates/ecstore/src/client/utils.rs | 4 -- 14 files changed, 106 insertions(+), 136 deletions(-) diff --git a/crates/ecstore/src/client/api_error_response.rs b/crates/ecstore/src/client/api_error_response.rs index f586113e3..8373517e7 100644 --- a/crates/ecstore/src/client/api_error_response.rs +++ b/crates/ecstore/src/client/api_error_response.rs @@ -229,17 +229,6 @@ pub fn http_resp_to_error_response( err_resp } -pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse { - ErrorResponse { - status_code: StatusCode::BAD_REQUEST, - code: S3ErrorCode::InvalidArgument, - message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’." - .to_string(), - bucket_name: bucket_name.to_string(), - ..Default::default() - } -} - pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { let msg = format!( "Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.", @@ -295,16 +284,6 @@ pub fn err_invalid_argument(message: &str) -> ErrorResponse { } } -pub fn err_api_not_supported(message: &str) -> ErrorResponse { - ErrorResponse { - status_code: StatusCode::NOT_IMPLEMENTED, - code: S3ErrorCode::Custom("APINotSupported".into()), - message: message.to_string(), - request_id: "rustfs".to_string(), - ..Default::default() - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/ecstore/src/client/api_get_object.rs b/crates/ecstore/src/client/api_get_object.rs index 14c9b44f4..a9c69aad0 100644 --- a/crates/ecstore/src/client/api_get_object.rs +++ b/crates/ecstore/src/client/api_get_object.rs @@ -135,6 +135,10 @@ impl Object { Self { ..Default::default() } } + #[allow( + dead_code, + reason = "MinIO-parity reader surface with no caller in this port (backlog#1823)" + )] fn do_get_request(&self, request: &GetRequest) -> Result { let _ = request.did_offset_change; let _ = request.offset; @@ -150,12 +154,20 @@ impl Object { )) } + #[allow( + dead_code, + reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)" + )] fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> { self.curr_offset += bytes_read; Ok(()) } + #[allow( + dead_code, + reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)" + )] fn read(&mut self, b: &[u8]) -> Result { let mut read_req = GetRequest { is_read_op: true, @@ -180,6 +192,10 @@ impl Object { Ok(response.size) } + #[allow( + dead_code, + reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)" + )] fn stat(&self) -> Result { if !self.is_started || !self.object_info_set { let _ = self.do_get_request(&GetRequest { @@ -192,6 +208,10 @@ impl Object { Ok(self.object_info.clone()) } + #[allow( + dead_code, + reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)" + )] fn read_at(&mut self, b: &[u8], offset: i64) -> Result { self.curr_offset = offset; @@ -219,6 +239,10 @@ impl Object { Ok(response.size) } + #[allow( + dead_code, + reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)" + )] fn seek(&mut self, offset: i64, whence: i64) -> Result { if !self.is_started || !self.object_info_set { let seek_req = GetRequest { @@ -253,6 +277,10 @@ impl Object { Ok(self.curr_offset) } + #[allow( + dead_code, + reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)" + )] fn close(&mut self) -> Result<(), std::io::Error> { self.is_closed = true; Ok(()) diff --git a/crates/ecstore/src/client/api_put_object.rs b/crates/ecstore/src/client/api_put_object.rs index cb5c98e97..5fbc3fd2c 100644 --- a/crates/ecstore/src/client/api_put_object.rs +++ b/crates/ecstore/src/client/api_put_object.rs @@ -37,7 +37,7 @@ use crate::client::{ api_put_object_common::optimal_part_info, api_put_object_multipart::UploadPartParams, api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart}, - constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS}, + constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE}, credentials::SignatureType, transition_api::{ReaderImpl, TransitionClient, UploadInfo}, utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header}, diff --git a/crates/ecstore/src/client/api_put_object_common.rs b/crates/ecstore/src/client/api_put_object_common.rs index 9a9d7dd7b..b0c405b43 100644 --- a/crates/ecstore/src/client/api_put_object_common.rs +++ b/crates/ecstore/src/client/api_put_object_common.rs @@ -30,10 +30,6 @@ pub fn is_object(reader: &ReaderImpl) -> bool { matches!(reader, ReaderImpl::ObjectBody(_)) } -pub fn is_read_at(reader: ReaderImpl) -> bool { - matches!(reader, ReaderImpl::ObjectBody(_)) -} - pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> { let unknown_size; let mut object_size = object_size; diff --git a/crates/ecstore/src/client/api_put_object_streaming.rs b/crates/ecstore/src/client/api_put_object_streaming.rs index 9d67f3ebb..b0a665707 100644 --- a/crates/ecstore/src/client/api_put_object_streaming.rs +++ b/crates/ecstore/src/client/api_put_object_streaming.rs @@ -81,18 +81,6 @@ async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result, diff --git a/crates/ecstore/src/client/api_s3_datatypes.rs b/crates/ecstore/src/client/api_s3_datatypes.rs index 158637cfd..6703fc4b3 100644 --- a/crates/ecstore/src/client/api_s3_datatypes.rs +++ b/crates/ecstore/src/client/api_s3_datatypes.rs @@ -29,10 +29,6 @@ use crate::client::utils::base64_decode; use super::transition_api; -pub struct ListAllMyBucketsResult { - pub owner: Owner, -} - #[derive(Debug, Default, Serialize, Deserialize)] pub struct CommonPrefix { pub prefix: String, @@ -89,6 +85,10 @@ pub struct ListVersionsResult { pub next_version_id_marker: String, } +#[allow( + dead_code, + reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)" +)] pub struct ListBucketResult { common_prefixes: Vec, contents: Vec, @@ -102,6 +102,10 @@ pub struct ListBucketResult { prefix: String, } +#[allow( + dead_code, + reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)" +)] pub struct ListMultipartUploadsResult { bucket: String, key_marker: String, @@ -117,16 +121,15 @@ pub struct ListMultipartUploadsResult { common_prefixes: Vec, } +#[allow( + dead_code, + reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)" +)] pub struct Initiator { id: String, display_name: String, } -pub struct CopyObjectResult { - pub etag: String, - pub last_modified: OffsetDateTime, -} - #[derive(Debug, Clone)] pub struct ObjectPart { pub etag: String, @@ -260,6 +263,7 @@ pub struct CompletePart { } impl CompletePart { + #[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")] fn checksum(&self, t: &ChecksumMode) -> String { match t { ChecksumMode::ChecksumCRC32C => { @@ -284,11 +288,6 @@ impl CompletePart { } } -pub struct CopyObjectPartResult { - pub etag: String, - pub last_modified: OffsetDateTime, -} - #[derive(Debug, Default, serde::Serialize)] #[serde(rename = "CompleteMultipartUpload")] pub struct CompleteMultipartUpload { @@ -357,10 +356,10 @@ impl CompleteMultipartUpload { } } -pub struct CreateBucketConfiguration { - pub location: String, -} - +#[allow( + dead_code, + reason = "live via quick_xml::de::from_str in bucket_cache.rs; serde deserialization is not a construction (backlog#1823)" +)] #[derive(serde::Serialize)] pub struct DeleteObject { //api has @@ -368,21 +367,6 @@ pub struct DeleteObject { pub version_id: String, } -pub struct DeletedObject { - //s3s has - pub key: String, - pub version_id: String, - pub deletemarker: bool, - pub deletemarker_version_id: String, -} - -pub struct NonDeletedObject { - pub key: String, - pub code: String, - pub message: String, - pub version_id: String, -} - #[derive(serde::Serialize)] pub struct DeleteMultiObjects { pub quiet: bool, @@ -402,6 +386,7 @@ impl DeleteMultiObjects { Ok(buf) } + #[allow(dead_code, reason = "MinIO-parity XML helper with no caller in this port (backlog#1823)")] pub fn unmarshal(buf: &[u8]) -> Result { #[derive(Debug, Deserialize)] struct WireDeleteObject { @@ -436,8 +421,3 @@ impl DeleteMultiObjects { }) } } - -pub struct DeleteMultiObjectsResult { - pub deleted_objects: Vec, - pub undeleted_objects: Vec, -} diff --git a/crates/ecstore/src/client/checksum.rs b/crates/ecstore/src/client/checksum.rs index 09f438822..c71394210 100644 --- a/crates/ecstore/src/client/checksum.rs +++ b/crates/ecstore/src/client/checksum.rs @@ -365,6 +365,10 @@ mod tests { pub struct Checksum { checksum_type: ChecksumMode, r: Vec, + #[allow( + dead_code, + reason = "checksum bookkeeping field kept beside the value it guards (backlog#1823)" + )] computed: bool, } diff --git a/crates/ecstore/src/client/constants.rs b/crates/ecstore/src/client/constants.rs index 5c5a02482..d4a1cd001 100644 --- a/crates/ecstore/src/client/constants.rs +++ b/crates/ecstore/src/client/constants.rs @@ -32,8 +32,5 @@ pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"; -pub const TOTAL_WORKERS: i64 = 4; - -pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] = format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z"); diff --git a/crates/ecstore/src/client/credentials.rs b/crates/ecstore/src/client/credentials.rs index 26b773690..2c81e690f 100644 --- a/crates/ecstore/src/client/credentials.rs +++ b/crates/ecstore/src/client/credentials.rs @@ -67,6 +67,10 @@ impl Credentials

{ Ok(self.creds.clone()) } + #[allow( + dead_code, + reason = "MinIO-parity credential surface with no caller in this port (backlog#1823)" + )] fn expire(&mut self) { self.force_refresh = true; } @@ -133,6 +137,10 @@ impl Provider for Static { #[derive(Debug, Clone, Default)] pub struct STSError { + #[allow( + dead_code, + reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)" + )] pub r#type: String, pub code: String, pub message: String, @@ -141,6 +149,10 @@ pub struct STSError { #[derive(Debug, Clone, thiserror::Error)] pub struct ErrorResponse { pub sts_error: STSError, + #[allow( + dead_code, + reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)" + )] pub request_id: String, } @@ -158,22 +170,3 @@ impl ErrorResponse { return self.sts_error.message.clone(); } } - -pub fn xml_decoder(body: &[u8]) -> Result -where - for<'de> T: Deserialize<'de>, -{ - match std::str::from_utf8(body) { - Ok(xml_body) => quick_xml::de::from_str::(xml_body).map_err(|err| Error::new(ErrorKind::InvalidData, err.to_string())), - Err(err) => Err(Error::new(ErrorKind::InvalidData, err.to_string())), - } -} - -pub fn xml_decode_and_body(body_reader: &[u8]) -> Result<(Vec, T), std::io::Error> -where - for<'de> T: Deserialize<'de>, -{ - let body = body_reader.to_vec(); - let parsed = xml_decoder(&body)?; - Ok((body, parsed)) -} diff --git a/crates/ecstore/src/client/mod.rs b/crates/ecstore/src/client/mod.rs index 39902cb8a..1b9977a97 100644 --- a/crates/ecstore/src/client/mod.rs +++ b/crates/ecstore/src/client/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: S3 client compatibility models are kept while ECStore callers move to narrower facades. -#![allow(dead_code)] pub mod admin_handler_utils; pub mod api_error_response; diff --git a/crates/ecstore/src/client/object_api_utils.rs b/crates/ecstore/src/client/object_api_utils.rs index 6099095a9..484233fad 100644 --- a/crates/ecstore/src/client/object_api_utils.rs +++ b/crates/ecstore/src/client/object_api_utils.rs @@ -77,39 +77,6 @@ fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option (i64, i64, i64, i64, u64) { - let mut skip_length: i64 = 0; - let mut cumulative_actual_size: i64 = 0; - let mut first_part_idx: i64 = 0; - let mut compressed_offset: i64 = 0; - let mut part_skip: i64 = 0; - let mut decrypt_skip: i64 = 0; - let mut seq_num: u64 = 0; - for (i, part) in oi.parts.iter().enumerate() { - cumulative_actual_size += part.actual_size as i64; - if cumulative_actual_size <= offset { - compressed_offset += part.size as i64; - } else { - first_part_idx = i as i64; - skip_length = cumulative_actual_size - part.actual_size as i64; - break; - } - } - skip_length = offset - skip_length; - - let parts: &[ObjectPartInfo] = &oi.parts; - if skip_length > 0 - && parts.len() > first_part_idx as usize - && parts[first_part_idx as usize].index.as_ref().is_some_and(|idx| idx.len() > 0) - { - let _ = part_skip; - let _ = decrypt_skip; - let _ = seq_num; - } - - (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num) -} - pub fn new_getobjectreader<'a>( rs: &Option, oi: &'a ObjectInfo, diff --git a/crates/ecstore/src/client/provider_versions.rs b/crates/ecstore/src/client/provider_versions.rs index b3e4ceb9d..ceb30bb9c 100644 --- a/crates/ecstore/src/client/provider_versions.rs +++ b/crates/ecstore/src/client/provider_versions.rs @@ -23,6 +23,7 @@ const X_OBS_VERSION_ID: &str = "x-obs-version-id"; const MAX_REMOTE_VERSION_ID_LEN: usize = 1024; #[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")] pub(crate) enum BucketVersioningState { Unknown, Disabled, @@ -47,6 +48,7 @@ impl RemoteVersion { } } + #[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")] pub(crate) fn exact_request_id(&self) -> Result, Error> { match self { Self::Unknown => Err(Error::new( diff --git a/crates/ecstore/src/client/transition_api.rs b/crates/ecstore/src/client/transition_api.rs index 38e65b7ab..6ad802418 100644 --- a/crates/ecstore/src/client/transition_api.rs +++ b/crates/ecstore/src/client/transition_api.rs @@ -101,6 +101,10 @@ where const C_UNKNOWN: i32 = -1; const C_OFFLINE: i32 = 0; +#[allow( + dead_code, + reason = "reachable only from the unused transition client methods below (backlog#1823)" +)] const C_ONLINE: i32 = 1; fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error { @@ -320,6 +324,10 @@ impl TransitionClient { Ok(client) } + #[allow( + dead_code, + reason = "MinIO-parity transition client surface with no caller in this port (backlog#1823)" + )] fn endpoint_url(&self) -> Url { self.endpoint_url.clone() } @@ -348,12 +356,20 @@ impl TransitionClient { .to_string()) } + #[allow( + dead_code, + reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)" + )] fn trace_errors_only_off(&self) { if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() { *trace_errors_only = false; } } + #[allow( + dead_code, + reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)" + )] fn trace_off(&self) { if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() { *is_trace_enabled = false; @@ -363,12 +379,20 @@ impl TransitionClient { } } + #[allow( + dead_code, + reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)" + )] fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) { if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() { *endpoint = accelerate_endpoint.to_string(); } } + #[allow( + dead_code, + reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)" + )] fn set_s3_enable_dual_stack(&self, enabled: bool) { if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() { *dual_stack = enabled; @@ -398,10 +422,18 @@ impl TransitionClient { (hash_algos, hash_sums) } + #[allow( + dead_code, + reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)" + )] fn is_online(&self) -> bool { !self.is_offline() } + #[allow( + dead_code, + reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)" + )] fn mark_offline(&self) { self.health_status .compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst); @@ -411,10 +443,18 @@ impl TransitionClient { self.health_status.load(Ordering::SeqCst) == C_OFFLINE } + #[allow( + dead_code, + reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)" + )] fn health_check(hc_duration: Duration) { let _ = hc_duration; } + #[allow( + dead_code, + reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)" + )] fn dump_http(&self, req: &Request, resp: &Response) -> Result<(), std::io::Error> { let mut resp_trace: Vec; @@ -1102,6 +1142,7 @@ impl Default for ObjectInfo { } impl ObjectInfo { + #[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")] pub(crate) fn remote_version( &self, capabilities: ProviderVersionCapabilities, diff --git a/crates/ecstore/src/client/utils.rs b/crates/ecstore/src/client/utils.rs index 3a05e7cf3..3a03449fb 100644 --- a/crates/ecstore/src/client/utils.rs +++ b/crates/ecstore/src/client/utils.rs @@ -48,10 +48,6 @@ lazy_static! { }; } -pub fn is_standard_query_value(qs_key: &str) -> bool { - SUPPORTED_QUERY_VALUES[qs_key] -} - pub fn is_storageclass_header(header_key: &str) -> bool { header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase() } From 4392f94e1a6c4bbbea7d456b6aad72cada1fed18 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 16 Aug 2026 13:35:41 +0800 Subject: [PATCH 37/71] chore(deps): update flake.lock (#6142) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/70ce234' (2026-08-06) → 'github:NixOS/nixpkgs/8be7bd0' (2026-08-14) • Updated input 'rust-overlay': 'github:oxalica/rust-overlay/6df7076' (2026-08-09) → 'github:oxalica/rust-overlay/b211ead' (2026-08-16) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index f2baddbb6..27fdbf27e 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1785975029, - "narHash": "sha256-X44cn5rzytELc3NNoQsh0aLkjWA/QzPfc6HPQmsG3sU=", + "lastModified": 1786719841, + "narHash": "sha256-QcpQOT0NQEFkI77t+YXPZqDJc35iIodG7zinieOwFUg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "70ce234312134a463ba7728e94da2486a1d237ac", + "rev": "8be7bd0c83f12e2e3bbba07c9044d6fed9e66f7f", "type": "github" }, "original": { @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1786247265, - "narHash": "sha256-cVTcTqAhzSNMOVddtBhFpuv+Vx6/SgrhgZWJiI61H24=", + "lastModified": 1786849542, + "narHash": "sha256-MS8cOa/ii1+QA8R3JWVzqEIoXimu9n50GwQDiBVTFOM=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "6df7076ea0f5697e3242719a4c71211f00411cf5", + "rev": "b211eadeba8b180da9453ec3413a8a3535c85b3f", "type": "github" }, "original": { From ed1bedf1fbccb5c9b506d55f7ccaf015e70441d6 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 14:34:46 +0800 Subject: [PATCH 38/71] fix(storage): skip table guards for multipart parts (#6143) --- rustfs/src/storage/access.rs | 119 +++++++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 5 deletions(-) diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 2d62f16c9..d69bde85f 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -111,6 +111,9 @@ pub(crate) struct PostObjectRequestMarker; #[derive(Clone, Debug)] struct InternalObjectAuthorization; +#[derive(Clone, Debug)] +struct StagedMultipartPartAuthorization; + #[derive(Clone, Default)] struct TableDataPlanePublicationGuards { state: Arc>, @@ -1333,6 +1336,15 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R Err(denial.deny("no_policy_allows_action", action)) } +// Multipart parts are staged and become visible only after CompleteMultipartUpload, +// which uses the normal publication-fenced authorization path. +async fn authorize_staged_multipart_part(req: &mut S3Request) -> S3Result<()> { + req.extensions.insert(StagedMultipartPartAuthorization); + let result = authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await; + req.extensions.remove::(); + result +} + /// Check if the request has the x-amz-bypass-governance-retention header set to true pub fn has_bypass_governance_header(headers: &http::HeaderMap) -> bool { headers @@ -1435,6 +1447,10 @@ fn table_data_plane_content_mutation(action: Action) -> bool { ) } +fn table_data_plane_publication_fence_required(req: &S3Request, action: Action) -> bool { + req.extensions.get::().is_none() && table_data_plane_content_mutation(action) +} + fn table_catalog_backend_for_data_plane( req: &S3Request, ) -> S3Result> { @@ -1613,7 +1629,7 @@ async fn authorize_table_data_plane_if_needed( let Some(admin_action) = table_data_plane_admin_action(action) else { return Ok(()); }; - if table_data_plane_content_mutation(action) { + if table_data_plane_publication_fence_required(req, action) { retain_table_bucket_publication_guard(req, bucket).await?; } let table_bucket_enabled = table_bucket_enabled_for_data_plane(req, bucket).await?; @@ -1639,7 +1655,7 @@ async fn authorize_table_data_plane_if_needed( }) .await; if allowed { - if table_data_plane_content_mutation(action) { + if table_data_plane_publication_fence_required(req, action) { retain_table_publication_guard(req, &resource).await?; } return Ok(()); @@ -1675,7 +1691,7 @@ async fn deny_anonymous_table_data_plane_if_needed( if table_data_plane_admin_action(action).is_none() { return Ok(()); } - if table_data_plane_content_mutation(action) { + if table_data_plane_publication_fence_required(req, action) { retain_table_bucket_publication_guard(req, bucket).await?; } let table_bucket_enabled = table_bucket_enabled_for_data_plane(req, bucket).await?; @@ -2906,7 +2922,7 @@ impl S3Access for FS { req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); - authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?; + authorize_staged_multipart_part(req).await?; req.extensions.insert(bucket_generation?); Ok(()) } @@ -2943,7 +2959,7 @@ impl S3Access for FS { req_info.object = Some(req.input.key.clone()); req_info.version_id = None; - authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?; + authorize_staged_multipart_part(req).await?; req.extensions.insert(bucket_generation?); Ok(()) } @@ -3159,6 +3175,99 @@ mod tests { } } + #[tokio::test] + #[serial] + async fn multipart_parts_skip_table_publication_fence_but_completion_retains_it() { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let server_ctx = ServerContextSlot::new(); + assert!(server_ctx.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms),)))); + let fs = FS::with_server_ctx(server_ctx); + let bucket = format!("multipart-table-fence-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("test bucket should be created"); + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket) + .await + .expect("new bucket metadata should be cached")) + .clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test bucket policy should parse")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("test bucket policy should be published"); + + let mut part_req = build_request( + UploadPartInput::builder() + .bucket(bucket.clone()) + .key("object".to_string()) + .upload_id("upload-id".to_string()) + .part_number(1) + .build() + .expect("upload part input should build"), + Method::PUT, + ); + ensure_req_info(&mut part_req); + part_req.extensions.insert(fs.server_ctx().clone()); + fs.upload_part(&mut part_req) + .await + .expect("anonymous UploadPart should be authorized by the test policy"); + assert!( + part_req.extensions.get::().is_none(), + "staged part data must not retain a table publication fence" + ); + + let mut copy_part_req = build_request( + UploadPartCopyInput::builder() + .bucket(bucket.clone()) + .key("object".to_string()) + .upload_id("upload-id".to_string()) + .part_number(2) + .copy_source(CopySource::Bucket { + bucket: bucket.clone().into(), + key: "source".into(), + version_id: None, + }) + .build() + .expect("upload part copy input should build"), + Method::PUT, + ); + ensure_req_info(&mut copy_part_req); + copy_part_req.extensions.insert(fs.server_ctx().clone()); + fs.upload_part_copy(&mut copy_part_req) + .await + .expect("anonymous UploadPartCopy should be authorized by the test policy"); + assert!( + copy_part_req.extensions.get::().is_none(), + "staged copied part data must not retain a table publication fence" + ); + + let mut complete_req = build_request( + CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key("object".to_string()) + .upload_id("upload-id".to_string()) + .multipart_upload(Some(CompletedMultipartUpload::default())) + .build() + .expect("complete multipart input should build"), + Method::POST, + ); + ensure_req_info(&mut complete_req); + complete_req.extensions.insert(fs.server_ctx().clone()); + fs.complete_multipart_upload(&mut complete_req) + .await + .expect("anonymous CompleteMultipartUpload should be authorized by the test policy"); + let bucket_fence = (bucket, crate::table_catalog::default_table_bucket_publication_lock_path()); + let guards = complete_req + .extensions + .get::() + .expect("completion should retain a table publication fence"); + assert!(guards.state.lock().keys.contains(&bucket_fence)); + } + #[tokio::test] async fn table_data_plane_request_reuses_table_resource_for_distinct_object_while_commit_waits() { let resource = crate::table_catalog::TableDataPlaneResource { From a118d7e4fd07be9aa2ccc705f463c639308981a8 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 16 Aug 2026 14:38:51 +0800 Subject: [PATCH 39/71] perf(ecstore): enable inline data read early-stop by default (#6140) * perf(ecstore): enable inline data read early-stop by default Co-Authored-By: heihutu * test(scanner): box large ILM transition flow future Co-Authored-By: heihutu * test(ecstore): align internal meta early-stop miss Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- .../src/set_disk/core/io_primitives.rs | 4 +- crates/ecstore/src/set_disk/mod.rs | 43 +- crates/ecstore/src/set_disk/ops/object.rs | 22 +- crates/ecstore/src/set_disk/read.rs | 36 +- .../tests/lifecycle_integration_test.rs | 563 +++++++++--------- 5 files changed, 349 insertions(+), 319 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index a251644ea..750c9d0ba 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -7141,7 +7141,7 @@ mod tests { } #[tokio::test] - async fn bounded_metadata_early_stop_defaults_keep_data_get_full_fanout() { + async fn bounded_metadata_early_stop_defaults_keep_non_inline_data_get_full_fanout() { const DISKS: usize = 4; let bucket = "bounded-data-get-default-bucket"; let object = "bounded-data-get-default-object"; @@ -7164,7 +7164,7 @@ mod tests { assert_eq!( calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64, - "default GET data-read metadata must keep full fanout for read-failure tolerance" + "default non-inline GET data-read metadata must keep full fanout for read-failure tolerance" ); assert_eq!(diagnostics.total_responses(), DISKS); assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 7f3a37ea0..5ac05f207 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -692,8 +692,9 @@ const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: usize = 128 * 102 const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE"; // Enabled by default (backlog#872): the early-stop path only engages for // requests `should_allow_metadata_early_stop` classifies as safe (latest-version -// metadata-only reads by default, without version_id / healing / free-version -// needs) and still requires a full read-quorum agreement before stopping. Set +// reads by default, without version_id / healing / free-version needs) and still +// requires a full read-quorum agreement before stopping. Data-read requests add +// a separate inline-shard verifier before cancelling the remaining fanout. Set // the env var to `false` to fall back to full-wait metadata fanout. const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true; @@ -704,7 +705,7 @@ const ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_META const DEFAULT_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: bool = false; const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE"; -const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = false; +const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = true; const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT"; const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = false; @@ -915,12 +916,6 @@ mod prepared_get_object_metadata_tests { .expect("4-disk test geometry should leave one bounded spare disk") } - fn bounded_slow_initial_disk_index(bucket: &str, object: &str) -> usize { - *bounded_metadata_fanout_order(bucket, object, 4, 2) - .get(2) - .expect("4-disk test geometry should include a third initial metadata disk") - } - #[tokio::test] async fn prepared_metadata_is_consumed_exactly_once() { let snapshot = GetObjectFileInfo::owned(FileInfo::default(), Vec::new(), Vec::new()); @@ -1039,7 +1034,7 @@ mod prepared_get_object_metadata_tests { #[test] #[serial_test::serial(body_cache_hook)] - fn inline_data_read_early_stop_reader_returns_exact_body() { + fn inline_data_read_early_stop_defaults_return_exact_body() { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -1071,14 +1066,14 @@ mod prepared_get_object_metadata_tests { temp_env::async_with_vars( [ - ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), - ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), - ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>), ], async { - let slow_initial_disk = bounded_slow_initial_disk_index(bucket, &object); + let slow_parity_disk = bounded_spare_disk_index(bucket, &object); let barrier = - rename_fanout_barrier::arm(&object, slow_initial_disk, rename_fanout_barrier::PHASE_READ_VERSION); + rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION); let calls = disk_call_counters::observe(&object); let set_disks_for_read = Arc::clone(&set_disks); let opts_for_read = opts.clone(); @@ -1091,10 +1086,10 @@ mod prepared_get_object_metadata_tests { tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused()) .await - .expect("bounded inline GET should pause a slow initial metadata read"); + .expect("default inline GET should pause a slow parity metadata read"); let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut open_reader) .await - .expect("production inline GET should return before the paused metadata response") + .expect("default production inline GET should return before the paused parity metadata response") .expect("inline GET reader task should not panic") .expect("inline GET reader should open"); let object_size = reader.object_info.size; @@ -1115,14 +1110,14 @@ mod prepared_get_object_metadata_tests { assert_eq!(object_size, payload.len() as i64); assert_eq!(restored, payload); - assert_eq!(calls_total, 4, "bounded production GET should schedule the initial quorum plus one spare"); + assert_eq!(calls_total, 4, "default production GET should eagerly schedule the full metadata fanout"); assert_eq!( recorder.histogram_values( "rustfs_io_get_object_metadata_fanout_scheduled", &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] ), vec![4.0], - "bounded production GET should record all scheduled metadata tasks" + "default production GET should record all scheduled metadata tasks" ); assert_eq!( recorder.histogram_values( @@ -1130,7 +1125,7 @@ mod prepared_get_object_metadata_tests { &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] ), vec![3.0], - "bounded production GET should record only observed metadata responses as completed" + "default production GET should record only observed metadata responses as completed" ); assert_eq!( recorder.histogram_values( @@ -1138,7 +1133,7 @@ mod prepared_get_object_metadata_tests { &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] ), vec![1.0], - "bounded production GET should record the aborted slow metadata task" + "default production GET should record the aborted slow parity metadata task" ); } @@ -1285,9 +1280,9 @@ mod prepared_get_object_metadata_tests { temp_env::async_with_vars( [ - ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), - ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), - ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>), ], async { let calls = disk_call_counters::observe(&object); diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 1c64de30d..3a2949140 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -7505,7 +7505,7 @@ mod get_object_downstream_close_accounting_tests { use super::hermetic_set_disks_support::hermetic_set_disks; use super::*; use crate::diagnostics::get::{ - GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT, + GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT, GetObjectFailureReason, }; use crate::disk::RUSTFS_META_BUCKET; @@ -7637,8 +7637,8 @@ mod get_object_downstream_close_accounting_tests { legacy_completed, internal_cancelled, legacy_cancelled, - internal_unsafe_miss, - legacy_unsafe_miss, + internal_not_found_miss, + legacy_not_found_miss, internal_saved, legacy_saved, ) = metrics::with_local_recorder(&recorder, || { @@ -7714,7 +7714,7 @@ mod get_object_downstream_close_accounting_tests { &[ ("path", GET_OBJECT_PATH_INTERNAL_META), ("decision", "miss"), - ("reason", GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST), + ("reason", GET_METADATA_EARLY_STOP_REASON_NOT_FOUND), ], ), recorder.counter_value( @@ -7722,7 +7722,7 @@ mod get_object_downstream_close_accounting_tests { &[ ("path", GET_OBJECT_PATH_LEGACY_DUPLEX), ("decision", "miss"), - ("reason", GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST), + ("reason", GET_METADATA_EARLY_STOP_REASON_NOT_FOUND), ], ), recorder.histogram_values( @@ -7773,21 +7773,21 @@ mod get_object_downstream_close_accounting_tests { "internal metadata lifecycle cancelled count must not leak into legacy_duplex" ); assert_eq!( - internal_unsafe_miss, 1, - "internal metadata unsafe early-stop miss must retain its path label" + internal_not_found_miss, 1, + "internal metadata not-found early-stop miss must retain its path label" ); assert_eq!( - legacy_unsafe_miss, 0, - "internal metadata unsafe early-stop miss must not leak into legacy_duplex" + legacy_not_found_miss, 0, + "internal metadata not-found early-stop miss must not leak into legacy_duplex" ); assert_eq!( internal_saved, vec![0.0], - "internal metadata unsafe miss must record zero saved responses on internal_meta" + "internal metadata not-found miss must record zero saved responses on internal_meta" ); assert!( legacy_saved.is_empty(), - "internal metadata unsafe miss saved responses must not leak into legacy_duplex" + "internal metadata not-found miss saved responses must not leak into legacy_duplex" ); } } diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index d0011cd11..4a3de7e93 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -3886,13 +3886,15 @@ mod tests { assert!(metadata_early_stop_permitted(true, true, false, "", false, false)); // observe=false (non-observed fanout) also disables early-stop. assert!(!metadata_early_stop_permitted(true, false, false, "", false, false)); - assert!(!metadata_early_stop_permitted(true, true, true, "", false, false)); + // Whole/latest data-read metadata is now allowed by default; + // the inline verifier still decides whether it can stop early. + assert!(metadata_early_stop_permitted(true, true, true, "", false, false)); }, ); } #[test] - fn metadata_early_stop_keeps_data_reads_opt_in_by_default() { + fn metadata_early_stop_allows_safe_data_reads_by_default() { temp_env::with_vars( [ (ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")), @@ -3900,7 +3902,7 @@ mod tests { (ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None), ], || { - assert!(!should_allow_metadata_early_stop(true, "", false, false)); + assert!(should_allow_metadata_early_stop(true, "", false, false)); assert!(!should_allow_metadata_early_stop(true, "version-id", false, false)); assert!(should_allow_metadata_early_stop(false, "", false, false)); assert!(!should_allow_metadata_early_stop(false, "version-id", false, false)); @@ -3932,6 +3934,34 @@ mod tests { ); } + #[test] + fn metadata_early_stop_bounded_fanout_defaults_to_disabled() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")), + (ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None), + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, None), + ], + || { + assert!(is_get_metadata_data_read_early_stop_enabled()); + assert!(!is_get_metadata_early_stop_bounded_fanout_enabled()); + }, + ); + temp_env::with_vars([(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true"))], || { + assert!(is_get_metadata_early_stop_bounded_fanout_enabled()); + }); + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("false")), + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")), + ], + || { + assert!(!is_get_metadata_data_read_early_stop_enabled()); + assert!(!is_get_metadata_early_stop_bounded_fanout_enabled()); + }, + ); + } + #[test] fn metadata_early_stop_rejects_healing_and_free_version_requests() { temp_env::with_vars( diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 0a43a8060..7b2130316 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -1069,309 +1069,314 @@ mod serial_tests { #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] async fn test_transition_and_restore_flows() { - let (disk_paths, ecstore) = setup_test_env().await; + async move { + let (disk_paths, ecstore) = setup_test_env().await; - let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); - let backend = register_mock_tier(&tier_name).await; + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&tier_name).await; - let put_bucket = format!("test-immediate-put-{}", &Uuid::new_v4().simple().to_string()[..8]); - let put_object = "test/object.txt"; - let put_payload = b"Hello, immediate transition!"; + let put_bucket = format!("test-immediate-put-{}", &Uuid::new_v4().simple().to_string()[..8]); + let put_object = "test/object.txt"; + let put_payload = b"Hello, immediate transition!"; - create_test_bucket(&ecstore, put_bucket.as_str()).await; - set_bucket_lifecycle_transition_with_tier(put_bucket.as_str(), &tier_name) - .await - .expect("Failed to set lifecycle configuration"); - - let mut reader = PutObjReader::from_vec(put_payload.to_vec()); - let mut metadata = HashMap::new(); - metadata.insert("content-type".to_string(), "text/plain".to_string()); - ecstore - .put_object( - put_bucket.as_str(), - put_object, - &mut reader, - &ObjectOptions { - user_defined: metadata, - ..Default::default() - }, - ) - .await - .expect("Failed to upload transition metadata test object"); - - enqueue_transition_for_existing_objects(ecstore.clone(), put_bucket.as_str()) - .await - .expect("Failed to enqueue transitioned put object"); - - let put_info = wait_for_transition(&ecstore, put_bucket.as_str(), put_object, TRANSITION_WAIT_TIMEOUT) - .await - .expect("object should transition after enqueueing existing objects"); - - assert_eq!(put_info.transitioned_object.status, "complete"); - assert_eq!(put_info.transitioned_object.tier, tier_name); - assert!(backend.contains(&put_info.transitioned_object.name).await); - { - let transitioned = backend - .stored(&put_info.transitioned_object.name) + create_test_bucket(&ecstore, put_bucket.as_str()).await; + set_bucket_lifecycle_transition_with_tier(put_bucket.as_str(), &tier_name) .await - .expect("transitioned object should be present in mock backend"); - assert_eq!(transitioned.metadata.get("content-type"), Some(&"text/plain".to_string())); - assert!( - !transitioned.metadata.contains_key("x-amz-replication-status"), - "transitioned objects must not inherit replication status defaults" - ); - assert!( - !transitioned.metadata.contains_key("x-amz-object-lock-legal-hold"), - "transitioned objects must not invent object lock headers" - ); - } + .expect("Failed to set lifecycle configuration"); - // Cross-shard xl.meta transition assertion helper (rustfs/backlog#1148 ilm-6): - // every disk must agree on the transition tuple for the object. - let put_meta = assert_transition_meta_consistent(&disk_paths, put_bucket.as_str(), put_object).await; - assert_eq!(put_meta.status, "complete"); - assert_eq!(put_meta.tier, tier_name); + let mut reader = PutObjReader::from_vec(put_payload.to_vec()); + let mut metadata = HashMap::new(); + metadata.insert("content-type".to_string(), "text/plain".to_string()); + ecstore + .put_object( + put_bucket.as_str(), + put_object, + &mut reader, + &ObjectOptions { + user_defined: metadata, + ..Default::default() + }, + ) + .await + .expect("Failed to upload transition metadata test object"); - let multipart_bucket = format!("test-immediate-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]); - let multipart_object = "test/multipart.txt"; + enqueue_transition_for_existing_objects(ecstore.clone(), put_bucket.as_str()) + .await + .expect("Failed to enqueue transitioned put object"); - create_test_bucket(&ecstore, multipart_bucket.as_str()).await; - set_bucket_lifecycle_transition_with_tier(multipart_bucket.as_str(), &tier_name) - .await - .expect("Failed to set lifecycle configuration"); + let put_info = wait_for_transition(&ecstore, put_bucket.as_str(), put_object, TRANSITION_WAIT_TIMEOUT) + .await + .expect("object should transition after enqueueing existing objects"); - let upload = ecstore - .new_multipart_upload(multipart_bucket.as_str(), multipart_object, &ObjectOptions::default()) - .await - .expect("Failed to create multipart upload"); + assert_eq!(put_info.transitioned_object.status, "complete"); + assert_eq!(put_info.transitioned_object.tier, tier_name); + assert!(backend.contains(&put_info.transitioned_object.name).await); + { + let transitioned = backend + .stored(&put_info.transitioned_object.name) + .await + .expect("transitioned object should be present in mock backend"); + assert_eq!(transitioned.metadata.get("content-type"), Some(&"text/plain".to_string())); + assert!( + !transitioned.metadata.contains_key("x-amz-replication-status"), + "transitioned objects must not inherit replication status defaults" + ); + assert!( + !transitioned.metadata.contains_key("x-amz-object-lock-legal-hold"), + "transitioned objects must not invent object lock headers" + ); + } - let part_data = b"multipart immediate transition"; - let mut reader = PutObjReader::from_vec(part_data.to_vec()); - let part = ecstore - .put_object_part( - multipart_bucket.as_str(), - multipart_object, - &upload.upload_id, - 1, - &mut reader, - &ObjectOptions::default(), - ) - .await - .expect("Failed to upload multipart part"); + // Cross-shard xl.meta transition assertion helper (rustfs/backlog#1148 ilm-6): + // every disk must agree on the transition tuple for the object. + let put_meta = assert_transition_meta_consistent(&disk_paths, put_bucket.as_str(), put_object).await; + assert_eq!(put_meta.status, "complete"); + assert_eq!(put_meta.tier, tier_name); - ecstore - .clone() - .complete_multipart_upload( - multipart_bucket.as_str(), - multipart_object, - &upload.upload_id, - vec![CompletePart { - part_num: 1, - etag: part.etag.clone(), - ..Default::default() - }], - &ObjectOptions::default(), - ) - .await - .expect("Failed to complete multipart upload"); + let multipart_bucket = format!("test-immediate-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]); + let multipart_object = "test/multipart.txt"; - enqueue_transition_for_existing_objects(ecstore.clone(), multipart_bucket.as_str()) - .await - .expect("Failed to enqueue transitioned multipart object"); + create_test_bucket(&ecstore, multipart_bucket.as_str()).await; + set_bucket_lifecycle_transition_with_tier(multipart_bucket.as_str(), &tier_name) + .await + .expect("Failed to set lifecycle configuration"); - let multipart_info = wait_for_transition(&ecstore, multipart_bucket.as_str(), multipart_object, TRANSITION_WAIT_TIMEOUT) - .await - .expect("object should transition after enqueueing existing objects"); + let upload = ecstore + .new_multipart_upload(multipart_bucket.as_str(), multipart_object, &ObjectOptions::default()) + .await + .expect("Failed to create multipart upload"); - assert_eq!(multipart_info.transitioned_object.status, "complete"); - assert_eq!(multipart_info.transitioned_object.tier, tier_name); - assert!(backend.contains(&multipart_info.transitioned_object.name).await); + let part_data = b"multipart immediate transition"; + let mut reader = PutObjReader::from_vec(part_data.to_vec()); + let part = ecstore + .put_object_part( + multipart_bucket.as_str(), + multipart_object, + &upload.upload_id, + 1, + &mut reader, + &ObjectOptions::default(), + ) + .await + .expect("Failed to upload multipart part"); - let src_bucket = format!("test-immediate-copy-src-{}", &Uuid::new_v4().simple().to_string()[..8]); - let dst_bucket = format!("test-immediate-copy-dst-{}", &Uuid::new_v4().simple().to_string()[..8]); - let src_object = "test/source.txt"; - let dst_object = "test/copied.txt"; - let payload = b"copy object immediate transition"; - - create_test_bucket(&ecstore, src_bucket.as_str()).await; - create_test_bucket(&ecstore, dst_bucket.as_str()).await; - set_bucket_lifecycle_transition_with_tier(dst_bucket.as_str(), &tier_name) - .await - .expect("Failed to set destination lifecycle configuration"); - - upload_test_object(&ecstore, src_bucket.as_str(), src_object, payload).await; - - let mut src_info = ecstore - .get_object_info(src_bucket.as_str(), src_object, &ObjectOptions::default()) - .await - .expect("Failed to load source object info"); - src_info.put_object_reader = Some(PutObjReader::from_vec(payload.to_vec())); - - ecstore - .copy_object( - src_bucket.as_str(), - src_object, - dst_bucket.as_str(), - dst_object, - &mut src_info, - &ObjectOptions::default(), - &ObjectOptions::default(), - ) - .await - .expect("Failed to copy object"); - - enqueue_transition_for_existing_objects(ecstore.clone(), dst_bucket.as_str()) - .await - .expect("Failed to enqueue transitioned copied object"); - - let copy_info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT) - .await - .expect("copied object should transition after enqueueing existing objects"); - - assert_eq!(copy_info.transitioned_object.status, "complete"); - assert_eq!(copy_info.transitioned_object.tier, tier_name); - assert!(backend.contains(©_info.transitioned_object.name).await); - - let bucket_name = format!("test-lifecycle-update-{}", &Uuid::new_v4().simple().to_string()[..8]); - let object_name = "test/existing.txt"; - let payload = b"existing object before lifecycle"; - - create_test_bucket(&ecstore, bucket_name.as_str()).await; - upload_test_object(&ecstore, bucket_name.as_str(), object_name, payload).await; - - set_bucket_lifecycle_transition_with_tier(bucket_name.as_str(), &tier_name) - .await - .expect("Failed to set lifecycle configuration"); - - enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str()) - .await - .expect("Failed to enqueue transition for existing objects"); - - let info = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT) - .await - .expect("existing object should transition after lifecycle update"); - - assert_eq!(info.transitioned_object.status, "complete"); - assert_eq!(info.transitioned_object.tier, tier_name); - assert!(backend.contains(&info.transitioned_object.name).await); - - let bucket_name = format!("test-restore-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]); - let object_name = "test/restore.txt"; - let part1 = vec![b'a'; 5 * 1024 * 1024]; - let part2 = b"restored-tail".to_vec(); - let expected = [part1.clone(), part2.clone()].concat(); - - create_test_bucket(&ecstore, bucket_name.as_str()).await; - set_bucket_lifecycle_transition_with_tier(bucket_name.as_str(), &tier_name) - .await - .expect("Failed to set lifecycle configuration"); - - let upload = ecstore - .new_multipart_upload(bucket_name.as_str(), object_name, &ObjectOptions::default()) - .await - .expect("Failed to create multipart upload"); - - let mut part1_reader = PutObjReader::from_vec(part1); - let uploaded_part1 = ecstore - .put_object_part( - bucket_name.as_str(), - object_name, - &upload.upload_id, - 1, - &mut part1_reader, - &ObjectOptions::default(), - ) - .await - .expect("Failed to upload first multipart part"); - - let mut part2_reader = PutObjReader::from_vec(part2); - let uploaded_part2 = ecstore - .put_object_part( - bucket_name.as_str(), - object_name, - &upload.upload_id, - 2, - &mut part2_reader, - &ObjectOptions::default(), - ) - .await - .expect("Failed to upload second multipart part"); - - ecstore - .clone() - .complete_multipart_upload( - bucket_name.as_str(), - object_name, - &upload.upload_id, - vec![ - CompletePart { + ecstore + .clone() + .complete_multipart_upload( + multipart_bucket.as_str(), + multipart_object, + &upload.upload_id, + vec![CompletePart { part_num: 1, - etag: uploaded_part1.etag.clone(), + etag: part.etag.clone(), ..Default::default() - }, - CompletePart { - part_num: 2, - etag: uploaded_part2.etag.clone(), - ..Default::default() - }, - ], - &ObjectOptions::default(), - ) - .await - .expect("Failed to complete multipart upload"); + }], + &ObjectOptions::default(), + ) + .await + .expect("Failed to complete multipart upload"); - enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str()) - .await - .expect("Failed to enqueue transitioned restore object"); + enqueue_transition_for_existing_objects(ecstore.clone(), multipart_bucket.as_str()) + .await + .expect("Failed to enqueue transitioned multipart object"); - let transitioned = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT) - .await - .expect("multipart object should transition after enqueueing existing objects"); - assert_eq!(transitioned.parts.len(), 2); + let multipart_info = + wait_for_transition(&ecstore, multipart_bucket.as_str(), multipart_object, TRANSITION_WAIT_TIMEOUT) + .await + .expect("object should transition after enqueueing existing objects"); - ecstore - .clone() - .restore_transitioned_object( - bucket_name.as_str(), - object_name, - &ObjectOptions { - transition: TransitionOptions { - restore_request: RestoreRequest { - days: Some(1), - description: None, - glacier_job_parameters: None, - output_location: None, - select_parameters: None, - tier: None, - type_: None, + assert_eq!(multipart_info.transitioned_object.status, "complete"); + assert_eq!(multipart_info.transitioned_object.tier, tier_name); + assert!(backend.contains(&multipart_info.transitioned_object.name).await); + + let src_bucket = format!("test-immediate-copy-src-{}", &Uuid::new_v4().simple().to_string()[..8]); + let dst_bucket = format!("test-immediate-copy-dst-{}", &Uuid::new_v4().simple().to_string()[..8]); + let src_object = "test/source.txt"; + let dst_object = "test/copied.txt"; + let payload = b"copy object immediate transition"; + + create_test_bucket(&ecstore, src_bucket.as_str()).await; + create_test_bucket(&ecstore, dst_bucket.as_str()).await; + set_bucket_lifecycle_transition_with_tier(dst_bucket.as_str(), &tier_name) + .await + .expect("Failed to set destination lifecycle configuration"); + + upload_test_object(&ecstore, src_bucket.as_str(), src_object, payload).await; + + let mut src_info = ecstore + .get_object_info(src_bucket.as_str(), src_object, &ObjectOptions::default()) + .await + .expect("Failed to load source object info"); + src_info.put_object_reader = Some(PutObjReader::from_vec(payload.to_vec())); + + ecstore + .copy_object( + src_bucket.as_str(), + src_object, + dst_bucket.as_str(), + dst_object, + &mut src_info, + &ObjectOptions::default(), + &ObjectOptions::default(), + ) + .await + .expect("Failed to copy object"); + + enqueue_transition_for_existing_objects(ecstore.clone(), dst_bucket.as_str()) + .await + .expect("Failed to enqueue transitioned copied object"); + + let copy_info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT) + .await + .expect("copied object should transition after enqueueing existing objects"); + + assert_eq!(copy_info.transitioned_object.status, "complete"); + assert_eq!(copy_info.transitioned_object.tier, tier_name); + assert!(backend.contains(©_info.transitioned_object.name).await); + + let bucket_name = format!("test-lifecycle-update-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object_name = "test/existing.txt"; + let payload = b"existing object before lifecycle"; + + create_test_bucket(&ecstore, bucket_name.as_str()).await; + upload_test_object(&ecstore, bucket_name.as_str(), object_name, payload).await; + + set_bucket_lifecycle_transition_with_tier(bucket_name.as_str(), &tier_name) + .await + .expect("Failed to set lifecycle configuration"); + + enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str()) + .await + .expect("Failed to enqueue transition for existing objects"); + + let info = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT) + .await + .expect("existing object should transition after lifecycle update"); + + assert_eq!(info.transitioned_object.status, "complete"); + assert_eq!(info.transitioned_object.tier, tier_name); + assert!(backend.contains(&info.transitioned_object.name).await); + + let bucket_name = format!("test-restore-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object_name = "test/restore.txt"; + let part1 = vec![b'a'; 5 * 1024 * 1024]; + let part2 = b"restored-tail".to_vec(); + let expected = [part1.clone(), part2.clone()].concat(); + + create_test_bucket(&ecstore, bucket_name.as_str()).await; + set_bucket_lifecycle_transition_with_tier(bucket_name.as_str(), &tier_name) + .await + .expect("Failed to set lifecycle configuration"); + + let upload = ecstore + .new_multipart_upload(bucket_name.as_str(), object_name, &ObjectOptions::default()) + .await + .expect("Failed to create multipart upload"); + + let mut part1_reader = PutObjReader::from_vec(part1); + let uploaded_part1 = ecstore + .put_object_part( + bucket_name.as_str(), + object_name, + &upload.upload_id, + 1, + &mut part1_reader, + &ObjectOptions::default(), + ) + .await + .expect("Failed to upload first multipart part"); + + let mut part2_reader = PutObjReader::from_vec(part2); + let uploaded_part2 = ecstore + .put_object_part( + bucket_name.as_str(), + object_name, + &upload.upload_id, + 2, + &mut part2_reader, + &ObjectOptions::default(), + ) + .await + .expect("Failed to upload second multipart part"); + + ecstore + .clone() + .complete_multipart_upload( + bucket_name.as_str(), + object_name, + &upload.upload_id, + vec![ + CompletePart { + part_num: 1, + etag: uploaded_part1.etag.clone(), + ..Default::default() + }, + CompletePart { + part_num: 2, + etag: uploaded_part2.etag.clone(), + ..Default::default() + }, + ], + &ObjectOptions::default(), + ) + .await + .expect("Failed to complete multipart upload"); + + enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str()) + .await + .expect("Failed to enqueue transitioned restore object"); + + let transitioned = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT) + .await + .expect("multipart object should transition after enqueueing existing objects"); + assert_eq!(transitioned.parts.len(), 2); + + ecstore + .clone() + .restore_transitioned_object( + bucket_name.as_str(), + object_name, + &ObjectOptions { + transition: TransitionOptions { + restore_request: RestoreRequest { + days: Some(1), + description: None, + glacier_job_parameters: None, + output_location: None, + select_parameters: None, + tier: None, + type_: None, + }, + ..Default::default() }, ..Default::default() }, - ..Default::default() - }, - ) - .await - .expect("Failed to restore transitioned multipart object"); + ) + .await + .expect("Failed to restore transitioned multipart object"); - let restored = ecstore - .get_object_info(bucket_name.as_str(), object_name, &ObjectOptions::default()) - .await - .expect("Failed to load restored object info"); - assert_eq!(restored.parts.len(), 2); - assert!(restored.restore_expires.is_some()); - assert!(!restored.restore_ongoing); + let restored = ecstore + .get_object_info(bucket_name.as_str(), object_name, &ObjectOptions::default()) + .await + .expect("Failed to load restored object info"); + assert_eq!(restored.parts.len(), 2); + assert!(restored.restore_expires.is_some()); + assert!(!restored.restore_ongoing); - let mut reader = ecstore - .get_object_reader(bucket_name.as_str(), object_name, None, http::HeaderMap::new(), &ObjectOptions::default()) - .await - .expect("Failed to read restored object"); - let mut data = Vec::new(); - reader - .stream - .read_to_end(&mut data) - .await - .expect("Failed to consume restored object stream"); - assert_eq!(data, expected); + let mut reader = ecstore + .get_object_reader(bucket_name.as_str(), object_name, None, http::HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("Failed to read restored object"); + let mut data = Vec::new(); + reader + .stream + .read_to_end(&mut data) + .await + .expect("Failed to consume restored object stream"); + assert_eq!(data, expected); + } + .boxed_local() + .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] From 1eef0de0036089a78901e5a5b2f01b06efebf613 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 21:38:37 +0800 Subject: [PATCH 40/71] chore(ecstore): drop the disk dead_code blanket (#6139) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(ecstore): drop the disk dead_code blanket Removing the blanket exposes 36 items in the lowest storage layer: 7 deleted, 29 kept with reasoned item-level allows. That is the smallest deletion share of this burn-down, and the reason is a verification limit rather than a judgement call. disk/local.rs carries 141 `#[cfg(target_os = "linux")]` sites — the densest platform gating in the tree, because O_DIRECT and io_uring only exist there. The direct-I/O cluster (six ENV_RUSTFS_OBJECT_DIRECT_IO_* constants plus is_direct_io_read_enabled, is_direct_io_write_enabled, get_direct_io_read_threshold, direct_write_staging_capacity, direct_write_tail_split and DIRECT_WRITE_STAGING_BYTES) reads as dead on macOS purely because its production callers at local.rs:1766, 3114 and 4605 sit inside Linux-gated blocks. direct_write_staging_capacity even documents itself as "Platform-independent (no O_DIRECT), so it is unit-tested on any host". Deleting those would leave every local check green — 4096 tests pass, clippy is clean, make pre-commit exits 0 — and break the Linux build in CI, because all four local lanes compile for aarch64-apple-darwin. Cross-checking locally is not available either: cargo check --target x86_64-unknown-linux-gnu fails in the aws-lc-sys build script for want of a Linux C cross-compiler. Their allows name the platform reason so the next reader on a non-Linux host does not repeat the investigation. Deleted, all in files with no target_os gating at all (os.rs, disk_store.rs): - HealthDiskCtxKey and HealthDiskCtxValue with its private log_success. Note that DiskHealthTracker::log_success is a different method of the same name and is live from cluster/rpc/peer_s3_client.rs and remote_disk.rs — the two have to be told apart by type, not by name. - LocalDiskWrapper::new_with_health and check_id. - os.rs file_exists and lock_destination_directory_for_path_access. Kept with allows: DiskHealthTracker's set_faulty, mark_offline, waiting_count and last_success have test callers in remote_disk.rs, so they only look dead in the lib target. to_disk_error, remove_all and sync_dir_files are asserted by their own files' tests. The reclaim, mmap and path-cache field groups are written but never read back. Placement follows the same rule as the earlier roots: per-method allows inside impl DiskHealthTracker and impl LocalDisk, since both are mostly live and a block-level allow would be a smaller version of the blanket this issue removes. Struct-level allows are used only where the warning covers that struct's own fields. The three cached_read_env! functions take their allow inside the macro invocation, before the fn line, because the macro forwards $(#[$meta:meta])* onto the generated item. Verification, four lanes warning-free: default, --tests, --features rio-v2 --tests, --features test-util --tests. cargo nextest run -p rustfs-ecstore 4096 passed; clippy --lib --tests -D warnings clean; make pre-commit exit 0. The Linux lane is not covered locally and is left to CI. Ref rustfs/backlog#1823 (step 2). * chore(ecstore): correct two dead_code reasons in the disk root check_valid_path and reject_symlink_components have no caller at all - not even a test - so 'asserted by this file's tests' misreads them as covered. Both are method wrappers over live free functions; say that instead. Ref rustfs/backlog#1823. --- crates/ecstore/src/disk/disk_store.rs | 46 +++++------------- crates/ecstore/src/disk/error_conv.rs | 1 + crates/ecstore/src/disk/fs.rs | 1 + crates/ecstore/src/disk/local.rs | 69 +++++++++++++++++++++++++++ crates/ecstore/src/disk/mod.rs | 5 +- crates/ecstore/src/disk/os.rs | 16 ++----- 6 files changed, 93 insertions(+), 45 deletions(-) diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 4ad8b22dc..29a8037dd 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -637,14 +637,23 @@ impl Default for DiskOperationMetrics { } impl DiskOperationMetrics { + #[allow( + dead_code, + reason = "internal metrics recorder reached only from record() below (backlog#1823)" + )] fn record_call(&mut self) { self.lifetime_calls.fetch_add(1, Ordering::Relaxed); } + #[allow( + dead_code, + reason = "internal metrics recorder reached only from record() below (backlog#1823)" + )] fn record_latency(&mut self, now_sec: u64, elapsed: Duration) { self.record_latency_atomic(now_sec, elapsed); } + #[allow(dead_code, reason = "metrics roll-up with no caller in this port (backlog#1823)")] fn record(&mut self, now_sec: u64, elapsed: Duration) { self.record_call(); self.record_latency(now_sec, elapsed); @@ -770,6 +779,7 @@ impl DiskHealthTracker { } /// Set disk as faulty + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn set_faulty(&self) { self.status.store(DISK_HEALTH_FAULTY, Ordering::Release); } @@ -850,6 +860,7 @@ impl DiskHealthTracker { became_offline } + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool { let current = self.runtime_state(); if current == RuntimeDriveHealthState::Offline { @@ -980,11 +991,13 @@ impl DiskHealthTracker { } /// Get waiting operations count + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn waiting_count(&self) -> u32 { self.waiting.load(Ordering::Relaxed) } /// Get last success timestamp + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn last_success(&self) -> i64 { self.last_success.load(Ordering::Acquire) } @@ -1026,21 +1039,6 @@ impl Default for DiskHealthTracker { } } -/// Health check context key for tracking disk operations -#[derive(Debug, Clone)] -struct HealthDiskCtxKey; - -#[derive(Debug)] -struct HealthDiskCtxValue { - last_success: Arc, -} - -impl HealthDiskCtxValue { - fn log_success(&self) { - self.last_success.store(current_unix_nanos(), Ordering::Relaxed); - } -} - /// LocalDiskWrapper wraps a DiskStore with health tracking capabilities. /// This is similar to Go's xlStorageDiskIDCheck. #[derive(Debug, Clone)] @@ -1072,10 +1070,6 @@ impl LocalDiskWrapper { ) } - pub(crate) fn new_with_health(disk: Arc, health_check: bool, health: Arc) -> Self { - Self::new_with_health_and_metrics(disk, health_check, health, Arc::new(DiskHealthMetricEpoch::default())) - } - pub(crate) fn new_with_reconnect_state( disk: Arc, health_check: bool, @@ -1438,20 +1432,6 @@ impl LocalDiskWrapper { } } - async fn check_id(&self, want_id: Option) -> Result<()> { - if want_id.is_none() { - return Ok(()); - } - - let stored_disk_id = self.disk.get_disk_id().await?; - - if stored_disk_id != want_id { - return Err(Error::other(format!("Disk ID mismatch wanted {want_id:?}, got {stored_disk_id:?}"))); - } - - Ok(()) - } - /// Check if disk ID is stale async fn check_disk_stale(&self) -> Result<()> { let Some(current_disk_id) = *self.disk_id.read().await else { diff --git a/crates/ecstore/src/disk/error_conv.rs b/crates/ecstore/src/disk/error_conv.rs index 82e5386f0..966ace2e6 100644 --- a/crates/ecstore/src/disk/error_conv.rs +++ b/crates/ecstore/src/disk/error_conv.rs @@ -48,6 +48,7 @@ pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error { } } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error { match io_err.kind() { std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(), diff --git a/crates/ecstore/src/disk/fs.rs b/crates/ecstore/src/disk/fs.rs index dd41551c5..e2473a1d4 100644 --- a/crates/ecstore/src/disk/fs.rs +++ b/crates/ecstore/src/disk/fs.rs @@ -178,6 +178,7 @@ pub async fn remove(path: impl AsRef) -> io::Result<()> { } } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub async fn remove_all(path: impl AsRef) -> io::Result<()> { // Try remove_file first; fall back to remove_dir_all if it's a directory match fs::remove_file(path.as_ref()).await { diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 1dc3a79c4..2cab189aa 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -665,6 +665,7 @@ async fn remove_empty_directory_tree_under_mount_lease( } #[cfg(unix)] +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] async fn remove_empty_directory_tree_with( root: &Path, before_descend: impl FnMut(&Path) -> std::io::Result<()>, @@ -1016,13 +1017,29 @@ fn record_direct_read_page_fault_delta(path: &'static str, stage: &'static str, /// When enabled, shard reads bypass the page cache using O_DIRECT flag. /// Requires aligned buffers (typically 512 bytes or 4096 bytes). /// Default: false (uses page cache via mmap/pread). +#[allow( + dead_code, + reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" +)] const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE"; +#[allow( + dead_code, + reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" +)] const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false; /// Minimum shard size threshold for O_DIRECT reads. /// Only shards larger than this threshold will use O_DIRECT. /// Default: 4MB. +#[allow( + dead_code, + reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" +)] const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD"; +#[allow( + dead_code, + reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" +)] const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024; /// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only). @@ -1036,7 +1053,15 @@ const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024; /// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall /// back to buffered writes for the whole disk. Non-Linux always falls back. /// Default: false (buffered writes via the page cache, as before). +#[allow( + dead_code, + reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" +)] const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE"; +#[allow( + dead_code, + reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" +)] const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false; const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE"; const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false; @@ -1095,12 +1120,14 @@ macro_rules! cached_read_env { cached_read_env! { /// Check if O_DIRECT reads are enabled. + #[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")] fn is_direct_io_read_enabled() -> bool = rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE); } cached_read_env! { /// Check if O_DIRECT shard/part data writes are enabled. + #[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")] fn is_direct_io_write_enabled() -> bool = rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE); } @@ -1456,6 +1483,7 @@ pub(crate) fn effective_durability(volume: &str) -> DurabilityMode { cached_read_env! { /// Get the O_DIRECT read threshold size. + #[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")] fn get_direct_io_read_threshold() -> usize = rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD); } @@ -1673,12 +1701,20 @@ impl DirectIoWriteState { /// Target staging size for O_DIRECT writes, rounded up to the DIO alignment. /// Bounds the per-writer aligned bounce buffer and batches many shard blocks /// into one positioned write to keep the syscall count low. +#[allow( + dead_code, + reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" +)] const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024; /// Aligned bounce-buffer capacity for a given DIO alignment: the target staging /// size rounded up to a whole multiple of `align` so the buffer address, every /// flushed batch length, and every write offset stay alignment-correct. /// Platform-independent (no O_DIRECT), so it is unit-tested on any host. +#[allow( + dead_code, + reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" +)] fn direct_write_staging_capacity(align: usize) -> usize { debug_assert!(align.is_power_of_two() && align >= 512); DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align @@ -1687,6 +1723,10 @@ fn direct_write_staging_capacity(align: usize) -> usize { /// Split `filled` staged bytes into the alignment-sized prefix written with /// O_DIRECT and the sub-alignment tail written buffered. Platform-independent, /// so the tail-boundary math is unit-tested on any host. +#[allow( + dead_code, + reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" +)] fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) { let aligned = filled - (filled % align); (aligned, filled - aligned) @@ -2142,6 +2182,7 @@ fn set_delete_version_fail_after_data_staged(path: &str) { } #[cfg(test)] +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) { DELETE_VERSION_FAIL_AFTER_COMMIT .lock() @@ -2447,6 +2488,10 @@ enum SyncMode { FileOnly, } +#[allow( + dead_code, + reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)" +)] struct FileCacheReclaimWriter { inner: File, reclaim_len: usize, @@ -2454,6 +2499,10 @@ struct FileCacheReclaimWriter { reclaimed: bool, } +#[allow( + dead_code, + reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)" +)] struct FileCacheReclaimReader { inner: File, reclaim_offset: u64, @@ -2519,6 +2568,10 @@ impl AsyncRead for StallTimeoutReader { } } +#[allow( + dead_code, + reason = "reclaim metrics emitter reached only from the Linux-gated reclaim paths (backlog#1823)" +)] fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) { // Runs per read-stream page-cache reclaim window; skip the whole emission // (three metric-key constructions) when general metrics are disabled. @@ -3071,6 +3124,7 @@ impl LocalIoBackend for StdBackend { use memmap2::MmapOptions; use std::time::{Duration as StdDuration, Instant as StdInstant}; + #[allow(dead_code, reason = "mmap copy result slot kept beside the mapping it owns (backlog#1823)")] struct MmapCopyReadResult { bytes: Bytes, access_check_duration: StdDuration, @@ -4704,6 +4758,10 @@ fn build_local_io_backend(root: PathBuf) -> Arc { Arc::new(StdBackend::new(root)) } +#[allow( + dead_code, + reason = "path cache and cwd slots retained beside the disk root they derive from (backlog#1823)" +)] pub struct LocalDisk { pub root: PathBuf, publication_root: os::PublicationRoot, @@ -5490,6 +5548,7 @@ impl LocalDisk { Ok(Self::resolve_abs_path_from(&self.root, path.as_ref())) } + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn io_resolve_abs_path(&self, path: impl AsRef) -> PathBuf { let path_ref = path.as_ref(); let path_str = path_ref.to_string_lossy(); @@ -5567,15 +5626,24 @@ impl LocalDisk { } // Check if a path is valid + #[allow( + dead_code, + reason = "method wrapper over the live free function check_local_disk_valid_path; no caller in this port (backlog#1823)" + )] fn check_valid_path>(&self, path: P) -> Result<()> { check_local_disk_valid_path(self.io_root(), path) } + #[allow( + dead_code, + reason = "method wrapper over the live free function reject_local_disk_symlink_components; no caller in this port (backlog#1823)" + )] fn reject_symlink_components(&self, path: &Path) -> Result<()> { reject_local_disk_symlink_components(self.io_root(), path) } // Batch path generation with single lock acquisition + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result> { let mut results = Vec::with_capacity(requests.len()); let mut cache_misses = Vec::new(); @@ -6488,6 +6556,7 @@ impl LocalDisk { Ok(f) } + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] async fn open_file_read_only(&self, path: impl AsRef) -> Result { let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?; Ok(f) diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 428bd61a9..3aaa54c8f 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: disk abstractions still carry staged health and direct-I/O migration paths. -#![allow(dead_code)] pub mod disk_store; pub mod endpoint; @@ -1114,6 +1113,10 @@ pub struct DiskInfo { } #[derive(Clone, Debug, Default)] +#[allow( + dead_code, + reason = "MinIO-parity disk info shape with no constructor in this port (backlog#1823)" +)] pub struct Info { pub total: u64, pub free: u64, diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index fc9c8027c..f5b1c5f66 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -571,6 +571,10 @@ fn regular_files(dir: &Path) -> io::Result> { /// Fdatasync every regular file directly inside `dir`, then fsync the directory /// itself. +#[allow( + dead_code, + reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)" +)] pub fn sync_dir_files_std(dir: impl AsRef) -> io::Result<()> { for entry in std::fs::read_dir(dir.as_ref())? { let entry = entry?; @@ -583,6 +587,7 @@ pub fn sync_dir_files_std(dir: impl AsRef) -> io::Result<()> { /// Async wrapper around [`sync_dir_files_std`]. Large directories flush files /// concurrently, bounded both per directory and process-wide. +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub async fn sync_dir_files(dir: impl AsRef) -> io::Result<()> { sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await } @@ -1809,10 +1814,6 @@ impl RenameCommitGuard { }) } - pub(crate) fn lock_destination_directory_for_path_access(&self, directory: &Path) -> io::Result { - self.destination_directory_guard(directory, false) - } - pub(crate) fn create_destination_directory_for_path_access( &self, directory: &Path, @@ -2858,13 +2859,6 @@ pub async fn os_mkdir_all(dir_path: impl AsRef, base_dir: impl AsRef Ok(()) } -/// Check if a file exists. -/// Returns true if the file exists, false otherwise. -#[tracing::instrument(level = "debug", skip_all)] -pub fn file_exists(path: impl AsRef) -> bool { - std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false) -} - /// Whether an [`io::Error`] means "the directory is not empty". /// /// POSIX lets `rmdir`/`rename` report a non-empty directory as either From f1f86ee9d05a30db6d4062726c7588f26c05cd18 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 21:38:46 +0800 Subject: [PATCH 41/71] chore(ecstore): drop the set_disk dead_code blanket (#6141) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(ecstore): drop the set_disk dead_code blanket Removing the blanket exposes 39 items; exactly one is deleted. The low share is a finding, not caution: unlike the disk root, where platform gating made local adjudication impossible, here the items were checked and nearly all of them are live. Deleted: HealEntryResult, the only item with no reference anywhere. What the checks turned up, in the order the warnings suggest deleting them: SetDisks::rename_data looked like the head of a dead chain feeding into_legacy_tuple and RenameDataLegacyTuple. It is not: production goes through rename_data_owned, and rename_data itself has test callers at mod.rs:5809 and 5880. The chain below it is therefore live through the tests, and inferring "this is dead, so its callee is dead" would have removed three working items. create_bitrot_readers_until_quorum, read_multiple_files and map_cleanup_join_result all have callers inside their files' test modules, so they only look dead in the lib target. TransitionCommitBarrier and TransitionUploadedSaveProbe, with their install/wait_until_paused/release surfaces, are installed by tests behind #[cfg(all(test, feature = "test-util"))]. ctx.rs's SetDisksCtx accessors are the split seam left by the SetDisks god-object break-up (backlog#815). heal_object_dir's two apparent references are comments, and they document an index-alignment contract that live code maintains for it, so they stay as they are. Worth a maintainer decision: the metadata early-stop switch has a complete percentage-rollout facet — ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT, get_metadata_early_stop_rollout_pct and should_use_metadata_early_stop — with no caller, no test and no documentation, while its sibling enable flag is live. It is kept with an allow that says so rather than removed, since a rollout knob is a product call. One placement note for anyone adding allows near heal code: check_logging_guardrails.sh requires #[instrument(level = "trace")] to sit immediately before async fn heal_object_dir, so the allow goes above the instrument attribute. Putting it between the two drops the guard's match count and fails the check. Verification, four lanes warning-free: default, --tests, --features rio-v2 --tests, --features test-util --tests. cargo nextest run -p rustfs-ecstore 4096 passed; clippy --lib --tests -D warnings clean; make pre-commit exit 0. Ref rustfs/backlog#1823 (step 2). * chore(ecstore): fix duplicated and inaccurate dead_code reasons in set_disk format_lock_error carried the same #[allow] twice. Five items in the locking/heal roots were labelled 'asserted by this file's tests' while having no reference at all - heal_object_dir's only two references are comments, as this branch's own notes point out. Say what each item actually is instead, so the next reader does not assume test coverage that is not there. Ref rustfs/backlog#1823. * chore(ecstore): correct the bounded_spare_disk_index dead_code reason The mod.rs copy is an unused test fixture, not something this module's tests assert; the namesake that is exercised lives in the io_primitives test module. Ref rustfs/backlog#1823. --- crates/ecstore/src/io_support/bitrot.rs | 1 + .../src/set_disk/core/io_primitives.rs | 10 +++++ crates/ecstore/src/set_disk/ctx.rs | 28 +++++++++++++ crates/ecstore/src/set_disk/mod.rs | 40 ++++++++++++++----- crates/ecstore/src/set_disk/ops/heal.rs | 4 ++ crates/ecstore/src/set_disk/ops/locking.rs | 15 +++++++ crates/ecstore/src/set_disk/ops/multipart.rs | 2 + crates/ecstore/src/set_disk/ops/object.rs | 36 +++++++++++++++++ crates/ecstore/src/set_disk/read.rs | 2 + 9 files changed, 128 insertions(+), 10 deletions(-) diff --git a/crates/ecstore/src/io_support/bitrot.rs b/crates/ecstore/src/io_support/bitrot.rs index 9a4d6dfab..05c81c048 100644 --- a/crates/ecstore/src/io_support/bitrot.rs +++ b/crates/ecstore/src/io_support/bitrot.rs @@ -704,6 +704,7 @@ pub(crate) async fn create_bitrot_reader_from_bytes_with_stage_metrics( } #[allow(clippy::too_many_arguments)] +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn create_deferred_bitrot_reader( inline_data: Option, disk: Option, diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 750c9d0ba..299299971 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -180,11 +180,13 @@ pub(in crate::set_disk) enum GetCodecStreamingReaderBuildOutcome { Fallback(GetCodecStreamingFallbackReason), } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(in crate::set_disk) struct MultipartCodecStreamingReader { pub(in crate::set_disk) readers: VecDeque>, } impl MultipartCodecStreamingReader { + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(in crate::set_disk) fn new(readers: Vec>) -> Self { Self { readers: VecDeque::from(readers), @@ -1836,6 +1838,7 @@ pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum_all_shards( } #[allow(clippy::too_many_arguments)] +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum( files: &[FileInfo], disks: &[Option], @@ -2126,6 +2129,7 @@ pub(in crate::set_disk) async fn create_data_block_bitrot_readers( setup } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(in crate::set_disk) async fn collect_read_multiple_results( tasks: Vec, read_quorum: usize, @@ -2955,6 +2959,7 @@ impl SetDisks { (meta_file_infos, errs) } + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(in crate::set_disk) async fn read_multiple_files( disks: &[Option], req: ReadMultipleReq, @@ -3134,6 +3139,7 @@ pub(in crate::set_disk) struct RenameDataCommit { pub(in crate::set_disk) committed_file_info: FileInfo, } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] type RenameDataLegacyTuple = ( Vec>, RenameConvergence, @@ -3143,6 +3149,7 @@ type RenameDataLegacyTuple = ( ); impl RenameDataCommit { + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn into_legacy_tuple(self) -> RenameDataLegacyTuple { ( self.online_disks, @@ -3261,6 +3268,7 @@ impl SetDisks { #[tracing::instrument(level = "debug", skip(disks, file_infos))] #[allow(clippy::type_complexity)] + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(in crate::set_disk) async fn rename_data( disks: &[Option], src_bucket: &str, @@ -5073,6 +5081,7 @@ fn is_cleanup_not_found(e: &DiskError) -> bool { /// normalized to `DiskNotFound`: a panic is not a "disk absent" condition and /// must not be silently swallowed as an ignorable error (fixes the historical /// `Unexpected`/`DiskNotFound` misclassification). +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn map_cleanup_join_result(joined: std::result::Result, tokio::task::JoinError>) -> Option { match joined { Ok(res) => res, @@ -5297,6 +5306,7 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase { /// The per-disk old-data-dir cleanup phase of the commit fan-out. pub const CLEANUP: &str = "cleanup"; /// The per-disk `read_version` phase of metadata read fan-out. + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub const READ_VERSION: &str = "read_version"; } diff --git a/crates/ecstore/src/set_disk/ctx.rs b/crates/ecstore/src/set_disk/ctx.rs index 9c601232f..3b7f681b3 100644 --- a/crates/ecstore/src/set_disk/ctx.rs +++ b/crates/ecstore/src/set_disk/ctx.rs @@ -42,12 +42,20 @@ impl<'a> SetDisksCtx<'a> { } /// The borrowed core, for state not yet fronted by a typed accessor. + #[allow( + dead_code, + reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)" + )] pub(crate) fn core(&self) -> &'a SetDisks { self.core } // --- Immutable topology / config (fixed after construction) --- + #[allow( + dead_code, + reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)" + )] pub(crate) fn set_index(&self) -> usize { self.core.set_index } @@ -56,14 +64,26 @@ impl<'a> SetDisksCtx<'a> { self.core.pool_index } + #[allow( + dead_code, + reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)" + )] pub(crate) fn set_drive_count(&self) -> usize { self.core.set_drive_count } + #[allow( + dead_code, + reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)" + )] pub(crate) fn default_parity_count(&self) -> usize { self.core.default_parity_count } + #[allow( + dead_code, + reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)" + )] pub(crate) fn set_endpoints(&self) -> &'a [Endpoint] { &self.core.set_endpoints } @@ -72,6 +92,10 @@ impl<'a> SetDisksCtx<'a> { &self.core.format } + #[allow( + dead_code, + reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)" + )] pub(crate) fn locker_owner(&self) -> &'a str { &self.core.locker_owner } @@ -84,6 +108,10 @@ impl<'a> SetDisksCtx<'a> { // --- Locker trio --- + #[allow( + dead_code, + reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)" + )] pub(crate) fn lockers(&self) -> &'a [Arc] { &self.core.lockers } diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 5ac05f207..dcc416d47 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -39,7 +39,6 @@ //! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers. // #730: SetDisks still hosts staged read/heal/write migration helpers. -#![allow(dead_code)] #![allow(unused_imports)] #![allow(unused_variables)] @@ -624,7 +623,9 @@ fn adaptive_duplex_buffer_size(object_size: i64) -> usize { // Each flag has a corresponding `*_ROLLOUT_PCT` for percentage-based gradual rollout. // ============================================================================ +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1); +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750); const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096 @@ -698,7 +699,15 @@ const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EAR // the env var to `false` to fall back to full-wait metadata fanout. const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true; +#[allow( + dead_code, + reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)" +)] const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT"; +#[allow( + dead_code, + reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)" +)] const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: u32 = 100; const ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE"; @@ -910,6 +919,10 @@ mod prepared_get_object_metadata_tests { .expect("test should find an object whose initial fanout covers both data shards") } + #[allow( + dead_code, + reason = "test fixture no assertion in this module uses today; the live namesake lives in io_primitives tests (backlog#1823)" + )] fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize { *bounded_metadata_fanout_order(bucket, object, 4, 2) .get(3) @@ -1709,6 +1722,10 @@ fn is_multipart_reader_setup_prefetch_enabled() -> bool { } } +#[allow( + dead_code, + reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)" +)] fn get_metadata_early_stop_rollout_pct() -> u32 { static CACHED: OnceLock = OnceLock::new(); *CACHED.get_or_init(|| { @@ -1748,6 +1765,10 @@ fn should_use_codec_streaming(config: GetCodecStreamingConfig, bucket: &str, obj } /// Should this specific request use metadata early-stop? +#[allow( + dead_code, + reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)" +)] pub fn should_use_metadata_early_stop(bucket: &str, object: &str) -> bool { let base = is_get_metadata_early_stop_enabled(); let pct = get_metadata_early_stop_rollout_pct(); @@ -2181,6 +2202,7 @@ fn classify_get_codec_streaming_object_class( GetCodecStreamingObjectClass::PlainSinglePart } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn is_get_small_object_direct_memory_eligible_with_threshold( range: &Option, object_info: &ObjectInfo, @@ -2786,6 +2808,7 @@ pub struct SetDisks { /// Stable namespace shared by every object lock created for this set. set_lock_namespace: Arc, pub format: FormatV3, + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] disk_health_cache: Arc>>>, get_object_metadata_cache: moka::future::Cache>, get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState, @@ -3061,11 +3084,13 @@ struct GetObjectMetadataCacheEntry { #[derive(Clone, Debug)] struct DiskHealthEntry { + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] last_check: Instant, online: bool, } impl DiskHealthEntry { + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn cached_value(&self) -> Option { if self.last_check.elapsed() <= DISK_HEALTH_CACHE_TTL { Some(self.online) @@ -3659,6 +3684,7 @@ fn multipart_put_large_batch_min_size_bytes() -> usize { }) } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath { if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) { SmallWritePath::Inline @@ -4237,6 +4263,7 @@ fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &Obj /// /// Fail closed: when bucket metadata cannot be resolved the check stays on, so /// object-lock protection is never skipped because of a metadata lookup miss. +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(crate) fn object_lock_delete_check_required(bucket_meta: Option<&crate::bucket::metadata::BucketMetadata>) -> bool { bucket_meta.is_none_or(|meta| meta.object_locking()) } @@ -4512,15 +4539,6 @@ impl Hash for ObjProps { } } -#[derive(Default, Clone, Debug)] -pub struct HealEntryResult { - pub bytes: usize, - pub success: bool, - pub skipped: bool, - pub entry_done: bool, - pub name: String, -} - fn is_object_dangling( meta_arr: &[FileInfo], errs: &[Option], @@ -5297,6 +5315,7 @@ pub fn is_valid_storage_class(storage_class: &str) -> bool { } /// Returns true if the storage class is a cold storage tier that requires special handling +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn is_cold_storage_class(storage_class: &str) -> bool { matches!( storage_class, @@ -5305,6 +5324,7 @@ pub fn is_cold_storage_class(storage_class: &str) -> bool { } /// Returns true if the storage class is an infrequent access tier +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn is_infrequent_access_class(storage_class: &str) -> bool { matches!( storage_class, diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 20e1f0fd6..24c46a7f5 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -1716,6 +1716,10 @@ impl SetDisks { Ok((result, None)) } + #[allow( + dead_code, + reason = "lock-taking wrapper over the live heal_object_dir_locked; only comments reference it (backlog#1823)" + )] #[tracing::instrument(level = "trace", skip(self), fields(bucket = %bucket, object = %object))] pub(in crate::set_disk) async fn heal_object_dir( &self, diff --git a/crates/ecstore/src/set_disk/ops/locking.rs b/crates/ecstore/src/set_disk/ops/locking.rs index d975d69cd..2c82c9150 100644 --- a/crates/ecstore/src/set_disk/ops/locking.rs +++ b/crates/ecstore/src/set_disk/ops/locking.rs @@ -66,6 +66,7 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks { } impl SetDisks { + #[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")] pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String { match err { LockResult::Timeout => { @@ -79,6 +80,7 @@ impl SetDisks { } } + #[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")] pub(in crate::set_disk) fn format_lock_error_from_error( &self, bucket: &str, @@ -143,6 +145,7 @@ impl SetDisks { disks } + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(in crate::set_disk) async fn get_online_disks(&self) -> Vec> { let snapshot = self.drive_membership_snapshot().await; let mut disks = snapshot.strict_online_candidates().into_iter().map(Some).collect::>(); @@ -153,6 +156,10 @@ impl SetDisks { disks } + #[allow( + dead_code, + reason = "local-only sibling of the test-covered get_online_disks; no caller in this port (backlog#1823)" + )] pub(in crate::set_disk) async fn get_online_local_disks(&self) -> Vec> { let snapshot = self.drive_membership_snapshot().await; let mut disks = snapshot @@ -432,6 +439,10 @@ impl SetDisks { Ok((disk, fm)) } + #[allow( + dead_code, + reason = "MinIO-parity healing-disk accessor with no caller in this port (backlog#1823)" + )] pub(in crate::set_disk) async fn get_online_disk_with_healing( &self, incl_healing: bool, @@ -440,6 +451,10 @@ impl SetDisks { Ok((new_disks, healing > 0)) } + #[allow( + dead_code, + reason = "reached only from get_online_disk_with_healing, itself uncalled in this port (backlog#1823)" + )] pub(in crate::set_disk) async fn get_online_disk_with_healing_and_info( &self, incl_healing: bool, diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 9684cc1f6..86bb41737 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -415,6 +415,7 @@ fn reduce_quorum_part_numbers(object_parts: Vec>, read_quorum: usize /// never returned, but flips `is_truncated` to `true` and yields a /// `next_upload_id_marker` pointing at the last returned upload so the caller can /// resume paging. +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn paginate_upload_page(remaining: &[MultipartInfo], max_uploads: usize) -> (Vec, bool, Option) { let is_truncated = remaining.len() > max_uploads; let page: Vec = remaining.iter().take(max_uploads).cloned().collect(); @@ -557,6 +558,7 @@ impl SetDisks { } #[tracing::instrument(level = "debug", skip(self))] + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(super) async fn check_upload_id_exists( &self, bucket: &str, diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 3a2949140..96ca009cf 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -3507,6 +3507,10 @@ struct TransitionUploadedSaveProbeState { } #[cfg(test)] +#[allow( + dead_code, + reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)" +)] struct TransitionUploadedSaveProbe { state: Arc, } @@ -3517,6 +3521,10 @@ static TRANSITION_UPLOADED_SAVE_PROBE: std::sync::OnceLock Self { let state = Arc::new(TransitionUploadedSaveProbeState { bucket: bucket.to_string(), @@ -3533,6 +3541,10 @@ impl TransitionUploadedSaveProbe { Self { state } } + #[allow( + dead_code, + reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)" + )] fn attempts(&self) -> usize { self.state.attempts.load(std::sync::atomic::Ordering::Acquire) } @@ -3738,6 +3750,10 @@ struct TransitionCommitBarrierState { } #[cfg(test)] +#[allow( + dead_code, + reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)" +)] struct TransitionCommitBarrier { state: Arc, } @@ -3748,14 +3764,26 @@ static TRANSITION_COMMIT_BARRIER: std::sync::OnceLock Self { Self::install_at(bucket, object, TransitionCommitPause::BeforeLockLost) } + #[allow( + dead_code, + reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)" + )] fn install(bucket: &str, object: &str) -> Self { Self::install_at(bucket, object, TransitionCommitPause::BeforeLeaseValidation) } + #[allow( + dead_code, + reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)" + )] fn install_after_lease_check(bucket: &str, object: &str) -> Self { Self::install_at(bucket, object, TransitionCommitPause::AfterLeaseValidation) } @@ -3778,12 +3806,20 @@ impl TransitionCommitBarrier { Self { state } } + #[allow( + dead_code, + reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)" + )] async fn wait_until_paused(&self) { tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) .await .expect("transition should reach the deterministic commit barrier"); } + #[allow( + dead_code, + reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)" + )] fn release(&self) { self.state.release.notify_one(); } diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 4a3de7e93..dd3a51f20 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -116,6 +116,7 @@ impl SetDisks { .then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE) } + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option> { match self.lookup_cached_get_object_fileinfo(bucket, object).await { MetadataCacheLookup::Hit(entry) => Some(entry), @@ -1826,6 +1827,7 @@ fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOp .then_some(GET_METADATA_CACHE_REASON_META_BUCKET) } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOptions, read_data: bool) -> bool { get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data).is_none() } From 6cf9cf7bb5e71f65756c486c5b61772fbf0c7683 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 21:39:04 +0800 Subject: [PATCH 42/71] chore(ecstore): drop the bucket dead_code blanket (#6147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(ecstore): drop the bucket dead_code blanket The last blanket of the backlog#1823 burn-down, and the largest: 71 items across lifecycle, replication, metadata, quota, object lock and bucket utils. Four are deleted. Deleted, all trivial: - check_valid_object_name and check_valid_object_name_prefix, a pair that only calls into each other with no external caller. Worth stating plainly so nobody reads this as a validation gap: object names are validated through check_object_name_for_length_and_slash, which is live; this pair is a second, unwired entry point. - DEFAULT_HEALTH_CHECK_RELOAD_DURATION, a lone unused constant. - The LifecycleReplicationConfig alias, which orphaned a re-export in replication/mod.rs that goes with it. Everything else is kept, in four groups, because the blanket here was hiding structure rather than rot: Windows platform gating. WINDOWS_RESERVED_NAMES, the two reason constants and object_name_has_windows_incompatible_segment are called from inside the #[cfg(target_os = "windows")] block in check_object_name_for_length_and_slash (utils.rs:228-255), so they only read as dead on non-Windows hosts. As with the Linux gating in the disk root, this cannot be adjudicated locally: cargo check for both x86_64-pc-windows-msvc and x86_64-unknown-linux-gnu fails in the aws-lc-sys build script for want of a cross C toolchain. CI covers both. Declared boundary surface. The *_boundary.rs and *_bridge.rs files carry the replication split plan's contracts, which scripts/check_architecture_migration_rules.sh pins through the EcstoreReplicationBoundaryImports section of the split-plan doc. Their unused items are declarations, not leftovers. test-util seams. ConfigWriteLockProbe with install/wait_until_attempted follows the same pattern as the barriers in the services and set_disk roots. MinIO-parity tier/lifecycle entry points that this port never wired: apply_lifecycle_action, get_transitioned_object_reader, recover_tier_free_versions, delete_object_from_remote_tier, abort_tier_delete_journal_entry and the replication pool's worker-management surface. These are complete, substantial machinery with no caller — the same shape as data_usage's local_snapshot feature. Removing them is a product decision, so they are made explicit here rather than deleted. Verification, four lanes warning-free: default, --tests, --features rio-v2 --tests, --features test-util --tests. cargo nextest run -p rustfs-ecstore 4096 passed; clippy --lib --tests -D warnings clean; make pre-commit exit 0. Note that clippy is what caught the orphaned re-export above: cargo check and pre-commit both treat unused_imports as a warning. Ref rustfs/backlog#1823 (step 2, final root). * chore(ecstore): correct inaccurate dead_code reasons in the bucket root Six items were labelled 'asserted by this file's tests' or as MinIO-parity entry points while having no caller at all - free get_bucket_acl_config and created_at only reach their own live methods (production goes through created_at_in), BucketVersioningSys::get_in, utils::serialize_content and ServiceType have no reference anywhere, and with_transition_queue_env_async is an unused test fixture, not a tier entry point. Name what each one is so the next reader does not assume coverage that is not there. Ref rustfs/backlog#1823. --- .../ecstore/src/bucket/bucket_target_sys.rs | 1 - .../bucket/lifecycle/bucket_lifecycle_ops.rs | 56 +++++++++++++++++++ .../bucket/lifecycle/manual_transition_job.rs | 16 ++++++ .../src/bucket/lifecycle/replication_sink.rs | 16 +++++- .../src/bucket/lifecycle/tagging_boundary.rs | 4 ++ .../bucket/lifecycle/tier_delete_journal.rs | 4 ++ .../lifecycle/tier_free_version_recovery.rs | 4 ++ .../src/bucket/lifecycle/tier_sweeper.rs | 16 ++++++ .../lifecycle/transition_transaction.rs | 8 +++ crates/ecstore/src/bucket/metadata_sys.rs | 13 +++++ crates/ecstore/src/bucket/mod.rs | 1 - .../src/bucket/object_lock/objectlock_sys.rs | 2 + crates/ecstore/src/bucket/quota/mod.rs | 2 + .../ecstore/src/bucket/quota/reservation.rs | 1 + crates/ecstore/src/bucket/replication/mod.rs | 2 +- .../replication/replication_config_store.rs | 4 ++ .../replication_lifecycle_bridge.rs | 20 +++++++ .../replication/replication_msgp_boundary.rs | 16 ++++++ .../replication/replication_object_bridge.rs | 4 ++ .../replication/replication_object_config.rs | 4 ++ .../bucket/replication/replication_pool.rs | 32 +++++++++++ .../replication_resync_boundary.rs | 20 +++++++ .../replication/replication_resyncer.rs | 12 ++++ .../bucket/replication/replication_state.rs | 4 ++ .../src/bucket/target/bucket_target.rs | 4 ++ crates/ecstore/src/bucket/utils.rs | 37 ++++++------ crates/ecstore/src/bucket/versioning_sys.rs | 4 ++ 27 files changed, 284 insertions(+), 23 deletions(-) diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index c6bcc5742..0a9b0f41c 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -82,7 +82,6 @@ use tracing::warn; use url::Url; use uuid::Uuid; -const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60); const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16; const REDACTED_CREDENTIAL: &str = ""; diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index e0849a637..c74451095 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -126,11 +126,23 @@ const EVENT_LIFECYCLE_EXPIRED_DETECTED: &str = "lifecycle_expired_detected"; const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued"; const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched"; const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed"; +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit"; const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed"; const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed"; +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub type TimeFn = Arc Pin + Send>> + Send + Sync + 'static>; +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub type TraceFn = Arc) -> Pin + Send>> + Send + Sync + 'static>; pub type ExpiryOpType = Box; @@ -140,9 +152,21 @@ static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new(); static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new(); pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging"; +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count"; +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive"; pub const AMZ_ENCRYPTION_AES: &str = "AES256"; +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms"; pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier."; @@ -280,6 +304,10 @@ impl LifecycleSys { } } + #[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" + )] pub fn trace(oi: &ObjectInfo) -> TraceFn { let bucket = oi.bucket.clone(); let name = oi.name.clone(); @@ -570,6 +598,10 @@ async fn delete_free_version_remote_object( Ok(()) } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] async fn delete_free_version_remote_object_then( oi: &ObjectInfo, tier_config_mgr: &Arc>, @@ -2868,6 +2900,10 @@ fn stale_upload_default_due(initiated: OffsetDateTime, default_expiry: StdDurati initiated + time::Duration::seconds(default_expiry.as_secs() as i64) } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] async fn stale_upload_current_size(set: &Arc, metadata: &HashMap, upload_dir: &str) -> Option { stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await } @@ -3352,6 +3388,10 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu Ok(()) } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) { if remote_delete_succeeded { opts.skip_decommissioned = true; @@ -4373,6 +4413,10 @@ pub async fn transition_object(api: Arc, oi: &ObjectInfo, lae: LcAuditE result } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn { let tier = _tier.to_string(); Arc::new(move || { @@ -4391,6 +4435,10 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn { }) } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub async fn get_transitioned_object_reader( bucket: &str, object: &str, @@ -5145,6 +5193,10 @@ async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Res ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { let mut success = false; match event.action { @@ -7422,6 +7474,10 @@ mod tests { // process environment while `env::set_var`/`env::remove_var` is active. // SAFETY: keep this note adjacent to the allowance for the repository guard. #[allow(unsafe_code)] + #[allow( + dead_code, + reason = "transition-queue env fixture kept for tests that scope those vars; no test uses it today (backlog#1823)" + )] async fn with_transition_queue_env_async(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F) where F: FnOnce() -> Fut, diff --git a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs index 448ae2230..b1fe45cad 100644 --- a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs +++ b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs @@ -759,6 +759,10 @@ pub struct ManualTransitionWorkerResultRecord { } impl ManualTransitionWorkerResultRecord { + #[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" + )] pub fn new(job_id: Uuid, task_key: impl Into, result: ManualTransitionWorkerResult) -> Self { Self::new_with_reason(job_id, task_key, result, None) } @@ -1257,6 +1261,10 @@ pub(crate) async fn save_manual_transition_task_if_absent( } } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub async fn load_manual_transition_task_record( api: Arc, job_id: Uuid, @@ -1320,6 +1328,10 @@ async fn scan_manual_transition_task_journal(api: Arc, job_id: Uuid) -> } } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub async fn load_manual_transition_worker_result_stats( api: Arc, job_id: Uuid, @@ -1455,6 +1467,10 @@ async fn scan_manual_transition_worker_result_journal( } } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub async fn reconcile_manual_transition_worker_results( api: Arc, job_id: Uuid, diff --git a/crates/ecstore/src/bucket/lifecycle/replication_sink.rs b/crates/ecstore/src/bucket/lifecycle/replication_sink.rs index a965988e1..32cd6b57a 100644 --- a/crates/ecstore/src/bucket/lifecycle/replication_sink.rs +++ b/crates/ecstore/src/bucket/lifecycle/replication_sink.rs @@ -15,25 +15,35 @@ use rustfs_common::metrics::IlmAction; use crate::bucket::lifecycle::lifecycle::ObjectOpts; +use crate::bucket::replication::ReplicationLifecycleBridge; pub(crate) use crate::bucket::replication::ReplicationStatusType; #[cfg(test)] pub(crate) use crate::bucket::replication::VersionPurgeStatusType; pub(crate) use crate::bucket::replication::{ DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta, }; -use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig}; use crate::storage_api_contracts::object::DeletedObject; -pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig; - +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool { obj.version_purge_status.is_pending() } +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool { replication_status_blocks_lifecycle(&obj.replication_status) } +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool { has_pending_object_replication(obj) || has_pending_version_purge(obj) } diff --git a/crates/ecstore/src/bucket/lifecycle/tagging_boundary.rs b/crates/ecstore/src/bucket/lifecycle/tagging_boundary.rs index 03e57def1..ff200c98a 100644 --- a/crates/ecstore/src/bucket/lifecycle/tagging_boundary.rs +++ b/crates/ecstore/src/bucket/lifecycle/tagging_boundary.rs @@ -14,6 +14,10 @@ use std::collections::HashMap; +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap { crate::bucket::tagging::decode_tags_to_map(tags) } diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 6542e88b8..ee05be7c7 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -331,6 +331,10 @@ where persist_tier_delete_journal_entry(api, &committed).await } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub async fn abort_tier_delete_journal_entry(api: Arc, je: &Jentry) -> std::io::Result<()> where S: ObjectOperations< diff --git a/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs b/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs index ac45de861..0bcae9dc6 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs @@ -148,6 +148,10 @@ struct RecoveryCursor { object: String, } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub async fn recover_tier_free_versions( api: Arc, limit: usize, diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index a934cdb11..2ce68fd03 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -385,6 +385,10 @@ impl ExpiryOp for Jentry { } } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> { let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await; if let Err(err) = &result @@ -395,6 +399,10 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na result } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> { #[cfg(test)] if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) { @@ -405,6 +413,10 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] async fn delete_object_from_remote_tier_raw_with_manager( obj_name: &str, rv_id: &str, @@ -485,6 +497,10 @@ pub enum RemoteTierDeleteOutcome { AlreadyRemoved, } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub async fn delete_object_from_remote_tier_idempotent( obj_name: &str, rv_id: &str, diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index 9549d231c..7fc231281 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -50,8 +50,16 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum TransitionTransactionError { #[error("transition transaction already exists")] + #[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" + )] AlreadyExists, #[error("transition transaction is not found")] + #[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" + )] NotFound, #[error("transition transaction is corrupt: {0}")] Corrupt(&'static str), diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 94709e4ed..96b35e206 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -60,12 +60,14 @@ struct ConfigWriteLockProbeState { static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock>>> = std::sync::OnceLock::new(); #[cfg(any(test, feature = "test-util"))] +#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")] pub struct ConfigWriteLockProbe { state: Arc, } #[cfg(any(test, feature = "test-util"))] impl ConfigWriteLockProbe { + #[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")] pub fn install(bucket: &str) -> Self { let state = Arc::new(ConfigWriteLockProbeState { bucket: bucket.to_string(), @@ -84,6 +86,7 @@ impl ConfigWriteLockProbe { Self { state } } + #[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")] pub async fn wait_until_attempted(&self) { tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) .await @@ -890,6 +893,10 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi bucket_meta_sys.get_bucket_policy_raw(bucket).await } +#[allow( + dead_code, + reason = "free-function facade over the live BucketMetadataSys::get_bucket_acl_config; no caller in this port (backlog#1823)" +)] pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; @@ -1104,6 +1111,10 @@ pub async fn get_config_from_disk(bucket: &str) -> Result { bucket_meta_sys.get_config_from_disk(bucket).await } +#[allow( + dead_code, + reason = "ambient-facade variant of the live created_at_in; no caller in this port (backlog#1823)" +)] pub async fn created_at(bucket: &str) -> Result { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; @@ -1617,6 +1628,7 @@ impl BucketMetadataSys { /// [`Self::update`], with the payload computed from the loaded metadata /// instead of supplied up front. Loads through this system's own store so /// the read and the persisted write target the same instance. + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] async fn update_config_with(&self, bucket: &str, config_file: &str, mutate: F) -> Result where F: FnOnce(&BucketMetadata) -> Result> + Send, @@ -1721,6 +1733,7 @@ impl BucketMetadataSys { /// A miss is never published as an authoritative default, and a snapshot /// read before delete plus same-name recreation cannot replace the new /// generation. + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> { if is_meta_bucketname(bucket) { return Err(Error::other("errInvalidArgument")); diff --git a/crates/ecstore/src/bucket/mod.rs b/crates/ecstore/src/bucket/mod.rs index 4f04755e1..54306247a 100644 --- a/crates/ecstore/src/bucket/mod.rs +++ b/crates/ecstore/src/bucket/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: bucket subsystems still contain staged ECStore migration code. -#![allow(dead_code)] pub mod bandwidth; pub mod bucket_target_sys; diff --git a/crates/ecstore/src/bucket/object_lock/objectlock_sys.rs b/crates/ecstore/src/bucket/object_lock/objectlock_sys.rs index b85c6976b..c8ca80288 100644 --- a/crates/ecstore/src/bucket/object_lock/objectlock_sys.rs +++ b/crates/ecstore/src/bucket/object_lock/objectlock_sys.rs @@ -136,6 +136,7 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime { /// Check if an object has legal hold enabled. /// Returns true if legal hold is ON. +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] fn has_legal_hold(user_defined: &std::collections::HashMap) -> bool { let lhold = objectlock::get_object_legalhold_meta(user_defined); matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON) @@ -151,6 +152,7 @@ fn has_legal_hold(user_defined: &std::collections::HashMap) -> b /// # Returns /// * `true` if the object is locked (cannot be deleted/modified) /// * `false` if the object is not locked +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap, is_delete_marker: bool) -> bool { // Delete markers are never locked if is_delete_marker { diff --git a/crates/ecstore/src/bucket/quota/mod.rs b/crates/ecstore/src/bucket/quota/mod.rs index 157c750e4..0ce5a3a52 100644 --- a/crates/ecstore/src/bucket/quota/mod.rs +++ b/crates/ecstore/src/bucket/quota/mod.rs @@ -193,6 +193,7 @@ pub enum QuotaError { } #[derive(Debug, Serialize)] +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub struct QuotaErrorResponse { #[serde(rename = "Code")] pub code: String, @@ -208,6 +209,7 @@ pub struct QuotaErrorResponse { } impl QuotaErrorResponse { + #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self { match quota_error { QuotaError::QuotaExceeded { .. } => Self { diff --git a/crates/ecstore/src/bucket/quota/reservation.rs b/crates/ecstore/src/bucket/quota/reservation.rs index 175785487..33aa5c876 100644 --- a/crates/ecstore/src/bucket/quota/reservation.rs +++ b/crates/ecstore/src/bucket/quota/reservation.rs @@ -899,6 +899,7 @@ async fn save_ledger_locked( } #[cfg(any(test, feature = "test-util"))] +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn fail_next_quota_ledger_save_for_test() { FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst); } diff --git a/crates/ecstore/src/bucket/replication/mod.rs b/crates/ecstore/src/bucket/replication/mod.rs index 1c4ac52d4..d46f78eb3 100644 --- a/crates/ecstore/src/bucket/replication/mod.rs +++ b/crates/ecstore/src/bucket/replication/mod.rs @@ -60,7 +60,7 @@ pub use replication_filemeta_boundary::{ pub(crate) use replication_filemeta_boundary::{ replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta, }; -pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig}; +pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge; pub(crate) use replication_migration_bridge::ReplicationMigrationBridge; pub use replication_object_bridge::ReplicationObjectBridge; pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig}; diff --git a/crates/ecstore/src/bucket/replication/replication_config_store.rs b/crates/ecstore/src/bucket/replication/replication_config_store.rs index de0e14c61..5ac6a48af 100644 --- a/crates/ecstore/src/bucket/replication/replication_config_store.rs +++ b/crates/ecstore/src/bucket/replication/replication_config_store.rs @@ -37,6 +37,10 @@ impl ReplicationConfigStore { com::read_config_limited(api, file, max_bytes).await } + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] pub(crate) async fn read_no_lock(api: Arc, file: &str) -> Result> where S: ReplicationObjectIO, diff --git a/crates/ecstore/src/bucket/replication/replication_lifecycle_bridge.rs b/crates/ecstore/src/bucket/replication/replication_lifecycle_bridge.rs index 3908389ba..c35bf99c1 100644 --- a/crates/ecstore/src/bucket/replication/replication_lifecycle_bridge.rs +++ b/crates/ecstore/src/bucket/replication/replication_lifecycle_bridge.rs @@ -24,15 +24,27 @@ use super::replication_storage_boundary::{ DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication, }; +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) type ReplicationLifecycleConfig = ReplicationConfig; pub(crate) struct ReplicationLifecycleBridge; impl ReplicationLifecycleBridge { + #[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" + )] pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig { ReplicationConfig::new(Some(config), None) } + #[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" + )] pub(crate) fn has_pending_version_purge( config: &ReplicationLifecycleConfig, object_name: &str, @@ -45,6 +57,10 @@ impl ReplicationLifecycleBridge { .is_some_and(|config| config.has_active_rules(object_name, true)) } + #[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" + )] pub(crate) async fn check_delete_replication( bucket: &str, object: &ObjectToDelete, @@ -54,6 +70,10 @@ impl ReplicationLifecycleBridge { check_replicate_delete(bucket, object, source, opts, None).await } + #[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" + )] pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState { let pending_status = decision.pending_status(); ReplicationState { diff --git a/crates/ecstore/src/bucket/replication/replication_msgp_boundary.rs b/crates/ecstore/src/bucket/replication/replication_msgp_boundary.rs index 48f0685b9..72622d6cf 100644 --- a/crates/ecstore/src/bucket/replication/replication_msgp_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_msgp_boundary.rs @@ -19,17 +19,33 @@ use time::OffsetDateTime; use super::replication_error_boundary::Result; use crate::bucket::msgp_decode; +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) struct ReplicationMsgpCodec; impl ReplicationMsgpCodec { + #[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" + )] pub(crate) fn read_ext8_time(rd: &mut R) -> Result { msgp_decode::read_msgp_ext8_time(rd) } + #[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" + )] pub(crate) fn skip_value(rd: &mut R) -> Result<()> { msgp_decode::skip_msgp_value(rd) } + #[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" + )] pub(crate) fn write_time(wr: &mut W, time: OffsetDateTime) -> Result<()> { msgp_decode::write_msgp_time(wr, time) } diff --git a/crates/ecstore/src/bucket/replication/replication_object_bridge.rs b/crates/ecstore/src/bucket/replication/replication_object_bridge.rs index 07b126c42..1043c22cd 100644 --- a/crates/ecstore/src/bucket/replication/replication_object_bridge.rs +++ b/crates/ecstore/src/bucket/replication/replication_object_bridge.rs @@ -77,6 +77,10 @@ impl ReplicationObjectBridge { load_delete_request_config_in(ctx, bucket).await } + #[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" + )] pub(crate) async fn delete_config_snapshot_in( ctx: &ReplicationInstanceContext, bucket: &str, diff --git a/crates/ecstore/src/bucket/replication/replication_object_config.rs b/crates/ecstore/src/bucket/replication/replication_object_config.rs index e8dbe33e4..3f44a9d10 100644 --- a/crates/ecstore/src/bucket/replication/replication_object_config.rs +++ b/crates/ecstore/src/bucket/replication/replication_object_config.rs @@ -231,6 +231,10 @@ pub(crate) async fn load_delete_replication_config( delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?) } +#[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" +)] pub(crate) async fn load_delete_replication_config_in( ctx: &ReplicationInstanceContext, bucket: &str, diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index 76d807d75..58b3efdca 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -217,6 +217,10 @@ impl DurableMrfBacklogTracker { } } +#[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" +)] fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker { let mut tracker = DurableMrfBacklogTracker { available: true, @@ -712,6 +716,10 @@ pub struct ReplicationPool { // MRF worker lifecycle mrf_worker_cancellations: Mutex>, + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] mrf_stop_tx: Sender<()>, // Worker size tracking @@ -940,6 +948,10 @@ impl ReplicationPool { } /// Resizes worker priority and counts + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] pub async fn resize_worker_priority( &self, pri: ReplicationPriority, @@ -1180,6 +1192,10 @@ impl ReplicationPool { } /// Queues an MRF save operation + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] async fn queue_mrf_save(&self, entry: MrfReplicateEntry) { let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await; } @@ -1651,6 +1667,10 @@ impl ReplicationPool { } /// Worker function for handling regular replication operations + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] async fn add_worker( &self, mut rx: Receiver, @@ -1664,6 +1684,10 @@ impl ReplicationPool { } /// Worker function for handling large object replication operations + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] async fn add_large_worker( &self, mut rx: Receiver, @@ -1678,6 +1702,10 @@ impl ReplicationPool { } /// Worker function for handling MRF (Most Recent Failures) operations + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] async fn add_mrf_worker( &self, mut rx: Receiver, @@ -1691,6 +1719,10 @@ impl ReplicationPool { } /// Delete resync metadata from replication resync state in memory + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] pub async fn delete_resync_metadata(&self, bucket: &str) { let mut status_map = self.resyncer.status_map.write().await; status_map.remove(bucket); diff --git a/crates/ecstore/src/bucket/replication/replication_resync_boundary.rs b/crates/ecstore/src/bucket/replication/replication_resync_boundary.rs index c41c6ab79..3b6dcdf5d 100644 --- a/crates/ecstore/src/bucket/replication/replication_resync_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_resync_boundary.rs @@ -21,11 +21,31 @@ pub(crate) use rustfs_replication::{ should_count_head_proxy_failure, }; +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT; +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION; pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES; +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX; +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT; +#[allow( + dead_code, + reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)" +)] pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION; fn map_replication_error(err: rustfs_replication::Error) -> Error { diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index c6cdbcb03..30198a75b 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -122,6 +122,10 @@ const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[ "tcp connect error", ]; +#[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" +)] const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60); static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new(); @@ -328,6 +332,10 @@ fn bounded_resync_max_jobs(value: usize) -> usize { #[derive(Debug)] pub struct ReplicationResyncer { pub status_map: Arc>>, + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] pub worker_size: usize, pub(crate) cancel_tokens: Arc>>, resync_admission: Arc, @@ -544,6 +552,10 @@ impl ReplicationResyncer { .is_some_and(|status| status.failed_count > 0) } + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] pub async fn persist_to_disk(&self, cancel_token: CancellationToken, api: Arc) where S: ReplicationObjectIO, diff --git a/crates/ecstore/src/bucket/replication/replication_state.rs b/crates/ecstore/src/bucket/replication/replication_state.rs index cd0e303f3..acd4b696a 100644 --- a/crates/ecstore/src/bucket/replication/replication_state.rs +++ b/crates/ecstore/src/bucket/replication/replication_state.rs @@ -340,6 +340,10 @@ impl ReplicationStats { } /// Site replication update replica statistics + #[allow( + dead_code, + reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)" + )] fn sr_update_replica_stat(&self, size: i64) { self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed); self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed); diff --git a/crates/ecstore/src/bucket/target/bucket_target.rs b/crates/ecstore/src/bucket/target/bucket_target.rs index f78a2e0ef..55e3d3e8a 100644 --- a/crates/ecstore/src/bucket/target/bucket_target.rs +++ b/crates/ecstore/src/bucket/target/bucket_target.rs @@ -59,6 +59,10 @@ impl fmt::Debug for Credentials { } #[derive(Debug, Deserialize, Serialize, Default, Clone)] +#[allow( + dead_code, + reason = "MinIO-parity bucket-target service discriminator with no caller in this port (backlog#1823)" +)] pub enum ServiceType { #[default] Replication, diff --git a/crates/ecstore/src/bucket/utils.rs b/crates/ecstore/src/bucket/utils.rs index fdd934dc6..d24ef95b5 100644 --- a/crates/ecstore/src/bucket/utils.rs +++ b/crates/ecstore/src/bucket/utils.rs @@ -73,23 +73,6 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> { check_bucket_name_common(bucket_name, true) } -pub fn check_valid_object_name_prefix(object_name: &str) -> Result<()> { - if object_name.len() > 1024 { - return Err(Error::other("Object name cannot be longer than 1024 characters")); - } - if !object_name.is_ascii() { - return Err(Error::other("Object name with non-UTF-8 strings are not supported")); - } - Ok(()) -} - -pub fn check_valid_object_name(object_name: &str) -> Result<()> { - if object_name.trim().is_empty() { - return Err(Error::other("Object name cannot be empty")); - } - check_valid_object_name_prefix(object_name) -} - pub fn deserialize(input: &[u8]) -> xml::DeResult where T: for<'xml> xml::Deserialize<'xml>, @@ -100,6 +83,10 @@ where Ok(ans) } +#[allow( + dead_code, + reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)" +)] pub fn serialize_content(val: &T) -> xml::SerResult { let mut buf = Vec::with_capacity(256); { @@ -186,15 +173,27 @@ pub fn is_valid_object_name(object: &str) -> bool { /// Client-facing reason attached to rejections of object keys that Win32/NTFS /// cannot represent as file paths (issue #3299). Deployments on Linux/macOS /// accept the full S3 key character set. +#[allow( + dead_code, + reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)" +)] pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str = "object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')"; /// Client-facing reason for path segments Windows can store but not address /// afterwards (issue #3449): trailing dot/space or reserved DOS device names. +#[allow( + dead_code, + reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)" +)] pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)"; /// Reserved DOS device names that shadow regular files on Windows, even when /// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device). +#[allow( + dead_code, + reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)" +)] const WINDOWS_RESERVED_NAMES: &[&str] = &[ "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", @@ -204,6 +203,10 @@ const WINDOWS_RESERVED_NAMES: &[&str] = &[ /// the Win32 API cannot address afterwards (issue #3449): segments ending in a /// dot or a space, and reserved DOS device names — bare or with an extension /// (`NUL.txt`), matching classic Win32 path resolution semantics. +#[allow( + dead_code, + reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)" +)] pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool { object.split(['/', '\\']).any(|segment| { if segment.ends_with('.') || segment.ends_with(' ') { diff --git a/crates/ecstore/src/bucket/versioning_sys.rs b/crates/ecstore/src/bucket/versioning_sys.rs index 685e95af9..abc6fee6d 100644 --- a/crates/ecstore/src/bucket/versioning_sys.rs +++ b/crates/ecstore/src/bucket/versioning_sys.rs @@ -90,6 +90,10 @@ impl BucketVersioningSys { /// caller's own instance context so a second in-process store never /// answers with the first instance's versioning state; falls back to the /// ambient system when the instance cell is not initialized. + #[allow( + dead_code, + reason = "instance-scoped seam (backlog#1052) with no caller in this port (backlog#1823)" + )] pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result { if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) { return Ok(VersioningConfiguration::default()); From cd0ac02879fdb11e65be7983df8f96e1977804df Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 22:05:27 +0800 Subject: [PATCH 43/71] test(interop): let the MinIO fixture lab build from registry mirrors (#6148) --- crates/rio-v2/tests/minio_fixture_lab/Dockerfile | 13 +++++++++++-- crates/rio-v2/tests/minio_fixture_lab/README.md | 12 ++++++++++++ .../tests/minio_fixture_lab/capture_via_docker.sh | 13 ++++++++++++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/crates/rio-v2/tests/minio_fixture_lab/Dockerfile b/crates/rio-v2/tests/minio_fixture_lab/Dockerfile index 8e6e50069..98c02c829 100644 --- a/crates/rio-v2/tests/minio_fixture_lab/Dockerfile +++ b/crates/rio-v2/tests/minio_fixture_lab/Dockerfile @@ -6,9 +6,18 @@ # # The MinIO release is pinned so the captured fixture format is reproducible; # this is the release the interop tests were validated against. -FROM minio/minio:RELEASE.2025-09-07T16-13-09Z AS minio +# +# Both base images are build args so a network that cannot reach Docker Hub can +# point them at a mirror carrying the same content — quay.io publishes the MinIO +# releases, and public.ecr.aws mirrors the official Python images. CI keeps the +# Docker Hub defaults. Override with: +# --build-arg MINIO_IMAGE=quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z \ +# --build-arg PYTHON_IMAGE=public.ecr.aws/docker/library/python:3.12-slim +ARG MINIO_IMAGE=minio/minio:RELEASE.2025-09-07T16-13-09Z +ARG PYTHON_IMAGE=python:3.12-slim +FROM ${MINIO_IMAGE} AS minio -FROM python:3.12-slim +FROM ${PYTHON_IMAGE} RUN apt-get update \ && apt-get install -y --no-install-recommends openssl ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/crates/rio-v2/tests/minio_fixture_lab/README.md b/crates/rio-v2/tests/minio_fixture_lab/README.md index 81ae85ef6..a003bab26 100644 --- a/crates/rio-v2/tests/minio_fixture_lab/README.md +++ b/crates/rio-v2/tests/minio_fixture_lab/README.md @@ -22,6 +22,18 @@ Use the automated path when you want the lab to: - upload a predefined SSE fixture case - export the generated backend tree into the lab layout +## Networks without Docker Hub access + +`capture_via_docker.sh` pulls its two base images from Docker Hub by default. Where that registry is unreachable, point the build at mirrors carrying the same content — quay.io publishes the MinIO releases and public.ecr.aws mirrors the official Python images: + +```bash +MINIO_LAB_MINIO_IMAGE=quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z \ +MINIO_LAB_PYTHON_IMAGE=public.ecr.aws/docker/library/python:3.12-slim \ +./capture_via_docker.sh +``` + +Pin the MinIO tag to the same release the Dockerfile names; an unpinned `:latest` captures whatever format that day's build writes, which is not what the interop tests were validated against. + ## Layout The default root is `artifacts/minio-fixture-lab`, which is already ignored by the repository. diff --git a/crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh b/crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh index 77f6e4687..6c1bec2f1 100755 --- a/crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh +++ b/crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh @@ -34,8 +34,19 @@ if [ "${cases[0]}" != "all" ]; then done fi +# Base images are overridable so a network without Docker Hub access can point +# them at a mirror (see the Dockerfile header). Unset by default, which keeps the +# Dockerfile's Docker Hub defaults for CI. +build_args=() +if [ -n "${MINIO_LAB_MINIO_IMAGE:-}" ]; then + build_args+=(--build-arg "MINIO_IMAGE=${MINIO_LAB_MINIO_IMAGE}") +fi +if [ -n "${MINIO_LAB_PYTHON_IMAGE:-}" ]; then + build_args+=(--build-arg "PYTHON_IMAGE=${MINIO_LAB_PYTHON_IMAGE}") +fi + echo ">> building ${IMAGE}" -docker build -f "${SCRIPT_DIR}/Dockerfile" -t "${IMAGE}" "${SCRIPT_DIR}" +docker build -f "${SCRIPT_DIR}/Dockerfile" -t "${IMAGE}" "${build_args[@]}" "${SCRIPT_DIR}" echo ">> capturing fixtures into ${FIXTURE_REL}" docker run --rm -v "${REPO_ROOT}:/repo" "${IMAGE}" \ From 1862112d0cfa2282bea6edf89eb718f02abfdde0 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 22:18:00 +0800 Subject: [PATCH 44/71] chore: drop the remaining product-code dead_code blankets (#6149) --- crates/checksums/src/base64.rs | 1 - crates/kms/src/encryption/dek.rs | 2 - rustfs/src/storage/deadlock_detector.rs | 5 +- rustfs/src/storage/lock_optimizer.rs | 1 - rustfs/src/storage/s3_api/mod.rs | 14 ----- rustfs/src/table_catalog/iceberg/manifest.rs | 16 ++++++ .../src/table_catalog/iceberg/validation.rs | 28 ++++++++++ rustfs/src/table_catalog/identifier.rs | 48 +++++++++++++++++ rustfs/src/table_catalog/mod.rs | 30 ++++++++++- rustfs/src/table_catalog/model.rs | 52 +++++++++++++++++++ rustfs/src/table_catalog/store/mod.rs | 36 +++++++++++++ rustfs/src/table_catalog/store/object.rs | 24 +++++++++ 12 files changed, 236 insertions(+), 21 deletions(-) diff --git a/crates/checksums/src/base64.rs b/crates/checksums/src/base64.rs index dcace1ac7..da7774eee 100644 --- a/crates/checksums/src/base64.rs +++ b/crates/checksums/src/base64.rs @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] use base64_simd::STANDARD; diff --git a/crates/kms/src/encryption/dek.rs b/crates/kms/src/encryption/dek.rs index f8f2ae275..2b411128d 100644 --- a/crates/kms/src/encryption/dek.rs +++ b/crates/kms/src/encryption/dek.rs @@ -18,8 +18,6 @@ //! data encryption keys using master keys. It abstracts the encryption //! operations so that different backends can share the same encryption logic. -#![allow(dead_code)] // Trait methods may be used by implementations - use crate::error::{KmsError, Result}; use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary}; use async_trait::async_trait; diff --git a/rustfs/src/storage/deadlock_detector.rs b/rustfs/src/storage/deadlock_detector.rs index 3955ee1b3..8cae936a4 100644 --- a/rustfs/src/storage/deadlock_detector.rs +++ b/rustfs/src/storage/deadlock_detector.rs @@ -56,7 +56,6 @@ //! ``` // Allow dead_code for public API that may be used by external modules or future features -#![allow(dead_code)] use parking_lot::{Mutex, RwLock}; use std::collections::{HashMap, HashSet}; @@ -264,6 +263,10 @@ pub struct ResourceUsage { /// Deadlock detector. pub struct DeadlockDetector { /// Configuration. + #[allow( + dead_code, + reason = "policy snapshot retained beside the detector it configures (backlog#1823)" + )] config: RequestHangDetectionPolicy, /// Shared concurrency facade policy. policy: DeadlockMonitorPolicy, diff --git a/rustfs/src/storage/lock_optimizer.rs b/rustfs/src/storage/lock_optimizer.rs index bff2356cf..3636c50c3 100644 --- a/rustfs/src/storage/lock_optimizer.rs +++ b/rustfs/src/storage/lock_optimizer.rs @@ -31,7 +31,6 @@ //! ``` // Allow dead_code for public API that may be used by external modules or future features -#![allow(dead_code)] //! # Key Features //! //! - Early lock release after metadata read diff --git a/rustfs/src/storage/s3_api/mod.rs b/rustfs/src/storage/s3_api/mod.rs index af30113db..6773d7716 100644 --- a/rustfs/src/storage/s3_api/mod.rs +++ b/rustfs/src/storage/s3_api/mod.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Facade modules for incremental S3 API extraction from `ecfs.rs`. //! //! This file intentionally starts as skeleton-only. Behavior remains in place @@ -29,18 +27,6 @@ pub(crate) mod common; pub(crate) mod multipart; pub(crate) mod tagging; -pub(crate) fn default_bucket_usecase() -> DefaultBucketUsecase { - DefaultBucketUsecase::from_global() -} - -pub(crate) fn default_multipart_usecase() -> DefaultMultipartUsecase { - DefaultMultipartUsecase::from_global() -} - -pub(crate) fn default_object_usecase() -> DefaultObjectUsecase { - DefaultObjectUsecase::from_global() -} - /// Resolve the object use-case for a server's request path (backlog#1052 S6): /// bind it to the server's own application context so it resolves that /// server's store instead of the ambient process default. diff --git a/rustfs/src/table_catalog/iceberg/manifest.rs b/rustfs/src/table_catalog/iceberg/manifest.rs index 82960d255..437c156a2 100644 --- a/rustfs/src/table_catalog/iceberg/manifest.rs +++ b/rustfs/src/table_catalog/iceberg/manifest.rs @@ -71,6 +71,10 @@ pub(crate) struct DecodedManifest { pub partition_spec_id: Option, } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn manifest_paths_from_manifest_list_avro(data: &[u8]) -> TableCatalogStoreResult> { Ok(manifest_list_references_from_manifest_list_avro(data)? .into_iter() @@ -78,6 +82,10 @@ pub(crate) fn manifest_paths_from_manifest_list_avro(data: &[u8]) -> TableCatalo .collect()) } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn manifest_list_references_from_manifest_list_avro( data: &[u8], ) -> TableCatalogStoreResult> { @@ -157,6 +165,10 @@ pub(crate) async fn decode_manifest_list_avro_async(data: Vec) -> TableCatal .map_err(|err| TableCatalogStoreError::Internal(format!("manifest-list parser task failed: {err}")))? } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn file_references_from_manifest_avro( data: &[u8], ) -> TableCatalogStoreResult> { @@ -166,6 +178,10 @@ pub(crate) fn file_references_from_manifest_avro( .collect()) } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn data_file_references_from_manifest_avro(data: &[u8]) -> TableCatalogStoreResult> { Ok(decode_manifest_avro(data)?.references) } diff --git a/rustfs/src/table_catalog/iceberg/validation.rs b/rustfs/src/table_catalog/iceberg/validation.rs index c10c30237..0783b5f78 100644 --- a/rustfs/src/table_catalog/iceberg/validation.rs +++ b/rustfs/src/table_catalog/iceberg/validation.rs @@ -103,6 +103,10 @@ pub(crate) fn table_warehouse_index_entry(entry: &TableEntry) -> TableCatalogSto }) } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] fn table_warehouse_data_dir_path(entry: &TableEntry) -> TableCatalogStoreResult { Ok(format!("{}{}", table_warehouse_object_prefix(entry)?, DATA_DIR)) } @@ -2414,11 +2418,35 @@ struct SnapshotGraphManifestLocation { sequence_number: Option, min_sequence_number: Option, added_snapshot_id: Option, + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] added_files_count: Option, + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] existing_files_count: Option, + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] deleted_files_count: Option, + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] added_rows_count: Option, + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] existing_rows_count: Option, + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] deleted_rows_count: Option, from_manifest_list: bool, } diff --git a/rustfs/src/table_catalog/identifier.rs b/rustfs/src/table_catalog/identifier.rs index e6c769d91..05555c188 100644 --- a/rustfs/src/table_catalog/identifier.rs +++ b/rustfs/src/table_catalog/identifier.rs @@ -87,6 +87,10 @@ impl Namespace { } #[derive(Debug, Clone, PartialEq, Eq)] +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub struct TableIdentifier { warehouse: IdentifierSegment, namespace: Namespace, @@ -94,6 +98,10 @@ pub struct TableIdentifier { } impl TableIdentifier { + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub fn new(warehouse: IdentifierSegment, namespace: Namespace, name: IdentifierSegment) -> Self { Self { warehouse, @@ -116,6 +124,10 @@ impl TableIdentifier { } #[derive(Debug, Clone, PartialEq, Eq)] +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub struct TablePathResolver { reserved_prefix: &'static str, } @@ -129,14 +141,26 @@ impl Default for TablePathResolver { } impl TablePathResolver { + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub fn current_pointer_path(&self, table: &TableIdentifier) -> String { format!("{}/{}", self.table_root(table), CURRENT_POINTER_FILE) } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub fn metadata_dir_path(&self, table: &TableIdentifier) -> String { format!("{}/{}", self.table_root(table), METADATA_DIR) } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub fn metadata_file_path(&self, table: &TableIdentifier, metadata_file_name: &str) -> String { format!("{}/{}", self.metadata_dir_path(table), metadata_file_name) } @@ -169,6 +193,10 @@ pub(crate) fn default_namespace_root_prefix() -> String { ) } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn default_namespace_marker_path(namespace: &Namespace) -> String { format!("{}{}/{}", default_namespace_root_prefix(), namespace.storage_id(), NAMESPACE_MARKER_FILE) } @@ -185,6 +213,10 @@ pub(crate) fn default_table_bucket_publication_lock_path() -> String { rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH.to_string() } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn default_table_marker_path(namespace: &Namespace, table: &IdentifierSegment) -> String { format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), TABLE_MARKER_FILE) } @@ -225,14 +257,26 @@ pub(crate) fn default_table_metadata_file_path( format!("{}/{}", default_table_metadata_dir_path(namespace, table), metadata_file_name) } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn default_table_current_pointer_path(namespace: &Namespace, table: &IdentifierSegment) -> String { format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), CURRENT_POINTER_FILE) } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn default_table_lifecycle_path(namespace: &Namespace, table: &IdentifierSegment) -> String { format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), LIFECYCLE_FILE) } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn namespace_name_from_marker_path(object_key: &str) -> Option { let prefix = default_namespace_root_prefix(); let suffix = format!("/{NAMESPACE_MARKER_FILE}"); @@ -244,6 +288,10 @@ pub(crate) fn namespace_name_from_marker_path(object_key: &str) -> Option Option { let prefix = default_table_root_prefix(namespace); let suffix = format!("/{TABLE_MARKER_FILE}"); diff --git a/rustfs/src/table_catalog/mod.rs b/rustfs/src/table_catalog/mod.rs index 4011639ce..c43571f31 100644 --- a/rustfs/src/table_catalog/mod.rs +++ b/rustfs/src/table_catalog/mod.rs @@ -18,8 +18,6 @@ //! S3 object behavior. It defines the stable internal boundary that later //! catalog routes and object guards can share. -#![allow(dead_code)] - use std::{ collections::{BTreeMap, BTreeSet}, num::NonZeroUsize, @@ -87,8 +85,20 @@ pub(crate) const RESERVED_CATALOG_OBJECT_MESSAGE: &str = "Object key is reserved pub(crate) const TABLE_BUCKET_CATALOG_TYPE: &str = "iceberg-rest"; pub(crate) const TABLE_BUCKET_CONFIG_VERSION: u16 = 1; pub(crate) const DEFAULT_WAREHOUSE_ID: &str = "default"; +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) const TABLE_NAMESPACE_MARKER_VERSION: u16 = 1; +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) const TABLE_RESOURCE_MARKER_VERSION: u16 = 1; +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1; pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1; pub(crate) const TABLE_WAREHOUSE_INDEX_STATE_VERSION: u16 = 2; @@ -123,9 +133,25 @@ const WAREHOUSE_ROOT: &str = "warehouses"; const NAMESPACE_ROOT: &str = "namespaces"; const TABLE_ROOT: &str = "tables"; const VIEW_ROOT: &str = "views"; +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] const NAMESPACE_MARKER_FILE: &str = "namespace.json"; +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] const TABLE_MARKER_FILE: &str = "table.json"; +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] const CURRENT_POINTER_FILE: &str = "current.json"; +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] const LIFECYCLE_FILE: &str = "lifecycle.json"; const METADATA_DIR: &str = "metadata"; const DATA_DIR: &str = "data"; diff --git a/rustfs/src/table_catalog/model.rs b/rustfs/src/table_catalog/model.rs index 3cef76469..d091dd849 100644 --- a/rustfs/src/table_catalog/model.rs +++ b/rustfs/src/table_catalog/model.rs @@ -1076,6 +1076,10 @@ pub(crate) enum TableCatalogBackingKind { #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub(crate) enum TableCatalogAuthority { RustfsSysObject, + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] LinearizableMetadataKv, } @@ -1083,6 +1087,10 @@ pub(crate) enum TableCatalogAuthority { #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub(crate) enum TableCatalogConsistencyMode { ConditionalObjectCas, + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] LinearizableCas, } @@ -1090,6 +1098,10 @@ pub(crate) enum TableCatalogConsistencyMode { #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub(crate) enum TableCatalogDurabilityMode { StagedCommitLogBeforePointerUpdate, + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] WalBeforeStateMachineApply, } @@ -1409,12 +1421,20 @@ pub(crate) struct TableCommitRecoveryReport { } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) struct NamespaceMarker { pub version: u16, pub namespace: String, } impl NamespaceMarker { + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub fn new(namespace: &Namespace) -> Self { Self { version: TABLE_NAMESPACE_MARKER_VERSION, @@ -1423,11 +1443,19 @@ impl NamespaceMarker { } } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn namespace_marker_json(namespace: &Namespace) -> Result, serde_json::Error> { serde_json::to_vec(&NamespaceMarker::new(namespace)) } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) struct TableMarker { pub version: u16, pub namespace: String, @@ -1436,6 +1464,10 @@ pub(crate) struct TableMarker { } impl TableMarker { + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub fn new(namespace: &Namespace, table: &IdentifierSegment) -> Self { Self { version: TABLE_RESOURCE_MARKER_VERSION, @@ -1446,17 +1478,29 @@ impl TableMarker { } } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn table_marker_json(namespace: &Namespace, table: &IdentifierSegment) -> Result, serde_json::Error> { serde_json::to_vec(&TableMarker::new(namespace, table)) } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) struct TableMetadataPointer { pub version: u16, pub metadata_location: String, } impl TableMetadataPointer { + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub fn new(metadata_location: String) -> Self { Self { version: TABLE_METADATA_POINTER_VERSION, @@ -1465,10 +1509,18 @@ impl TableMetadataPointer { } } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn table_metadata_pointer_json(metadata_location: String) -> Result, serde_json::Error> { serde_json::to_vec(&TableMetadataPointer::new(metadata_location)) } +#[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn parse_table_metadata_pointer(data: &[u8]) -> Result { serde_json::from_slice(data) } diff --git a/rustfs/src/table_catalog/store/mod.rs b/rustfs/src/table_catalog/store/mod.rs index 9e8d57129..3856588a3 100644 --- a/rustfs/src/table_catalog/store/mod.rs +++ b/rustfs/src/table_catalog/store/mod.rs @@ -164,6 +164,10 @@ pub(crate) trait TableCatalogStore: Send + Sync { )) } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] async fn list_namespaces_page( &self, table_bucket: &str, @@ -193,8 +197,16 @@ pub(crate) trait TableCatalogStore: Send + Sync { async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult<()>; + #[allow( + dead_code, + reason = "declared trait method: implementors provide it but no caller dispatches through the trait yet (backlog#1823)" + )] async fn create_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()>; + #[allow( + dead_code, + reason = "declared trait method: implementors provide it but no caller dispatches through the trait yet (backlog#1823)" + )] async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()>; async fn register_table_with_publication( @@ -250,6 +262,10 @@ pub(crate) trait TableCatalogStore: Send + Sync { /// /// Callers publishing client-supplied Iceberg metadata must validate its logical shape and the physical graph of /// newly introduced or changed snapshots before invoking this persistence boundary. + #[allow( + dead_code, + reason = "declared trait method: implementors provide it but no caller dispatches through the trait yet (backlog#1823)" + )] async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult; async fn commit_table_with_publication( @@ -495,6 +511,10 @@ pub(crate) struct TableCatalogLockGuard { } impl TableCatalogLockGuard { + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub(crate) fn stable(guard: impl Send + 'static) -> Self { Self { _guard: Box::new(guard), @@ -615,6 +635,10 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult; + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] async fn object_exists_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { self.object_exists(bucket, object).await } @@ -1006,6 +1030,10 @@ where } } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub(crate) fn backing_mode(&self) -> TableCatalogBackingMode { match self { Self::ObjectBacked(_) => TableCatalogBackingMode::ObjectBacked, @@ -1529,6 +1557,10 @@ where } } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub(crate) async fn get_external_catalog_bridge( &self, table_bucket: &str, @@ -1541,6 +1573,10 @@ where } } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub(crate) async fn put_external_catalog_bridge( &self, entry: ExternalCatalogBridgeEntry, diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index 3cd08f321..3f668b945 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -1270,6 +1270,10 @@ where Ok(Some((entry, etag))) } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] async fn write_table_entry( &self, entry: TableEntry, @@ -1690,6 +1694,10 @@ where Ok(config) } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub(crate) async fn put_table_bucket_maintenance_config( &self, table_bucket: &str, @@ -3006,6 +3014,10 @@ where table_compaction_planning_report(&self.backend, table_bucket, &namespace, &table, &entry, ¤t_metadata, config).await } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub(crate) async fn commit_table_compaction( &self, table_bucket: &str, @@ -3559,6 +3571,10 @@ where Ok(report) } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub(crate) async fn delete_table_metadata_maintenance_candidates( &self, table_bucket: &str, @@ -3573,6 +3589,10 @@ where .await } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub(crate) async fn run_table_metadata_maintenance( &self, table_bucket: &str, @@ -3754,6 +3774,10 @@ where Ok(report) } + #[allow( + dead_code, + reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)" + )] pub(in crate::table_catalog) async fn delete_table_metadata_maintenance_report( &self, table_bucket: &str, From 3272730c130479c417d51bb0be11f0beef5fd008 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 22:42:45 +0800 Subject: [PATCH 45/71] fix(ecstore): silence two dead_code warnings left on main (#6153) --- crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs | 4 ++++ crates/ecstore/src/disk/mod.rs | 1 + crates/ecstore/src/set_disk/mod.rs | 4 +--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index c74451095..0366551e0 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -4379,6 +4379,10 @@ pub async fn expire_transitioned_object( Ok(dobj) } +#[allow( + dead_code, + reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)" +)] pub fn gen_transition_objname(bucket: &str) -> Result { let us = Uuid::new_v4().to_string(); let mut hasher = Sha256::new(); diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 3aaa54c8f..ecd0179e0 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -1375,6 +1375,7 @@ pub fn conv_part_err_to_int(err: &Option) -> usize { } } +#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn has_part_err(part_errs: &[usize]) -> bool { part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS) } diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index dcc416d47..ad5db566b 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -102,9 +102,7 @@ use crate::storage_api_contracts::{ }; use crate::store::utils::is_reserved_or_invalid_bucket; use crate::{ - bucket::lifecycle::bucket_lifecycle_ops::{ - LifecycleOps, gen_transition_objname, get_transitioned_object_reader_with_tier_manager, put_restore_opts, - }, + bucket::lifecycle::bucket_lifecycle_ops::{LifecycleOps, get_transitioned_object_reader_with_tier_manager, put_restore_opts}, cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, config::storageclass, disk::{ From 4c8b9f87e14e6dc31095c2cf12d17758a029fb17 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 22:42:51 +0800 Subject: [PATCH 46/71] chore(ecstore): annotate the two dead fns the blanket removals missed (#6152) From a2f16aa066df60c484367482fc17371a680f745e Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 16 Aug 2026 23:27:59 +0800 Subject: [PATCH 47/71] test(tier): pin the compressed transitioned read against its stored bytes (#6151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #6107 routed the transitioned read through the object's own ReadPlan so a tiered SSE object stops serving ciphertext. Compression rides that same plan and got fixed with it, but nothing pins it: revert the routing and a compressed object that ILM moved to a warm tier returns its stored (compressed) bytes under the compressed size, with every existing test still green. The gap is easy to reopen because transition genuinely uploads the stored representation — the upload side is correct and the read side is the only place that can decode it. These tests state that contract at the boundary where it broke. Four cases, all through SetDisks::get_object_reader against a mock warm tier: - a full GET of a transitioned compressed object returns the plaintext and publishes the plaintext size (the test also asserts the remote copy holds the compressed bytes, so it fails loudly if the upload side ever changes instead); - a ranged GET returns that plaintext slice, with the range deliberately starting past the compressed size so a range still measured in stored coordinates cannot produce it; - a restore read still receives the stored bytes under the stored size — restore_request_active holds it on the Plain branch, and decompressing there would write plaintext under compressed metadata; - a plain transitioned object still reads back byte-identical, full and ranged. Verified as guards, not decoration: forcing the tiered read back onto the Plain branch turns the two compressed tests red and leaves the plain and restore tests green. What these do not pin, so the gap stays recorded rather than implied covered: the fixture carries no compression index, so part.index stays None and the plan's storage offset is always 0 — the compressed-offset translation itself is still untested, as are multipart compressed objects, partNumber reads, and the encrypted tiered read that #6107 targeted. --- crates/ecstore/src/set_disk/ops/object.rs | 282 ++++++++++++++++++++++ 1 file changed, 282 insertions(+) diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 96ca009cf..670397c12 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -10206,6 +10206,288 @@ mod transition_upload_integrity_tests { assert!(backend.contains(remote_object).await, "committed remote object should remain available"); } + /// Compresses `plaintext` with the codec the PUT path uses, so the stored + /// bytes round-trip through the read path's decompressor. + async fn compress_for_storage(plaintext: &[u8]) -> Vec { + let mut reader = crate::io_support::rio::compression_reader( + Cursor::new(plaintext.to_vec()), + rustfs_utils::CompressionAlgorithm::default(), + false, + ); + let mut compressed = Vec::new(); + reader.read_to_end(&mut compressed).await.expect("plaintext should compress"); + assert!(compressed.len() < plaintext.len(), "test payload must actually compress"); + compressed + } + + /// Writes a genuinely compressed object: stored data is `compressed`, and the + /// metadata marks it compressed with the plaintext length as its actual size, + /// exactly as the app-layer compress path records it. + async fn write_compressed_source( + set_disks: &Arc, + disk_stores: &[DiskStore], + bucket: &str, + object: &str, + plaintext: &[u8], + compressed: &[u8], + ) -> ObjectInfo { + for disk in disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut user_defined = HashMap::new(); + rustfs_utils::http::insert_str( + &mut user_defined, + rustfs_utils::http::SUFFIX_COMPRESSION, + crate::io_support::rio::compression_metadata_value(rustfs_utils::CompressionAlgorithm::default()), + ); + rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string()); + let stream = crate::io_support::rio::HashReader::from_stream( + Cursor::new(compressed.to_vec()), + compressed.len() as i64, + plaintext.len() as i64, + None, + None, + false, + ) + .expect("hash reader over compressed bytes"); + let mut reader = PutObjReader::new(stream); + set_disks + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + no_lock: true, + user_defined, + ..Default::default() + }, + ) + .await + .expect("compressed object should be written") + } + + async fn read_transitioned( + set_disks: &Arc, + bucket: &str, + object: &str, + range: Option, + opts: &ObjectOptions, + ) -> (Vec, i64) { + let mut reader = set_disks + .get_object_reader(bucket, object, range, HeaderMap::new(), opts) + .await + .expect("transitioned object reader should open"); + let published_size = reader.object_info.size; + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("transitioned body should drain"); + (body, published_size) + } + + /// Transition uploads the object's STORED bytes, so a tiered read has to + /// apply the same transform an erasure read would. #6107 routed this path + /// through `ReadPlan` to stop serving an encrypted object's ciphertext; + /// compression rides the same plan, and nothing pinned it (backlog#1851). + /// Without the transform this GET returns the compressed bytes under the + /// compressed size — silent corruption for every client of a compressed + /// object that ILM has moved to a warm tier. + #[tokio::test] + #[serial_test::serial] + async fn transitioned_compressed_object_get_returns_plaintext() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transitioned-compressed-get-bucket"; + let object = "object.txt"; + let plaintext = b"transitioned compressed objects must decompress on read ".repeat(20_000); + let compressed = compress_for_storage(&plaintext).await; + let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await; + + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + let (local_body, local_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await; + assert_eq!(local_body, plaintext, "control: the pre-transition read must decompress"); + assert_eq!( + local_size, + plaintext.len() as i64, + "control: the pre-transition read publishes the plaintext size" + ); + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + set_disks + .transition_object(bucket, object, &transition_options(&original, tier_name)) + .await + .expect("transition should commit"); + + let put_versions = backend.put_versions().await; + assert_eq!(put_versions.len(), 1, "transition should upload one remote candidate"); + let remote_bytes = backend + .bytes(&put_versions[0].0) + .await + .expect("remote candidate should be stored"); + assert_eq!( + remote_bytes, compressed, + "transition uploads the stored representation; the read side is what has to decode it" + ); + + let (body, published_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await; + assert_eq!(body, plaintext, "a tiered read must return the object's content, not its stored bytes"); + assert_eq!( + published_size, + plaintext.len() as i64, + "a tiered read must publish the plaintext size, not the compressed one" + ); + } + + /// A ranged tiered read is expressed in plaintext coordinates, so the plan + /// has to translate it into the remote copy's compressed extent and skip + /// into the decompressed stream — the same translation the erasure path does. + #[tokio::test] + #[serial_test::serial] + async fn transitioned_compressed_object_range_get_returns_plaintext_slice() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transitioned-compressed-range-bucket"; + let object = "object.txt"; + let plaintext = b"ranged reads of transitioned compressed objects must land in plaintext ".repeat(20_000); + let compressed = compress_for_storage(&plaintext).await; + let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await; + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + set_disks + .transition_object(bucket, object, &transition_options(&original, tier_name)) + .await + .expect("transition should commit"); + + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + // Deliberately past the compressed size, so a range still measured in + // stored coordinates could not produce this slice. + let start = compressed.len() as i64 + 4096; + let end = start + 511; + let range = HTTPRangeSpec { + is_suffix_length: false, + start, + end, + }; + let (body, published_size) = read_transitioned(&set_disks, bucket, object, Some(range), &opts).await; + + let expected = &plaintext[start as usize..=end as usize]; + assert_eq!(body, expected, "a ranged tiered read must return that plaintext slice"); + assert_eq!(published_size, expected.len() as i64, "a ranged tiered read publishes the slice length"); + } + + /// The restore copy-back re-writes the object under its original metadata, + /// which still says "compressed". It therefore has to keep receiving the + /// STORED bytes: `restore_request_active` holds it on the plan's `Plain` + /// branch, and decompressing there would write plaintext under compressed + /// metadata. + #[tokio::test] + #[serial_test::serial] + async fn restore_read_of_transitioned_compressed_object_keeps_stored_bytes() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transitioned-compressed-restore-bucket"; + let object = "object.txt"; + let plaintext = b"restore copy-back must keep the stored representation intact ".repeat(20_000); + let compressed = compress_for_storage(&plaintext).await; + let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await; + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + set_disks + .transition_object(bucket, object, &transition_options(&original, tier_name)) + .await + .expect("transition should commit"); + + let oi = set_disks + .get_object_info( + bucket, + object, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("transitioned metadata should resolve"); + let restore_opts = ObjectOptions { + no_lock: true, + part_number: Some(1), + transition: TransitionOptions { + restore_request: s3s::dto::RestoreRequest { + days: Some(1), + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() + }; + let mut reader = get_transitioned_object_reader_with_tier_manager( + bucket, + object, + &None, + &HeaderMap::new(), + &oi, + &restore_opts, + &set_disks.ctx.tier_config_mgr(), + set_disks.ctx.object_encryption_resolver(), + ) + .await + .expect("restore read of the tiered copy should open"); + let published_size = reader.object_info.size; + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("restore body should drain"); + + assert_eq!(body, compressed, "a restore read must copy the stored bytes back verbatim"); + assert_eq!( + published_size, + compressed.len() as i64, + "a restore read must keep publishing the stored size" + ); + } + + /// Plain objects must keep streaming the remote bytes through untouched: + /// their plan is `Plain`, so the tiered read stays byte-identical. + #[tokio::test] + #[serial_test::serial] + async fn transitioned_plain_object_get_is_unchanged() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transitioned-plain-get-bucket"; + let object = "object.bin"; + let payload = b"plain transitioned objects must keep reading back byte-identical ".repeat(1024); + let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await; + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + set_disks + .transition_object(bucket, object, &transition_options(&original, tier_name)) + .await + .expect("transition should commit"); + + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + let (body, published_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await; + assert_eq!(body, payload); + assert_eq!(published_size, payload.len() as i64); + + let range = HTTPRangeSpec { + is_suffix_length: false, + start: 100, + end: 611, + }; + let (ranged_body, ranged_size) = read_transitioned(&set_disks, bucket, object, Some(range), &opts).await; + assert_eq!(ranged_body, &payload[100..=611]); + assert_eq!(ranged_size, payload.len() as i64, "a plain ranged read keeps publishing the object size"); + } + async fn corrupt_beyond_read_quorum( temp_dirs: &[tempfile::TempDir], bucket: &str, From 33eff4c3c4514ee299c4895cbdcdeb20e38cfcce Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 16 Aug 2026 23:36:46 +0800 Subject: [PATCH 48/71] test(ecstore): add metadata slow-tail fault hook (#6150) Add a diagnostic metadata-only read_version delay hook for GET data-read fanout so bounded/default behavior can be compared under controlled slow-tail metadata responses. Co-authored-by: heihutu --- .../src/set_disk/core/io_primitives.rs | 134 ++++++++++++++++++ crates/ecstore/src/set_disk/mod.rs | 97 ++++++++++++- 2 files changed, 229 insertions(+), 2 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 299299971..4e0e0e759 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -2446,6 +2446,7 @@ impl SetDisks { let bucket: Arc = Arc::from(bucket); let object: Arc = Arc::from(object); let version_id: Arc = Arc::from(version_id); + let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data); let futures = disks.iter().enumerate().map(|(disk_index, disk)| { let disk = disk.clone(); let task_opts = opts; @@ -2453,10 +2454,14 @@ impl SetDisks { let bucket = bucket.clone(); let object = object.clone(); let version_id = version_id.clone(); + let slowtail_fault = slowtail_fault.clone(); tokio::spawn(async move { let response_start = observe.then(Instant::now); let result = if let Some(disk) = disk { Self::record_read_version_call(&object, disk_index); + if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) { + tokio::time::sleep(delay).await; + } disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts) .await } else { @@ -2552,6 +2557,7 @@ impl SetDisks { let mut scheduled_count = 0usize; let mut force_full_wait = false; let mut final_miss_reason_override = None; + let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data); let spawn_read_version = |join_set: &mut JoinSet<(usize, disk::error::Result, Duration)>, index: usize, disk: Option| { let task_opts = opts; @@ -2559,6 +2565,7 @@ impl SetDisks { let bucket = bucket.clone(); let object = object.clone(); let version_id = version_id.clone(); + let slowtail_fault = slowtail_fault.clone(); join_set.spawn(async move { let response_start = Instant::now(); let result = if let Some(disk) = disk { @@ -2567,6 +2574,9 @@ impl SetDisks { Self::record_read_version_call(&object, index); #[cfg(test)] Self::read_version_fanout_barrier(&object, index).await; + if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) { + tokio::time::sleep(delay).await; + } disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts) .await } else { @@ -5744,6 +5754,130 @@ mod tests { (dirs, disks) } + #[test] + fn metadata_slowtail_fault_delay_parses_and_filters_request() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,3")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some("bench-bucket")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")), + ], + || { + assert_eq!( + get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, true), + Some(Duration::from_millis(25)) + ); + assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 2, true).is_none()); + assert!(get_metadata_slowtail_fault_delay("other-bucket", "objects/000001", 3, true).is_none()); + assert!(get_metadata_slowtail_fault_delay("bench-bucket", "other/000001", 3, true).is_none()); + assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, false).is_none()); + }, + ); + } + + #[test] + fn metadata_slowtail_fault_delay_disables_invalid_disk_list() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,nope")), + ], + || { + assert!(get_metadata_slowtail_fault_delay("bucket", "object", 1, true).is_none()); + }, + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn metadata_slowtail_fault_delays_only_data_read_metadata_task() { + const DISKS: usize = 4; + let bucket = "metadata-slowtail-fault-bucket"; + let object = "objects/metadata-slowtail-fault-object"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_metadata_fanout_fileinfo(&disks, bucket, object, None).await; + + temp_env::async_with_vars( + [ + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("false")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")), + ], + async { + let read_without_data = + SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", false, false, false, true, 2); + tokio::time::timeout(Duration::from_millis(100), read_without_data) + .await + .expect("non-data metadata fanout must not be delayed by the data-read slowtail hook") + .expect("metadata fanout without read_data should resolve"); + + let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed( + &disks, bucket, bucket, object, "", true, false, false, true, 2, + )); + assert!( + tokio::time::timeout(Duration::from_millis(40), &mut read_with_data) + .await + .is_err(), + "data-read metadata fanout must wait for the injected slow read_version response" + ); + let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data) + .await + .expect("injected slowtail should eventually complete") + .expect("data-read metadata fanout should resolve"); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); + assert_eq!(diagnostics.total_responses(), DISKS); + }, + ) + .await; + + drop(dirs); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn metadata_slowtail_fault_delays_early_stop_metadata_task() { + const DISKS: usize = 4; + let bucket = "metadata-slowtail-early-stop-bucket"; + let object = "objects/metadata-slowtail-early-stop-object"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_metadata_fanout_fileinfo(&disks, bucket, object, None).await; + + temp_env::async_with_vars( + [ + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")), + (ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")), + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")), + ], + async { + let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed( + &disks, bucket, bucket, object, "", true, false, false, true, 2, + )); + assert!( + tokio::time::timeout(Duration::from_millis(40), &mut read_with_data) + .await + .is_err(), + "early-stop metadata fanout must still wait for the injected slow response after fallback to full wait" + ); + let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data) + .await + .expect("injected early-stop slowtail should eventually complete") + .expect("early-stop metadata fanout should resolve"); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); + assert_eq!(diagnostics.total_responses(), DISKS); + }, + ) + .await; + + drop(dirs); + } + /// Demo / regression guard for the backlog#1325 per-disk call counters. /// /// The metadata fan-out issues each `read_version` inside its own diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index ad5db566b..a6a875a7b 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -174,15 +174,14 @@ use std::future::Future; use std::hash::{BuildHasher, Hash, Hasher}; use std::mem::{self}; use std::pin::Pin; -use std::sync::OnceLock; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock}; use std::task::{Context, Poll}; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use std::{ collections::{HashMap, HashSet}, io::{Cursor, Write}, path::Path, - sync::Arc, time::Duration, }; use time::OffsetDateTime; @@ -717,6 +716,11 @@ const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = true; const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT"; const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = false; +const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS"; +const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS"; +const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET"; +const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX"; + // --- Multipart Reader-Setup Prefetch Configuration (backlog#870) --- const ENV_RUSTFS_GET_MULTIPART_READER_SETUP_PREFETCH: &str = "RUSTFS_GET_MULTIPART_READER_SETUP_PREFETCH"; @@ -1695,6 +1699,95 @@ fn is_get_metadata_early_stop_bounded_fanout_enabled() -> bool { } } +#[derive(Debug)] +struct GetMetadataSlowtailFaultConfig { + delay: Duration, + disks: Arc<[usize]>, + bucket: Option, + object_prefix: Option, +} + +#[derive(Clone, Debug)] +struct GetMetadataSlowtailFaultRequest { + delay: Duration, + disks: Arc<[usize]>, +} + +impl GetMetadataSlowtailFaultRequest { + fn delay_for_disk(&self, disk_index: usize) -> Option { + self.disks.contains(&disk_index).then_some(self.delay) + } +} + +fn parse_get_metadata_slowtail_fault_disks(raw: &str) -> Option> { + let mut disks = Vec::new(); + for item in raw.split(',').map(str::trim).filter(|item| !item.is_empty()) { + let Ok(index) = item.parse::() else { + return None; + }; + if !disks.contains(&index) { + disks.push(index); + } + } + (!disks.is_empty()).then_some(disks) +} + +fn load_get_metadata_slowtail_fault_config() -> Option { + let delay_ms = rustfs_utils::get_env_u64(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, 0); + if delay_ms == 0 { + return None; + } + let disks = parse_get_metadata_slowtail_fault_disks(&std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS).ok()?)?; + let bucket = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET) + .ok() + .filter(|value| !value.is_empty()); + let object_prefix = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX) + .ok() + .filter(|value| !value.is_empty()); + Some(GetMetadataSlowtailFaultConfig { + delay: Duration::from_millis(delay_ms), + disks: Arc::from(disks.into_boxed_slice()), + bucket, + object_prefix, + }) +} + +fn get_metadata_slowtail_fault_request(bucket: &str, object: &str, read_data: bool) -> Option { + if !read_data { + return None; + } + + #[cfg(test)] + let config = load_get_metadata_slowtail_fault_config(); + #[cfg(test)] + let config = config.as_ref()?; + #[cfg(not(test))] + let config = ({ + static CACHED: OnceLock> = OnceLock::new(); + CACHED.get_or_init(load_get_metadata_slowtail_fault_config).as_ref() + })?; + + if let Some(expected_bucket) = &config.bucket + && expected_bucket != bucket + { + return None; + } + if let Some(expected_prefix) = &config.object_prefix + && !object.starts_with(expected_prefix) + { + return None; + } + Some(GetMetadataSlowtailFaultRequest { + delay: config.delay, + disks: config.disks.clone(), + }) +} + +#[cfg(test)] +fn get_metadata_slowtail_fault_delay(bucket: &str, object: &str, disk_index: usize, read_data: bool) -> Option { + get_metadata_slowtail_fault_request(bucket, object, read_data)?.delay_for_disk(disk_index) +} + /// Check if multipart reads prefetch the next part's bitrot reader setup /// while the current part decodes (backlog#870). /// From 39274fc37ca389f2311fbbca77c4d7602b809bab Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 00:56:20 +0800 Subject: [PATCH 49/71] feat(ecstore): default bounded metadata fanout (#6156) Co-authored-by: heihutu --- .../src/set_disk/core/io_primitives.rs | 148 ++++++++++++------ crates/ecstore/src/set_disk/mod.rs | 7 +- crates/ecstore/src/set_disk/read.rs | 12 +- 3 files changed, 107 insertions(+), 60 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 4e0e0e759..5f4471f13 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -668,6 +668,60 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( parts_metadata: &[FileInfo], disks: &[Option], ) -> Option<&'static str> { + if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) { + return Some(reason); + } + + let Ok(erasure) = coding::Erasure::try_new_with_options( + candidate.erasure.data_blocks, + candidate.erasure.parity_blocks, + candidate.erasure.block_size, + candidate.uses_legacy_checksum, + ) else { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); + }; + let data_files = + match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| { + disks.get(index).is_some_and(Option::is_some) + }) { + Ok(data_files) => data_files, + Err(reason) => return Some(reason), + }; + + let Some(part) = candidate.parts.first() else { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); + }; + let Ok(object_size) = usize::try_from(candidate.size) else { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); + }; + let checksum_info = candidate.erasure.get_checksum_info(part.number); + let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { + HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; + let read_length = inline_erasure_shard_file_offset( + 0, + object_size, + object_size, + candidate.erasure.block_size, + erasure.data_shards, + candidate.uses_legacy_checksum, + ); + let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum); + let Ok(mut readers) = + build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await + else { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY); + }; + + match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await { + Some(body) if body.len() == object_size => None, + _ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), + } +} + +fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> { // `inline_data` excludes remote objects; this diagnostic reports them separately. if !rustfs_utils::http::contains_key_str(&candidate.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA) { return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE); @@ -705,51 +759,7 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) { return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); } - - let Ok(erasure) = coding::Erasure::try_new_with_options( - candidate.erasure.data_blocks, - candidate.erasure.parity_blocks, - candidate.erasure.block_size, - candidate.uses_legacy_checksum, - ) else { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); - }; - let data_files = - match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| { - disks.get(index).is_some_and(Option::is_some) - }) { - Ok(data_files) => data_files, - Err(reason) => return Some(reason), - }; - - let Some(part) = candidate.parts.first() else { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); - }; - let checksum_info = candidate.erasure.get_checksum_info(part.number); - let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { - HashAlgorithm::HighwayHash256SLegacy - } else { - checksum_info.algorithm - }; - let read_length = inline_erasure_shard_file_offset( - 0, - object_size, - object_size, - candidate.erasure.block_size, - erasure.data_shards, - candidate.uses_legacy_checksum, - ); - let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum); - let Ok(mut readers) = - build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await - else { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY); - }; - - match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await { - Some(body) if body.len() == object_size => None, - _ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), - } + None } fn data_read_inline_missing_shards_are_pending( @@ -2610,6 +2620,14 @@ impl SetDisks { Ok(file_info) => { observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed)); accumulator.observe_file_info(&file_info); + if bounded_fanout + && read_data + && !force_full_wait + && let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(&file_info) + { + force_full_wait = true; + final_miss_reason_override.get_or_insert(reason); + } if let Some(slot) = ress.get_mut(index) { *slot = file_info; } @@ -7225,7 +7243,7 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn bounded_non_inline_data_get_hedges_then_waits_for_full_fanout() { + async fn bounded_non_inline_data_get_immediately_forces_full_fanout() { const DISKS: usize = 4; let bucket = "bounded-data-get-hedge-bucket"; let object = "bounded-data-get-hedge-object"; @@ -7256,7 +7274,7 @@ mod tests { } }) .await - .expect("bounded data-read fanout should hedge by starting the spare disk"); + .expect("bounded non-inline data-read fanout should immediately schedule the spare disk"); let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await; assert!( @@ -7272,7 +7290,7 @@ mod tests { assert_eq!( calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64, - "bounded data-read fanout should issue the paused disk plus one spare hedge" + "bounded non-inline data-read fanout should issue the paused disk plus the remaining spare" ); assert_eq!(diagnostics.total_responses(), DISKS); assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); @@ -7299,16 +7317,42 @@ mod tests { ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>), ], async { + let barrier = rename_fanout_barrier::arm(object, 2, rename_fanout_barrier::PHASE_READ_VERSION); let calls = disk_call_counters::observe(object); - let (parts_metadata, errs, diagnostics) = - SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2) + let disks_for_read = disks.clone(); + let mut read = tokio::spawn(async move { + SetDisks::read_all_fileinfo_observed(&disks_for_read, bucket, bucket, object, "", true, false, false, true, 2) .await - .expect("default data-read metadata should resolve"); + }); + + tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused()) + .await + .expect("default bounded non-inline read should schedule the paused metadata task"); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + while calls.for_disk(disk_call_counters::KIND_READ_VERSION, 3) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect( + "default bounded non-inline read should immediately force full fanout after the first non-inline response", + ); + + let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await; + assert!( + pending.is_err(), + "default non-inline data reads must not return before the paused metadata response" + ); + barrier.release(); + let (parts_metadata, errs, diagnostics) = read + .await + .expect("metadata read task should not panic") + .expect("default data-read metadata should resolve"); assert_eq!( calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64, - "default non-inline GET data-read metadata must keep full fanout for read-failure tolerance" + "default non-inline GET data-read metadata must keep full fanout without waiting for a quorum miss first" ); assert_eq!(diagnostics.total_responses(), DISKS); assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index a6a875a7b..a04c5db82 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -714,7 +714,7 @@ const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_ME const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = true; const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT"; -const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = false; +const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = true; const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS"; const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS"; @@ -1125,7 +1125,10 @@ mod prepared_get_object_metadata_tests { assert_eq!(object_size, payload.len() as i64); assert_eq!(restored, payload); - assert_eq!(calls_total, 4, "default production GET should eagerly schedule the full metadata fanout"); + assert_eq!( + calls_total, 4, + "default production inline GET should schedule the initial bounded quorum plus one hedge" + ); assert_eq!( recorder.histogram_values( "rustfs_io_get_object_metadata_fanout_scheduled", diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index dd3a51f20..d59fa3e7e 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -3937,7 +3937,7 @@ mod tests { } #[test] - fn metadata_early_stop_bounded_fanout_defaults_to_disabled() { + fn metadata_early_stop_bounded_fanout_defaults_to_enabled() { temp_env::with_vars( [ (ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")), @@ -3946,20 +3946,20 @@ mod tests { ], || { assert!(is_get_metadata_data_read_early_stop_enabled()); - assert!(!is_get_metadata_early_stop_bounded_fanout_enabled()); + assert!(is_get_metadata_early_stop_bounded_fanout_enabled()); }, ); - temp_env::with_vars([(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true"))], || { - assert!(is_get_metadata_early_stop_bounded_fanout_enabled()); + temp_env::with_vars([(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false"))], || { + assert!(!is_get_metadata_early_stop_bounded_fanout_enabled()); }); temp_env::with_vars( [ (ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("false")), - (ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")), + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")), ], || { assert!(!is_get_metadata_data_read_early_stop_enabled()); - assert!(!is_get_metadata_early_stop_bounded_fanout_enabled()); + assert!(is_get_metadata_early_stop_bounded_fanout_enabled()); }, ); } From 9e6e02ea09c86bedf44c7bd64a74ea02a0cff1de Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Mon, 17 Aug 2026 01:06:25 +0800 Subject: [PATCH 50/71] fix(table-catalog): assign fresh schema IDs on create (#6146) * fix(table-catalog): assign fresh schema IDs on create * fix(table-catalog): accept negative create schema IDs --------- Co-authored-by: Henry Guo --- .../src/admin/handlers/table_catalog/mod.rs | 9 +- .../src/admin/handlers/table_catalog/tests.rs | 202 +++++++++++++++++- .../src/table_catalog/iceberg/validation.rs | 183 ++++++++++++++++ 3 files changed, 390 insertions(+), 4 deletions(-) diff --git a/rustfs/src/admin/handlers/table_catalog/mod.rs b/rustfs/src/admin/handlers/table_catalog/mod.rs index 1dd03db42..d3138e96e 100644 --- a/rustfs/src/admin/handlers/table_catalog/mod.rs +++ b/rustfs/src/admin/handlers/table_catalog/mod.rs @@ -2995,9 +2995,9 @@ fn table_entry_from_create_table_request( let CreateTableRequest { name, location, - schema, - partition_spec, - write_order, + mut schema, + mut partition_spec, + mut write_order, stage_create, mut properties, } = request; @@ -3031,6 +3031,9 @@ fn table_entry_from_create_table_request( let metadata_location = crate::table_catalog::default_table_metadata_file_path(namespace, &table, &next_metadata_file_name(1, &table_id)); + crate::table_catalog::assign_fresh_create_schema_ids(&mut schema, partition_spec.as_mut(), write_order.as_mut()) + .map_err(catalog_store_error)?; + let entry = crate::table_catalog::TableEntry { version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, table_bucket: bucket.to_string(), diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index 013ae9304..969378424 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -1873,6 +1873,66 @@ fn create_table_request_accepts_standard_iceberg_rest_shape() { assert_eq!(request.name, "events"); } +#[test] +fn create_table_assigns_positive_ids_to_spark_schema() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 0, "name": "id", "required": false, "type": "long"}, + {"id": 1, "name": "payload", "required": false, "type": "string"} + ] + }, + "partition-spec": {"spec-id": 0, "fields": []}, + "properties": {"owner": "spark"} + })) + .expect("Spark create table request should parse"); + + let (_, metadata) = table_entry_from_create_table_request("warehouse", &namespace, request) + .expect("catalog should assign positive field IDs"); + + assert_eq!(metadata["schemas"][0]["fields"][0]["id"], 1); + assert_eq!(metadata["schemas"][0]["fields"][1]["id"], 2); + assert_eq!(metadata["last-column-id"], 2); +} + +#[test] +fn create_table_assigns_fresh_id_to_negative_temporary_field_id() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "identifier-field-ids": [-1], + "fields": [{"id": -1, "name": "id", "required": true, "type": "long"}] + }, + "partition-spec": { + "fields": [{"source-id": -1, "name": "id", "transform": "identity"}] + }, + "write-order": { + "fields": [{ + "source-id": -1, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + } + })) + .expect("create table request with a negative temporary field ID should parse"); + + let (_, metadata) = table_entry_from_create_table_request("warehouse", &namespace, request) + .expect("catalog should replace the negative temporary field ID"); + + assert_eq!(metadata["schemas"][0]["fields"][0]["id"], 1); + assert_eq!(metadata["schemas"][0]["identifier-field-ids"], serde_json::json!([1])); + assert_eq!(metadata["partition-specs"][0]["fields"][0]["source-id"], 1); + assert_eq!(metadata["sort-orders"][0]["fields"][0]["source-id"], 1); + assert_eq!(metadata["last-column-id"], 1); +} + #[test] fn create_table_request_honors_supported_format_version_property() { let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -1990,6 +2050,142 @@ fn catalog_assigns_read_only_schema_spec_and_sort_order_ids() { assert_eq!(updated["default-sort-order-id"], 0); } +#[test] +fn create_table_assigns_fresh_schema_field_ids_and_rewrites_references() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 41, + "identifier-field-ids": [0], + "fields": [ + {"id": 0, "name": "id", "required": true, "type": "long"}, + { + "id": 10, + "name": "details", + "required": false, + "type": { + "type": "struct", + "fields": [{"id": 11, "name": "category", "required": false, "type": "string"}] + } + }, + { + "id": 20, + "name": "tags", + "required": false, + "type": { + "type": "list", + "element-id": 21, + "element-required": false, + "element": "string" + } + }, + { + "id": 30, + "name": "attributes", + "required": false, + "type": { + "type": "map", + "key-id": 31, + "key": "string", + "value-id": 32, + "value-required": false, + "value": { + "type": "struct", + "fields": [{"id": 33, "name": "score", "required": false, "type": "int"}] + } + } + } + ] + }, + "partition-spec": { + "spec-id": 42, + "fields": [{"source-id": 0, "name": "id", "transform": "identity"}] + }, + "write-order": { + "order-id": 43, + "fields": [{ + "source-id": 11, + "transform": "identity", + "direction": "asc", + "null-order": "nulls-first" + }] + } + })) + .expect("create table request should parse"); + + let (_, metadata) = + table_entry_from_create_table_request("warehouse", &namespace, request).expect("catalog should assign fresh field IDs"); + + let schema = &metadata["schemas"][0]; + assert_eq!(schema["fields"][0]["id"], 1); + assert_eq!(schema["fields"][1]["id"], 2); + assert_eq!(schema["fields"][2]["id"], 3); + assert_eq!(schema["fields"][3]["id"], 4); + assert_eq!(schema["fields"][1]["type"]["fields"][0]["id"], 5); + assert_eq!(schema["fields"][2]["type"]["element-id"], 6); + assert_eq!(schema["fields"][3]["type"]["key-id"], 7); + assert_eq!(schema["fields"][3]["type"]["value-id"], 8); + assert_eq!(schema["fields"][3]["type"]["value"]["fields"][0]["id"], 9); + assert_eq!(schema["identifier-field-ids"], serde_json::json!([1])); + assert_eq!(metadata["last-column-id"], 9); + assert_eq!(metadata["partition-specs"][0]["fields"][0]["source-id"], 1); + assert_eq!(metadata["sort-orders"][0]["fields"][0]["source-id"], 5); +} + +#[test] +fn create_table_rejects_duplicate_temporary_schema_field_ids() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "fields": [ + {"id": 0, "name": "id", "required": false, "type": "long"}, + {"id": 0, "name": "payload", "required": false, "type": "string"} + ] + } + })) + .expect("create table request should parse"); + + let error = table_entry_from_create_table_request("warehouse", &namespace, request) + .expect_err("duplicate temporary field IDs must be rejected"); + + assert_eq!(error.message(), Some("duplicate create schema field id 0")); +} + +#[test] +fn create_table_rejects_excessive_schema_nesting() { + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let mut field_type = serde_json::Value::from("long"); + for element_id in 1..=crate::table_catalog::ICEBERG_MAX_SCHEMA_NESTING_DEPTH + 1 { + field_type = serde_json::json!({ + "type": "list", + "element-id": element_id, + "element-required": false, + "element": field_type + }); + } + let request = CreateTableRequest { + name: "events".to_string(), + location: None, + schema: serde_json::json!({ + "type": "struct", + "fields": [{"id": 0, "name": "nested", "required": false, "type": field_type}] + }), + partition_spec: None, + write_order: None, + stage_create: false, + properties: BTreeMap::new(), + }; + + let error = table_entry_from_create_table_request("warehouse", &namespace, request) + .expect_err("excessively nested create schemas must be rejected"); + + assert_eq!(error.message(), Some("create schema exceeds the maximum nesting depth")); +} + #[test] fn standard_commit_binds_new_specs_and_sort_orders_to_current_schema() { let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -2247,7 +2443,11 @@ fn create_table_counts_collection_ids_in_last_column_id() { let (_, metadata) = table_entry_from_create_table_request("warehouse", &namespace, request).expect("table metadata should be created"); - assert_eq!(metadata["last-column-id"], 9); + let schema = &metadata["schemas"][0]; + assert_eq!(schema["fields"][0]["type"]["element-id"], 3); + assert_eq!(schema["fields"][1]["type"]["key-id"], 4); + assert_eq!(schema["fields"][1]["type"]["value-id"], 5); + assert_eq!(metadata["last-column-id"], 5); } #[test] diff --git a/rustfs/src/table_catalog/iceberg/validation.rs b/rustfs/src/table_catalog/iceberg/validation.rs index 0783b5f78..8e2455b19 100644 --- a/rustfs/src/table_catalog/iceberg/validation.rs +++ b/rustfs/src/table_catalog/iceberg/validation.rs @@ -19,6 +19,7 @@ use futures::{StreamExt, TryStreamExt, stream}; use super::super::*; const ICEBERG_MAX_USER_FIELD_ID: i32 = i32::MAX - 200; +pub(crate) const ICEBERG_MAX_SCHEMA_NESTING_DEPTH: usize = 128; fn normalize_warehouse_object_prefix(object_prefix: &str, max_prefix_depth: Option) -> TableCatalogStoreResult { let object_prefix = object_prefix.strip_suffix('/').unwrap_or(object_prefix); @@ -1361,6 +1362,188 @@ fn validate_iceberg_schema(schema: &serde_json::Value, label: &str) -> TableCata Ok(validate_iceberg_schema_fields(schema, label)?.field_ids) } +pub(crate) fn assign_fresh_create_schema_ids( + schema: &mut serde_json::Value, + partition_spec: Option<&mut serde_json::Value>, + sort_order: Option<&mut serde_json::Value>, +) -> TableCatalogStoreResult<()> { + let mut assigner = FreshCreateSchemaIdAssigner::new(); + assigner.assign_schema(schema)?; + assigner.remap_identifier_field_ids(schema)?; + if let Some(partition_spec) = partition_spec { + assigner.remap_source_ids(partition_spec, "partition spec")?; + } + if let Some(sort_order) = sort_order { + assigner.remap_source_ids(sort_order, "sort order")?; + } + Ok(()) +} + +struct FreshCreateSchemaIdAssigner { + next_id: i32, + old_to_new: BTreeMap, +} + +impl FreshCreateSchemaIdAssigner { + fn new() -> Self { + Self { + next_id: 1, + old_to_new: BTreeMap::new(), + } + } + + fn assign_schema(&mut self, schema: &mut serde_json::Value) -> TableCatalogStoreResult<()> { + let schema = schema + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("create schema must be a JSON object".to_string()))?; + if schema.get("type").and_then(serde_json::Value::as_str) != Some("struct") { + return Err(TableCatalogStoreError::Invalid("create schema type must be struct".to_string())); + } + let fields = schema + .get_mut("fields") + .and_then(serde_json::Value::as_array_mut) + .ok_or_else(|| TableCatalogStoreError::Invalid("create schema fields must be an array".to_string()))?; + self.assign_struct_fields(fields, 0) + } + + fn assign_struct_fields(&mut self, fields: &mut [serde_json::Value], depth: usize) -> TableCatalogStoreResult<()> { + for field in fields.iter_mut() { + let field = field + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("create schema fields must be JSON objects".to_string()))?; + self.assign_object_id(field, "id", "create schema field id")?; + } + for field in fields { + let field = field + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("create schema fields must be JSON objects".to_string()))?; + let field_type = field + .get_mut("type") + .ok_or_else(|| TableCatalogStoreError::Invalid("create schema field type is required".to_string()))?; + self.assign_type_ids(field_type, depth)?; + } + Ok(()) + } + + fn assign_type_ids(&mut self, field_type: &mut serde_json::Value, depth: usize) -> TableCatalogStoreResult<()> { + if field_type.is_string() { + return Ok(()); + } + if depth >= ICEBERG_MAX_SCHEMA_NESTING_DEPTH { + return Err(TableCatalogStoreError::Invalid( + "create schema exceeds the maximum nesting depth".to_string(), + )); + } + let nested_depth = depth + 1; + let field_type = field_type.as_object_mut().ok_or_else(|| { + TableCatalogStoreError::Invalid("create schema field type must be a string or JSON object".to_string()) + })?; + match field_type.get("type").and_then(serde_json::Value::as_str) { + Some("struct") => { + let fields = field_type + .get_mut("fields") + .and_then(serde_json::Value::as_array_mut) + .ok_or_else(|| TableCatalogStoreError::Invalid("create schema struct fields must be an array".to_string()))?; + self.assign_struct_fields(fields, nested_depth) + } + Some("list") => { + self.assign_object_id(field_type, "element-id", "create schema list element-id")?; + let element = field_type + .get_mut("element") + .ok_or_else(|| TableCatalogStoreError::Invalid("create schema list element is required".to_string()))?; + self.assign_type_ids(element, nested_depth) + } + Some("map") => { + self.assign_object_id(field_type, "key-id", "create schema map key-id")?; + self.assign_object_id(field_type, "value-id", "create schema map value-id")?; + let key = field_type + .get_mut("key") + .ok_or_else(|| TableCatalogStoreError::Invalid("create schema map key is required".to_string()))?; + self.assign_type_ids(key, nested_depth)?; + let value = field_type + .get_mut("value") + .ok_or_else(|| TableCatalogStoreError::Invalid("create schema map value is required".to_string()))?; + self.assign_type_ids(value, nested_depth) + } + _ => Err(TableCatalogStoreError::Invalid( + "create schema contains an unsupported field type".to_string(), + )), + } + } + + fn assign_object_id( + &mut self, + object: &mut serde_json::Map, + field: &str, + label: &str, + ) -> TableCatalogStoreResult<()> { + let old_id = required_i32_value(object, field, label)?; + let entry = match self.old_to_new.entry(old_id) { + std::collections::btree_map::Entry::Occupied(_) => { + return Err(TableCatalogStoreError::Invalid(format!("duplicate create schema field id {old_id}"))); + } + std::collections::btree_map::Entry::Vacant(entry) => entry, + }; + let new_id = self.next_id; + if new_id > ICEBERG_MAX_USER_FIELD_ID { + return Err(TableCatalogStoreError::Invalid( + "create schema exceeds the available Iceberg field ID range".to_string(), + )); + } + self.next_id = new_id.checked_add(1).ok_or_else(|| { + TableCatalogStoreError::Invalid("create schema exceeds the available Iceberg field ID range".to_string()) + })?; + entry.insert(new_id); + object.insert(field.to_string(), serde_json::Value::from(new_id)); + Ok(()) + } + + fn remap_identifier_field_ids(&self, schema: &mut serde_json::Value) -> TableCatalogStoreResult<()> { + let Some(identifier_field_ids) = schema + .as_object_mut() + .and_then(|schema| schema.get_mut("identifier-field-ids")) + else { + return Ok(()); + }; + let identifier_field_ids = identifier_field_ids + .as_array_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid("create schema identifier-field-ids must be an array".to_string()))?; + for field_id in identifier_field_ids { + let old_id = required_i32(field_id, "create schema identifier field id")?; + let new_id = self.old_to_new.get(&old_id).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "create schema identifier field id {old_id} does not reference a schema field" + )) + })?; + *field_id = serde_json::Value::from(*new_id); + } + Ok(()) + } + + fn remap_source_ids(&self, value: &mut serde_json::Value, label: &str) -> TableCatalogStoreResult<()> { + let value = value + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} must be a JSON object")))?; + let Some(fields) = value.get_mut("fields") else { + return Ok(()); + }; + let fields = fields + .as_array_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} fields must be an array")))?; + for field in fields { + let field = field + .as_object_mut() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{label} fields must be JSON objects")))?; + let old_id = required_i32_value(field, "source-id", &format!("{label} source-id"))?; + let new_id = self.old_to_new.get(&old_id).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!("{label} source-id {old_id} does not reference the create schema")) + })?; + field.insert("source-id".to_string(), serde_json::Value::from(*new_id)); + } + Ok(()) + } +} + fn validate_iceberg_schema_fields(schema: &serde_json::Value, label: &str) -> TableCatalogStoreResult { let schema = schema .as_object() From d7957295859486cb02ab91089253ea2e27d7a016 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 17 Aug 2026 07:11:25 +0800 Subject: [PATCH 51/71] fix(scanner): canonicalize temp_dir for drive field assertion on macOS (#6159) --- crates/scanner/src/scanner_folder.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 7e353bf75..25f1e758d 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -4621,6 +4621,8 @@ mod tests { let _subscriber_guard = tracing::subscriber::set_default(subscriber); let (mut scanner, temp_dir) = build_test_scanner().await; + // Canonicalize for the "drive" field comparison (scanner resolves symlinks). + let canonical_temp_dir = std::fs::canonicalize(&temp_dir).unwrap_or_else(|_| temp_dir.clone()); let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); let object_dir = temp_dir.join("bucket").join("object"); @@ -4689,7 +4691,7 @@ mod tests { let fields = &events[0]["fields"]; assert_eq!(fields["component"], LOG_COMPONENT_SCANNER); assert_eq!(fields["subsystem"], LOG_SUBSYSTEM_FOLDER); - assert_eq!(fields["drive"], temp_dir.to_string_lossy().as_ref()); + assert_eq!(fields["drive"], canonical_temp_dir.to_string_lossy().as_ref()); assert_eq!(fields["bucket"], "bucket"); assert_eq!(fields["object"], "object"); assert_eq!(fields["metadata_path"], metadata_path.to_string_lossy().as_ref()); From 3ff250f1cd850bbd6b1a4262e34e2226e4e3a236 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 17 Aug 2026 08:03:29 +0800 Subject: [PATCH 52/71] chore(protocols): drop 43 no-op dead_code allows from swift (#6157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backlog#1823 step 8, partial. The swift module carries 43 #[allow(dead_code)] attributes, most with a comment naming a consumer: "Used by handler", "Handler integration: GET container", "Used by handler and object.rs". Every one of them suppresses nothing. crates/protocols/src/lib.rs declares `pub mod swift`, and swift/mod.rs declares all 22 submodules `pub mod`, so every item is publicly reachable and dead_code never applied to it. Removing all 43 leaves the warning count at zero, in both the default and --features swift lanes. That is also why those comments survived. They assert who calls the item — a claim the compiler normally settles on its own — and the compiler had been silenced by the visibility chain. The rest of step 8 needs a decision this PR does not make. Downgrading the 22 submodules to `pub(crate) mod` does restore detection, and it surfaces 39 real items, 16 of them the whole of sync.rs: SyncConfig, SyncStatus, SyncQueueEntry, ConflictResolution and every function and constant around them, i.e. Swift container sync is built and never wired. But the `pub mod` chain is load-bearing. Six integration tests under crates/protocols/tests are separate crates that import the submodules directly (swift::quota, swift::slo, swift::symlink, swift::sync, swift::tempurl, swift::container), and the downgrade fails to compile them. Restoring dead-code detection for this module therefore depends on first deciding whether those tests move in-crate — which is a testing-strategy call, not a cleanup one. Verification: cargo check -p rustfs-protocols warning-free in the default lane and with --features swift (lib and --tests); clippy --features swift --lib --tests -D warnings clean; cargo nextest run -p rustfs-protocols --features swift 441 passed; make pre-commit exit 0. Ref rustfs/backlog#1823 (step 8). --- crates/protocols/src/swift/account.rs | 2 -- crates/protocols/src/swift/container.rs | 14 -------------- crates/protocols/src/swift/errors.rs | 1 - crates/protocols/src/swift/object.rs | 21 --------------------- crates/protocols/src/swift/router.rs | 2 -- crates/protocols/src/swift/types.rs | 3 --- 6 files changed, 43 deletions(-) diff --git a/crates/protocols/src/swift/account.rs b/crates/protocols/src/swift/account.rs index 8abaa849f..3cdab2114 100644 --- a/crates/protocols/src/swift/account.rs +++ b/crates/protocols/src/swift/account.rs @@ -38,7 +38,6 @@ use std::collections::HashMap; /// - Account format is invalid /// - Credentials don't contain project_id /// - Account project_id doesn't match credentials project_id -#[allow(dead_code)] // Used by Swift implementation pub fn validate_account_access(account: &str, credentials: &Credentials) -> SwiftResult { // Extract project_id from account (strip "AUTH_" prefix) let account_project_id = account @@ -70,7 +69,6 @@ pub fn validate_account_access(account: &str, credentials: &Credentials) -> Swif /// /// Admin users (with "admin" or "reseller_admin" roles) can perform /// cross-tenant operations and administrative tasks. -#[allow(dead_code)] // Used by Swift implementation pub fn is_admin_user(credentials: &Credentials) -> bool { credentials .claims diff --git a/crates/protocols/src/swift/container.rs b/crates/protocols/src/swift/container.rs index 495664525..cd4bb0724 100644 --- a/crates/protocols/src/swift/container.rs +++ b/crates/protocols/src/swift/container.rs @@ -144,7 +144,6 @@ impl ContainerMapper { /// - S3 bucket name compatible (only uses [a-z0-9-]) /// - Deterministic mapping (same input always produces same bucket name) /// - Fixed-length prefix (16 hex chars = 8 bytes) - #[allow(dead_code)] // Used in: create/delete container operations pub fn swift_to_s3_bucket(&self, container: &str, project_id: &str) -> String { if self.config.tenant_prefix_enabled { let hash = self.hash_project_id(project_id); @@ -216,7 +215,6 @@ pub fn bucket_info_to_container(info: &BucketInfo, mapper: &ContainerMapper, pro /// 2. Lists all S3 buckets /// 3. Filters to buckets belonging to this tenant (using tenant prefix) /// 4. Converts BucketInfo to Swift Container format -#[allow(dead_code)] // Used by handler: list containers pub async fn list_containers(account: &str, credentials: &Credentials) -> SwiftResult> { // Validate account access and extract project_id let project_id = validate_account_access(account, credentials)?; @@ -279,7 +277,6 @@ pub async fn list_containers(account: &str, credentials: &Credentials) -> SwiftR /// - Returns 201 Created on success /// - Returns 202 Accepted if container already exists /// - Returns 400 Bad Request for invalid container names -#[allow(dead_code)] // Used by handler pub async fn create_container(account: &str, container: &str, credentials: &Credentials) -> SwiftResult { // Validate account access and extract project_id let project_id = validate_account_access(account, credentials)?; @@ -348,7 +345,6 @@ fn validate_container_name(container: &str) -> SwiftResult<()> { } /// Container metadata for HEAD response -#[allow(dead_code)] // TODO: Remove once Swift API integration is complete #[derive(Debug, Clone)] pub struct ContainerMetadata { /// Number of objects in container @@ -411,7 +407,6 @@ pub(crate) async fn get_container_custom_metadata( /// - HEAD /v1/{account}/{container} returns container metadata /// - Returns 204 No Content on success with headers /// - Returns 404 Not Found if container doesn't exist -#[allow(dead_code)] // Used by handler pub async fn get_container_metadata(account: &str, container: &str, credentials: &Credentials) -> SwiftResult { let (bucket_name, bucket_info, custom_metadata) = get_container_metadata_base(account, container, credentials).await?; @@ -448,7 +443,6 @@ pub async fn get_container_metadata(account: &str, container: &str, credentials: /// - The update is additive: items the request does not name keep their stored /// value, and removal is explicit, via `X-Remove-Container-Meta-{name}` or an /// empty value -#[allow(dead_code)] // Used by handler pub async fn update_container_metadata( account: &str, container: &str, @@ -520,7 +514,6 @@ pub async fn update_container_metadata( /// - Returns 204 No Content on success /// - Returns 404 Not Found if container doesn't exist /// - Returns 409 Conflict if container is not empty -#[allow(dead_code)] // Used by handler pub async fn delete_container(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<()> { // Validate account access and extract project_id let project_id = validate_account_access(account, credentials)?; @@ -603,7 +596,6 @@ pub async fn delete_container(account: &str, container: &str, credentials: &Cred /// - Account validation fails /// - Container doesn't exist /// - Storage layer errors occur -#[allow(dead_code)] // Handler integration: GET container pub async fn list_objects( account: &str, container: &str, @@ -706,7 +698,6 @@ pub async fn list_objects( /// Versioning configuration is stored as an S3 bucket tag: /// - Tag key: `swift-versions-location` /// - Tag value: archive container name -#[allow(dead_code)] // Used by handler pub async fn enable_versioning( account: &str, container: &str, @@ -795,7 +786,6 @@ pub async fn enable_versioning( /// * `account` - Account identifier /// * `container` - Container name to disable versioning on /// * `credentials` - Keystone credentials -#[allow(dead_code)] // Used by handler pub async fn disable_versioning(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<()> { // Validate account access let project_id = validate_account_access(account, credentials)?; @@ -855,7 +845,6 @@ pub async fn disable_versioning(account: &str, container: &str, credentials: &Cr /// # Returns /// - Some(archive_container_name) if versioning is enabled /// - None if versioning is not enabled -#[allow(dead_code)] // Used by handler and object.rs pub async fn get_versions_location(account: &str, container: &str, credentials: &Credentials) -> SwiftResult> { // Validate account access let project_id = validate_account_access(account, credentials)?; @@ -918,7 +907,6 @@ pub async fn get_versions_location(account: &str, container: &str, credentials: /// &credentials /// ).await?; /// ``` -#[allow(dead_code)] // Used by handler pub async fn set_container_acl( account: &str, container: &str, @@ -1022,7 +1010,6 @@ pub async fn set_container_acl( /// println!("Container is publicly readable"); /// } /// ``` -#[allow(dead_code)] // Used by handler pub async fn get_container_acl( account: &str, container: &str, @@ -1083,7 +1070,6 @@ pub async fn get_container_acl( /// /// # Returns /// Ok(()) if ACLs were deleted successfully -#[allow(dead_code)] // Used by handler pub async fn delete_container_acl(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<()> { // Setting both ACLs to None removes them set_container_acl(account, container, None, None, credentials).await diff --git a/crates/protocols/src/swift/errors.rs b/crates/protocols/src/swift/errors.rs index 25275dc41..ff1c49501 100644 --- a/crates/protocols/src/swift/errors.rs +++ b/crates/protocols/src/swift/errors.rs @@ -20,7 +20,6 @@ use std::fmt; /// Swift-specific error type #[derive(Debug)] -#[allow(dead_code)] // Error variants used by Swift implementation pub enum SwiftError { /// 400 Bad Request BadRequest(String), diff --git a/crates/protocols/src/swift/object.rs b/crates/protocols/src/swift/object.rs index 7169b152a..7cb388f4f 100644 --- a/crates/protocols/src/swift/object.rs +++ b/crates/protocols/src/swift/object.rs @@ -122,12 +122,10 @@ fn swift_user_metadata(headers: &HeaderMap) -> Option> { /// /// Handles URL encoding/decoding and path normalization for Swift object keys. /// Swift object names can contain any UTF-8 characters except null bytes. -#[allow(dead_code)] // Used in: object operations pub struct ObjectKeyMapper; impl ObjectKeyMapper { /// Create a new object key mapper - #[allow(dead_code)] // Used in: object operations pub fn new() -> Self { Self } @@ -140,7 +138,6 @@ impl ObjectKeyMapper { /// - Not contain null bytes /// - Not contain '..' path segments (directory traversal) /// - Not start with '/' (leading slash handled by routing) - #[allow(dead_code)] // Used in: object operations pub fn validate_object_name(object: &str) -> SwiftResult<()> { if object.is_empty() { return Err(SwiftError::BadRequest("Object name cannot be empty".to_string())); @@ -183,7 +180,6 @@ impl ObjectKeyMapper { /// Example: /// - Swift: "photos/vacation/beach photo.jpg" /// - S3: "photos/vacation/beach photo.jpg" - #[allow(dead_code)] // Used in: object operations pub fn swift_to_s3_key(object: &str) -> SwiftResult { Self::validate_object_name(object)?; Ok(object.to_string()) @@ -193,7 +189,6 @@ impl ObjectKeyMapper { /// /// This is essentially an identity transformation since we store /// Swift object names as-is in S3. - #[allow(dead_code)] // Used in: object operations pub fn s3_to_swift_name(key: &str) -> String { key.to_string() } @@ -208,7 +203,6 @@ impl ObjectKeyMapper { /// - Object: "vacation/beach.jpg" /// - Bucket: "abc123:photos" /// - Key: "vacation/beach.jpg" - #[allow(dead_code)] // Used in: object operations pub fn build_s3_key(object: &str) -> SwiftResult { Self::swift_to_s3_key(object) } @@ -220,7 +214,6 @@ impl ObjectKeyMapper { /// /// Example URL: /v1/AUTH_abc/container/path%2Fto%2Ffile.txt /// Decoded: "path/to/file.txt" - #[allow(dead_code)] // Used in: object operations pub fn decode_object_from_url(encoded: &str) -> SwiftResult { // Decode percent-encoding let decoded = urlencoding::decode(encoded).map_err(|e| SwiftError::BadRequest(format!("Invalid URL encoding: {}", e)))?; @@ -233,7 +226,6 @@ impl ObjectKeyMapper { /// /// When constructing URLs (e.g., for redirect responses), we need to /// percent-encode object names. - #[allow(dead_code)] // Used in: object operations pub fn encode_object_for_url(object: &str) -> String { urlencoding::encode(object).to_string() } @@ -241,7 +233,6 @@ impl ObjectKeyMapper { /// Check if object name represents a directory (pseudo-directory) /// /// In Swift, objects ending with '/' are treated as directory markers. - #[allow(dead_code)] // Used in: object operations pub fn is_directory_marker(object: &str) -> bool { object.ends_with('/') } @@ -250,7 +241,6 @@ impl ObjectKeyMapper { /// /// Removes redundant slashes and normalizes the path while preserving /// trailing slashes for directory markers. - #[allow(dead_code)] // Used in: object operations pub fn normalize_path(object: &str) -> String { // Split by '/', filter out empty segments (except if it's the end) let has_trailing_slash = object.ends_with('/'); @@ -324,7 +314,6 @@ fn sanitize_storage_error(operation: &str, error: E) -> Sw /// # Returns /// * `Ok(etag)` - Object ETag on success /// * `Err(SwiftError)` - Error if validation fails or upload fails -#[allow(dead_code)] // Handler integration: PUT object pub async fn put_object( account: &str, container: &str, @@ -445,7 +434,6 @@ where /// /// Similar to put_object, but allows directly specifying metadata instead of extracting from headers. /// This is used internally for storing SLO manifests and marker objects. -#[allow(dead_code)] // Used by SLO implementation pub async fn put_object_with_metadata( account: &str, container: &str, @@ -549,7 +537,6 @@ where /// - `bytes=1000-1999` - Bytes 1000-1999 /// - `bytes=1000-` - From byte 1000 to end /// - `bytes=-500` - Last 500 bytes -#[allow(dead_code)] // Handler integration: GET object pub async fn get_object( account: &str, container: &str, @@ -608,7 +595,6 @@ pub async fn get_object( /// # Returns /// * `Ok(object_info)` - Object metadata (ObjectInfo) /// * `Err(SwiftError)` - Error if validation fails or object not found -#[allow(dead_code)] // Handler integration: HEAD object pub async fn head_object( account: &str, container: &str, @@ -671,7 +657,6 @@ pub async fn head_object( /// # Returns /// * `Ok(())` - Object deleted successfully (or didn't exist) /// * `Err(SwiftError)` - Error if validation fails or deletion fails -#[allow(dead_code)] // Handler integration: DELETE object pub async fn delete_object(account: &str, container: &str, object: &str, credentials: &Credentials) -> SwiftResult<()> { // 1. Validate account access and get project_id let project_id = validate_account_access(account, credentials)?; @@ -732,7 +717,6 @@ pub async fn delete_object(account: &str, container: &str, object: &str, credent /// # Returns /// * `Ok(())` - Metadata updated successfully /// * `Err(SwiftError)` - Error if validation fails, object not found, or update fails -#[allow(dead_code)] // Handler integration: POST object pub async fn update_object_metadata( account: &str, container: &str, @@ -846,7 +830,6 @@ pub async fn update_object_metadata( /// # Handler Integration Note /// The current handler architecture needs to be updated to pass headers through /// to support COPY method and X-Copy-From header detection. See handler.rs for details. -#[allow(dead_code)] // Handler integration: COPY object #[allow(clippy::too_many_arguments)] // Necessary for full copy functionality pub async fn copy_object( src_account: &str, @@ -979,7 +962,6 @@ pub async fn copy_object( /// assert_eq!(container, "my-container"); /// assert_eq!(object, "path/to/file.txt"); /// ``` -#[allow(dead_code)] // Handler integration: COPY method pub fn parse_destination_header(destination: &str) -> SwiftResult<(String, String)> { let destination = destination.trim_start_matches('/'); let parts: Vec<&str> = destination.splitn(2, '/').collect(); @@ -1013,7 +995,6 @@ pub fn parse_destination_header(destination: &str) -> SwiftResult<(String, Strin /// # Returns /// * `Ok((container, object))` - Parsed container and object names /// * `Err(SwiftError)` - Error if format is invalid -#[allow(dead_code)] // Handler integration: X-Copy-From pub fn parse_copy_from_header(copy_from: &str) -> SwiftResult<(String, String)> { // Same parsing logic as Destination header parse_destination_header(copy_from) @@ -1042,7 +1023,6 @@ pub fn parse_copy_from_header(copy_from: &str) -> SwiftResult<(String, String)> /// assert_eq!(range.start, 0); /// assert_eq!(range.end, 1023); /// ``` -#[allow(dead_code)] // Handler integration: Range header pub fn parse_range_header(range_str: &str) -> SwiftResult { if !range_str.starts_with("bytes=") { return Err(SwiftError::BadRequest("Range header must start with 'bytes='".to_string())); @@ -1124,7 +1104,6 @@ pub fn parse_range_header(range_str: &str) -> SwiftResult { /// let header = format_content_range(0, 1023, 5000); /// assert_eq!(header, "bytes 0-1023/5000"); /// ``` -#[allow(dead_code)] // Handler integration: Range header pub fn format_content_range(start: i64, end: i64, total: i64) -> String { format!("bytes {}-{}/{}", start, end, total) } diff --git a/crates/protocols/src/swift/router.rs b/crates/protocols/src/swift/router.rs index ca2740e72..82e400fb1 100644 --- a/crates/protocols/src/swift/router.rs +++ b/crates/protocols/src/swift/router.rs @@ -50,7 +50,6 @@ pub enum SwiftRoute { impl SwiftRoute { /// Get the account identifier from the route - #[allow(dead_code)] // Public API for future use pub fn account(&self) -> &str { match self { SwiftRoute::Account { account, .. } => account, @@ -60,7 +59,6 @@ impl SwiftRoute { } /// Extract project_id from account string (removes AUTH_ prefix) - #[allow(dead_code)] // Public API for future use pub fn project_id(&self) -> Option<&str> { let account = self.account(); ACCOUNT_PATTERN diff --git a/crates/protocols/src/swift/types.rs b/crates/protocols/src/swift/types.rs index 40df9fc6d..66de347a3 100644 --- a/crates/protocols/src/swift/types.rs +++ b/crates/protocols/src/swift/types.rs @@ -19,7 +19,6 @@ use std::collections::HashMap; /// Swift container metadata #[derive(Debug, Clone, Serialize, Deserialize)] -#[allow(dead_code)] // Used in container listing operations pub struct Container { /// Container name pub name: String, @@ -34,7 +33,6 @@ pub struct Container { /// Swift object metadata #[derive(Debug, Clone, Serialize, Deserialize)] -#[allow(dead_code)] // Used in object listing operations pub struct Object { /// Object name (key) pub name: String, @@ -50,7 +48,6 @@ pub struct Object { /// Swift metadata extracted from headers #[derive(Debug, Clone, Default)] -#[allow(dead_code)] // Used by Swift implementation pub struct SwiftMetadata { /// Custom metadata key-value pairs (from X-Container-Meta-* or X-Object-Meta-*) pub metadata: HashMap, From 33cd11472afb939be9d406f1185baa30cca5fb03 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 17 Aug 2026 08:05:35 +0800 Subject: [PATCH 53/71] refactor(rustfs): move module switches below the layer boundary (#6154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backlog#1834 PR5. Whether the scanner, heal, audit and notify modules are on gets read from infra (storage helpers, node-service RPC) and from interface (admin handlers), but the switches lived in startup_background (composition) and server (interface). Every one of those reads was an upward edge carried in the layer-dependency baseline. The env-derived scanner/heal predicates and the audit/notify state cells now live in rustfs/src/module_switches.rs, at the bottom of the layer order, so the same reads are ordinary downward edges. startup_background and server import from there; server keeps re-exporting the getters for its own consumers. The issue's plan was to move is/refresh_audit/notify_module_enabled as a group. Moving refresh_* wholesale would have dragged resolve_audit_module_state and resolve_notify_module_state — server-side configuration logic — down into infra, which breaks more layering than it fixes. State and resolution are split instead: module_switches owns the atomics plus is_*/set_* accessors, and server's refresh_* keeps the configuration logic and publishes through the setter. That leaves storage/helper.rs's test module importing refresh_* from server, so two infra->interface edges stay. Those tests assert that a configuration change takes effect through refresh, which a plain setter would no longer exercise; the edges are worth more than the two baseline lines. Baseline drops 44 -> 36 lines, deletions only: - 4 interface/infra -> composition edges for ENV_SCANNER_ENABLED, scanner_enabled_from_env and heal_enabled_from_env - 2 infra -> interface edges for is_audit_module_enabled and is_notify_module_enabled - cycle|composition<->infra and cycle|composition<->interface The two cycles were not expected to go until whole subsystems moved out; clearing composition's inbound upward edges dissolved both, leaving three of the original five. Verification: scripts/check_layer_dependencies.sh passes, cargo check -p rustfs warning-free, make pre-commit exit 0. --- rustfs/src/admin/handlers/scanner.rs | 2 +- rustfs/src/lib.rs | 1 + rustfs/src/module_switches.rs | 68 +++++++++++++++++++++ rustfs/src/server/audit.rs | 9 +-- rustfs/src/server/event.rs | 9 +-- rustfs/src/startup_background.rs | 14 +---- rustfs/src/storage/helper.rs | 2 +- rustfs/src/storage/rpc/node_service/heal.rs | 2 +- scripts/layer-dependency-baseline.txt | 8 --- 9 files changed, 78 insertions(+), 37 deletions(-) create mode 100644 rustfs/src/module_switches.rs diff --git a/rustfs/src/admin/handlers/scanner.rs b/rustfs/src/admin/handlers/scanner.rs index b9ff901c5..fde894e9c 100644 --- a/rustfs/src/admin/handlers/scanner.rs +++ b/rustfs/src/admin/handlers/scanner.rs @@ -16,8 +16,8 @@ use crate::admin::auth::validate_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::current_scanner_metrics_report; use crate::auth::{check_key_valid, get_session_token}; +use crate::module_switches::{ENV_SCANNER_ENABLED, scanner_enabled_from_env}; use crate::server::{ADMIN_PREFIX, RemoteAddr}; -use crate::startup_background::{ENV_SCANNER_ENABLED, scanner_enabled_from_env}; use chrono::Utc; use http::{HeaderMap, HeaderValue}; use hyper::{Method, StatusCode}; diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index 077365569..f8e9d893d 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -88,6 +88,7 @@ pub mod inspect; pub(crate) mod kms_deletion_gate; pub mod license; pub mod memory_observability; +pub mod module_switches; pub mod profiling; #[cfg(any(feature = "ftps", feature = "webdav", feature = "sftp"))] pub mod protocols; diff --git a/rustfs/src/module_switches.rs b/rustfs/src/module_switches.rs new file mode 100644 index 000000000..fcb0ffcef --- /dev/null +++ b/rustfs/src/module_switches.rs @@ -0,0 +1,68 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Layer-neutral module switches (backlog#1834). +//! +//! Whether the scanner, heal, audit and notify modules are on is read from the +//! infra layer (storage helpers, node-service RPC) and from the interface layer +//! (admin handlers), but the switches used to live in `startup_background` +//! (composition) and `server` (interface). Every lower-layer read was therefore +//! an upward edge that had to be baselined by the layer-dependency guard. +//! +//! The env-derived scanner/heal predicates and the audit/notify state cells now +//! live here, at the bottom of the layer order, so those reads are ordinary +//! downward edges. Resolving the audit/notify state still needs server-side +//! configuration, so `server::refresh_audit_module_enabled` and its notify twin +//! keep that logic and publish the result through the setters below. + +use rustfs_utils::get_env_bool_with_aliases; +use std::sync::atomic::{AtomicBool, Ordering}; + +pub(crate) const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED"; +pub(crate) const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER"; +pub(crate) const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED"; +pub(crate) const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL"; + +static AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE); +static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE); + +/// Whether the data scanner is enabled, defaulting to on. +pub(crate) fn scanner_enabled_from_env() -> bool { + get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true) +} + +/// Whether background heal is enabled, defaulting to on. +pub(crate) fn heal_enabled_from_env() -> bool { + get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true) +} + +/// Last published audit-module state. +pub fn is_audit_module_enabled() -> bool { + AUDIT_MODULE_ENABLED.load(Ordering::Relaxed) +} + +/// Publish the audit-module state resolved by `server::refresh_audit_module_enabled`. +pub(crate) fn set_audit_module_enabled(enabled: bool) { + AUDIT_MODULE_ENABLED.store(enabled, Ordering::Relaxed); +} + +/// Last published notify-module state. +pub fn is_notify_module_enabled() -> bool { + NOTIFY_MODULE_ENABLED.load(Ordering::Relaxed) +} + +/// Publish the notify-module state resolved by `server::refresh_notify_module_enabled`. +pub(crate) fn set_notify_module_enabled(enabled: bool) { + NOTIFY_MODULE_ENABLED.store(enabled, Ordering::Relaxed); +} diff --git a/rustfs/src/server/audit.rs b/rustfs/src/server/audit.rs index 6fb01a8a5..72a0b21c6 100644 --- a/rustfs/src/server/audit.rs +++ b/rustfs/src/server/audit.rs @@ -19,11 +19,8 @@ use super::{ use crate::runtime_sources::AppContext; use rustfs_audit::{AuditError, AuditResult, audit_system, init_audit_system, system::AuditSystemState}; use std::collections::HashSet; -use std::sync::atomic::{AtomicBool, Ordering}; use tracing::{info, warn}; -static AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE); - fn server_config_from_context() -> Option { runtime_sources::current_server_config() } @@ -37,13 +34,11 @@ fn server_config_for_context(context: Option<&AppContext>) -> Option bool { let enabled = resolve_audit_module_state().enabled; - AUDIT_MODULE_ENABLED.store(enabled, Ordering::Relaxed); + crate::module_switches::set_audit_module_enabled(enabled); enabled } -pub fn is_audit_module_enabled() -> bool { - AUDIT_MODULE_ENABLED.load(Ordering::Relaxed) -} +pub use crate::module_switches::is_audit_module_enabled; fn has_any_persisted_audit_targets(config: &rustfs_config::server_config::Config) -> bool { for &subsystem in rustfs_config::audit::AUDIT_SUB_SYSTEMS { diff --git a/rustfs/src/server/event.rs b/rustfs/src/server/event.rs index b9ca8e6e5..6f0769774 100644 --- a/rustfs/src/server/event.rs +++ b/rustfs/src/server/event.rs @@ -34,7 +34,6 @@ use tokio::time::{Instant, MissedTickBehavior}; use tokio_util::sync::CancellationToken; use tracing::{info, instrument, warn}; -static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE); static NOTIFY_RUNTIME_RECONCILED: AtomicBool = AtomicBool::new(false); static NOTIFY_BUCKET_RULES_RECONCILED: AtomicBool = AtomicBool::new(false); static ECSTORE_EVENT_DISPATCH_HOOK: OnceLock<()> = OnceLock::new(); @@ -70,13 +69,11 @@ fn should_reconcile_bucket_notification_rules(runtime_changed: bool, notify_enab pub fn refresh_notify_module_enabled() -> bool { let enabled = resolve_notify_module_state().enabled; - NOTIFY_MODULE_ENABLED.store(enabled, Ordering::Relaxed); + crate::module_switches::set_notify_module_enabled(enabled); enabled } -pub fn is_notify_module_enabled() -> bool { - NOTIFY_MODULE_ENABLED.load(Ordering::Relaxed) -} +pub use crate::module_switches::is_notify_module_enabled; pub(crate) use crate::shared_types::convert_ecstore_object_info; @@ -171,7 +168,7 @@ pub(crate) async fn reconcile_event_notifier_from_store( let transition_system = system.clone(); let transition_store = store.clone(); let transition = with_refreshed_notify_module_state_from(store.clone(), move |resolution| async move { - NOTIFY_MODULE_ENABLED.store(resolution.enabled, Ordering::Relaxed); + crate::module_switches::set_notify_module_enabled(resolution.enabled); let read_store = transition_store.clone(); let config_system = transition_system.clone(); with_server_config_read_lock(transition_store, move || async move { diff --git a/rustfs/src/startup_background.rs b/rustfs/src/startup_background.rs index dfbdc257e..8cee2c502 100644 --- a/rustfs/src/startup_background.rs +++ b/rustfs/src/startup_background.rs @@ -12,32 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::module_switches::{heal_enabled_from_env, scanner_enabled_from_env}; use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider}; use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; use rustfs_heal::{ create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager_with_workload_provider, }; -use rustfs_utils::get_env_bool_with_aliases; use std::{io::Result, sync::Arc}; use tracing::{debug, info}; -pub(crate) const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED"; -pub(crate) const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER"; -pub(crate) const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED"; -pub(crate) const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL"; const LOG_COMPONENT_MAIN: &str = "main"; const LOG_SUBSYSTEM_STARTUP: &str = "startup"; const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured"; -pub(crate) fn scanner_enabled_from_env() -> bool { - get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true) -} - -pub(crate) fn heal_enabled_from_env() -> bool { - get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true) -} - pub(crate) async fn init_background_service_runtime(store: Arc) -> Result { let _ = create_ahm_services_cancel_token(); diff --git a/rustfs/src/storage/helper.rs b/rustfs/src/storage/helper.rs index 074d30f55..26f5476c0 100644 --- a/rustfs/src/storage/helper.rs +++ b/rustfs/src/storage/helper.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::server::{is_audit_module_enabled, is_notify_module_enabled}; +use crate::module_switches::{is_audit_module_enabled, is_notify_module_enabled}; use crate::shared_types::convert_ecstore_object_info; use crate::storage::access::{ReqInfo, request_context_from_req}; use crate::storage::request_context::RequestContext; diff --git a/rustfs/src/storage/rpc/node_service/heal.rs b/rustfs/src/storage/rpc/node_service/heal.rs index ea398125e..f9b990bf3 100644 --- a/rustfs/src/storage/rpc/node_service/heal.rs +++ b/rustfs/src/storage/rpc/node_service/heal.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::startup_background::{heal_enabled_from_env, scanner_enabled_from_env}; +use crate::module_switches::{heal_enabled_from_env, scanner_enabled_from_env}; use crate::storage::storage_api::runtime_sources_consumer::EndpointServerPools; use jiff::Timestamp; use rmp_serde::Deserializer; diff --git a/scripts/layer-dependency-baseline.txt b/scripts/layer-dependency-baseline.txt index 08aea30d2..c705152a4 100644 --- a/scripts/layer-dependency-baseline.txt +++ b/scripts/layer-dependency-baseline.txt @@ -16,11 +16,7 @@ # cycle|left_layer<->right_layer cycle|app<->infra cycle|app<->interface -cycle|composition<->infra -cycle|composition<->interface cycle|infra<->interface -dep|rustfs/src/admin/handlers/scanner.rs|interface->composition|crate::startup_background::ENV_SCANNER_ENABLED -dep|rustfs/src/admin/handlers/scanner.rs|interface->composition|crate::startup_background::scanner_enabled_from_env dep|rustfs/src/app/admin_usecase.rs|app->interface|crate::server::collect_dependency_readiness_report dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_bucket_meta_hook dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_delete_bucket_hook @@ -29,8 +25,6 @@ dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::snapshot_depe dep|rustfs/src/runtime_sources.rs|infra->app|crate::app::context dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::server::cors dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::storage::ecfs::ListObjectUnorderedQuery -dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::is_audit_module_enabled -dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::is_notify_module_enabled dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::refresh_audit_module_enabled dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::refresh_notify_module_enabled dep|rustfs/src/storage/rpc/http_service.rs|infra->interface|crate::server::RPC_PREFIX @@ -40,5 +34,3 @@ dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::servic dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::config::reload_runtime_config_snapshot dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::site_replication::reload_site_replication_runtime_state dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::server::MODULE_SWITCHES_SIGNAL_SUBSYSTEM -dep|rustfs/src/storage/rpc/node_service/heal.rs|infra->composition|crate::startup_background::heal_enabled_from_env -dep|rustfs/src/storage/rpc/node_service/heal.rs|infra->composition|crate::startup_background::scanner_enabled_from_env From 9f02ca6c36fcbaee3ad0ec9f5a2190b880c0bfae Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 17 Aug 2026 08:06:37 +0800 Subject: [PATCH 54/71] fix(ecstore): resolve nine unused bindings in set_disk write and heal paths (#6158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backlog#1823 step 1, the diagnosis half. Temporarily removing set_disk/mod.rs's #![allow(unused_variables)] surfaced nine bindings. The issue asks that values computed and then dropped on write/quorum paths be diagnosed before being underscored, and that turned out to matter: only four were plain leftovers. Two errors were bound and then left out of the log they were bound for. complete_multipart_upload's checksum failures read `if let Err(err) = ...` and then log part_id, bucket and object with no `err` anywhere in the message, so a checksum failure in production told you which part failed but not why. Both messages now carry the error. One is a lock guard. heal's write_lock_guard holds a namespace write lock for the rest of the scope; renaming it to a bare `_` would drop it immediately and release the lock. It is now `_write_lock_guard`, with a comment saying why it must not be `_`. One was kept alive by a corpse. `errors` in read_multiple_files is read by nothing except two commented-out debug! lines directly below it; the binding and the commented lines go together. One is a cfg split. heal's disk_index is read only inside the #[cfg(test)] fault-injection branch, so underscoring it would break the test build; a `#[cfg(not(test))] let _ = disk_index;` covers the non-test lane instead. The remaining four are genuine leftovers: an unused enumerate index in list_object_parts, a discarded error in a heal reader loop, an inner binding shadowing its own iterator variable, and delete_object's write_quorum. That last one is worth a separate look: delete_object asks get_object_info_and_quorum for a write quorum and never uses it, because delete_object_version below recomputes its own as disks.len() / 2 + 1. The two are not the same number — one comes from the object's erasure configuration, the other is a plain majority of the disk array. Pre-existing behaviour, untouched here. The blankets stay for now. Removing #![allow(unused_imports)] exposes 76 unused imports in set_disk/mod.rs, and they cannot be removed per-lane: cargo fix, working from the lib lane, produced 54 compile errors in the test lane. That needs its own pass with both lanes checked per import. Verification: cargo check -p rustfs-ecstore --tests and --features test-util --tests both warning-free; clippy --lib --tests -D warnings clean; cargo nextest run -p rustfs-ecstore 4101 passed; make pre-commit exit 0. Ref rustfs/backlog#1823 (step 1). --- crates/ecstore/src/set_disk/core/io_primitives.rs | 5 +---- crates/ecstore/src/set_disk/ops/heal.rs | 9 +++++++-- crates/ecstore/src/set_disk/ops/multipart.rs | 10 +++++----- crates/ecstore/src/set_disk/ops/object.rs | 6 ++++-- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 5f4471f13..41bd6dac7 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3021,14 +3021,11 @@ impl SetDisks { }); } - let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await { + let (ress, _errors) = match collect_read_multiple_results(futures, read_quorum).await { Ok(collected) => collected, Err(()) => return empty_quorum_result(), }; - // debug!("ReadMultipleResp ress {:?}", ress); - // debug!("ReadMultipleResp errors {:?}", errors); - let mut ret = Vec::with_capacity(req.files.len()); for want in req.files.iter() { diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 24c46a7f5..274b33f43 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -453,7 +453,9 @@ impl SetDisks { ..Default::default() }; - let write_lock_guard = if !opts.no_lock { + // Bound, not `_`: this guard must live to the end of the scope. A bare + // `_` would drop it here and release the namespace write lock. + let _write_lock_guard = if !opts.no_lock { let ns_lock = self.new_ns_lock(bucket, object).await?; Some( ns_lock @@ -996,7 +998,7 @@ impl SetDisks { readers.push(None); continue; } - Err(e) => { + Err(_e) => { readers.push(None); continue; } @@ -1545,6 +1547,9 @@ impl SetDisks { for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) { for (disk_index, disk) in disks.iter().enumerate() { + // Only the #[cfg(test)] fault-injection branch below reads this. + #[cfg(not(test))] + let _ = disk_index; let Some(disk) = disk else { return Ok(DanglingDeleteSafety::UnsafeToDelete); }; diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 86bb41737..35be8062c 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -1400,7 +1400,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let mut count = max_parts; - for (i, part) in object_parts.iter().enumerate() { + for part in object_parts.iter() { if let Some(err) = &part.error { warn!("list_object_parts part error: {:?}", &err); } @@ -2043,8 +2043,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { && let Err(err) = checksum.add_part(&cs, ext_part.actual_size) { error!( - "complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}", - p.part_num, bucket, object + "complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}, err={}", + p.part_num, bucket, object, err ); return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default())); } @@ -2089,8 +2089,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } } else if let Err(err) = wtcs.matches(&checksum_combined, uploaded_parts.len() as i32) { error!( - "complete_multipart_upload checksum matches failed want={}, got={}", - wtcs.encoded, checksum.encoded + "complete_multipart_upload checksum matches failed want={}, got={}, err={}", + wtcs.encoded, checksum.encoded, err ); return Err(Error::other(format!( "complete_multipart_upload checksum matches failed want={}, got={}", diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 670397c12..e7498ee9d 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -5656,7 +5656,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // TODO: Lifecycle let mut version_found = true; - let (mut goi, write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await; + // delete_object_version below derives its own majority quorum from the + // disk array, so the object-derived quorum here is unused. + let (mut goi, _write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await; if let Some(err) = &gerr && goi.name.is_empty() { @@ -6410,7 +6412,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks); for disk in disks.iter() { - if let Some(disk) = disk { + if disk.is_some() { continue; } let _ = self From 890ddea94b4b82eda7c8794d341370e265deed11 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Mon, 17 Aug 2026 08:06:53 +0800 Subject: [PATCH 55/71] fix(table-catalog): prevent object catalog lock reentry (#6144) * fix(table-catalog): prevent object catalog lock reentry * test(table-catalog): distinguish unlocked object reads --------- Co-authored-by: Henry Guo --- rustfs/src/table_catalog/store/object.rs | 53 ++++++++++++------- rustfs/src/table_catalog/test_support.rs | 15 ++++++ rustfs/src/table_catalog/tests.rs | 10 +++- scripts/table-catalog/pyiceberg_smoke.py | 1 + scripts/table-catalog/test_pyiceberg_smoke.py | 1 + 5 files changed, 60 insertions(+), 20 deletions(-) diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index 3f668b945..bade7b02d 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -387,6 +387,7 @@ where async fn has_active_namespace_descendant(&self, table_bucket: &str, namespace: &Namespace) -> TableCatalogStoreResult { let parent = namespace.public_name(); + let namespace_path = self.paths.namespace_entry_path(table_bucket, namespace); let descendant_prefix = format!("{}{}/", self.paths.namespace_entries_prefix(table_bucket), namespace.storage_id()); let scan_limit = NonZeroUsize::new(TABLE_CATALOG_LIST_MAX_KEYS) .ok_or_else(|| TableCatalogStoreError::Internal("catalog object scan limit must be positive".to_string()))?; @@ -398,6 +399,9 @@ where .await?; let last_scanned = page.objects.last().cloned(); for object in page.objects { + if object == namespace_path { + continue; + } if self .read_active_namespace_evidence(&object) .await? @@ -1814,7 +1818,6 @@ where &self, report: &TableMetadataMaintenanceReport, ) -> TableCatalogStoreResult<()> { - let report = table_maintenance_report_with_recommended_actions(report.clone()); let namespace = parse_namespace_for_store(&report.job.namespace)?; let table = parse_table_for_store(&report.job.table)?; let Some((entry, _)) = self @@ -1828,20 +1831,27 @@ where table.as_str() ))); }; - validate_table_maintenance_report_owner(&report, &report.job.table_bucket, &namespace, &table, &entry.table_id)?; - let job_path = self.paths.table_maintenance_job_path( - &report.job.table_bucket, - &namespace, - &table, - &report.job.table_id, - &report.job.job_id, - ); + self.put_table_metadata_maintenance_report_for_entry(report, &entry).await + } + + async fn put_table_metadata_maintenance_report_for_entry( + &self, + report: &TableMetadataMaintenanceReport, + entry: &TableEntry, + ) -> TableCatalogStoreResult<()> { + let report = table_maintenance_report_with_recommended_actions(report.clone()); + let namespace = parse_namespace_for_store(&entry.namespace)?; + let table = parse_table_for_store(&entry.table)?; + validate_table_maintenance_report_owner(&report, &entry.table_bucket, &namespace, &table, &entry.table_id)?; + let job_path = + self.paths + .table_maintenance_job_path(&entry.table_bucket, &namespace, &table, &entry.table_id, &report.job.job_id); let latest_job_path = self.paths - .table_maintenance_latest_job_path(&report.job.table_bucket, &namespace, &table, &report.job.table_id); + .table_maintenance_latest_job_path(&entry.table_bucket, &namespace, &table, &entry.table_id); let current_job_path = self.paths - .table_maintenance_current_job_path(&report.job.table_bucket, &namespace, &table, &report.job.table_id); + .table_maintenance_current_job_path(&entry.table_bucket, &namespace, &table, &entry.table_id); self.write_entry(self.catalog_bucket(), &job_path, &report, TableCatalogPutPrecondition::Any) .await?; self.write_entry(self.catalog_bucket(), &latest_job_path, &report, TableCatalogPutPrecondition::Any) @@ -2157,7 +2167,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report_unfenced(&report).await?; + self.put_table_metadata_maintenance_report_for_entry(&report, &entry).await?; report } TableMaintenanceSchedulerPreflight::Complete(report) => *report, @@ -2270,7 +2280,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report_unfenced(&report).await?; + self.put_table_metadata_maintenance_report_for_entry(&report, &entry).await?; report }; @@ -2381,6 +2391,7 @@ where } self.expire_table_maintenance_job( current, + context.entry, now, "maintenance worker lease expired", TableMaintenanceAuditAction::WorkerLeaseExpired, @@ -2392,6 +2403,7 @@ where } self.expire_table_maintenance_job( current, + context.entry, now, "maintenance scheduler lease expired", TableMaintenanceAuditAction::SchedulerLeaseExpired, @@ -2555,7 +2567,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report_unfenced(&report).await?; + self.put_table_metadata_maintenance_report_for_entry(&report, &entry).await?; let delete = matches!(report.job.operation, TableMetadataMaintenanceOperation::Delete); (report, effective, delete) @@ -2621,6 +2633,7 @@ where } self.expire_table_maintenance_job( current, + context.entry, now, "maintenance worker lease expired", TableMaintenanceAuditAction::WorkerLeaseExpired, @@ -2635,6 +2648,7 @@ where } self.expire_table_maintenance_job( current, + context.entry, now, "maintenance scheduler lease expired", TableMaintenanceAuditAction::SchedulerLeaseExpired, @@ -2737,13 +2751,14 @@ where Some(TableMetadataMaintenanceJobStatus::Running), before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report_unfenced(&report).await?; + self.put_table_metadata_maintenance_report_for_entry(&report, &entry).await?; Ok(report) } async fn expire_table_maintenance_job( &self, mut report: TableMetadataMaintenanceReport, + entry: &TableEntry, now: OffsetDateTime, reason: &str, action: TableMaintenanceAuditAction, @@ -2763,7 +2778,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report_unfenced(&report).await?; + self.put_table_metadata_maintenance_report_for_entry(&report, entry).await?; Ok(report) } @@ -2840,7 +2855,8 @@ where None, None, ); - self.put_table_metadata_maintenance_report_unfenced(&report).await?; + self.put_table_metadata_maintenance_report_for_entry(&report, control.entry) + .await?; Ok(report) } @@ -2917,7 +2933,8 @@ where None, None, ); - self.put_table_metadata_maintenance_report_unfenced(&report).await?; + self.put_table_metadata_maintenance_report_for_entry(&report, control.entry) + .await?; Ok(report) } diff --git a/rustfs/src/table_catalog/test_support.rs b/rustfs/src/table_catalog/test_support.rs index 9e49a39a2..38725d854 100644 --- a/rustfs/src/table_catalog/test_support.rs +++ b/rustfs/src/table_catalog/test_support.rs @@ -356,6 +356,7 @@ pub(crate) struct TestCatalogObjectBackend { pub(crate) missing_read_object_path: Arc>>, pub(crate) fail_read_object_path: Arc>>, pub(crate) lock_attempts: Arc>>, + pub(crate) reject_reads_while_write_locked: bool, /// Content-addressed (sha256) etags instead of the store fake's counter. /// The admin handler tests observe an object's etag and expect rewriting /// identical bytes to reproduce it, so their fixtures set this. @@ -689,6 +690,14 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { drop(fail_read_object_path); let key = (bucket.to_string(), object.to_string()); + if self.reject_reads_while_write_locked { + let lock = self.locks.lock().await.get(&key).cloned(); + if lock.as_ref().is_some_and(|lock| lock.try_read().is_err()) { + return Err(TableCatalogStoreError::Internal(format!( + "catalog read attempted while its write lock is held: {object}" + ))); + } + } let (attempt, pause_before) = { let mut state = self.state.lock().await; state.read_calls += 1; @@ -737,6 +746,12 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { Ok(result) } + async fn read_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + let mut backend = self.clone(); + backend.reject_reads_while_write_locked = false; + backend.read_object(bucket, object).await + } + async fn read_object_limited( &self, bucket: &str, diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs index cfc050343..bd83f9c41 100644 --- a/rustfs/src/table_catalog/tests.rs +++ b/rustfs/src/table_catalog/tests.rs @@ -3630,7 +3630,10 @@ async fn configured_table_catalog_store_uses_durable_strong_snapshot() { #[tokio::test] async fn object_table_catalog_store_persists_view_entries_and_blocks_non_empty_namespace_drop() { - let backend = TestCatalogObjectBackend::default(); + let backend = TestCatalogObjectBackend { + reject_reads_while_write_locked: true, + ..Default::default() + }; let store = ObjectTableCatalogStore::new(backend.clone()); let bucket = "analytics"; let namespace = Namespace::parse("sales").unwrap(); @@ -6442,7 +6445,10 @@ async fn maintenance_scheduler_report_marks_disabled_default() { #[tokio::test] async fn maintenance_scheduler_run_queues_one_durable_job() { - let backend = TestCatalogObjectBackend::default(); + let backend = TestCatalogObjectBackend { + reject_reads_while_write_locked: true, + ..Default::default() + }; let store = ObjectTableCatalogStore::new(backend.clone()); let bucket = "analytics"; let namespace = Namespace::parse("sales").expect("namespace should parse"); diff --git a/scripts/table-catalog/pyiceberg_smoke.py b/scripts/table-catalog/pyiceberg_smoke.py index 0fc13c1d5..f11d81f28 100755 --- a/scripts/table-catalog/pyiceberg_smoke.py +++ b/scripts/table-catalog/pyiceberg_smoke.py @@ -1207,6 +1207,7 @@ def smoke_view_request(args: argparse.Namespace, view_name: str, version_id: int "schema-id": 0, "timestamp-ms": int(time.time() * 1000), "summary": {"operation": "replace"}, + "default-namespace": [args.namespace], "representations": [ { "type": "sql", diff --git a/scripts/table-catalog/test_pyiceberg_smoke.py b/scripts/table-catalog/test_pyiceberg_smoke.py index 43cb49136..6fd1e5b58 100644 --- a/scripts/table-catalog/test_pyiceberg_smoke.py +++ b/scripts/table-catalog/test_pyiceberg_smoke.py @@ -749,6 +749,7 @@ class PyIcebergSmokeConfigTest(unittest.TestCase): self.assertEqual(request["name"], "orders_view") self.assertEqual(request["schema"]["type"], "struct") self.assertEqual(request["view-version"]["version-id"], 7) + self.assertEqual(request["view-version"]["default-namespace"], ["sales"]) self.assertEqual(request["view-version"]["representations"][0]["dialect"], "spark") self.assertEqual(request["properties"]["rustfs.smoke.table"], "orders") From 3377688dab6ce355bd21811c8e536ca712b10a1b Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 17 Aug 2026 08:08:36 +0800 Subject: [PATCH 56/71] ci(arch): ratchet ecstore module-level lint blankets (#6155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backlog#1823 step 9. The step 2 burn-down cleared every module-level #![allow(dead_code)] from ecstore, but nothing stops the next PR from adding one back, and the other module-level blankets (unused_variables, unused_must_use, clippy::all) were never counted at all. Two rules land in check_architecture_migration_rules.sh: ecstore must carry zero module-level #![allow(dead_code)]. The count is asserted at zero rather than registered, since there is nothing left to grandfather; a genuinely unused item takes an item-level allow with a reason, which is what step 2 produced roughly 350 times. Every other module-level blanket must match scripts/ecstore-module-lint-register.txt exactly — 94 entries across 33 files, nearly all of them in the MinIO-ported client module. The exact match is the point. A "no new entries" rule lets the register rot into an amnesty list, which is the failure mode backlog#1834 found in the layer-dependency baseline: rebuilt at 29 entries, 2 more added by a later PR, zero retired. Here, removing a blanket costs one line in the register, so it can only shrink, and adding one shows up as a register line a reviewer has to accept. Verified by injection, since a guard that cannot fail is worse than no guard: adding a dead_code blanket, adding an unregistered clippy::all blanket, and deleting a registered blanket without updating the register each produce the expected failure, and the tree passes once reverted. make pre-commit exit 0. Ref rustfs/backlog#1823 (step 9). --- scripts/check_architecture_migration_rules.sh | 51 ++++++++ scripts/ecstore-module-lint-register.txt | 113 ++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 scripts/ecstore-module-lint-register.txt diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 476fbce90..08c79d4ef 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -5450,6 +5450,57 @@ if [[ -s "$LEAF_CRATE_DEP_HITS_FILE" ]]; then report_failure "leaf crates (config/credentials/crypto/io-metrics/madmin) must not depend on internal rustfs-* crates (allowlist: io-metrics -> rustfs-s3-ops, backlog#1834): $(paste -sd '; ' "$LEAF_CRATE_DEP_HITS_FILE")" fi +# --- ecstore module-level lint blankets (backlog#1823 step 9) --- +# +# Two rules: +# 1. ecstore carries zero `#![allow(dead_code)]` after the step 2 burn-down; +# the count may only stay at zero. +# 2. Every other module-level blanket (unused_variables / unused_must_use / +# clippy::all) must match scripts/ecstore-module-lint-register.txt exactly. +# Exact match, not "no additions": a one-way rule lets the register rot +# into an amnesty list, which is what backlog#1834 found in the +# layer-dependency baseline. + +ECSTORE_LINT_REGISTER="${ROOT_DIR}/scripts/ecstore-module-lint-register.txt" +ECSTORE_DEAD_CODE_HITS="${TMP_DIR}/ecstore_dead_code_blankets.txt" +ECSTORE_LINT_ACTUAL="${TMP_DIR}/ecstore_lint_actual.txt" +ECSTORE_LINT_EXPECTED="${TMP_DIR}/ecstore_lint_expected.txt" + +( + cd "$ROOT_DIR" + rg -n '^#!\[allow\(dead_code\)\]' crates/ecstore/src/ 2>/dev/null || true +) >"$ECSTORE_DEAD_CODE_HITS" + +if [[ -s "$ECSTORE_DEAD_CODE_HITS" ]]; then + report_failure "ecstore must carry no module-level #![allow(dead_code)] (backlog#1823 step 2 cleared them; re-add an item-level allow with a reason instead): $(paste -sd '; ' "$ECSTORE_DEAD_CODE_HITS")" +fi + +( + cd "$ROOT_DIR" + rg -n '^#!\[allow\((unused_variables|unused_must_use|clippy::all)\)\]' crates/ecstore/src/ 2>/dev/null | + sed -E 's#^([^:]+):[0-9]+:\#!\[allow\(([^)]+)\)\]#\1|\2#' | sort -u +) >"$ECSTORE_LINT_ACTUAL" + +if [[ ! -f "$ECSTORE_LINT_REGISTER" ]]; then + report_failure "missing scripts/ecstore-module-lint-register.txt (backlog#1823 step 9)" +else + rg -v '^\s*(#|$)' "$ECSTORE_LINT_REGISTER" | sort -u >"$ECSTORE_LINT_EXPECTED" + + while IFS= read -r entry; do + [[ -z "$entry" ]] && continue + if ! rg -qxF "$entry" "$ECSTORE_LINT_EXPECTED"; then + report_failure "new ecstore module-level lint blanket '${entry}' is not in scripts/ecstore-module-lint-register.txt; prefer an item-level allow with a reason, or add the line with a rationale in the PR description (backlog#1823 step 9)" + fi + done <"$ECSTORE_LINT_ACTUAL" + + while IFS= read -r entry; do + [[ -z "$entry" ]] && continue + if ! rg -qxF "$entry" "$ECSTORE_LINT_ACTUAL"; then + report_failure "scripts/ecstore-module-lint-register.txt lists '${entry}' but the blanket is gone; delete the line in the same PR so the register can only shrink (backlog#1823 step 9)" + fi + done <"$ECSTORE_LINT_EXPECTED" +fi + if (( FAILURES > 0 )); then exit 1 fi diff --git a/scripts/ecstore-module-lint-register.txt b/scripts/ecstore-module-lint-register.txt new file mode 100644 index 000000000..a6d2ac8d4 --- /dev/null +++ b/scripts/ecstore-module-lint-register.txt @@ -0,0 +1,113 @@ +# ecstore module-level lint blanket register (backlog#1823 step 9) +# +# Each line is `path|lint` for a `#![allow()]` at the top of an ecstore +# source file. These blankets disable the lint for a whole module, so a real +# defect introduced anywhere in the file goes unreported. +# +# The guard in scripts/check_architecture_migration_rules.sh compares this file +# against the tree and fails on ANY difference, in either direction: +# +# * a blanket not listed here -> new suppression, justify it in the PR +# * a line here with no blanket -> delete the line in the same PR +# +# Exact match is deliberate. A "no new entries" rule would let the register rot +# into an amnesty list, which is the failure mode backlog#1834 found in the +# layer-dependency baseline. Removing a blanket must cost one line here, so the +# register can only shrink. +# +# `#![allow(dead_code)]` is NOT listed: ecstore carries zero of those after the +# step 2 burn-down, and the guard asserts that count stays at zero. +crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|clippy::all +crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_must_use +crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_variables +crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|clippy::all +crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_must_use +crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_variables +crates/ecstore/src/client/api_error_response.rs|clippy::all +crates/ecstore/src/client/api_error_response.rs|unused_must_use +crates/ecstore/src/client/api_error_response.rs|unused_variables +crates/ecstore/src/client/api_get_object.rs|clippy::all +crates/ecstore/src/client/api_get_object.rs|unused_must_use +crates/ecstore/src/client/api_get_object.rs|unused_variables +crates/ecstore/src/client/api_get_options.rs|clippy::all +crates/ecstore/src/client/api_get_options.rs|unused_must_use +crates/ecstore/src/client/api_get_options.rs|unused_variables +crates/ecstore/src/client/api_list.rs|clippy::all +crates/ecstore/src/client/api_list.rs|unused_must_use +crates/ecstore/src/client/api_list.rs|unused_variables +crates/ecstore/src/client/api_put_object.rs|clippy::all +crates/ecstore/src/client/api_put_object.rs|unused_must_use +crates/ecstore/src/client/api_put_object.rs|unused_variables +crates/ecstore/src/client/api_put_object_common.rs|clippy::all +crates/ecstore/src/client/api_put_object_common.rs|unused_must_use +crates/ecstore/src/client/api_put_object_common.rs|unused_variables +crates/ecstore/src/client/api_put_object_multipart.rs|clippy::all +crates/ecstore/src/client/api_put_object_multipart.rs|unused_must_use +crates/ecstore/src/client/api_put_object_multipart.rs|unused_variables +crates/ecstore/src/client/api_put_object_streaming.rs|clippy::all +crates/ecstore/src/client/api_put_object_streaming.rs|unused_must_use +crates/ecstore/src/client/api_put_object_streaming.rs|unused_variables +crates/ecstore/src/client/api_remove.rs|clippy::all +crates/ecstore/src/client/api_remove.rs|unused_must_use +crates/ecstore/src/client/api_remove.rs|unused_variables +crates/ecstore/src/client/api_s3_datatypes.rs|clippy::all +crates/ecstore/src/client/api_s3_datatypes.rs|unused_must_use +crates/ecstore/src/client/api_s3_datatypes.rs|unused_variables +crates/ecstore/src/client/api_stat.rs|clippy::all +crates/ecstore/src/client/api_stat.rs|unused_must_use +crates/ecstore/src/client/api_stat.rs|unused_variables +crates/ecstore/src/client/bucket_cache.rs|clippy::all +crates/ecstore/src/client/bucket_cache.rs|unused_must_use +crates/ecstore/src/client/bucket_cache.rs|unused_variables +crates/ecstore/src/client/checksum.rs|clippy::all +crates/ecstore/src/client/checksum.rs|unused_must_use +crates/ecstore/src/client/checksum.rs|unused_variables +crates/ecstore/src/client/constants.rs|unused_must_use +crates/ecstore/src/client/constants.rs|unused_variables +crates/ecstore/src/client/credentials.rs|clippy::all +crates/ecstore/src/client/credentials.rs|unused_must_use +crates/ecstore/src/client/credentials.rs|unused_variables +crates/ecstore/src/client/object_api_utils.rs|clippy::all +crates/ecstore/src/client/object_api_utils.rs|unused_must_use +crates/ecstore/src/client/object_api_utils.rs|unused_variables +crates/ecstore/src/client/transition_api.rs|clippy::all +crates/ecstore/src/client/transition_api.rs|unused_must_use +crates/ecstore/src/client/transition_api.rs|unused_variables +crates/ecstore/src/services/event_notification.rs|unused_variables +crates/ecstore/src/services/tier/tier.rs|clippy::all +crates/ecstore/src/services/tier/tier.rs|unused_must_use +crates/ecstore/src/services/tier/tier.rs|unused_variables +crates/ecstore/src/services/tier/tier_admin.rs|clippy::all +crates/ecstore/src/services/tier/tier_admin.rs|unused_must_use +crates/ecstore/src/services/tier/tier_admin.rs|unused_variables +crates/ecstore/src/services/tier/warm_backend.rs|clippy::all +crates/ecstore/src/services/tier/warm_backend.rs|unused_must_use +crates/ecstore/src/services/tier/warm_backend.rs|unused_variables +crates/ecstore/src/services/tier/warm_backend_aliyun.rs|clippy::all +crates/ecstore/src/services/tier/warm_backend_aliyun.rs|unused_must_use +crates/ecstore/src/services/tier/warm_backend_aliyun.rs|unused_variables +crates/ecstore/src/services/tier/warm_backend_azure.rs|clippy::all +crates/ecstore/src/services/tier/warm_backend_azure.rs|unused_must_use +crates/ecstore/src/services/tier/warm_backend_azure.rs|unused_variables +crates/ecstore/src/services/tier/warm_backend_gcs.rs|clippy::all +crates/ecstore/src/services/tier/warm_backend_gcs.rs|unused_must_use +crates/ecstore/src/services/tier/warm_backend_gcs.rs|unused_variables +crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|clippy::all +crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|unused_must_use +crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs|unused_variables +crates/ecstore/src/services/tier/warm_backend_minio.rs|clippy::all +crates/ecstore/src/services/tier/warm_backend_minio.rs|unused_must_use +crates/ecstore/src/services/tier/warm_backend_minio.rs|unused_variables +crates/ecstore/src/services/tier/warm_backend_r2.rs|clippy::all +crates/ecstore/src/services/tier/warm_backend_r2.rs|unused_must_use +crates/ecstore/src/services/tier/warm_backend_r2.rs|unused_variables +crates/ecstore/src/services/tier/warm_backend_rustfs.rs|clippy::all +crates/ecstore/src/services/tier/warm_backend_rustfs.rs|unused_must_use +crates/ecstore/src/services/tier/warm_backend_rustfs.rs|unused_variables +crates/ecstore/src/services/tier/warm_backend_s3.rs|clippy::all +crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_must_use +crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_variables +crates/ecstore/src/services/tier/warm_backend_tencent.rs|clippy::all +crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_must_use +crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_variables +crates/ecstore/src/set_disk/mod.rs|unused_variables From 01e0af6312dda757adb5b6a983dd2dfbdbd23338 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 08:26:23 +0800 Subject: [PATCH 57/71] perf(io-metrics): avoid get handoff label allocations (#6160) Co-authored-by: heihutu --- crates/io-metrics/src/lib.rs | 63 +++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 1d99f50e5..3f021772d 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -487,22 +487,21 @@ pub fn record_get_object_completion(total_duration_secs: f64, response_size_byte /// Record the streaming strategy chosen for a GetObject response body. #[inline(always)] -pub fn record_get_object_stream_strategy(strategy: &str, buffer_size_bytes: usize, response_size_bytes: i64) { +pub fn record_get_object_stream_strategy(strategy: &'static str, buffer_size_bytes: usize, response_size_bytes: i64) { if !get_stage_metrics_enabled() { return; } - counter!("rustfs_io_get_object_stream_strategy_total", "strategy" => strategy.to_string()).increment(1); - histogram!("rustfs_io_get_object_stream_buffer_size_bytes", "strategy" => strategy.to_string()) - .record(usize_to_f64(buffer_size_bytes)); - histogram!("rustfs_io_get_object_stream_response_size_bytes", "strategy" => strategy.to_string()) + counter!("rustfs_io_get_object_stream_strategy_total", "strategy" => strategy).increment(1); + histogram!("rustfs_io_get_object_stream_buffer_size_bytes", "strategy" => strategy).record(usize_to_f64(buffer_size_bytes)); + histogram!("rustfs_io_get_object_stream_response_size_bytes", "strategy" => strategy) .record(i64_non_negative_to_f64(response_size_bytes)); } /// Record the response-body handoff shape from a GetObject reader into the S3 streaming body. #[inline(always)] pub fn record_get_object_response_handoff( - strategy: &str, - buffer_source: &str, + strategy: &'static str, + buffer_source: &'static str, buffer_size_bytes: usize, response_size_bytes: i64, duration_secs: f64, @@ -512,26 +511,26 @@ pub fn record_get_object_response_handoff( } counter!( "rustfs_io_get_object_response_handoff_total", - "strategy" => strategy.to_string(), - "buffer_source" => buffer_source.to_string() + "strategy" => strategy, + "buffer_source" => buffer_source ) .increment(1); histogram!( "rustfs_io_get_object_response_handoff_buffer_size_bytes", - "strategy" => strategy.to_string(), - "buffer_source" => buffer_source.to_string() + "strategy" => strategy, + "buffer_source" => buffer_source ) .record(usize_to_f64(buffer_size_bytes)); histogram!( "rustfs_io_get_object_response_handoff_response_size_bytes", - "strategy" => strategy.to_string(), - "buffer_source" => buffer_source.to_string() + "strategy" => strategy, + "buffer_source" => buffer_source ) .record(i64_non_negative_to_f64(response_size_bytes)); histogram!( "rustfs_io_get_object_response_handoff_duration_seconds", - "strategy" => strategy.to_string(), - "buffer_source" => buffer_source.to_string() + "strategy" => strategy, + "buffer_source" => buffer_source ) .record(duration_secs); record_get_object_response_handoff_duration("s3_handler", duration_secs); @@ -539,14 +538,18 @@ pub fn record_get_object_response_handoff( /// Record ReaderStream capacity chosen for GetObject handoff. #[inline(always)] -pub fn record_get_object_reader_stream_buffer_size(strategy: &str, buffer_source: &str, buffer_size_bytes: usize) { +pub fn record_get_object_reader_stream_buffer_size( + strategy: &'static str, + buffer_source: &'static str, + buffer_size_bytes: usize, +) { if !get_stage_metrics_enabled() { return; } histogram!( "rustfs_io_get_object_reader_stream_buffer_size_bytes", - "strategy" => strategy.to_string(), - "buffer_source" => buffer_source.to_string() + "strategy" => strategy, + "buffer_source" => buffer_source ) .record(usize_to_f64(buffer_size_bytes)); } @@ -554,8 +557,8 @@ pub fn record_get_object_reader_stream_buffer_size(strategy: &str, buffer_source /// Record ReaderStream poll outcomes for GetObject handoff attribution. #[inline(always)] pub fn record_get_object_reader_stream_poll( - strategy: &str, - buffer_source: &str, + strategy: &'static str, + buffer_source: &'static str, outcome: &'static str, remaining_before: usize, bytes: usize, @@ -567,36 +570,36 @@ pub fn record_get_object_reader_stream_poll( let bytes = u64::try_from(bytes).unwrap_or(u64::MAX); counter!( "rustfs_io_get_object_reader_stream_poll_total", - "strategy" => strategy.to_string(), - "buffer_source" => buffer_source.to_string(), + "strategy" => strategy, + "buffer_source" => buffer_source, "outcome" => outcome ) .increment(1); counter!( "rustfs_io_get_object_reader_stream_poll_bytes_total", - "strategy" => strategy.to_string(), - "buffer_source" => buffer_source.to_string(), + "strategy" => strategy, + "buffer_source" => buffer_source, "outcome" => outcome ) .increment(bytes); histogram!( "rustfs_io_get_object_reader_stream_poll_remaining_bytes", - "strategy" => strategy.to_string(), - "buffer_source" => buffer_source.to_string(), + "strategy" => strategy, + "buffer_source" => buffer_source, "outcome" => outcome ) .record(usize_to_f64(remaining_before)); histogram!( "rustfs_io_get_object_reader_stream_poll_bytes", - "strategy" => strategy.to_string(), - "buffer_source" => buffer_source.to_string(), + "strategy" => strategy, + "buffer_source" => buffer_source, "outcome" => outcome ) .record(usize_to_f64(bytes as usize)); histogram!( "rustfs_io_get_object_reader_stream_poll_duration_seconds", - "strategy" => strategy.to_string(), - "buffer_source" => buffer_source.to_string(), + "strategy" => strategy, + "buffer_source" => buffer_source, "outcome" => outcome ) .record(duration_secs); From 3f3e3f4f0513a48c01a4e187463120c948f4ada2 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 10:45:24 +0800 Subject: [PATCH 58/71] perf(get): avoid memory body stream wrapper (#6163) Use MemoryTrackedBytesStream directly as an s3s ByteStream so in-memory GET bodies avoid the generic StreamingBlob::wrap adapter while preserving exact remaining length, request lifecycle tracking, and length-mismatch failure semantics. Co-authored-by: heihutu --- rustfs/src/app/object_usecase.rs | 39 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 913d6eb52..643b4fb6e 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -1074,7 +1074,7 @@ where } impl futures::Stream for MemoryTrackedBytesStream { - type Item = std::io::Result; + type Item = Result; fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); @@ -1105,7 +1105,8 @@ impl futures::Stream for MemoryTrackedBytesStream { return Poll::Ready(Some(Err(std::io::Error::new( std::io::ErrorKind::InvalidData, format!("materialized GET body length mismatch: expected {}, got {}", this.expected, actual), - )))); + ) + .into()))); } let Some(bytes) = this.bytes.take() else { @@ -1132,6 +1133,16 @@ impl futures::Stream for MemoryTrackedBytesStream { } } +impl ByteStream for MemoryTrackedBytesStream { + fn remaining_length(&self) -> RemainingLength { + if self.emitted || self.bytes.is_none() { + RemainingLength::new_exact(0) + } else { + RemainingLength::new_exact(self.expected) + } + } +} + impl Drop for MemoryTrackedBytesStream { fn drop(&mut self) { if self.lifecycle.is_finished() { @@ -4149,7 +4160,7 @@ impl DefaultObjectUsecase { let bytes_len = bytes.len(); let guard = rustfs_io_metrics::track_get_object_buffered_bytes(bytes_len); let remaining = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX); - let blob = StreamingBlob::wrap(MemoryTrackedBytesStream::new(bytes, remaining, source, guard, lifecycle)); + let blob = StreamingBlob::new(MemoryTrackedBytesStream::new(bytes, remaining, source, guard, lifecycle)); if let Some(handoff_start) = handoff_start { rustfs_io_metrics::record_get_object_response_handoff( "single_chunk", @@ -12882,7 +12893,10 @@ mod tests { .await .expect("mismatched memory body must yield an item") .expect_err("a short memory body must fail the stream instead of serving a truncated body"); - assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert_eq!( + err.downcast_ref::().map(std::io::Error::kind), + Some(std::io::ErrorKind::InvalidData) + ); assert!(stream.next().await.is_none(), "stream must terminate after the error"); } @@ -12901,7 +12915,22 @@ mod tests { .await .expect("mismatched memory body must yield an item") .expect_err("an over-long memory body must fail the stream instead of serving mismatched bytes"); - assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert_eq!( + err.downcast_ref::().map(std::io::Error::kind), + Some(std::io::ErrorKind::InvalidData) + ); + } + + #[test] + fn memory_blob_preserves_exact_remaining_length() { + let blob = DefaultObjectUsecase::build_memory_bytes_blob( + Bytes::from_static(b"hello"), + 5, + GET_MEMORY_BODY_SOURCE_BUFFERED_BODY, + GetObjectBodyLifecycle::disabled(), + ); + + assert_eq!(blob.remaining_length().exact(), Some(5)); } #[tokio::test] From 7db38827776f2efb4a879a192242b35aa8a30067 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 17 Aug 2026 11:32:23 +0800 Subject: [PATCH 59/71] chore(obs): adjudicate 19 bare dead_code allows (#6162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backlog#1823 step 10, batch 2. Eighteen of the nineteen suppress nothing and are deleted; one was real and keeps an allow that now says why. Rotation::Never is constructed only by the rolling-appender tests at rolling.rs:456, 477 and 498, so the lib target reports it as never constructed. Its allow is restored with that reason. Finding it corrected the method used for batch 1. Removing all nineteen and running cargo check -p rustfs-obs --tests reported zero warnings even after touching every source file, while clippy --lib --tests -D warnings caught Rotation::Never. cargo's warning output is not a reliable completeness check — it does not re-emit for cached compilations, and touching the sources did not cover the lib target here. Later batches should treat clippy -D warnings as the gate; batch 1's six crates were re-checked under clippy and are clean. Taken with #6086, which cleared this crate's 44 module-level blankets and left six real items, obs has now had 63 dead-code suppressions examined, of which seven were suppressing anything at all. The rest sat on items that are publicly reachable, where dead_code never applied — the same shape as the swift module and kms's dek.rs. Verification: clippy --lib --tests -D warnings clean in the default, gpu and pyroscope lanes; cargo nextest run -p rustfs-obs 324 passed; make pre-commit exit 0. Ref rustfs/backlog#1823 (step 10). --- crates/obs/src/metrics/config.rs | 1 - crates/obs/src/metrics/report.rs | 3 --- crates/obs/src/metrics/schema/entry/descriptor.rs | 3 --- crates/obs/src/metrics/schema/entry/metric_name.rs | 2 -- crates/obs/src/metrics/schema/entry/metric_type.rs | 3 --- crates/obs/src/metrics/schema/entry/mod.rs | 1 - crates/obs/src/metrics/schema/entry/namespace.rs | 1 - crates/obs/src/metrics/schema/entry/path_utils.rs | 1 - crates/obs/src/metrics/schema/entry/subsystem.rs | 3 --- crates/obs/src/telemetry/rolling.rs | 5 ++++- 10 files changed, 4 insertions(+), 19 deletions(-) diff --git a/crates/obs/src/metrics/config.rs b/crates/obs/src/metrics/config.rs index 26b73d6b1..c43e0c1e1 100644 --- a/crates/obs/src/metrics/config.rs +++ b/crates/obs/src/metrics/config.rs @@ -17,7 +17,6 @@ use std::time::Duration; /// Environment variable key for the global default metrics interval (seconds). pub const ENV_DEFAULT_METRICS_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL_SEC"; /// Default interval for metrics collection if not specified otherwise. -#[allow(dead_code)] pub const DEFAULT_METRICS_INTERVAL: Duration = Duration::from_secs(60); /// Environment variable key for cluster metrics interval (seconds). diff --git a/crates/obs/src/metrics/report.rs b/crates/obs/src/metrics/report.rs index b03630272..b3a421f55 100644 --- a/crates/obs/src/metrics/report.rs +++ b/crates/obs/src/metrics/report.rs @@ -145,21 +145,18 @@ impl PrometheusMetric { } #[inline] - #[allow(dead_code)] pub fn with_label(mut self, key: &'static str, value: impl Into>) -> Self { self.labels.push((key, value.into())); self } #[inline] - #[allow(dead_code)] pub fn with_label_owned(mut self, key: &'static str, value: String) -> Self { self.labels.push((key, Cow::Owned(value))); self } #[inline] - #[allow(dead_code)] pub fn with_labels(mut self, labels: Vec<(&'static str, Cow<'static, str>)>) -> Self { self.labels = labels; self diff --git a/crates/obs/src/metrics/schema/entry/descriptor.rs b/crates/obs/src/metrics/schema/entry/descriptor.rs index c4612e1f8..fce80d998 100644 --- a/crates/obs/src/metrics/schema/entry/descriptor.rs +++ b/crates/obs/src/metrics/schema/entry/descriptor.rs @@ -16,7 +16,6 @@ use crate::{MetricName, MetricNamespace, MetricSubsystem, MetricType}; use std::collections::HashSet; /// MetricDescriptor - Metric descriptors -#[allow(dead_code)] #[derive(Debug, Clone)] pub struct MetricDescriptor { pub name: MetricName, @@ -52,7 +51,6 @@ impl MetricDescriptor { } /// Get the full metric name in Prometheus style: __ - #[allow(dead_code)] pub fn get_full_metric_name(&self) -> String { let namespace = self.namespace.as_str(); let formatted_subsystem = self.subsystem.as_str(); @@ -61,7 +59,6 @@ impl MetricDescriptor { } /// check whether the label is in the label set - #[allow(dead_code)] pub fn has_label(&mut self, label: &str) -> bool { self.get_label_set().contains(label) } diff --git a/crates/obs/src/metrics/schema/entry/metric_name.rs b/crates/obs/src/metrics/schema/entry/metric_name.rs index 7d22eaa62..d9ec407b3 100644 --- a/crates/obs/src/metrics/schema/entry/metric_name.rs +++ b/crates/obs/src/metrics/schema/entry/metric_name.rs @@ -13,7 +13,6 @@ // limitations under the License. /// The metric name is the individual name of the metric -#[allow(dead_code)] #[derive(Debug, Clone, PartialEq, Eq)] pub enum MetricName { // The generic metric name @@ -443,7 +442,6 @@ pub enum MetricName { } impl MetricName { - #[allow(dead_code)] pub fn as_str(&self) -> String { match self { Self::AuthTotal => "auth_total".to_string(), diff --git a/crates/obs/src/metrics/schema/entry/metric_type.rs b/crates/obs/src/metrics/schema/entry/metric_type.rs index 33d8a78dd..b16f4cd12 100644 --- a/crates/obs/src/metrics/schema/entry/metric_type.rs +++ b/crates/obs/src/metrics/schema/entry/metric_type.rs @@ -13,7 +13,6 @@ // limitations under the License. /// MetricType - Indicates the type of indicator -#[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MetricType { Counter, @@ -23,7 +22,6 @@ pub enum MetricType { impl MetricType { /// convert the metric type to a string representation - #[allow(dead_code)] pub fn as_str(&self) -> &'static str { match self { Self::Counter => "counter", @@ -34,7 +32,6 @@ impl MetricType { /// Convert the metric type to the Prometheus value type /// In a Rust implementation, this might return the corresponding Prometheus Rust client type - #[allow(dead_code)] pub fn as_prom(&self) -> &'static str { match self { Self::Counter => "counter.", diff --git a/crates/obs/src/metrics/schema/entry/mod.rs b/crates/obs/src/metrics/schema/entry/mod.rs index 87215d15c..c10b48b45 100644 --- a/crates/obs/src/metrics/schema/entry/mod.rs +++ b/crates/obs/src/metrics/schema/entry/mod.rs @@ -56,7 +56,6 @@ pub fn new_gauge_md( } /// create a new histogram indicator descriptor -#[allow(dead_code)] pub fn new_histogram_md( name: impl Into, help: impl Into, diff --git a/crates/obs/src/metrics/schema/entry/namespace.rs b/crates/obs/src/metrics/schema/entry/namespace.rs index 31c3ce590..fe9a5df1f 100644 --- a/crates/obs/src/metrics/schema/entry/namespace.rs +++ b/crates/obs/src/metrics/schema/entry/namespace.rs @@ -19,7 +19,6 @@ pub enum MetricNamespace { } impl MetricNamespace { - #[allow(dead_code)] pub fn as_str(&self) -> &'static str { match self { Self::RustFS => "rustfs", diff --git a/crates/obs/src/metrics/schema/entry/path_utils.rs b/crates/obs/src/metrics/schema/entry/path_utils.rs index f8b63da28..d2abc7bfd 100644 --- a/crates/obs/src/metrics/schema/entry/path_utils.rs +++ b/crates/obs/src/metrics/schema/entry/path_utils.rs @@ -14,7 +14,6 @@ /// Format the path to the metric name format /// Replace '/' and '-' with '_' -#[allow(dead_code)] pub fn format_path_to_metric_name(path: &str) -> String { path.trim_start_matches('/').replace(['/', '-'], "_") } diff --git a/crates/obs/src/metrics/schema/entry/subsystem.rs b/crates/obs/src/metrics/schema/entry/subsystem.rs index 1c313f6e8..7b0606890 100644 --- a/crates/obs/src/metrics/schema/entry/subsystem.rs +++ b/crates/obs/src/metrics/schema/entry/subsystem.rs @@ -102,7 +102,6 @@ impl MetricSubsystem { } /// Get the formatted metric name format string - #[allow(dead_code)] pub fn as_str(&self) -> String { format_path_to_metric_name(self.path()) } @@ -151,7 +150,6 @@ impl MetricSubsystem { } /// A convenient way to create custom subsystems directly - #[allow(dead_code)] pub fn new(path: impl Into) -> Self { Self::Custom(path.into()) } @@ -176,7 +174,6 @@ impl std::fmt::Display for MetricSubsystem { } } -#[allow(dead_code)] pub mod subsystems { use super::MetricSubsystem; diff --git a/crates/obs/src/telemetry/rolling.rs b/crates/obs/src/telemetry/rolling.rs index 3ff661bfd..c5680ac61 100644 --- a/crates/obs/src/telemetry/rolling.rs +++ b/crates/obs/src/telemetry/rolling.rs @@ -38,7 +38,10 @@ pub enum Rotation { Minutely, Hourly, Daily, - #[allow(dead_code)] + #[allow( + dead_code, + reason = "constructed only by this file's rolling-appender tests; the lib target cannot see them (backlog#1823)" + )] Never, } From a9691b67978c56cb38081f453fc4959e568275a6 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 17 Aug 2026 11:34:47 +0800 Subject: [PATCH 60/71] chore: adjudicate 19 bare dead_code allows across six leaf crates (#6161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backlog#1823 step 10, batch 1 of the repo-wide item-allow sweep. 227 bare #[allow(dead_code)] remain across 83 files; this takes the 19 in utils, notify, checksums, policy, keystone and trusted-proxies, which are small enough to verify end to end. Removing all 19 first, before writing any reason, matters: 8 of them suppress nothing. Every allow in utils, one in policy and three in notify sit on items that are publicly reachable, so dead_code never applied to them — the same shape as the swift module and kms's dek.rs. Writing a reason onto a no-op allow would dress noise up as considered judgement, so those are simply deleted. Three items are genuinely dead and go with their allows: notify's new_target_id_set, the AWS metadata fetcher's get_metadata_token, and policy's empty `pub struct Value;`, none of which is referenced anywhere in the tree. The remaining eight keep an allow, now saying why the item survives rather than who calls it. Two are exercised only by their own crate's tests (checksums' MD5_HEADER_NAME, policy's is_match_as_pattern_prefix). Four are fields written but never read back: keystone's verify_ssl, parsed from config after the reqwest client is already built; keystone's client handle, which keeps the Keystone client alive for the mapper's lifetime; the AWS IMDS endpoint, kept beside the client while requests build their own URLs; and notify's rules_map, whose own comment retains it for snapshot-time judgements no code performs. checksums' Md5 needed the most care. Crc32, Sha256 and seven others each have an arm in ChecksumAlgorithm::into_impl, and Md5 has none, which reads like a missing algorithm. It is not: ChecksumAlgorithm has no Md5 variant at all. S3 carries Content-MD5 as its own header, separate from the x-amz-checksum-* family, and this impl exists so both paths share the Checksum trait. The reason records that, so the next reader does not re-derive it. One measurement note for anyone continuing this sweep: cargo does not re-emit warnings for cached compilations, so a per-crate loop of `cargo check -p ` under-reports. checksums showed zero that way while actually carrying three. Touch the sources and check the crates in one invocation, then attribute by path. Verification: the six crates are warning-free under cargo check --tests; clippy --lib --tests -D warnings clean; cargo nextest run 1096 passed; make pre-commit exit 0. Ref rustfs/backlog#1823 (step 10). --- crates/checksums/src/http.rs | 5 +- crates/checksums/src/lib.rs | 10 +++- crates/keystone/src/client.rs | 5 +- crates/keystone/src/identity.rs | 5 +- crates/notify/src/rules/config.rs | 5 +- crates/notify/src/rules/rules_map.rs | 3 -- crates/notify/src/rules/target_id_set.rs | 6 --- crates/policy/src/policy/function.rs | 4 -- crates/policy/src/policy/utils/wildcard.rs | 6 ++- .../trusted-proxies/src/cloud/metadata/aws.rs | 53 ++----------------- crates/trusted-proxies/src/config/env.rs | 1 - crates/utils/src/io.rs | 1 - crates/utils/src/net.rs | 1 - crates/utils/src/os/fs_type.rs | 1 - crates/utils/src/path.rs | 1 - 15 files changed, 32 insertions(+), 75 deletions(-) diff --git a/crates/checksums/src/http.rs b/crates/checksums/src/http.rs index 1a369a42d..ef21bc3ba 100644 --- a/crates/checksums/src/http.rs +++ b/crates/checksums/src/http.rs @@ -38,7 +38,10 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3"; pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64"; pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128"; -#[allow(dead_code)] +#[allow( + dead_code, + reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)" +)] pub(crate) static MD5_HEADER_NAME: &str = "content-md5"; pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] = diff --git a/crates/checksums/src/lib.rs b/crates/checksums/src/lib.rs index a8da44545..5b566fe83 100644 --- a/crates/checksums/src/lib.rs +++ b/crates/checksums/src/lib.rs @@ -476,13 +476,19 @@ impl Checksum for Xxhash64 { } } -#[allow(dead_code)] #[derive(Debug, Default)] +#[allow( + dead_code, + reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)" +)] struct Md5 { hasher: md5::Md5, } -#[allow(dead_code)] +#[allow( + dead_code, + reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)" +)] impl Md5 { fn update(&mut self, bytes: &[u8]) { use md5::Digest; diff --git a/crates/keystone/src/client.rs b/crates/keystone/src/client.rs index d5df169e8..bd1d36d30 100644 --- a/crates/keystone/src/client.rs +++ b/crates/keystone/src/client.rs @@ -31,7 +31,10 @@ pub struct KeystoneClient { admin_password: Option, admin_project: Option, admin_domain: String, - #[allow(dead_code)] + #[allow( + dead_code, + reason = "TLS verification flag parsed from config; the reqwest client is built before it is consulted, so nothing reads it back (backlog#1823)" + )] verify_ssl: bool, /// Request timeout applied to the underlying HTTP client. timeout: std::time::Duration, diff --git a/crates/keystone/src/identity.rs b/crates/keystone/src/identity.rs index 96e45de17..f61ca5c2b 100644 --- a/crates/keystone/src/identity.rs +++ b/crates/keystone/src/identity.rs @@ -20,7 +20,10 @@ use tracing::{debug, info}; /// Maps Keystone identities to RustFS concepts pub struct KeystoneIdentityMapper { - #[allow(dead_code)] + #[allow( + dead_code, + reason = "keeps the Keystone client alive for the mapper's lifetime; the mapping paths do not call through it yet (backlog#1823)" + )] client: Arc, role_policy_map: HashMap, enable_tenant_prefix: bool, diff --git a/crates/notify/src/rules/config.rs b/crates/notify/src/rules/config.rs index 42ca136fa..624f99787 100644 --- a/crates/notify/src/rules/config.rs +++ b/crates/notify/src/rules/config.rs @@ -40,7 +40,10 @@ impl RuleEvents for RuleView { #[derive(Debug)] struct CompiledRules { // Keep RulesMap (can be used later if you want to make more complex judgments during the snapshot reading phase) - #[allow(dead_code)] + #[allow( + dead_code, + reason = "speculative retention: the comment above keeps it for richer snapshot-time judgements that no code performs yet (backlog#1823)" + )] rules_map: RulesMap, // for RulesContainer::iter_rules rule_views: Vec, diff --git a/crates/notify/src/rules/rules_map.rs b/crates/notify/src/rules/rules_map.rs index 9ae40ad85..7d1314893 100644 --- a/crates/notify/src/rules/rules_map.rs +++ b/crates/notify/src/rules/rules_map.rs @@ -187,7 +187,6 @@ impl RulesMap { /// # Parameters /// * `event_name` - The EventName from which to remove the rule. /// * `pattern` - The pattern of the rule to be removed. - #[allow(dead_code)] pub fn remove_rule(&mut self, event_name: &EventName, pattern: &str) { let mut remove_event = false; @@ -209,7 +208,6 @@ impl RulesMap { /// /// # Parameters /// * `event_names` - A slice of EventNames to be removed. - #[allow(dead_code)] pub fn remove_rules(&mut self, event_names: &[EventName]) { for event_name in event_names { self.map.remove(event_name); @@ -223,7 +221,6 @@ impl RulesMap { /// * `event_name` - The EventName to update. /// * `pattern` - The pattern of the rule to be updated. /// * `target_id` - The TargetID to be added. - #[allow(dead_code)] pub fn update_rule(&mut self, event_name: EventName, pattern: String, target_id: TargetID) { self.map.entry(event_name).or_default().add(pattern, target_id); self.total_events_mask |= event_name.mask(); // Update only the relevant bitmask diff --git a/crates/notify/src/rules/target_id_set.rs b/crates/notify/src/rules/target_id_set.rs index d5036975c..06c2c2f87 100644 --- a/crates/notify/src/rules/target_id_set.rs +++ b/crates/notify/src/rules/target_id_set.rs @@ -18,12 +18,6 @@ use rustfs_targets::arn::TargetID; /// TargetIDSet - A collection representation of TargetID. pub type TargetIdSet = HashSet; -/// Provides a Go-like method for TargetIdSet (can be implemented as trait if needed) -#[allow(dead_code)] -pub(crate) fn new_target_id_set(target_ids: Vec) -> TargetIdSet { - target_ids.into_iter().collect() -} - // HashSet has built-in clone, union, difference and other operations. // But the Go version of the method returns a new Set, and the HashSet method is usually iterator or modify itself. // If you need to exactly match Go's API style, you can add wrapper functions. diff --git a/crates/policy/src/policy/function.rs b/crates/policy/src/policy/function.rs index b1fa5d1ee..55bf7e13a 100644 --- a/crates/policy/src/policy/function.rs +++ b/crates/policy/src/policy/function.rs @@ -219,10 +219,6 @@ impl PartialEq for Functions { } } -#[derive(Clone, Serialize, Deserialize)] -#[allow(dead_code)] -pub struct Value; - #[cfg(test)] mod tests { use crate::policy::Functions; diff --git a/crates/policy/src/policy/utils/wildcard.rs b/crates/policy/src/policy/utils/wildcard.rs index 915cb153e..0b8fa3bb2 100644 --- a/crates/policy/src/policy/utils/wildcard.rs +++ b/crates/policy/src/policy/utils/wildcard.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[allow(dead_code)] pub fn is_simple_match(pattern: P, name: N) -> bool where P: AsRef, @@ -29,7 +28,10 @@ where inner_match(pattern, name, false) } -#[allow(dead_code)] +#[allow( + dead_code, + reason = "prefix-matcher asserted by this file's tests; no production caller yet (backlog#1823)" +)] pub fn is_match_as_pattern_prefix(pattern: P, text: N) -> bool where P: AsRef, diff --git a/crates/trusted-proxies/src/cloud/metadata/aws.rs b/crates/trusted-proxies/src/cloud/metadata/aws.rs index 506eca925..c971cfa4d 100644 --- a/crates/trusted-proxies/src/cloud/metadata/aws.rs +++ b/crates/trusted-proxies/src/cloud/metadata/aws.rs @@ -27,6 +27,10 @@ use crate::CloudMetadataFetcher; #[derive(Debug, Clone)] pub struct AwsMetadataFetcher { client: Client, + #[allow( + dead_code, + reason = "IMDS endpoint retained beside the client it configures; requests build their own URLs (backlog#1823)" + )] metadata_endpoint: String, } @@ -46,55 +50,6 @@ impl AwsMetadataFetcher { metadata_endpoint: "http://169.254.169.254".to_string(), } } - - /// Retrieves an IMDSv2 token for secure metadata access. - #[allow(dead_code)] - async fn get_metadata_token(&self) -> Result { - let url = format!("{}/latest/api/token", self.metadata_endpoint); - - match self - .client - .put(&url) - .header("X-aws-ec2-metadata-token-ttl-seconds", "21600") - .send() - .await - { - Ok(response) => { - if response.status().is_success() { - let token = response - .text() - .await - .map_err(|e| AppError::cloud(format!("Failed to read IMDSv2 token: {}", e)))?; - Ok(token) - } else { - debug!( - event = "trusted_proxies.cloud_metadata", - component = "trusted_proxies", - subsystem = "aws_metadata", - provider = "aws", - operation = "imdsv2_token", - result = "http_error", - status = %response.status(), - "trusted proxy cloud metadata request failed" - ); - Err(AppError::cloud("Failed to obtain IMDSv2 token")) - } - } - Err(e) => { - debug!( - event = "trusted_proxies.cloud_metadata", - component = "trusted_proxies", - subsystem = "aws_metadata", - provider = "aws", - operation = "imdsv2_token", - result = "request_failed", - error = %e, - "trusted proxy cloud metadata request failed" - ); - Err(AppError::cloud(format!("IMDSv2 request failed: {}", e))) - } - } - } } #[async_trait] diff --git a/crates/trusted-proxies/src/config/env.rs b/crates/trusted-proxies/src/config/env.rs index a982ae0d8..2a8a3ff6a 100644 --- a/crates/trusted-proxies/src/config/env.rs +++ b/crates/trusted-proxies/src/config/env.rs @@ -68,7 +68,6 @@ pub fn is_env_set(key: &str) -> bool { } /// Returns a list of all proxy-related environment variables and their current values. -#[allow(dead_code)] pub fn get_all_proxy_env_vars() -> Vec<(String, String)> { let vars = [ ENV_TRUSTED_PROXY_ENABLED, diff --git a/crates/utils/src/io.rs b/crates/utils/src/io.rs index 92e69e5db..44388b0f5 100644 --- a/crates/utils/src/io.rs +++ b/crates/utils/src/io.rs @@ -68,7 +68,6 @@ pub async fn read_full_or_eof( /// Read exactly buf.len() bytes into buf, or return an error if EOF is reached before any bytes are read. /// Like Go's io.ReadFull. -#[allow(dead_code)] pub async fn read_full(reader: R, buf: &mut [u8]) -> std::io::Result { match read_full_or_eof(reader, buf).await? { Some(n) => Ok(n), diff --git a/crates/utils/src/net.rs b/crates/utils/src/net.rs index 873a6d940..3489ba41c 100644 --- a/crates/utils/src/net.rs +++ b/crates/utils/src/net.rs @@ -431,7 +431,6 @@ pub fn parse_and_resolve_address(addr_str: &str) -> std::io::Result Ok(resolved_addr) } -#[allow(dead_code)] pub fn bytes_stream(stream: S, content_length: usize) -> impl Stream> + Send + 'static where S: Stream> + Send + 'static, diff --git a/crates/utils/src/os/fs_type.rs b/crates/utils/src/os/fs_type.rs index a3ae1045c..3650cb793 100644 --- a/crates/utils/src/os/fs_type.rs +++ b/crates/utils/src/os/fs_type.rs @@ -16,7 +16,6 @@ /// /// The table follows Linux `include/uapi/linux/magic.h`; filesystem magic /// values without a stable Linux uapi source stay `UNKNOWN`. -#[allow(dead_code)] pub(crate) fn get_fs_type(fs_type: u64) -> &'static str { // Magic numbers for various filesystems. match fs_type { diff --git a/crates/utils/src/path.rs b/crates/utils/src/path.rs index c383dc822..8b53e764b 100644 --- a/crates/utils/src/path.rs +++ b/crates/utils/src/path.rs @@ -70,7 +70,6 @@ pub fn is_dir_object(object: &str) -> bool { /// /// If the object name ends with `GLOBAL_DIR_SUFFIX`, it is replaced with a slash. /// Otherwise, the name is returned as is. -#[allow(dead_code)] pub fn decode_dir_object(object: &str) -> String { if has_suffix(object, GLOBAL_DIR_SUFFIX) { format!("{}{}", object.trim_end_matches(GLOBAL_DIR_SUFFIX), SLASH_SEPARATOR) From c04ee41cf088b86150b6bb75f13fed8913fcf0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Mon, 17 Aug 2026 14:12:04 +0800 Subject: [PATCH 61/71] feat(site-replication): drain the retry queue from the reconcile tick (#6131) --- crates/madmin/src/site_replication.rs | 6 +- rustfs/src/admin/handlers/site_replication.rs | 1057 ++++++++++++++++- 2 files changed, 1051 insertions(+), 12 deletions(-) diff --git a/crates/madmin/src/site_replication.rs b/crates/madmin/src/site_replication.rs index a8813f99c..740924405 100644 --- a/crates/madmin/src/site_replication.rs +++ b/crates/madmin/src/site_replication.rs @@ -258,7 +258,7 @@ pub struct SRLDAPUser { pub api_version: Option, } -#[derive(Debug, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct SRIAMUser { #[serde(rename = "accessKey", default)] pub access_key: String, @@ -270,7 +270,7 @@ pub struct SRIAMUser { pub api_version: Option, } -#[derive(Debug, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct SRGroupInfo { #[serde(rename = "updateReq", default)] pub update_req: GroupAddRemove, @@ -346,7 +346,7 @@ pub struct SRCredInfo { pub api_version: Option, } -#[derive(Debug, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct SRIAMItem { #[serde(default)] pub r#type: String, diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index d9604fe6f..fba311ba9 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -2986,6 +2986,18 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin { if state.pending_endpoint_refresh.is_some() || state.pending_remove.is_some() || state.pending_rotation.is_some() @@ -3017,6 +3029,9 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None), None => { - dequeue_site_replication_retry_events(&mut state.retry_queue, &peer, &path); + dequeue_site_replication_retry_events_including_escalated(&mut state.retry_queue, &peer, &path); } } Ok(()) @@ -6071,10 +6086,96 @@ fn retry_event_matches(event: &SiteReplicationRetryEvent, peer: &PeerInfo, path: (event.peer_deployment_id == peer.deployment_id || event.peer_endpoint == peer.endpoint) && event.path == path } +const SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH: &str = "internal:retry-snapshot:iam"; +const SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH: &str = "internal:retry-snapshot:bucket-metadata"; + +fn collapsed_retry_queue_path(path: &str) -> Option<&'static str> { + let base_path = path.split_once('?').map(|(base, _)| base).unwrap_or(path); + match base_path { + "/rustfs/admin/v3/site-replication/peer/iam-item" | SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH => { + Some(SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH) + } + "/rustfs/admin/v3/site-replication/peer/bucket-meta" | SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH => { + Some(SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH) + } + _ => None, + } +} + +fn normalize_collapsed_retry_queue_paths(queue: &mut Vec) -> bool { + let mut changed = false; + let mut normalized: Vec = Vec::with_capacity(queue.len()); + for mut event in queue.drain(..) { + if let Some(path) = collapsed_retry_queue_path(&event.path) + && event.path != path + { + event.path = path.to_string(); + changed = true; + } + + let duplicate = normalized.iter().position(|existing| { + existing.path == event.path + && (existing.peer_deployment_id == event.peer_deployment_id || existing.peer_endpoint == event.peer_endpoint) + }); + let Some(index) = duplicate else { + normalized.push(event); + continue; + }; + + changed = true; + let existing = &mut normalized[index]; + let event_is_newer = match (event.updated_at, existing.updated_at) { + (Some(event), Some(existing)) => event >= existing, + (Some(_), None) => true, + _ => false, + }; + if event_is_newer { + let retry_count = existing.retry_count.max(event.retry_count); + *existing = event; + existing.retry_count = retry_count; + } else { + existing.retry_count = existing.retry_count.max(event.retry_count); + } + existing.failed = existing.retry_count >= SITE_REPLICATION_RETRY_FAILED_AFTER; + } + *queue = normalized; + changed +} + +async fn migrate_collapsed_retry_queue_paths() -> S3Result<()> { + update_site_replication_state_when_changed(|state| { + Ok(if normalize_collapsed_retry_queue_paths(&mut state.retry_queue) { + StateCommit::Changed(()) + } else { + StateCommit::Unchanged(()) + }) + }) + .await +} + +#[cfg(test)] fn dequeue_site_replication_retry_events(queue: &mut Vec, peer: &PeerInfo, path: &str) -> usize { settle_site_replication_retry_events(queue, peer, path, None) } +/// Repair-path settlement: also clears snapshot-escalated entries. Running a +/// repair is the operator's explicit accountability transfer for the +/// possibly-unreplayed deletion the marker records; ordinary delivery +/// successes must not clear it (see [`settle_site_replication_retry_events`]). +fn dequeue_site_replication_retry_events_including_escalated( + queue: &mut Vec, + peer: &PeerInfo, + path: &str, +) -> usize { + let before = queue.len(); + let collapsed_path = collapsed_retry_queue_path(path); + queue.retain(|event| { + !retry_event_matches(event, peer, path) + && !collapsed_path.is_some_and(|collapsed_path| retry_event_matches(event, peer, collapsed_path)) + }); + before.saturating_sub(queue.len()) +} + /// Remove the retry events for (peer, path) that `generation` is entitled to /// settle. A successful delivery only proves the peer reached the state the /// delivery carried: while it was in flight another edit can commit, fail its @@ -6090,10 +6191,24 @@ fn settle_site_replication_retry_events( generation: Option, ) -> usize { let before = queue.len(); + let collapsed_path = collapsed_retry_queue_path(path); queue.retain(|event| { if !retry_event_matches(event, peer, path) { return true; } + // A wire-path success identifies no IAM or bucket-metadata entity. + // This also protects legacy rows until the startup migration moves + // them under their internal snapshot path. + if collapsed_path.is_some() { + return true; + } + // A snapshot-escalated entry records a possibly-unreplayed deletion. + // Collapsed paths are shared by every entity, so a later successful + // delivery of a DIFFERENT item proves nothing about the deleted one — + // only a repair settles it (dequeue_..._including_escalated). + if event.last_error == SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + return true; + } match (generation, event.edit_generation) { (Some(settled), Some(failed)) => failed > settled, _ => false, @@ -6109,6 +6224,7 @@ fn upsert_site_replication_retry_event( error: &str, generation: Option, ) { + let path = collapsed_retry_queue_path(path).unwrap_or(path); let now = OffsetDateTime::now_utc(); let detail = summarize_peer_error_detail(error); if let Some(event) = queue.iter_mut().find(|event| retry_event_matches(event, peer, path)) { @@ -6171,7 +6287,12 @@ async fn enqueue_site_replication_retry_event_for_generation( let path_owned = path.to_string(); let error_text = error.to_string(); let result = update_site_replication_state(move |state| { - upsert_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &path_owned, &error_text, generation); + // A peer that left the state can never drain its entries again + // (remove_sites already pruned them); recording a late failure for it + // would only pollute retry_stats until the queue cap evicts it. + if state.peers.contains_key(&peer_owned.deployment_id) { + upsert_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &path_owned, &error_text, generation); + } Ok(()) }) .await; @@ -6205,6 +6326,595 @@ fn retry_event_replayed_by_bootstrap(event: &SiteReplicationRetryEvent) -> bool ) } +/// Exponential backoff base for the background retry drain, aligned with the +/// reconcile cadence (`site_replication_reconcile::RECONCILE_INTERVAL`). +const SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS: i64 = 600; +/// Backoff ceiling: a permanently failed peer is still probed daily. +const SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS: i64 = 86_400; + +/// What the background drain may do for one retry event. Everything not +/// representable here is operator territory (manual repair). +#[derive(Debug, Clone, PartialEq, Eq)] +enum RetryDrainAction { + /// Constant-path IAM item deliveries collapse into one queue entry per + /// peer and their bodies are not persisted; the only faithful replay is + /// the current IAM snapshot from the bootstrap plan. + IamSnapshot, + /// Same collapse for bucket-meta deliveries: replay the bucket metadata + /// snapshot from the bootstrap plan. + BucketMetadataSnapshot, + /// A self-contained bucket op the bootstrap plan can re-derive for its + /// bucket (`make-with-versioning` / `configure-replication`). + BucketOpReplay { operation: String, bucket: String }, + /// Re-send the current peer records under a fresh edit generation. + PeerEdit, +} + +#[derive(Clone)] +enum RetrySnapshot { + Iam(Vec), + BucketMetadata(Vec), +} + +impl RetrySnapshot { + fn from_plan(action: &RetryDrainAction, plan: &SiteReplicationBootstrapPlan) -> Option { + match action { + RetryDrainAction::IamSnapshot => Some(Self::Iam(plan.iam_items.clone())), + RetryDrainAction::BucketMetadataSnapshot => Some(Self::BucketMetadata(plan.bucket_items.clone())), + _ => None, + } + } + + fn fingerprint(&self) -> S3Result>> { + let mut payloads = match self { + Self::Iam(items) => items.iter().map(serde_json::to_vec).collect::, _>>(), + Self::BucketMetadata(items) => items.iter().map(serde_json::to_vec).collect::, _>>(), + } + .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry snapshot failed: {err}")))?; + payloads.sort_unstable(); + Ok(payloads) + } + + fn replay_after_change(previous: &Self, fresh: &Self, observed_at: OffsetDateTime) -> Self { + match (previous, fresh) { + (Self::Iam(previous), Self::Iam(fresh)) => { + let fresh_keys: HashSet = fresh.iter().filter_map(iam_snapshot_key).collect(); + let mut replay = fresh.clone(); + for item in previous { + if iam_snapshot_key(item).is_some_and(|key| !fresh_keys.contains(&key)) { + replay.extend(iam_snapshot_tombstones(item, observed_at)); + } + } + Self::Iam(replay) + } + (Self::BucketMetadata(previous), Self::BucketMetadata(fresh)) => { + let fresh_keys: HashSet<(&str, &str)> = fresh + .iter() + .map(|item| (item.bucket.as_str(), item.r#type.as_str())) + .collect(); + let mut replay = fresh.clone(); + for item in previous { + if !fresh_keys.contains(&(item.bucket.as_str(), item.r#type.as_str())) { + replay.push(bucket_metadata_snapshot_tombstone(item, observed_at)); + } + } + Self::BucketMetadata(replay) + } + _ => fresh.clone(), + } + } + + async fn send(&self, transport: &PeerTransport, access_key: &str, secret_key: &str) -> S3Result<()> { + match self { + Self::Iam(items) => { + for item in items { + SiteReplicationRepairTask::Iam(item) + .send(transport, access_key, secret_key) + .await?; + } + } + Self::BucketMetadata(items) => { + for item in items { + SiteReplicationRepairTask::BucketMetadata(item) + .send(transport, access_key, secret_key) + .await?; + } + } + } + Ok(()) + } +} + +#[derive(Hash, PartialEq, Eq)] +enum IamSnapshotKey { + Policy(String), + User(String), + Group(String), + PolicyMapping { target: String, user_type: i64, is_group: bool }, +} + +fn iam_snapshot_key(item: &SRIAMItem) -> Option { + match item.r#type.as_str() { + "policy" => Some(IamSnapshotKey::Policy(item.name.clone())), + "iam-user" => item + .iam_user + .as_ref() + .map(|user| IamSnapshotKey::User(user.access_key.clone())), + "group-info" => item + .group_info + .as_ref() + .map(|group| IamSnapshotKey::Group(group.update_req.group.clone())), + "policy-mapping" => item.policy_mapping.as_ref().map(|mapping| IamSnapshotKey::PolicyMapping { + target: mapping.user_or_group.clone(), + user_type: mapping.user_type, + is_group: mapping.is_group, + }), + _ => None, + } +} + +fn iam_snapshot_tombstones(item: &SRIAMItem, observed_at: OffsetDateTime) -> Vec { + let mut tombstone = item.clone(); + tombstone.updated_at = Some(observed_at); + match item.r#type.as_str() { + "policy" => tombstone.policy = None, + "iam-user" => { + if let Some(user) = tombstone.iam_user.as_mut() { + user.is_delete_req = true; + user.user_req = None; + } + } + "group-info" => { + let Some(group) = tombstone.group_info.as_mut() else { + return Vec::new(); + }; + group.update_req.is_remove = true; + if group.update_req.members.is_empty() { + return vec![tombstone]; + } + let mut delete = tombstone.clone(); + if let Some(group) = delete.group_info.as_mut() { + group.update_req.members.clear(); + } + return vec![tombstone, delete]; + } + "policy-mapping" => { + if let Some(mapping) = tombstone.policy_mapping.as_mut() { + mapping.policy.clear(); + } + } + _ => return Vec::new(), + } + vec![tombstone] +} + +fn bucket_metadata_snapshot_tombstone(item: &SRBucketMeta, observed_at: OffsetDateTime) -> SRBucketMeta { + SRBucketMeta { + r#type: item.r#type.clone(), + bucket: item.bucket.clone(), + updated_at: Some(observed_at), + expiry_updated_at: Some(observed_at), + api_version: item.api_version.clone(), + ..Default::default() + } +} + +const SITE_REPLICATION_RETRY_SNAPSHOT_STABILITY_ATTEMPTS: usize = 3; + +fn classify_site_replication_retry_event(event: &SiteReplicationRetryEvent) -> Option { + let snapshot_action = match event.path.as_str() { + SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH => Some(RetryDrainAction::IamSnapshot), + SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH => Some(RetryDrainAction::BucketMetadataSnapshot), + _ => None, + }; + if snapshot_action.is_some() && event.last_error != SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + return snapshot_action; + } + if event.path.starts_with("internal:") { + // Marker records store payloads in `last_error` (legacy + // pending-endpoint-refresh backup and snapshot liabilities); they are + // not drainable delivery failures. + return None; + } + if event.last_error == SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + // Already snapshot-replayed once for this failure episode; a possible + // deletion cannot be replayed from a snapshot, so re-sending daily + // proves nothing. A new hook failure overwrites the marker. + return None; + } + let base_path = event.path.split_once('?').map(|(base, _)| base).unwrap_or(&event.path); + match base_path { + "/rustfs/admin/v3/site-replication/peer/iam-item" => Some(RetryDrainAction::IamSnapshot), + "/rustfs/admin/v3/site-replication/peer/bucket-meta" => Some(RetryDrainAction::BucketMetadataSnapshot), + SITE_REPLICATION_PEER_EDIT_PATH => Some(RetryDrainAction::PeerEdit), + SITE_REPLICATION_PEER_BUCKET_OPS_PATH => { + let operation = retry_bucket_operation(&event.path)?; + if !matches!( + operation.as_str(), + SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING | SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION + ) { + // Destructive ops (delete-bucket / force-delete-bucket) are + // operator territory: replaying them against a peer whose + // bucket was since recreated is irreversible. + return None; + } + let bucket = retry_bucket_name(&event.path)?; + Some(RetryDrainAction::BucketOpReplay { operation, bucket }) + } + _ => None, + } +} + +fn retry_bucket_name(path: &str) -> Option { + let (_, query) = path.split_once('?')?; + form_urlencoded::parse(query.as_bytes()) + .find_map(|(key, value)| (key == "bucket" && !value.is_empty()).then(|| value.into_owned())) +} + +/// A collapsed retry event after a stable snapshot resend is escalated with +/// this marker instead of being cleared: the snapshot contains no task for a +/// failed deletion, so remote absence remains operator-visible. Collapsed +/// failures use an internal queue path so ordinary successes and older nodes +/// cannot settle an unrelated entity's liability. +const SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER: &str = "snapshot replayed; a failed deletion cannot be replayed from a snapshot — run site replication repair or re-deliver to settle"; + +/// Escalate a collapsed retry event after its snapshot resend succeeded, +/// unless a newer failure was recorded after `snapshot_updated_at` (that +/// failure belongs to a newer local commit the snapshot did not contain and +/// must keep the entry drain-eligible). +fn escalate_site_replication_retry_events_up_to( + queue: &mut Vec, + peer: &PeerInfo, + path: &str, + snapshot_updated_at: Option, +) -> usize { + let Some(marker_path) = collapsed_retry_queue_path(path) else { + return 0; + }; + + if path != marker_path { + queue.retain(|event| { + if !retry_event_matches(event, peer, path) { + return true; + } + matches!((event.updated_at, snapshot_updated_at), (Some(current), Some(seen)) if current > seen) + || matches!((event.updated_at, snapshot_updated_at), (Some(_), None)) + }); + } + + let marker_index = queue.iter().position(|event| retry_event_matches(event, peer, marker_path)); + let marker_index = marker_index.unwrap_or_else(|| { + queue.push(SiteReplicationRetryEvent { + id: Uuid::new_v4().to_string(), + peer_deployment_id: peer.deployment_id.clone(), + peer_endpoint: peer.endpoint.clone(), + path: marker_path.to_string(), + updated_at: snapshot_updated_at, + ..Default::default() + }); + queue.len() - 1 + }); + let event = &mut queue[marker_index]; + let newer_failure_recorded = match (event.updated_at, snapshot_updated_at) { + (Some(current), Some(seen)) => current > seen, + (Some(_), None) => true, + (None, _) => false, + }; + if newer_failure_recorded && event.last_error != SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + return 0; + } + event.failed = true; + event.retry_count = event.retry_count.max(SITE_REPLICATION_RETRY_FAILED_AFTER); + event.last_error = SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER.to_string(); + event.updated_at = Some(OffsetDateTime::now_utc()); + 1 +} + +async fn escalate_site_replication_retry_event_up_to(peer: &PeerInfo, path: &str, snapshot_updated_at: Option) { + let peer_owned = peer.clone(); + let path_owned = path.to_string(); + let result = update_site_replication_state(move |state| { + escalate_site_replication_retry_events_up_to(&mut state.retry_queue, &peer_owned, &path_owned, snapshot_updated_at); + Ok(()) + }) + .await; + + if let Err(err) = result { + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + peer = %peer.endpoint, + deployment_id = %peer.deployment_id, + path, + error = ?err, + "failed to escalate site replication retry event" + ); + } +} + +/// Whether the drain may attempt this event now. +fn site_replication_retry_backoff_elapsed(event: &SiteReplicationRetryEvent, now: OffsetDateTime) -> bool { + let Some(updated_at) = event.updated_at else { + return true; + }; + // 600 * 2^8 already exceeds the daily ceiling; capping the shift keeps + // the arithmetic overflow-free for any persisted retry_count. + let exponent = event.retry_count.saturating_sub(1).min(8); + let delay = (SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS << exponent).min(SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS); + now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) >= delay +} + +/// The subset of the retry queue the background drain is allowed to touch. +fn actionable_site_replication_retry_events(state: &SiteReplicationState, now: OffsetDateTime) -> Vec { + state + .retry_queue + .iter() + .filter(|event| classify_site_replication_retry_event(event).is_some()) + .filter(|event| state.peers.contains_key(&event.peer_deployment_id)) + .filter(|event| site_replication_retry_backoff_elapsed(event, now)) + .cloned() + .collect() +} + +/// Background consumer for the retry queue, run from the reconcile tick. +/// +/// Scope: this settles "delivered once and failed" entries whose replay is +/// faithful (bucket ops, peer edits). Collapsed iam-item / bucket-meta +/// entries are snapshot-resent and then *escalated*, not cleared — a failed +/// deletion leaves no task in the snapshot, so remote absence stays unproven +/// until a later delivery or a manual repair. A hook that never fired (crash +/// between the local commit and the send) leaves no entry at all, so the +/// drain is not a full cross-site diff-heal; manual repair remains the +/// authoritative catch-all. +async fn drain_site_replication_retry_queue() { + if let Err(err) = drain_site_replication_retry_queue_inner().await { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_drain_failed", + error = ?err, + "admin site replication state" + ); + } +} + +async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + let actionable = actionable_site_replication_retry_events(&runtime.state, OffsetDateTime::now_utc()); + if actionable.is_empty() { + return Ok(()); + } + let Some(store) = current_object_store_handle() else { + return Ok(()); + }; + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + // The tick-level gate ran before the reconcilers; a multi-step flow + // (endpoint refresh commits its pending marker without the lifecycle + // guard) may have started since. Re-check on the fresh state. + return Ok(()); + } + // Serialize against operator repair execution. This does NOT close the + // dry-run -> execute window (dry-run takes no lock): a drain settling a + // replayable bucket-op entry in that window changes the preflight token + // and execute fails safe with "preflight is stale" — the operator + // re-runs the dry-run. Lock order matches repair: lifecycle guard (held + // by the reconcile tick) -> repair execution lock -> state object lock + // inside the send bookkeeping. An operator repair holding the lock makes + // this tick skip after the lock-acquire timeout. + with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { + drain_site_replication_retry_queue_locked(runtime, actionable).await + }) + .await + .map_err(ApiError::from)? +} + +async fn drain_site_replication_retry_queue_locked( + runtime: SiteReplicationRuntime, + events: Vec, +) -> S3Result<()> { + let needs_plan = events + .iter() + .any(|event| !matches!(classify_site_replication_retry_event(event), Some(RetryDrainAction::PeerEdit))); + // The plan is a full local snapshot (buckets + IAM); build it once per + // tick and only when a snapshot resend is actually due. + let plan = if needs_plan { + let info = build_sr_info(&runtime.state, &runtime.local_peer).await?; + Some(site_replication_bootstrap_plan(&info)?) + } else { + None + }; + + let mut events_by_peer: BTreeMap> = BTreeMap::new(); + for event in events { + events_by_peer + .entry(event.peer_deployment_id.clone()) + .or_default() + .push(event); + } + + let mut settled = 0usize; + let mut failures = 0usize; + for (deployment_id, peer_events) in events_by_peer { + let Some(peer) = runtime.state.peers.get(&deployment_id) else { + continue; + }; + if deployment_id == runtime.local_peer.deployment_id + || same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) + { + continue; + } + let transport = match PeerTransport::for_runtime_peer(peer).await { + Ok(transport) => transport, + Err(err) => { + // Record the attempt so backoff advances for an unreachable + // peer instead of re-dialing it every tick. + for event in &peer_events { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + } + failures += peer_events.len(); + continue; + } + }; + for event in peer_events { + let Some(action) = classify_site_replication_retry_event(&event) else { + continue; + }; + match drain_one_site_replication_retry_event(&runtime, peer, &transport, &event, action, plan.as_ref()).await { + Ok(true) => settled += 1, + Ok(false) => {} + Err(_) => failures += 1, + } + } + } + + if settled > 0 || failures > 0 { + info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_drain_settled", + settled, + failures, + "admin site replication state" + ); + } + Ok(()) +} + +/// Replay one retry event against its peer. Returns `Ok(true)` when the +/// event was settled (delivered, or provably stale), `Ok(false)` when it was +/// skipped, and `Err` after a failed delivery (already re-queued with an +/// incremented retry count). +async fn drain_one_site_replication_retry_event( + runtime: &SiteReplicationRuntime, + peer: &PeerInfo, + transport: &PeerTransport, + event: &SiteReplicationRetryEvent, + action: RetryDrainAction, + plan: Option<&SiteReplicationBootstrapPlan>, +) -> S3Result { + let access_key = &runtime.state.service_account_access_key; + let secret_key = &runtime.service_account_secret_key; + match action.clone() { + RetryDrainAction::IamSnapshot | RetryDrainAction::BucketMetadataSnapshot => { + let Some(plan) = plan else { + return Ok(false); + }; + let mut current_snapshot = RetrySnapshot::from_plan(&action, plan).expect("snapshot action has a snapshot"); + let mut replay = current_snapshot.clone(); + for _ in 0..SITE_REPLICATION_RETRY_SNAPSHOT_STABILITY_ATTEMPTS { + let current_fingerprint = current_snapshot.fingerprint()?; + if let Err(err) = replay.send(transport, access_key, secret_key).await { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } + let fresh_info = build_sr_info(&runtime.state, &runtime.local_peer).await?; + let fresh_plan = site_replication_bootstrap_plan(&fresh_info)?; + let fresh_snapshot = RetrySnapshot::from_plan(&action, &fresh_plan).expect("snapshot action has a snapshot"); + if fresh_snapshot.fingerprint()? == current_fingerprint { + escalate_site_replication_retry_event_up_to(peer, &event.path, event.updated_at).await; + return Ok(true); + } + replay = RetrySnapshot::replay_after_change(¤t_snapshot, &fresh_snapshot, OffsetDateTime::now_utc()); + current_snapshot = fresh_snapshot; + } + Ok(false) + } + RetryDrainAction::BucketOpReplay { operation, bucket } => { + let Some(plan) = plan else { + return Ok(false); + }; + // Replay from the CURRENT plan, never the recorded path: the + // recorded query can carry an expired one-shot bootstrap token or + // a stale createdAt. + let make_op = operation == SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING; + let paths = if make_op { + &plan.bucket_make_ops + } else { + &plan.bucket_configure_ops + }; + let tasks: Vec> = paths + .iter() + .filter(|path| retry_bucket_name(path).as_deref() == Some(bucket.as_str())) + .map(|path| { + if make_op { + SiteReplicationRepairTask::BucketMake(path) + } else { + SiteReplicationRepairTask::Replication(path) + } + }) + .collect(); + if tasks.is_empty() { + // The bucket left the plan (deleted, or replication no longer + // configured): the recorded intent is stale, settle it. + dequeue_site_replication_retry_event(peer, &event.path).await; + return Ok(true); + } + for task in &tasks { + if let Err(err) = task.send(transport, access_key, secret_key).await { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } + } + dequeue_site_replication_retry_event(peer, &event.path).await; + Ok(true) + } + RetryDrainAction::PeerEdit => { + // The recorded generation is stale by definition — the receiver + // fences it. Allocate a fresh generation and re-send the current + // peer records (a superset of the failed body; the receiver + // upserts), all inside one state transaction so the fence and the + // bodies agree. + let target_id = peer.deployment_id.clone(); + let (generation, bodies) = update_site_replication_state(move |state| { + if !state.peers.contains_key(&target_id) { + return Ok((None, Vec::new())); + } + Ok((Some(next_peer_edit_generation(state)), state.peers.values().cloned().collect::>())) + }) + .await?; + let Some(generation) = generation else { + // Peer left between the snapshot and now; the queue entry was + // already pruned by remove_sites. + return Ok(false); + }; + let local_deployment_id = Some(runtime.local_peer.deployment_id.as_str()).filter(|id| !id.is_empty()); + let edit_path = peer_edit_path_with_fence(local_deployment_id, generation); + let delivery_fence = local_deployment_id.is_some().then_some(generation); + for body in &bodies { + if let Err(err) = send_peer_admin_request_with_client( + &transport.client, + &transport.connection, + &edit_path, + access_key, + secret_key, + body, + ) + .await + { + enqueue_site_replication_retry_event_for_generation( + peer, + SITE_REPLICATION_PEER_EDIT_PATH, + &err, + delivery_fence, + ) + .await; + return Err(err); + } + } + dequeue_site_replication_retry_event_for_generation(peer, SITE_REPLICATION_PEER_EDIT_PATH, delivery_fence).await; + Ok(true) + } + } +} + /// Remove a retry event for (peer, path) from the queue on successful delivery. /// This is a no-op (load + no-op persist skipped) when no matching entry exists, /// avoiding unnecessary I/O on the common path. @@ -11839,6 +12549,320 @@ mod tests { assert!(target_state.peers["remote"].skip_tls_verify); } + fn drain_event(peer: &str, path: &str, retry_count: u32, updated_at: Option) -> SiteReplicationRetryEvent { + SiteReplicationRetryEvent { + id: format!("evt-{peer}"), + peer_deployment_id: peer.to_string(), + peer_endpoint: format!("https://{peer}.example.com"), + path: path.to_string(), + retry_count, + failed: retry_count >= SITE_REPLICATION_RETRY_FAILED_AFTER, + last_error: "remote-operation-failed".to_string(), + updated_at, + edit_generation: None, + } + } + + /// P1-3 red-light: the drain must only ever act on deliveries it can + /// replay faithfully. IAM / bucket-meta entries collapse per (peer, path) + /// with no body persisted — only a snapshot resend is truthful; bucket + /// makes/replication configs are re-derivable; destructive bucket ops and + /// unrelated `internal:` marker records are never background-replayed. + #[test] + fn test_classify_site_replication_retry_event_actions() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let classify = |path: &str| classify_site_replication_retry_event(&drain_event("remote", path, 1, Some(now))); + + assert_eq!( + classify("/rustfs/admin/v3/site-replication/peer/iam-item"), + Some(RetryDrainAction::IamSnapshot) + ); + assert_eq!( + classify("/rustfs/admin/v3/site-replication/peer/bucket-meta"), + Some(RetryDrainAction::BucketMetadataSnapshot) + ); + assert_eq!(classify(SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH), Some(RetryDrainAction::IamSnapshot)); + assert_eq!( + classify(SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH), + Some(RetryDrainAction::BucketMetadataSnapshot) + ); + assert_eq!(classify(SITE_REPLICATION_PEER_EDIT_PATH), Some(RetryDrainAction::PeerEdit)); + assert_eq!( + classify( + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning&createdAt=1" + ), + Some(RetryDrainAction::BucketOpReplay { + operation: SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING.to_string(), + bucket: "photos".to_string(), + }) + ); + assert_eq!( + classify("/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication"), + Some(RetryDrainAction::BucketOpReplay { + operation: SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION.to_string(), + bucket: "photos".to_string(), + }) + ); + // Destructive ops are operator territory: replaying a bucket delete + // against a peer whose bucket was since recreated is irreversible. + assert_eq!( + classify("/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=delete-bucket"), + None + ); + assert_eq!( + classify("/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=force-delete-bucket"), + None + ); + // `internal:` records store payloads in `last_error`, not failures. + assert_eq!(classify(SITE_REPLICATION_ENDPOINT_REFRESH_RETRY_PATH), None); + assert_eq!(classify("internal:some-future-marker"), None); + assert_eq!(classify("/rustfs/admin/v3/site-replication/peer/unknown"), None); + } + + #[test] + fn test_retry_snapshot_fingerprint_detects_concurrent_iam_change() { + let old = SRIAMItem { + r#type: "policy".to_string(), + name: "readwrite".to_string(), + updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")), + ..Default::default() + }; + let mut new = old.clone(); + new.updated_at = Some(OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("timestamp")); + + let sent = RetrySnapshot::Iam(vec![old]); + let changed = RetrySnapshot::Iam(vec![new]); + assert_ne!(sent.fingerprint().unwrap(), changed.fingerprint().unwrap()); + } + + #[test] + fn test_retry_snapshot_replays_a_concurrent_deletion_as_a_tombstone() { + let observed_at = OffsetDateTime::from_unix_timestamp(1_700_000_010).expect("timestamp"); + let policy = SRIAMItem { + r#type: "policy".to_string(), + name: "readwrite".to_string(), + policy: Some(serde_json::json!({"Version": "2012-10-17"})), + ..Default::default() + }; + let replay = + RetrySnapshot::replay_after_change(&RetrySnapshot::Iam(vec![policy]), &RetrySnapshot::Iam(Vec::new()), observed_at); + let RetrySnapshot::Iam(items) = replay else { + panic!("IAM snapshot expected"); + }; + assert_eq!(items.len(), 1); + assert_eq!(items[0].name, "readwrite"); + assert!(items[0].policy.is_none()); + assert_eq!(items[0].updated_at, Some(observed_at)); + + let bucket = SRBucketMeta { + r#type: "tags".to_string(), + bucket: "photos".to_string(), + tags: Some("encoded-tags".to_string()), + ..Default::default() + }; + let replay = RetrySnapshot::replay_after_change( + &RetrySnapshot::BucketMetadata(vec![bucket]), + &RetrySnapshot::BucketMetadata(Vec::new()), + observed_at, + ); + let RetrySnapshot::BucketMetadata(items) = replay else { + panic!("bucket metadata snapshot expected"); + }; + assert_eq!(items.len(), 1); + assert_eq!(items[0].bucket, "photos"); + assert_eq!(items[0].r#type, "tags"); + assert!(items[0].tags.is_none()); + assert_eq!(items[0].updated_at, Some(observed_at)); + } + + /// Exponential backoff gates every attempt: without it a dead peer's + /// entries hit `failed` (retry_count >= 3) within 30 minutes of reconcile + /// ticks and the retry stats lose their signal. + #[test] + fn test_site_replication_retry_backoff_schedule() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let at = |secs_ago: i64| Some(now - time::Duration::seconds(secs_ago)); + let elapsed = |retry_count: u32, secs_ago: i64| { + site_replication_retry_backoff_elapsed(&drain_event("remote", "/p", retry_count, at(secs_ago)), now) + }; + + // No record of when it failed: attempt now. + assert!(site_replication_retry_backoff_elapsed(&drain_event("remote", "/p", 1, None), now)); + // First failure: one reconcile interval. + assert!(!elapsed(1, 599)); + assert!(elapsed(1, 601)); + // Third failure: 600 * 2^2 = 2400s. + assert!(!elapsed(3, 1200)); + assert!(elapsed(3, 2401)); + // Ceiling: a long-dead peer is still probed daily, never less often. + assert!(!elapsed(30, 86_000)); + assert!(elapsed(30, 86_401)); + } + + /// The actionable subset respects classification, peer membership and + /// backoff; everything else stays untouched in the queue. + #[test] + fn test_actionable_site_replication_retry_events_filters() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let old = Some(now - time::Duration::seconds(700)); + let mut state = SiteReplicationState::default(); + state + .peers + .insert("remote".to_string(), peer("remote", "https://remote.example.com")); + + state.retry_queue = vec![ + // Eligible: known peer, replayable, past backoff. + drain_event("remote", SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, 1, old), + // Not yet due. + drain_event("remote", "/rustfs/admin/v3/site-replication/peer/bucket-meta", 2, Some(now)), + // Unknown peer (removed since the failure was recorded). + drain_event("gone", "/rustfs/admin/v3/site-replication/peer/iam-item", 1, old), + // Marker record, not a delivery failure. + drain_event("remote", SITE_REPLICATION_ENDPOINT_REFRESH_RETRY_PATH, 0, old), + // Destructive op: operator-only. + drain_event( + "remote", + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=delete-bucket", + 1, + old, + ), + ]; + + let actionable = actionable_site_replication_retry_events(&state, now); + assert_eq!(actionable.len(), 1, "only the due, replayable, known-peer event is actionable"); + assert_eq!(actionable[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + } + + /// The drain settles a peer-edit success under a freshly allocated + /// generation; legacy queue entries carry `edit_generation: None` and + /// must be cleared by that generation-scoped settlement (`(Some, None)` + /// falls through to removal), or the drain would spin on them forever. + #[test] + fn test_settle_clears_legacy_none_generation_event_for_generation_scoped_success() { + let target = peer("remote", "https://remote.example.com"); + let mut queue = vec![drain_event("remote", SITE_REPLICATION_PEER_EDIT_PATH, 1, None)]; + assert!(queue[0].edit_generation.is_none()); + + let settled = settle_site_replication_retry_events(&mut queue, &target, SITE_REPLICATION_PEER_EDIT_PATH, Some(42)); + + assert_eq!(settled, 1, "a legacy None-generation event must settle under a newer generation"); + assert!(queue.is_empty()); + } + + /// A successful snapshot resend cannot prove a failed *deletion* was + /// replayed, so the collapsed entry is escalated (operator-visible, + /// drain-idle) instead of cleared — unless a newer failure was stamped + /// during the delivery window, which keeps the entry drain-eligible. + #[test] + fn test_escalate_up_to_marks_snapshot_replayed_and_keeps_newer_failures() { + let target = peer("remote", "https://remote.example.com"); + let path = "/rustfs/admin/v3/site-replication/peer/iam-item"; + let snapshot_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + + // Failure re-stamped after the snapshot: untouched, still eligible. + let mut queue = vec![drain_event( + "remote", + SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, + 2, + Some(snapshot_at + time::Duration::seconds(5)), + )]; + assert_eq!( + escalate_site_replication_retry_events_up_to( + &mut queue, + &target, + SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, + Some(snapshot_at), + ), + 0 + ); + assert!(!queue[0].failed); + assert!( + classify_site_replication_retry_event(&queue[0]).is_some(), + "a newer failure must stay drain-eligible" + ); + + // Unchanged since the snapshot: escalated, kept, drain-idle. + let mut queue = vec![drain_event( + "remote", + SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, + 2, + Some(snapshot_at), + )]; + assert_eq!( + escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)), + 1 + ); + assert_eq!(queue.len(), 1, "the entry must survive until remote absence is proven"); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + assert!(queue[0].failed); + assert_eq!(queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER); + assert!( + classify_site_replication_retry_event(&queue[0]).is_none(), + "a snapshot-replayed entry must not be re-sent daily" + ); + // Ordinary success dequeues must not clear the marker: collapsed + // paths are shared by every entity, so a successful Bob update + // proves nothing about a failed Alice deletion (second review + // round). + assert_eq!(dequeue_site_replication_retry_events(&mut queue, &target, path), 0); + assert_eq!(queue.len(), 1, "an escalated entry must survive an ordinary delivery success"); + // Only a repair — the operator's accountability transfer — settles it. + assert_eq!(dequeue_site_replication_retry_events_including_escalated(&mut queue, &target, path), 1); + assert!(queue.is_empty()); + + // A failed Alice deletion is stored under the internal path, so a + // successful Bob update on the shared wire path cannot erase it even + // before the drain runs. + let mut queue = Vec::new(); + upsert_site_replication_retry_event(&mut queue, &target, path, "alice delete failed", None); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + assert_eq!(dequeue_site_replication_retry_events(&mut queue, &target, path), 0); + assert_eq!(queue.len(), 1); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + + // A later hook failure overwrites the marker and re-arms the drain. + let mut queue = vec![drain_event("remote", path, 2, Some(snapshot_at))]; + escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)); + upsert_site_replication_retry_event(&mut queue, &target, path, "peer offline", None); + assert!(classify_site_replication_retry_event(&queue[0]).is_some()); + + // Legacy entry without a timestamp: escalated. + let mut queue = vec![drain_event("remote", path, 2, None)]; + assert_eq!( + escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)), + 1 + ); + + // A cloned event can disappear during replay; escalation recreates + // the internal liability while leaving another peer's row untouched. + let mut queue = vec![drain_event("other", path, 2, Some(snapshot_at))]; + assert_eq!( + escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)), + 1 + ); + assert!(!queue[0].failed); + assert_eq!(queue.len(), 2); + assert_eq!(queue[1].peer_deployment_id, target.deployment_id); + assert_eq!(queue[1].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + } + + #[test] + fn test_collapsed_retry_queue_migration_preserves_legacy_liability() { + let peer = PeerInfo { + deployment_id: "remote-dep".to_string(), + ..peer("remote", "https://remote.example.com") + }; + let wire_path = "/rustfs/admin/v3/site-replication/peer/iam-item"; + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let mut queue = vec![drain_event("remote-dep", wire_path, 2, Some(now))]; + + assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, wire_path), 0); + assert!(normalize_collapsed_retry_queue_paths(&mut queue)); + assert_eq!(queue.len(), 1); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + assert!(!normalize_collapsed_retry_queue_paths(&mut queue)); + } + #[test] fn test_pending_endpoint_refresh_retry_summary_redacts_pem() { let pem = "-----BEGIN CERTIFICATE-----\nsecret-marker\n-----END CERTIFICATE-----"; @@ -13545,6 +14569,7 @@ mod tests { upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "third", None); assert_eq!(queue.len(), 1); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); assert_eq!(queue[0].retry_count, SITE_REPLICATION_RETRY_FAILED_AFTER); assert!(queue[0].failed); assert_eq!(queue[0].last_error, "third"); @@ -13586,12 +14611,12 @@ mod tests { ); assert!(queue.is_empty()); - // Broadcast paths carry no generation and keep settling unconditionally - // — their retry events live under their own path and never collide - // with a peer-edit delivery. + // Collapsed broadcast failures live under an internal snapshot path; + // an unrelated success on their shared wire path cannot settle them. let iam_path = "/rustfs/admin/v3/site-replication/peer/iam-item"; upsert_site_replication_retry_event(&mut queue, &peer, iam_path, "peer offline", None); - assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, iam_path), 1); + assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, iam_path), 0); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); } /// P1-15 review follow-up: the receiving side of the ordering fence. Two @@ -13910,7 +14935,7 @@ mod tests { deployment_id: "current-dep".to_string(), ..peer("remote", "https://remote.example.com") }; - let path = "/rustfs/admin/v3/site-replication/peer/iam-item"; + let path = SITE_REPLICATION_PEER_EDIT_PATH; let mut queue = vec![ SiteReplicationRetryEvent { id: "same-endpoint".to_string(), @@ -17167,17 +18192,31 @@ mod tests { async fn test_retry_event_persist_must_not_wipe_concurrent_locked_rmw() { publish_ready_iam_context().await; + const ROUNDS: usize = 8; let seed = SiteReplicationState { pending_rotation: Some(PendingRotation { id: "rot-1".to_string(), access_key: "svc-account".to_string(), ..Default::default() }), + // Retry events are only recorded for current peers; seed them so + // the concurrency assertion below exercises the persist path. + peers: (0..ROUNDS) + .map(|round| { + let deployment_id = format!("peer-{round}-deployment"); + ( + deployment_id.clone(), + PeerInfo { + endpoint: format!("https://peer-{round}.example:9000"), + deployment_id, + ..Default::default() + }, + ) + }) + .collect(), ..Default::default() }; save_site_replication_state(&seed).await.expect("seed state"); - - const ROUNDS: usize = 8; for round in 0..ROUNDS { let peer = PeerInfo { endpoint: format!("https://peer-{round}.example:9000"), From d091554ffe73d7d82b29f4c32433f101cd7130fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Mon, 17 Aug 2026 14:12:39 +0800 Subject: [PATCH 62/71] fix(kms): resolve Vault auth from the environment at startup and add Kubernetes auth (#6095) --- crates/kms/src/api_types.rs | 58 +++ crates/kms/src/backends/vault.rs | 1 + crates/kms/src/backends/vault_credentials.rs | 276 ++++++++++++++- crates/kms/src/backends/vault_transit.rs | 1 + crates/kms/src/backup/vault_restore.rs | 4 + crates/kms/src/config.rs | 349 ++++++++++++++++--- docs/operations/kms-backend-security.md | 2 +- docs/operations/vault-kms-authentication.md | 54 ++- rustfs/src/admin/handlers/kms_backup.rs | 6 +- rustfs/src/init.rs | 215 ++++++++++-- 10 files changed, 862 insertions(+), 104 deletions(-) diff --git a/crates/kms/src/api_types.rs b/crates/kms/src/api_types.rs index b53aa4f34..6faf6cacc 100644 --- a/crates/kms/src/api_types.rs +++ b/crates/kms/src/api_types.rs @@ -293,6 +293,15 @@ enum StrictVaultAuthMethod { #[serde(default)] refresh_safety_window_secs: Option, }, + Kubernetes { + role: String, + #[serde(default)] + mount: Option, + #[serde(default)] + jwt_path: Option, + #[serde(default)] + refresh_safety_window_secs: Option, + }, TokenFile { path: std::path::PathBuf, #[serde(default)] @@ -319,6 +328,17 @@ impl From for VaultAuthMethod { mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()), refresh_safety_window_secs, }, + StrictVaultAuthMethod::Kubernetes { + role, + mount, + jwt_path, + refresh_safety_window_secs, + } => Self::Kubernetes { + role, + mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()), + jwt_path: jwt_path.unwrap_or_else(|| std::path::PathBuf::from(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)), + refresh_safety_window_secs, + }, StrictVaultAuthMethod::TokenFile { path, poll_interval_secs, @@ -499,6 +519,7 @@ impl From<&KmsConfig> for KmsConfigSummary { auth_method_type: match &vault_config.auth_method { VaultAuthMethod::Token { .. } => "token".to_string(), VaultAuthMethod::AppRole { .. } => "approle".to_string(), + VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(), VaultAuthMethod::TokenFile { .. } => "token_file".to_string(), }, has_stored_credentials: true, @@ -513,6 +534,7 @@ impl From<&KmsConfig> for KmsConfigSummary { auth_method_type: match &vault_config.auth_method { VaultAuthMethod::Token { .. } => "token".to_string(), VaultAuthMethod::AppRole { .. } => "approle".to_string(), + VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(), VaultAuthMethod::TokenFile { .. } => "token_file".to_string(), }, has_stored_credentials: true, @@ -901,6 +923,42 @@ mod tests { assert!(request.to_kms_config().validate().is_ok()); } + /// The admin API reaches Kubernetes auth with the role alone; the mount and + /// the projected token path fall back to the cluster defaults, so a Tenant + /// manifest carries no credential and no cluster-specific paths. + #[test] + fn test_deserialize_vault_configure_request_accepts_kubernetes_auth() { + let raw = serde_json::json!({ + "backend_type": "vault-transit", + "address": "https://vault.example.com:8200", + "mount_path": "rustfs", + "auth_method": { "Kubernetes": { "role": "rustfs" } } + }); + + let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("kubernetes auth should deserialize"); + let config = request.to_kms_config(); + config.validate().expect("kubernetes auth must validate"); + + let vault = config.vault_transit_config().expect("vault transit backend config"); + let VaultAuthMethod::Kubernetes { + role, mount, jwt_path, .. + } = &vault.auth_method + else { + panic!("expected Kubernetes auth, got {:?}", vault.auth_method); + }; + assert_eq!(role, "rustfs"); + assert_eq!(mount, crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT); + assert_eq!(jwt_path, std::path::Path::new(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)); + + let unknown_field = serde_json::json!({ + "backend_type": "vault-transit", + "address": "https://vault.example.com:8200", + "auth_method": { "Kubernetes": { "role": "rustfs", "service_account": "rustfs" } } + }); + serde_json::from_value::(unknown_field) + .expect_err("an unknown auth field must be rejected rather than silently dropped"); + } + #[test] fn test_deserialize_aws_configure_request_accepts_type_aliases() { for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] { diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 84ba8e6fa..56d2a856e 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -550,6 +550,7 @@ impl VaultKmsClient { address: config.address.clone(), namespace: config.namespace.clone(), attempt_timeout: kms_config.effective_timeout(), + skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify), }; let source = token_source_for(&config.auth_method, &settings)?; let policy = VaultCredentialPolicy::from_kms_config( diff --git a/crates/kms/src/backends/vault_credentials.rs b/crates/kms/src/backends/vault_credentials.rs index b8766442b..d5d8a564f 100644 --- a/crates/kms/src/backends/vault_credentials.rs +++ b/crates/kms/src/backends/vault_credentials.rs @@ -326,6 +326,97 @@ impl fmt::Debug for AppRoleLogin { } } +/// Token source for [`VaultAuthMethod::Kubernetes`]: exchanges the pod's +/// projected ServiceAccount token for a lease-bound Vault token. +/// +/// The JWT is re-read on every login because the kubelet rotates a projected +/// token well inside the pod's lifetime; caching it would strand the source on +/// an expired assertion once the current Vault token can no longer be renewed. +/// +/// Unlike [`TokenFileSource`], the file mode is not checked: the kubelet owns +/// the projected token and mounts it world-readable by default, so rejecting +/// group/other bits would refuse every standard pod rather than catch a +/// deployment error. +pub(crate) struct KubernetesLogin { + /// Unauthenticated client used only for the login exchange. + login_client: VaultClient, + mount: String, + role: String, + jwt_path: PathBuf, +} + +impl KubernetesLogin { + pub(crate) fn new(settings: &VaultConnectionSettings, mount: String, role: String, jwt_path: PathBuf) -> Result { + Ok(Self { + login_client: settings.build_login_client()?, + mount, + role, + jwt_path, + }) + } + + /// Read the ServiceAccount token for one login attempt. + /// + /// Mirrors [`AppRoleLogin::resolve_secret_id`]: a read failure is fatal for + /// the attempt but the refresh loop keeps retrying, so a token the kubelet + /// has not projected yet heals the source without a restart. + async fn resolve_jwt(&self) -> AttemptResult { + let mut raw = tokio::fs::read_to_string(&self.jwt_path) + .await + .map_err(|error| AttemptError { + class: ErrorClass::Fatal, + error: KmsError::configuration_error(format!( + "Failed to read Kubernetes ServiceAccount token {}: {error}", + self.jwt_path.display() + )), + })?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + raw.zeroize(); + return Err(AttemptError { + class: ErrorClass::Fatal, + error: KmsError::configuration_error(format!( + "Kubernetes ServiceAccount token {} is empty", + self.jwt_path.display() + )), + }); + } + let jwt = SecretString::new(trimmed.to_string()); + raw.zeroize(); + Ok(jwt) + } +} + +#[async_trait] +impl TokenSource for KubernetesLogin { + async fn acquire(&self) -> AttemptResult { + let jwt = self.resolve_jwt().await?; + let auth = vaultrs::auth::kubernetes::login(&self.login_client, &self.mount, &self.role, jwt.expose()) + .await + .map_err(|error| attempt_error("Kubernetes login", error))?; + Ok(TokenLease::from_auth(auth)) + } + + async fn renew(&self, client: &VaultClient) -> AttemptResult { + let auth = vaultrs::token::renew_self(client, None) + .await + .map_err(|error| attempt_error("token renewal", error))?; + Ok(TokenLease::from_auth(auth)) + } +} + +impl fmt::Debug for KubernetesLogin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // The login client embeds Vault client settings and must stay out of + // Debug output; the role name is not a secret, and the JWT is never held. + f.debug_struct("KubernetesLogin") + .field("mount", &self.mount) + .field("role", &self.role) + .field("jwt_path", &self.jwt_path) + .finish_non_exhaustive() + } +} + /// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed /// token file (for example a Vault Agent auto-auth sink). /// @@ -464,6 +555,9 @@ pub(crate) fn token_source_for( secret_id.clone(), secret_id_file.clone(), )?)), + VaultAuthMethod::Kubernetes { + role, mount, jwt_path, .. + } => Ok(Box::new(KubernetesLogin::new(settings, mount.clone(), role.clone(), jwt_path.clone())?)), VaultAuthMethod::TokenFile { path, poll_interval_secs, @@ -486,6 +580,9 @@ pub(crate) struct VaultConnectionSettings { pub(crate) namespace: Option, /// Per-attempt HTTP timeout applied to the underlying reqwest client. pub(crate) attempt_timeout: Duration, + /// Whether to accept an unverified Vault server certificate. Gated on + /// `allow_insecure_dev_defaults` by `KmsConfig::validate`. + pub(crate) skip_tls_verify: bool, } impl VaultConnectionSettings { @@ -499,6 +596,11 @@ impl VaultConnectionSettings { // operation-level retry policy. settings_builder.timeout(Some(self.attempt_timeout)); settings_builder.token(token); + // Always set explicitly: left unset, vaultrs derives this from its own + // VAULT_SKIP_VERIFY variable, so a stray value in the environment would + // disable certificate verification behind the KMS configuration and its + // insecure-defaults gate. + settings_builder.verify(!self.skip_tls_verify); if let Some(namespace) = &self.namespace { settings_builder.namespace(Some(namespace.clone())); @@ -551,6 +653,10 @@ impl VaultCredentialPolicy { refresh_safety_window_secs: Some(secs), .. } + | VaultAuthMethod::Kubernetes { + refresh_safety_window_secs: Some(secs), + .. + } | VaultAuthMethod::TokenFile { refresh_safety_window_secs: Some(secs), .. @@ -584,15 +690,25 @@ pub(crate) struct VaultClientHandle { impl VaultClientHandle { /// Absolute expiry of this generation's token. + /// + /// `lease.ttl` is built from the `lease_duration` the Vault server sent, so + /// a value too large to add to `issued_at` would panic on the bare `+`. A + /// TTL that cannot be represented is indistinguishable from no expiry, so it + /// collapses to `None` — the same answer already given for the zero-lease + /// tokens Vault issues, which keeps the token in use and still fully + /// validated by Vault on every call. fn expires_at(&self) -> Option { - self.lease.map(|lease| self.issued_at + lease.ttl) + self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl)) } /// When the renewal task should refresh this generation: half the TTL, /// leaving the second half as budget for retries before the fail-closed /// window is reached. + /// + /// Unrepresentable TTLs collapse to `None` as in [`Self::expires_at`], + /// leaving a token that never expires with nothing to renew. fn renew_at(&self) -> Option { - self.lease.map(|lease| self.issued_at + lease.ttl / 2) + self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl / 2)) } } @@ -662,7 +778,7 @@ impl VaultCredentialProvider { let handle = self.current.load_full(); if let Some(expires_at) = handle.expires_at() { let now = Instant::now(); - if now + self.policy.safety_window >= expires_at { + if self.inside_safety_window(now, expires_at) { return Err(KmsError::credentials_unavailable(format!( "Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it", handle.generation, self.policy.safety_window @@ -672,6 +788,18 @@ impl VaultCredentialProvider { Ok(handle) } + /// Whether the token expiring at `expires_at` is close enough to refuse. + /// + /// `safety_window` reaches here from persisted configuration, so it is not + /// guaranteed to have passed this version's validation: a window too large + /// to add to the current instant would panic on the bare `+`. Such a window + /// means every token is always inside it, so saturating to "refuse" is both + /// the fail-closed answer and the one the arithmetic was reaching for. + fn inside_safety_window(&self, now: Instant, expires_at: Instant) -> bool { + now.checked_add(self.policy.safety_window) + .is_none_or(|deadline| deadline >= expires_at) + } + /// Publish the credential gauges for the generation currently installed. /// /// The fail-closed gauge re-evaluates the very gate @@ -683,7 +811,7 @@ impl VaultCredentialProvider { let fail_closed = match handle.expires_at() { Some(expires_at) => { metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64()); - now + self.policy.safety_window >= expires_at + self.inside_safety_window(now, expires_at) } // A generation without an expiry has no remaining TTL to report // and can never lapse, so it can never fail closed either. @@ -860,7 +988,7 @@ impl Drop for CredentialTaskHandle { #[cfg(test)] mod tests { use super::*; - use crate::config::REDACTED_SECRET; + use crate::config::{DEFAULT_VAULT_KUBERNETES_MOUNT, REDACTED_SECRET}; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; const TEST_TOKEN: &str = "vault-token-debug-leak-canary"; @@ -871,6 +999,7 @@ mod tests { address: "http://127.0.0.1:8200".to_string(), namespace: Some("team-namespace".to_string()), attempt_timeout: Duration::from_secs(30), + skip_tls_verify: false, } } @@ -1057,6 +1186,143 @@ mod tests { assert!(format!("{source:?}").contains("AppRoleLogin")); } + #[tokio::test] + async fn test_kubernetes_auth_method_maps_to_login_source() { + let settings = test_settings(); + let source = token_source_for(&VaultAuthMethod::kubernetes("rustfs".to_string()), &settings) + .expect("kubernetes auth must map to a login source"); + + assert!(format!("{source:?}").contains("KubernetesLogin")); + } + + /// `refresh_safety_window_secs` is operator-supplied and reaches the request + /// path from persisted configuration, so the fail-closed comparison must + /// survive a window too large to add to the current instant. Before the + /// checked arithmetic this panicked with "overflow when adding duration to + /// instant" on the first request after a lease-bearing login. + #[tokio::test] + async fn test_current_refuses_rather_than_panics_on_an_unrepresentable_safety_window() { + let (provider, _state) = scripted_provider( + Duration::from_secs(60), + true, + test_policy(Duration::from_secs(u64::MAX), Duration::from_secs(5)), + ) + .await; + + let error = provider + .current() + .expect_err("a window wider than any lease must refuse the token"); + assert!( + matches!(error, KmsError::CredentialsUnavailable { .. }), + "expected CredentialsUnavailable, got {error:?}" + ); + } + + /// `lease_duration` is a bare u64 straight off the Vault response and forms + /// the other side of the same comparison, so an absurd one must not panic + /// either. It is indistinguishable from a non-expiring token, which is how + /// the zero-lease case already behaves. + #[tokio::test] + async fn test_an_unrepresentable_lease_is_treated_as_non_expiring() { + let (provider, _state) = scripted_provider( + Duration::from_secs(u64::MAX), + true, + test_policy(Duration::from_secs(30), Duration::from_secs(5)), + ) + .await; + + provider + .current() + .expect("a token whose expiry cannot be represented must stay usable"); + } + + /// The configured flag has to reach the HTTP client, not just the config + /// struct: every generation (authenticated and login) builds its own client, + /// and a Vault with a self-signed certificate fails the handshake unless + /// each one carries the setting. + #[test] + fn test_skip_tls_verify_reaches_every_vault_client_generation() { + for skip_tls_verify in [false, true] { + let settings = VaultConnectionSettings { + address: "https://vault.example.com:8200".to_string(), + namespace: None, + attempt_timeout: Duration::from_secs(30), + skip_tls_verify, + }; + + let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build"); + assert_eq!(authenticated.settings.verify, !skip_tls_verify); + + let login = settings.build_login_client().expect("login client must build"); + assert_eq!(login.settings.verify, !skip_tls_verify); + } + } + + /// vaultrs derives `verify` from its own VAULT_SKIP_VERIFY variable when the + /// builder leaves it unset, which would disable certificate verification + /// without passing the KMS insecure-defaults gate. + #[test] + fn test_vaultrs_skip_verify_env_cannot_override_the_configured_setting() { + temp_env::with_var("VAULT_SKIP_VERIFY", Some("true"), || { + let client = test_settings().build_client(TEST_TOKEN).expect("client must build"); + assert!( + client.settings.verify, + "a stray VAULT_SKIP_VERIFY must not disable verification behind the KMS configuration" + ); + }); + } + + /// The projected token is read fresh per login attempt and trimmed, so a + /// kubelet rotation is picked up without a restart and a trailing newline + /// does not corrupt the assertion sent to Vault. + #[tokio::test] + async fn test_kubernetes_login_rereads_and_trims_the_service_account_token() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("token"); + tokio::fs::write(&path, " first-jwt\n").await.expect("write token"); + + let login = KubernetesLogin::new( + &test_settings(), + DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), + "rustfs".to_string(), + path.clone(), + ) + .expect("login source must build"); + + assert_eq!(login.resolve_jwt().await.expect("first read").expose(), "first-jwt"); + + tokio::fs::write(&path, "rotated-jwt").await.expect("rotate token"); + assert_eq!( + login.resolve_jwt().await.expect("second read").expose(), + "rotated-jwt", + "a rotated projected token must be picked up without a restart" + ); + } + + /// The ServiceAccount token is re-read per attempt, so an unreadable or + /// empty one fails that attempt without reaching Vault; the refresh loop + /// keeps retrying, which is what lets a late projection heal the source. + #[tokio::test] + async fn test_kubernetes_login_rejects_an_unusable_service_account_token() { + let dir = tempfile::tempdir().expect("temp dir"); + let missing = dir.path().join("absent-token"); + let empty = dir.path().join("empty-token"); + tokio::fs::write(&empty, " \n").await.expect("write empty token"); + + for (path, expected) in [(missing, "Failed to read"), (empty, "is empty")] { + let login = + KubernetesLogin::new(&test_settings(), DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), "rustfs".to_string(), path) + .expect("login source must build"); + + let error = login + .acquire() + .await + .expect_err("an unusable ServiceAccount token must fail the attempt"); + assert!(matches!(error.class, ErrorClass::Fatal)); + assert!(error.error.to_string().contains(expected), "got {}", error.error); + } + } + #[tokio::test(start_paused = true)] async fn test_renewal_task_renews_at_half_ttl() { let (provider, state) = scripted_provider( diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 095fcf455..602c53154 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -415,6 +415,7 @@ impl VaultTransitKmsClient { address: config.address.clone(), namespace: config.namespace.clone(), attempt_timeout: kms_config.effective_timeout(), + skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify), }; let source = token_source_for(&config.auth_method, &settings)?; let policy = VaultCredentialPolicy::from_kms_config( diff --git a/crates/kms/src/backup/vault_restore.rs b/crates/kms/src/backup/vault_restore.rs index b7d4884cc..b033c4d5f 100644 --- a/crates/kms/src/backup/vault_restore.rs +++ b/crates/kms/src/backup/vault_restore.rs @@ -450,6 +450,10 @@ impl VaultRestoreClient { address: target.address.clone(), namespace: target.namespace.clone(), attempt_timeout: kms_config.effective_timeout(), + // A restore target carries no TLS settings, so certificates are + // always verified: recovery is the last path that should accept an + // unauthenticated Vault. + skip_tls_verify: false, }; let source = token_source_for(&target.auth_method, &settings)?; let policy = VaultCredentialPolicy::from_kms_config( diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index c33d1ae0a..bd6c4a0e1 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -25,6 +25,10 @@ use url::Url; pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS"; pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION"; +pub const ENV_KMS_VAULT_ADDRESS: &str = "RUSTFS_KMS_VAULT_ADDRESS"; +pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN"; +pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE"; +pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH"; pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY"; pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT"; pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX"; @@ -35,6 +39,9 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE"; pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT"; pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE"; +pub const ENV_KMS_VAULT_KUBERNETES_ROLE: &str = "RUSTFS_KMS_VAULT_KUBERNETES_ROLE"; +pub const ENV_KMS_VAULT_KUBERNETES_MOUNT: &str = "RUSTFS_KMS_VAULT_KUBERNETES_MOUNT"; +pub const ENV_KMS_VAULT_KUBERNETES_JWT_PATH: &str = "RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH"; pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION"; pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL"; /// Age in whole seconds beyond which a key is reported as due for rotation; @@ -45,6 +52,9 @@ pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata"; pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle"; +pub const DEFAULT_VAULT_KUBERNETES_MOUNT: &str = "kubernetes"; +/// Where the kubelet projects a pod's ServiceAccount token by default. +pub const DEFAULT_VAULT_KUBERNETES_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token"; /// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior. /// @@ -84,6 +94,14 @@ fn default_vault_approle_mount() -> String { DEFAULT_VAULT_APPROLE_MOUNT.to_string() } +fn default_vault_kubernetes_mount() -> String { + DEFAULT_VAULT_KUBERNETES_MOUNT.to_string() +} + +fn default_vault_kubernetes_jwt_path() -> PathBuf { + PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH) +} + pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[ RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"), RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"), @@ -490,6 +508,23 @@ pub enum VaultAuthMethod { #[serde(default)] refresh_safety_window_secs: Option, }, + /// Kubernetes authentication: the pod's ServiceAccount token is exchanged + /// for a lease-bound Vault token that is renewed in the background. + Kubernetes { + /// Vault role bound to this ServiceAccount. + role: String, + /// Kubernetes auth engine mount path. + #[serde(default = "default_vault_kubernetes_mount")] + mount: String, + /// Projected ServiceAccount token to present. Re-read on every login so + /// a token the kubelet rotates is picked up without a restart. + #[serde(default = "default_vault_kubernetes_jwt_path")] + jwt_path: PathBuf, + /// Fail-closed margin in seconds, as on `AppRole`. Defaults to the + /// per-attempt timeout. + #[serde(default)] + refresh_safety_window_secs: Option, + }, /// Agent-managed token file (for example a Vault Agent auto-auth sink): /// the token is read from `path` and re-read periodically so a token /// rotated by the agent is picked up without a restart. @@ -520,6 +555,16 @@ impl VaultAuthMethod { } } + /// Kubernetes authentication with the default mount and projected token path. + pub fn kubernetes(role: String) -> Self { + Self::Kubernetes { + role, + mount: default_vault_kubernetes_mount(), + jwt_path: default_vault_kubernetes_jwt_path(), + refresh_safety_window_secs: None, + } + } + /// Agent-managed token file with the default poll interval. pub fn token_file(path: PathBuf) -> Self { Self::TokenFile { @@ -548,6 +593,20 @@ impl fmt::Debug for VaultAuthMethod { .field("mount", mount) .field("refresh_safety_window_secs", refresh_safety_window_secs) .finish(), + // No redaction: the role and mount name a Vault binding, and the + // ServiceAccount token itself is never held on this type. + Self::Kubernetes { + role, + mount, + jwt_path, + refresh_safety_window_secs, + } => f + .debug_struct("Kubernetes") + .field("role", role) + .field("mount", mount) + .field("jwt_path", jwt_path) + .field("refresh_safety_window_secs", refresh_safety_window_secs) + .finish(), Self::TokenFile { path, poll_interval_secs, @@ -1028,50 +1087,12 @@ impl KmsConfig { }); } KmsBackend::VaultKv2 => { - let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200"); - let auth_method = vault_auth_method_from_env()?; - let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false); - - let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") { - Some(path) => { - tracing::warn!( - "RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused" - ); - path - } - None => default_vault_kv2_mount_path(), - }; - - config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig { - address, - auth_method, - namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"), - mount_path, - kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"), - key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"), - tls: vault_tls_config(skip_tls_verify), - })); + config.backend_config = + BackendConfig::VaultKv2(Box::new(vault_kv2_config_from_env(VaultCliOverrides::default())?)); } KmsBackend::VaultTransit => { - let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200"); - let auth_method = vault_auth_method_from_env()?; - let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false); - - config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig { - address, - auth_method, - namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"), - mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"), - metadata_kv_mount: get_env_str( - ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, - DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, - ), - metadata_key_prefix: get_env_str( - ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, - DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, - ), - tls: vault_tls_config(skip_tls_verify), - })); + config.backend_config = + BackendConfig::VaultTransit(Box::new(vault_transit_config_from_env(VaultCliOverrides::default())?)); } KmsBackend::Static => { // Read from file first, then fall back to direct env var @@ -1202,6 +1223,78 @@ fn is_under_temp_dir(path: &Path) -> bool { path.starts_with(std::env::temp_dir()) } +/// Command-line values that take precedence over the matching environment +/// variables when assembling a Vault backend configuration. +/// +/// Every field has a `RUSTFS_KMS_VAULT_*` equivalent that the CLI layer already +/// reads, so these are only set when the operator passed an explicit flag. +/// +/// Deliberately not `Debug`: `token` holds the raw Vault token, and the +/// redacting `Debug` impls elsewhere in this module exist because a derived one +/// would print it. Denying the derive makes a future `{overrides:?}` a compile +/// error instead of a leak. +#[derive(Default, Clone, Copy)] +pub struct VaultCliOverrides<'a> { + pub address: Option<&'a str>, + pub token: Option<&'a str>, + pub mount_path: Option<&'a str>, +} + +/// Assemble the Vault KV2 backend configuration from the environment. +/// +/// Shared by [`KmsConfig::from_env`] and the server's command-line startup path +/// so both resolve the same auth method, namespace, TLS and mount settings. +pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result { + let mount_path = match overrides + .mount_path + .map(str::to_string) + .or_else(|| get_env_opt_str(ENV_KMS_VAULT_MOUNT_PATH)) + { + Some(path) => { + tracing::warn!( + "RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused" + ); + path + } + None => default_vault_kv2_mount_path(), + }; + + Ok(VaultConfig { + address: vault_address_from_env(overrides.address), + auth_method: vault_auth_method_from_env(overrides.token)?, + namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE), + mount_path, + kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"), + key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"), + tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)), + }) +} + +/// Assemble the Vault Transit backend configuration from the environment. +/// +/// Companion to [`vault_kv2_config_from_env`]; see there for why both entry +/// points share it. +pub fn vault_transit_config_from_env(overrides: VaultCliOverrides<'_>) -> Result { + Ok(VaultTransitConfig { + address: vault_address_from_env(overrides.address), + auth_method: vault_auth_method_from_env(overrides.token)?, + namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE), + mount_path: overrides + .mount_path + .map(str::to_string) + .unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")), + metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT), + metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX), + tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)), + }) +} + +fn vault_address_from_env(override_value: Option<&str>) -> String { + override_value + .map(str::to_string) + .unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_ADDRESS, "http://localhost:8200")) +} + /// Resolve the Vault auth method from environment variables. /// /// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication; @@ -1209,27 +1302,59 @@ fn is_under_temp_dir(path: &Path) -> bool { /// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE` /// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the /// file taking precedence. Without a role id the legacy token flow applies. -fn vault_auth_method_from_env() -> Result { +/// +/// `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` selects Kubernetes authentication, which +/// presents the pod's projected ServiceAccount token. +/// +/// `token_override` carries a token supplied on the command line; it stands in +/// for `RUSTFS_KMS_VAULT_TOKEN` everywhere below, including the conflict checks, +/// so a flag and the variable it mirrors select the same method. +fn vault_auth_method_from_env(token_override: Option<&str>) -> Result { + let token = token_override + .map(str::to_string) + .or_else(|| get_env_opt_str(ENV_KMS_VAULT_TOKEN)); + let role_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID); + let kubernetes_role = get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_ROLE); + if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) { // A token file names one authoritative credential source; combining it // with another one would leave the effective identity ambiguous, so // that is a configuration error rather than a precedence rule. - if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() { - return Err(KmsError::configuration_error(format!( - "{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method" - ))); - } - if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() { - return Err(KmsError::configuration_error(format!( - "{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method" - ))); + for (name, configured) in [ + (ENV_KMS_VAULT_APPROLE_ROLE_ID, role_id.is_some()), + (ENV_KMS_VAULT_KUBERNETES_ROLE, kubernetes_role.is_some()), + (ENV_KMS_VAULT_TOKEN, token.is_some()), + ] { + if configured { + return Err(KmsError::configuration_error(format!( + "{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {name}; configure exactly one Vault auth method" + ))); + } } return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file))); } - let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else { + if let Some(role) = kubernetes_role { + // Unlike a leftover static token, a second login method is never a + // stale remnant: both were configured deliberately and neither can be + // ranked over the other. + if role_id.is_some() { + return Err(KmsError::configuration_error(format!( + "{ENV_KMS_VAULT_KUBERNETES_ROLE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method" + ))); + } + return Ok(VaultAuthMethod::Kubernetes { + role, + mount: get_env_str(ENV_KMS_VAULT_KUBERNETES_MOUNT, DEFAULT_VAULT_KUBERNETES_MOUNT), + jwt_path: get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_JWT_PATH) + .map_or_else(default_vault_kubernetes_jwt_path, PathBuf::from), + refresh_safety_window_secs: None, + }); + } + + let Some(role_id) = role_id else { return Ok(VaultAuthMethod::Token { - token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"), + token: token.unwrap_or_else(|| "dev-token".to_string()), }); }; @@ -1273,6 +1398,22 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod) } Ok(()) } + VaultAuthMethod::Kubernetes { + role, mount, jwt_path, .. + } => { + if role.is_empty() { + return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes role cannot be empty"))); + } + if mount.is_empty() { + return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes mount cannot be empty"))); + } + if jwt_path.as_os_str().is_empty() { + return Err(KmsError::configuration_error(format!( + "{backend_name} Kubernetes ServiceAccount token path cannot be empty" + ))); + } + Ok(()) + } VaultAuthMethod::TokenFile { path, poll_interval_secs, @@ -1976,6 +2117,106 @@ mod tests { .expect("well-formed token file auth must validate"); } + /// A Kubernetes role alone configures the method: the credential is the + /// pod's projected ServiceAccount token, so nothing secret is in the + /// environment and the mount and token path fall back to the cluster + /// defaults. + #[test] + fn test_from_env_selects_kubernetes() { + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault-transit")), + (ENV_KMS_VAULT_ADDRESS, Some("https://vault.example.com")), + (ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")), + (ENV_KMS_VAULT_KUBERNETES_MOUNT, None), + (ENV_KMS_VAULT_KUBERNETES_JWT_PATH, None), + (ENV_KMS_VAULT_TOKEN, None), + (ENV_KMS_VAULT_TOKEN_FILE, None), + (ENV_KMS_VAULT_APPROLE_ROLE_ID, None), + ], + || { + let config = KmsConfig::from_env().expect("kms config should load from env"); + let vault = config.vault_transit_config().expect("vault transit backend config"); + let VaultAuthMethod::Kubernetes { + role, + mount, + jwt_path, + refresh_safety_window_secs, + } = &vault.auth_method + else { + panic!( + "a kubernetes role in the environment must select Kubernetes auth, got {:?}", + vault.auth_method + ); + }; + assert_eq!(role, "rustfs"); + assert_eq!(mount, DEFAULT_VAULT_KUBERNETES_MOUNT); + assert_eq!(jwt_path, Path::new(DEFAULT_VAULT_KUBERNETES_JWT_PATH)); + assert_eq!(refresh_safety_window_secs, &None); + }, + ); + } + + #[test] + fn test_from_env_kubernetes_is_mutually_exclusive_with_other_auth() { + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault-transit")), + (ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")), + (ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")), + (ENV_KMS_VAULT_TOKEN, None), + (ENV_KMS_VAULT_TOKEN_FILE, None), + ], + || { + let error = KmsConfig::from_env().expect_err("kubernetes combined with approle must be rejected"); + assert!(error.to_string().contains(ENV_KMS_VAULT_KUBERNETES_ROLE)); + assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID)); + }, + ); + } + + #[test] + fn test_validate_rejects_bad_kubernetes_settings() { + let vault_config = |auth_method: VaultAuthMethod| KmsConfig { + backend: KmsBackend::VaultTransit, + backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig { + address: "https://vault.example.com:8200".to_string(), + auth_method, + ..Default::default() + })), + ..Default::default() + }; + + let error = vault_config(VaultAuthMethod::kubernetes(String::new())) + .validate() + .expect_err("an empty kubernetes role must be rejected"); + assert!(error.to_string().contains("role"), "got {error}"); + + let error = vault_config(VaultAuthMethod::Kubernetes { + role: "rustfs".to_string(), + mount: String::new(), + jwt_path: PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH), + refresh_safety_window_secs: None, + }) + .validate() + .expect_err("an empty kubernetes mount must be rejected"); + assert!(error.to_string().contains("mount"), "got {error}"); + + let error = vault_config(VaultAuthMethod::Kubernetes { + role: "rustfs".to_string(), + mount: DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), + jwt_path: PathBuf::new(), + refresh_safety_window_secs: None, + }) + .validate() + .expect_err("an empty ServiceAccount token path must be rejected"); + assert!(error.to_string().contains("token path"), "got {error}"); + + vault_config(VaultAuthMethod::kubernetes("rustfs".to_string())) + .validate() + .expect("well-formed kubernetes auth must validate"); + } + /// Every KV2 read, write and listing is routed through `kv_mount`, so an /// empty one names a path no Vault engine answers. The Transit backend /// already rejects its own empty mounts; this closes the same gap on the diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index 4fe4aad07..3f1dfdc7e 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -2,7 +2,7 @@ RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone. -For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first. +For how the Vault backends authenticate (static token, AppRole, Kubernetes, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first. ## Backend comparison diff --git a/docs/operations/vault-kms-authentication.md b/docs/operations/vault-kms-authentication.md index 6ee0a0c2c..adabd2918 100644 --- a/docs/operations/vault-kms-authentication.md +++ b/docs/operations/vault-kms-authentication.md @@ -8,9 +8,12 @@ This runbook covers how the RustFS Vault KMS backends (KV2 and Transit) authenti | --- | --- | --- | --- | --- | | Static token | `Token` | Whatever the operator provisioned; RustFS never renews it | None | Development; short-lived experiments | | AppRole | `AppRole` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production without a Vault Agent sidecar | +| Kubernetes | `Kubernetes` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production on Kubernetes, with no credential to distribute | | Agent token file | `TokenFile` | Owned by Vault Agent; RustFS only re-reads the sink file | File re-read once per poll interval | Production with a Vault Agent (or equivalent) managing auth | -Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` or an explicit `RUSTFS_KMS_VAULT_TOKEN` is rejected at startup with a configuration error, because the effective identity would be ambiguous. +Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with any other method, or `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID`, is rejected at startup with a configuration error, because the effective identity would be ambiguous. A leftover `RUSTFS_KMS_VAULT_TOKEN` alongside a configured login method is tolerated and ignored, so a stale variable cannot silently downgrade the identity. + +All of these are read the same way whether the service is started with `RUSTFS_KMS_ENABLE=true` or configured later through `POST /rustfs/admin/v3/kms/configure`. The default `dev-token` fallback for `RUSTFS_KMS_VAULT_TOKEN` is rejected outside explicit development mode (`RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS=true`), as are plain-HTTP Vault addresses and disabled TLS verification. @@ -56,7 +59,44 @@ Deliver the SecretID out of band — a secrets-manager-mounted file, an init-con The secret_id file is re-read on every login attempt, so rotating the SecretID is a two-step operation with no restart: generate a new SecretID (`vault write -f auth/approle/role/rustfs-kms/secret-id`), atomically replace the file, then revoke the old SecretID accessor. The already-issued token keeps renewing; the new SecretID is only needed at the next full re-login. -An empty or missing secret_id file fails the login attempt immediately (no Vault round trip) and is retried on the normal refresh cadence, so repairing the file heals the backend without a restart. +An empty or missing secret_id file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a file missing at boot is recovered by restarting the process, not by an in-process retry. Once RustFS is running, the same failure is retried on the normal refresh cadence, so repairing the file mid-run heals the backend without a restart. + +## Kubernetes authentication + +On Kubernetes this is the method to prefer: the pod's own ServiceAccount is the identity, so there is no credential to distribute, rotate, or leak into a Secret. + +### Vault-side setup + +```shell +vault auth enable kubernetes + +vault write auth/kubernetes/config \ + kubernetes_host="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT" + +vault write auth/kubernetes/role/rustfs \ + bound_service_account_names=rustfs \ + bound_service_account_namespaces=rustfs \ + token_policies=rustfs-kms \ + token_ttl=1h +``` + +As with AppRole, keep `token_ttl` comfortably above the RustFS per-attempt timeout (default 30s). + +### RustFS configuration + +```shell +RUSTFS_KMS_BACKEND=vault-transit # or "vault" for the KV2 backend +RUSTFS_KMS_VAULT_ADDRESS=https://vault.vault.svc.cluster.local:8200 +RUSTFS_KMS_VAULT_KUBERNETES_ROLE=rustfs +# Optional, defaults to "kubernetes": +# RUSTFS_KMS_VAULT_KUBERNETES_MOUNT=kubernetes +# Optional, defaults to the kubelet's projected token path: +# RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token +``` + +RustFS logs in at startup and renews the token at half its TTL, falling back to a fresh login exactly as AppRole does. The ServiceAccount token is re-read from disk on every login rather than cached, so a projected token the kubelet rotates is picked up without a restart. + +A missing or empty token file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a token projected late during a slow pod start is recovered by the pod restart loop, not by an in-process retry. Once RustFS is running, a token file that goes missing or turns empty is retried on the normal refresh cadence and heals the backend on its own. ## Vault Agent token file @@ -101,13 +141,13 @@ If the agent stops refreshing the file that is fine — RustFS re-reads the same ## Fail-closed window -For lease-bound credentials (AppRole tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side. +For lease-bound credentials (AppRole and Kubernetes tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side. - Default window: one per-attempt timeout (`RUSTFS_KMS_TIMEOUT_SECS`, default 30s) — a request issued now can legitimately stay in flight that long, so the token must outlive it. -- Override: `refresh_safety_window_secs` on the `AppRole` or `TokenFile` auth configuration. +- Override: `refresh_safety_window_secs` on the `AppRole`, `Kubernetes` or `TokenFile` auth configuration. - Static tokens never trip the window: they carry no lease and are assumed valid until Vault says otherwise. -The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole) or two poll intervals (token file). +The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole, Kubernetes) or two poll intervals (token file). ### Troubleshooting @@ -117,6 +157,8 @@ The window is a symptom threshold, not the fault itself: by the time it trips, r | Renewal succeeded but re-login later fails | `Vault token renewal failed; falling back to a fresh login` followed by login errors | SecretID expired/revoked or AppRole role changed; rotate the secret_id file | | Token file mode error at startup or during polls | `has insecure permissions` in the error | Fix the sink `mode` (0600) and the file owner; the next poll heals the provider | | Token file missing/empty errors | `Failed to read Vault token file` / `token file ... is empty` | Vault Agent down or sink misconfigured; restart the agent, the next poll heals the provider | -| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, token file | +| Kubernetes login fails with a permission error | `Vault Kubernetes login failed` | The pod's ServiceAccount is not in the role's `bound_service_account_names`/`_namespaces`, or `auth/kubernetes/config` names the wrong API server | +| Kubernetes ServiceAccount token errors | `Failed to read Kubernetes ServiceAccount token` / `ServiceAccount token ... is empty` | The token is not projected into the pod (check `automountServiceAccountToken` and the volume mount); the next refresh cycle heals the provider | +| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, Kubernetes, token file | When diagnosing, confirm three clocks/lifetimes in order: the Vault token TTL (`vault token lookup` with the token's accessor), the RustFS refresh cadence (half TTL or the poll interval), and the fail-closed window. The renewal task logs every failed cycle, so a silent gap in warnings combined with `CredentialsUnavailable` errors points at the process clock or a paused runtime rather than Vault. diff --git a/rustfs/src/admin/handlers/kms_backup.rs b/rustfs/src/admin/handlers/kms_backup.rs index 02384f487..02682e7da 100644 --- a/rustfs/src/admin/handlers/kms_backup.rs +++ b/rustfs/src/admin/handlers/kms_backup.rs @@ -286,6 +286,7 @@ fn auth_method_kind(auth: &VaultAuthMethod) -> String { match auth { VaultAuthMethod::Token { .. } => "token", VaultAuthMethod::AppRole { .. } => "approle", + VaultAuthMethod::Kubernetes { .. } => "kubernetes", VaultAuthMethod::TokenFile { .. } => "token-file", } .to_string() @@ -484,7 +485,10 @@ fn business_trust_root_secrets(config: &KmsConfig) -> Vec> { secrets.push(Zeroizing::new(role_id.clone())); secrets.push(Zeroizing::new(secret_id.clone())); } - VaultAuthMethod::TokenFile { .. } => {} + // Kubernetes and TokenFile hold no inline plaintext credential: the + // ServiceAccount token and the agent-managed token live in files, and + // the role names a Vault binding rather than half a credential pair. + VaultAuthMethod::Kubernetes { .. } | VaultAuthMethod::TokenFile { .. } => {} }; match &config.backend_config { diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index 67c889c1a..34a1d6ca0 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -304,30 +304,37 @@ fn build_local_kms_config(cfg: &config::Config) -> std::io::Result( + cfg: &'a config::Config, + backend_name: &str, +) -> std::io::Result> { + let address = cfg + .kms_vault_address + .as_deref() + .ok_or_else(|| Error::other(format!("Vault address is required for {backend_name} backend")))?; + + Ok(rustfs_kms::config::VaultCliOverrides { + address: Some(address), + token: cfg.kms_vault_token.as_deref(), + mount_path: cfg.kms_vault_mount_path.as_deref(), + }) +} + /// Build KMS configuration for Vault backend fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result { - let vault_address = cfg - .kms_vault_address - .as_ref() - .ok_or_else(|| Error::other("Vault address is required for vault backend"))?; - let vault_token = cfg - .kms_vault_token - .as_ref() - .ok_or_else(|| Error::other("Vault token is required for vault backend"))?; + let backend_config = rustfs_kms::config::vault_kv2_config_from_env(vault_cli_overrides(cfg, "vault")?) + .map_err(|e| Error::other(format!("Vault KMS configuration failed: {e}")))?; let kms_config = rustfs_kms::config::KmsConfig { backend: rustfs_kms::config::KmsBackend::VaultKv2, - backend_config: rustfs_kms::config::BackendConfig::VaultKv2(Box::new(rustfs_kms::config::VaultConfig { - address: vault_address.clone(), - auth_method: rustfs_kms::config::VaultAuthMethod::Token { - token: vault_token.clone(), - }, - namespace: None, - mount_path: cfg.kms_vault_mount_path.clone().unwrap_or_else(|| "transit".to_string()), - kv_mount: "secret".to_string(), - key_path_prefix: "rustfs/kms/keys".to_string(), - tls: None, - })), + backend_config: rustfs_kms::config::BackendConfig::VaultKv2(Box::new(backend_config)), allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults, allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(), default_key_id: cfg.kms_default_key_id.clone(), @@ -344,26 +351,12 @@ fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result std::io::Result { - let vault_address = cfg - .kms_vault_address - .as_ref() - .ok_or_else(|| Error::other("Vault address is required for vault-transit backend"))?; - let vault_token = cfg - .kms_vault_token - .as_ref() - .ok_or_else(|| Error::other("Vault token is required for vault-transit backend"))?; + let backend_config = rustfs_kms::config::vault_transit_config_from_env(vault_cli_overrides(cfg, "vault-transit")?) + .map_err(|e| Error::other(format!("Vault Transit KMS configuration failed: {e}")))?; let kms_config = rustfs_kms::config::KmsConfig { backend: rustfs_kms::config::KmsBackend::VaultTransit, - backend_config: rustfs_kms::config::BackendConfig::VaultTransit(Box::new(rustfs_kms::config::VaultTransitConfig { - address: vault_address.clone(), - auth_method: rustfs_kms::config::VaultAuthMethod::Token { - token: vault_token.clone(), - }, - namespace: None, - mount_path: cfg.kms_vault_mount_path.clone().unwrap_or_else(|| "transit".to_string()), - ..rustfs_kms::config::VaultTransitConfig::default() - })), + backend_config: rustfs_kms::config::BackendConfig::VaultTransit(Box::new(backend_config)), allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults, allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(), default_key_id: cfg.kms_default_key_id.clone(), @@ -1405,7 +1398,10 @@ pub async fn init_sftp_system() -> Result, Box crate::config::Config { + let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-vault-kms".to_string()]); + config.kms_enable = true; + config.kms_backend = backend.to_string(); + config.kms_vault_address = Some("https://vault.example.com:8200".to_string()); + config + } + + /// The Vault auth method and the settings the CLI has no flag for come from + /// the environment, so startup and `KmsConfig::from_env` cannot disagree. + /// Regression: startup used to hardcode token auth and require a token, + /// which made every non-token method unreachable through `RUSTFS_KMS_ENABLE`. + #[test] + fn build_vault_transit_kms_config_resolves_auth_and_mounts_from_env() { + let config = temp_env::with_vars( + [ + ("RUSTFS_KMS_VAULT_TOKEN", None), + ("RUSTFS_KMS_VAULT_TOKEN_FILE", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None), + ("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", Some("env-role-id")), + ("RUSTFS_KMS_VAULT_APPROLE_SECRET_ID", Some("env-secret-id")), + ("RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE", None), + ("RUSTFS_KMS_VAULT_NAMESPACE", Some("team-a")), + ("RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT", Some("rustfs-kv")), + ], + || { + build_vault_transit_kms_config(&vault_kms_test_config("vault-transit")) + .expect("vault transit KMS configuration should build") + }, + ); + + let vault = config.vault_transit_config().expect("vault transit backend config"); + let rustfs_kms::config::VaultAuthMethod::AppRole { role_id, secret_id, .. } = &vault.auth_method else { + panic!("approle in the environment must select AppRole auth, got {:?}", vault.auth_method); + }; + assert_eq!(role_id, "env-role-id"); + assert_eq!(secret_id, "env-secret-id"); + assert_eq!(vault.namespace.as_deref(), Some("team-a")); + assert_eq!(vault.metadata_kv_mount, "rustfs-kv"); + } + + /// Kubernetes auth needs no credential in the environment at all: the role + /// selects it and the pod's projected ServiceAccount token supplies the rest. + #[test] + fn build_vault_transit_kms_config_selects_kubernetes_auth() { + let config = temp_env::with_vars( + [ + ("RUSTFS_KMS_VAULT_TOKEN", None), + ("RUSTFS_KMS_VAULT_TOKEN_FILE", None), + ("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", Some("rustfs")), + ("RUSTFS_KMS_VAULT_KUBERNETES_MOUNT", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH", None), + ], + || { + build_vault_transit_kms_config(&vault_kms_test_config("vault-transit")) + .expect("vault transit KMS configuration should build") + }, + ); + + let vault = config.vault_transit_config().expect("vault transit backend config"); + let rustfs_kms::config::VaultAuthMethod::Kubernetes { + role, mount, jwt_path, .. + } = &vault.auth_method + else { + panic!( + "a kubernetes role in the environment must select Kubernetes auth, got {:?}", + vault.auth_method + ); + }; + assert_eq!(role, "rustfs"); + assert_eq!(mount, rustfs_kms::config::DEFAULT_VAULT_KUBERNETES_MOUNT); + assert_eq!(jwt_path, std::path::Path::new(rustfs_kms::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)); + } + + /// Two credential sources leave the effective identity ambiguous, so + /// startup refuses rather than picking one. + #[test] + fn build_vault_kms_config_refuses_two_auth_methods() { + temp_env::with_vars( + [ + ("RUSTFS_KMS_VAULT_TOKEN", None), + ("RUSTFS_KMS_VAULT_TOKEN_FILE", Some("/run/vault-agent/token")), + ("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", Some("rustfs")), + ], + || { + let error = build_vault_kms_config(&vault_kms_test_config("vault")) + .expect_err("two Vault auth methods must not start the server"); + assert!(error.to_string().contains("exactly one"), "unexpected error: {error}"); + }, + ); + } + + /// The KV2 backend has its own builder, so the key-location settings have + /// to be proven separately from the Transit one: pointing at the wrong KV + /// mount or prefix makes existing keys look absent. + #[test] + fn build_vault_kms_config_resolves_kv_mount_and_prefix_from_env() { + let config = temp_env::with_vars( + [ + ("RUSTFS_KMS_VAULT_TOKEN", Some("a-real-token")), + ("RUSTFS_KMS_VAULT_TOKEN_FILE", None), + ("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None), + ("RUSTFS_KMS_VAULT_KV_MOUNT", Some("rustfs-kv")), + ("RUSTFS_KMS_VAULT_KEY_PREFIX", Some("tenant/keys")), + ], + || build_vault_kms_config(&vault_kms_test_config("vault")).expect("vault KV2 KMS configuration should build"), + ); + + let vault = config.vault_config().expect("vault kv2 backend config"); + assert_eq!(vault.kv_mount, "rustfs-kv"); + assert_eq!(vault.key_path_prefix, "tenant/keys"); + } + + /// Skipping TLS verification was silently dropped on this path before, so + /// an operator who asked for it still got a verified connection. Now that it + /// is honoured it must fail closed without the development opt-in, rather + /// than quietly downgrading the Vault connection. + #[test] + fn build_vault_transit_kms_config_refuses_skip_tls_verify_without_opt_in() { + let vars = [ + ("RUSTFS_KMS_VAULT_TOKEN", Some("a-real-token")), + ("RUSTFS_KMS_VAULT_TOKEN_FILE", None), + ("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None), + ("RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY", Some("true")), + ]; + + temp_env::with_vars(vars, || { + let error = build_vault_transit_kms_config(&vault_kms_test_config("vault-transit")) + .expect_err("skipping TLS verification must not start the server"); + assert!(error.to_string().contains("TLS"), "unexpected error: {error}"); + }); + + temp_env::with_vars(vars, || { + let mut cfg = vault_kms_test_config("vault-transit"); + cfg.kms_allow_insecure_dev_defaults = true; + let config = build_vault_transit_kms_config(&cfg).expect("the development opt-in should accept skip-verify"); + let vault = config.vault_transit_config().expect("vault transit backend config"); + assert!(vault.tls.as_ref().is_some_and(|tls| tls.skip_verify)); + }); + } + fn aws_kms_test_config() -> crate::config::Config { let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-aws-kms".to_string()]); config.kms_enable = true; From 89e25132055efa3f316b237b404966cc52364fdb Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 15:04:23 +0800 Subject: [PATCH 63/71] feat(ecstore): pin bitrot algorithms with a startup self-test (HS-11) (#6165) feat(ecstore): pin bitrot algorithms with a startup self-test A drifted HighwayHash implementation fails silently: every shard reads back corrupt, heal rewrites healthy data, and cross-platform clusters disagree about which copy is good. Mirror MinIO's bitrotSelfTest by verifying, once at process start: - known-answer digests for HighwayHash256S / HighwayHash256SLegacy over a deterministic 4096-byte xorshift64* payload, plus the externally verifiable FIPS SHA-256 "abc" vector guarding the HashAlgorithm plumbing itself; - an end-to-end roundtrip per streaming variant (encode -> size formula -> bitrot_verify -> BitrotReader read-back), over full blocks and a partial tail; - tamper detection: one flipped byte in the final data block and one in the leading hash must both be rejected as a hash mismatch, not by an incidental read error. The check costs microseconds and runs inline in init_background_service_runtime before any shard can be written or verified. Outcome surfaces as one structured bitrot_selftest log event, the rustfs_bitrot_selftest_status gauge (1=passed / 0=failed / 2=skipped), a bitrotSelftest field on the admin server-info response, and RUSTFS_BITROT_SELFTEST_STRICT=on turns a failure into a startup error (MinIO Fatal parity; the default only degrades the status so a bad build cannot brick an existing fleet on upgrade). Closes rustfs/backlog#1873 (HS-11). Co-authored-by: heihutu --- crates/ecstore/src/api/mod.rs | 4 +- crates/ecstore/src/erasure/coding/bitrot.rs | 303 +++++++++++++++++++- rustfs/src/admin/handlers/system.rs | 29 ++ rustfs/src/bitrot_selftest.rs | 181 ++++++++++++ rustfs/src/lib.rs | 1 + rustfs/src/module_switches.rs | 14 + rustfs/src/startup_background.rs | 11 +- rustfs/src/storage/storage_api.rs | 4 + rustfs/src/storage_api.rs | 4 +- 9 files changed, 535 insertions(+), 16 deletions(-) create mode 100644 rustfs/src/bitrot_selftest.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index f18da095a..5526b7d00 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -373,8 +373,8 @@ pub mod error { pub mod erasure { pub use crate::erasure::coding::{ - BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder, - calc_shard_size, calc_shard_size_legacy, + BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, + ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy, }; } diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index 7b3166fbc..698b1b02c 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -820,10 +820,263 @@ impl BitrotWriterWrapper { } } +// --- startup bitrot self-test (rustfs/backlog#1873, MinIO bitrotSelfTest parity) --- +// +// A broken hash implementation (bad SIMD feature combination, platform drift, a +// key-handling regression) fails silently: every shard reads back "corrupt", +// heal rewrites data that was fine, and cross-platform clusters disagree about +// which copy is healthy. The self-test below pins the algorithms the moment a +// process starts, so a drifted build announces itself instead of quietly +// rewriting objects. See docs/rustfs-heal-scanner-vs-minio-comprehensive- +// analysis-2026-08-16.md §6 HS-11. + +/// Length of the deterministic self-test payload. +pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096; + +/// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S` +/// (the production default). Pinned so any platform or build where the +/// implementation drifts fails startup instead of mis-hashing shards. +const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [ + 0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea, + 0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06, +]; + +/// Known-answer digest of the same payload under `HighwayHash256SLegacy`. +const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY: [u8; 32] = [ + 0x98, 0x24, 0x71, 0x4f, 0x16, 0xbb, 0x48, 0x39, 0xed, 0x68, 0xfa, 0x63, 0x5e, 0xd9, 0x07, 0x61, 0xdf, 0x0a, 0xff, 0xcf, 0x7d, + 0x8c, 0xa8, 0xc7, 0xc0, 0xb6, 0x6f, 0x05, 0xdb, 0xda, 0x5a, 0x22, +]; + +/// FIPS 180-2 test vector: SHA-256 of the ASCII string "abc". Unlike the +/// Highway digests above this one is externally verifiable, so it guards the +/// whole `HashAlgorithm` plumbing even for readers who distrust pinned +/// self-computed constants. +const BITROT_SELF_TEST_KAT_SHA256_ABC: [u8; 32] = [ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, + 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad, +]; + +/// Deterministic self-test payload: xorshift64* from a fixed seed, so every +/// platform and every run hashes the same 4096 bytes. +fn bitrot_self_test_payload() -> [u8; BITROT_SELF_TEST_PAYLOAD_LEN] { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut payload = [0u8; BITROT_SELF_TEST_PAYLOAD_LEN]; + for byte in payload.iter_mut() { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + *byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D) as u8; + } + payload +} + +/// Why a bitrot self-test failed. +#[derive(Debug)] +pub enum BitrotSelfTestError { + /// A known-answer digest mismatched the pinned constant. + KnownAnswerMismatch { + algorithm: &'static str, + got: String, + want: String, + }, + /// A freshly encoded shard failed `bitrot_verify`. + RoundtripVerify { algorithm: &'static str, detail: String }, + /// A verified roundtrip read back different bytes than were written. + RoundtripReadback { algorithm: &'static str }, + /// A deliberately tampered shard was not rejected by `bitrot_verify`. + TamperNotRejected { + algorithm: &'static str, + tampered: &'static str, + }, +} + +impl std::fmt::Display for BitrotSelfTestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::KnownAnswerMismatch { algorithm, got, want } => { + write!(f, "known-answer mismatch for {algorithm}: got {got}, want {want}") + } + Self::RoundtripVerify { algorithm, detail } => write!(f, "{algorithm} roundtrip shard failed verification: {detail}"), + Self::RoundtripReadback { algorithm } => write!(f, "{algorithm} roundtrip read back different bytes"), + Self::TamperNotRejected { algorithm, tampered } => { + write!(f, "{algorithm} tampered shard ({tampered}) was not rejected") + } + } + } +} + +impl std::error::Error for BitrotSelfTestError {} + +fn self_test_hex(bytes: &[u8]) -> String { + rustfs_utils::hex(bytes) +} + +// (kept as a named one-liner so every KAT failure site reads the same; the +// underlying formatter is the shared `rustfs_utils::hex`) + +/// Compare a digest against its pinned constant. Split out so a test can drive +/// it with a wrong constant and prove the mismatch path fires. +fn bitrot_kat_check( + algorithm: &'static str, + algo: &HashAlgorithm, + payload: &[u8], + expected: &[u8; 32], +) -> Result<(), BitrotSelfTestError> { + let digest = algo.hash_encode(payload); + let digest = digest.as_ref(); + if digest.len() != expected.len() || digest != expected.as_slice() { + return Err(BitrotSelfTestError::KnownAnswerMismatch { + algorithm, + got: self_test_hex(digest), + want: self_test_hex(expected), + }); + } + Ok(()) +} + +/// Encode `payload` with `shard_size` blocks, verify it end to end, and read +/// every block back through `BitrotReader` comparing bytes. +async fn bitrot_roundtrip_check( + algorithm: &'static str, + algo: HashAlgorithm, + payload: &[u8], + shard_size: usize, +) -> Result<(), BitrotSelfTestError> { + let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo.clone()); + for chunk in payload.chunks(shard_size) { + writer + .write(chunk) + .await + .map_err(|err| BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("encode failed: {err}"), + })?; + } + let encoded = writer.into_inner().into_inner(); + + let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone()); + if encoded.len() != on_disk { + return Err(BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("encoded {} bytes, size formula says {on_disk}", encoded.len()), + }); + } + bitrot_verify(std::io::Cursor::new(encoded.clone()), on_disk, payload.len(), algo.clone(), shard_size) + .await + .map_err(|err| BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: err.to_string(), + })?; + + let mut reader = BitrotReader::new(std::io::Cursor::new(encoded), shard_size, algo, false); + let mut offset = 0usize; + while offset < payload.len() { + let want = shard_size.min(payload.len() - offset); + let mut buf = vec![0u8; want]; + let read = reader + .read(&mut buf) + .await + .map_err(|err| BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("read back failed at offset {offset}: {err}"), + })?; + if read != want || buf[..read] != payload[offset..offset + read] { + return Err(BitrotSelfTestError::RoundtripReadback { algorithm }); + } + offset += read; + } + Ok(()) +} + +/// Flip one byte and require `bitrot_verify` to reject the result. +async fn bitrot_tamper_check( + algorithm: &'static str, + algo: HashAlgorithm, + payload: &[u8], + shard_size: usize, + tampered: &'static str, + flip_at: usize, +) -> Result<(), BitrotSelfTestError> { + let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo.clone()); + for chunk in payload.chunks(shard_size) { + writer.write(chunk).await.expect("self-test encode should not fail"); + } + let mut corrupt = writer.into_inner().into_inner(); + let flip_index = flip_at % corrupt.len(); + corrupt[flip_index] ^= 0x80; + + let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone()); + match bitrot_verify(std::io::Cursor::new(corrupt), on_disk, payload.len(), algo, shard_size).await { + // The flipped byte must be rejected as a hash mismatch specifically, not + // by any incidental read error: an in-memory cursor cannot fail reads, + // so accepting any other failure here would mask a verify path that + // errors out before it ever compares hashes. + Err(err) if err.to_string().contains("hash mismatch") => Ok(()), + Ok(()) => Err(BitrotSelfTestError::TamperNotRejected { algorithm, tampered }), + Err(err) => Err(BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("tampered shard rejected with an unexpected error: {err}"), + }), + } +} + +/// Verify every bitrot algorithm this crate can write or verify in production: +/// both streaming Highway variants roundtrip end to end (encode → size formula +/// → `bitrot_verify` → read back) and reject a flipped byte in both the data +/// and the leading hash, while all three hashed algorithms reproduce their +/// pinned known-answer digests. +/// +/// Runs in well under a millisecond on 4 KiB of data; callers may run it inline +/// at startup. Pure CPU, no allocation beyond a few KiB of scratch. +pub async fn bitrot_self_test() -> Result<(), BitrotSelfTestError> { + let payload = bitrot_self_test_payload(); + + // Externally verifiable vector first: it guards the HashAlgorithm plumbing + // itself, before any self-pinned constants are consulted. + let abc = HashAlgorithm::SHA256.hash_encode(b"abc"); + if abc.as_ref() != BITROT_SELF_TEST_KAT_SHA256_ABC.as_slice() { + return Err(BitrotSelfTestError::KnownAnswerMismatch { + algorithm: "SHA256", + got: self_test_hex(abc.as_ref()), + want: self_test_hex(&BITROT_SELF_TEST_KAT_SHA256_ABC), + }); + } + + bitrot_kat_check( + "HighwayHash256S", + &HashAlgorithm::HighwayHash256S, + &payload, + &BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S, + )?; + bitrot_kat_check( + "HighwayHash256SLegacy", + &HashAlgorithm::HighwayHash256SLegacy, + &payload, + &BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY, + )?; + + for (algorithm, algo) in [ + ("HighwayHash256S", HashAlgorithm::HighwayHash256S), + ("HighwayHash256SLegacy", HashAlgorithm::HighwayHash256SLegacy), + ] { + // Full blocks plus a partial tail, exactly like a real part stripe. + let tail_len = 2 * 1024 + 333; + bitrot_roundtrip_check(algorithm, algo.clone(), &payload, 1024).await?; + bitrot_roundtrip_check(algorithm, algo.clone(), &payload[..tail_len], 1024).await?; + // One flipped byte in the final data block, one in the first leading + // hash: both must fail verification. + bitrot_tamper_check(algorithm, algo.clone(), &payload, 1024, "final data byte", payload.len() - 1).await?; + bitrot_tamper_check(algorithm, algo, &payload, 1024, "leading hash byte", 0).await?; + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::{ - BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored, + BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_kat_check, bitrot_self_test, + bitrot_self_test_payload, bitrot_shard_file_size, bitrot_verify, write_all_vectored, }; use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource}; use bytes::Bytes; @@ -1090,6 +1343,32 @@ mod tests { } } + #[test] + fn bitrot_self_test_payload_is_deterministic() { + // Two independent builds of the payload must agree byte for byte, or + // the pinned known-answer digests below would be meaningless. + assert_eq!(bitrot_self_test_payload(), bitrot_self_test_payload()); + } + + #[test] + fn bitrot_self_test_rejects_a_wrong_known_answer_digest() { + let payload = bitrot_self_test_payload(); + let wrong = [0u8; 32]; + let err = bitrot_kat_check("HighwayHash256S", &HashAlgorithm::HighwayHash256S, &payload, &wrong) + .expect_err("a zeroed digest must never match"); + match err { + super::BitrotSelfTestError::KnownAnswerMismatch { algorithm, .. } => assert_eq!(algorithm, "HighwayHash256S"), + other => panic!("expected KnownAnswerMismatch, got {other:?}"), + } + } + + #[tokio::test] + async fn bitrot_self_test_passes() { + bitrot_self_test() + .await + .expect("the pinned digests and roundtrip checks must all pass on this platform"); + } + #[tokio::test] async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() { let mut counting = VectoredCountingWriter::default(); @@ -1189,7 +1468,7 @@ mod tests { let last = corrupt.len() - 1; corrupt[last] ^= 0x80; let err = bitrot_verify( - Cursor::new(corrupt), + std::io::Cursor::new(corrupt), super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()), data.len(), algo, @@ -1282,7 +1561,7 @@ mod tests { #[tokio::test] async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() { - let mut reader = BitrotReader::new(Cursor::new(Vec::::new()), 4, HashAlgorithm::None, false); + let mut reader = BitrotReader::new(std::io::Cursor::new(Vec::::new()), 4, HashAlgorithm::None, false); let mut out = [0u8; 5]; let err = reader .read(&mut out) @@ -1407,7 +1686,7 @@ mod tests { (HashAlgorithm::HighwayHash256, true), ] { let label = format!("{algo:?}"); - let writer = Cursor::new(Vec::::new()); + let writer = std::io::Cursor::new(Vec::::new()); let mut w = BitrotWriter::new(writer, shard_size, algo.clone()); w.write(&[7u8; 16]).await.unwrap(); let written = w.into_inner().into_inner(); @@ -1492,7 +1771,7 @@ mod tests { } async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec { - let mut w = BitrotWriter::new(Cursor::new(Vec::::new()), shard_size, algo); + let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo); w.write(payload).await.unwrap(); w.into_inner().into_inner() } @@ -1600,7 +1879,7 @@ mod tests { for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] { for &size in &[1usize, 16, 17, 32, 40, 48] { let payload: Vec = (0..size).map(|i| i as u8).collect(); - let mut w = BitrotWriter::new(Cursor::new(Vec::::new()), shard_size, algo.clone()); + let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo.clone()); for chunk in payload.chunks(shard_size) { w.write(chunk).await.unwrap(); } @@ -1674,14 +1953,14 @@ mod tests { w.write(&data).await.expect("write shard"); let mut via_read = vec![0u8; SHARD]; - let n1 = BitrotReader::new(Cursor::new(encoded.clone()), SHARD, algo.clone(), false) + let n1 = BitrotReader::new(std::io::Cursor::new(encoded.clone()), SHARD, algo.clone(), false) .read(&mut via_read) .await .expect("read"); // A buffer with only capacity — no initialized bytes at all. let mut via_append: Vec = Vec::with_capacity(SHARD); - let n2 = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false) + let n2 = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false) .read_appending(&mut via_append, SHARD) .await .expect("read_appending"); @@ -1706,7 +1985,7 @@ mod tests { encoded.truncate(encoded.len() - 1); let mut out: Vec = Vec::with_capacity(SHARD); - let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false) + let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false) .read_appending(&mut out, SHARD) .await .expect_err("a truncated shard must not succeed"); @@ -1732,7 +2011,7 @@ mod tests { encoded[last] ^= 0xff; let mut out: Vec = Vec::with_capacity(SHARD); - let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo, false) + let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false) .read_appending(&mut out, SHARD) .await .expect_err("a corrupt shard must not verify"); @@ -1844,7 +2123,7 @@ mod tests { "Cursor must be able to hand out a block, otherwise the fast path is dead code" ); assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length"); - let mut streamed = Cursor::new(encoded.clone()); + let mut streamed = std::io::Cursor::new(encoded.clone()); assert!( ShardSource::try_take_block(&mut streamed, 8).is_none(), "a non-Bytes source must stay on the streaming path" @@ -1872,7 +2151,7 @@ mod tests { ); let mut via_stream: Vec = Vec::with_capacity(SHARD); - BitrotReader::new(Cursor::new(encoded), SHARD, algo, false) + BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false) .read_appending(&mut via_stream, SHARD) .await .expect("streaming read"); diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index 96054ada5..c7ad93b54 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -417,6 +417,13 @@ struct SystemAdminDiscovery { struct ServerInfoResponse { info: InfoMessage, admin_discovery: SystemAdminDiscovery, + /// Startup bitrot algorithm self-test outcome (rustfs/backlog#1873): + /// `passed` (algorithms verified at boot), `failed` (a drifted hash + /// implementation — the process is serving with degraded integrity + /// checking unless `RUSTFS_BITROT_SELFTEST_STRICT` aborted it), or + /// `unknown` (not yet run or disabled). + #[serde(rename = "bitrotSelftest")] + bitrot_selftest: &'static str, } #[derive(Serialize)] @@ -433,6 +440,14 @@ fn system_admin_discovery(usecase: &DefaultAdminUsecase) -> SystemAdminDiscovery } } +fn bitrot_selftest_status_str() -> &'static str { + match crate::bitrot_selftest::bitrot_selftest_passed() { + Some(true) => "passed", + Some(false) => "failed", + None => "unknown", + } +} + #[async_trait::async_trait] impl Operation for ServerInfoHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { @@ -464,6 +479,7 @@ impl Operation for ServerInfoHandler { let response = ServerInfoResponse { info, admin_discovery: system_admin_discovery(&usecase), + bitrot_selftest: bitrot_selftest_status_str(), }; let data = serde_json::to_vec(&response).map_err(|e| { @@ -1535,6 +1551,18 @@ mod tests { ); } + /// The startup bitrot self-test outcome must surface in server info as one + /// of three closed-set strings, never an internal enum or a null + /// (rustfs/backlog#1873). This test pins the string mapping; whether the + /// process-global cell holds Some(true)/Some(false)/None is owned by + /// `crate::bitrot_selftest`'s own tests. + #[test] + fn bitrot_selftest_status_str_is_a_closed_set_of_operators_strings() { + let rendered = super::bitrot_selftest_status_str(); + assert!(matches!(rendered, "passed" | "failed" | "unknown")); + assert_eq!(super::bitrot_selftest_status_str(), rendered); + } + #[test] fn server_info_response_exposes_admin_discovery_paths() { let usecase = DefaultAdminUsecase::without_context(); @@ -1556,6 +1584,7 @@ mod tests { pools: None, }, admin_discovery: system_admin_discovery(&usecase), + bitrot_selftest: super::bitrot_selftest_status_str(), }; let value = serde_json::to_value(response).expect("server info response should serialize"); diff --git a/rustfs/src/bitrot_selftest.rs b/rustfs/src/bitrot_selftest.rs new file mode 100644 index 000000000..4b8c1d1c4 --- /dev/null +++ b/rustfs/src/bitrot_selftest.rs @@ -0,0 +1,181 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Startup bitrot algorithm self-test (rustfs/backlog#1873). +//! +//! A drifted hash implementation fails silently in production: every shard +//! reads back "corrupt", heal rewrites healthy data, and cross-platform +//! clusters disagree about which copy is good. [`run_startup_bitrot_self_test`] +//! pins the algorithms once at process start — the check itself runs in well +//! under a millisecond on 4 KiB, so it executes inline before background +//! services come up and the result is published before the server accepts +//! traffic. +//! +//! Outcome surface: +//! - one structured `bitrot_selftest` log event (`passed`/`failed`/`skipped`), +//! - the `rustfs_bitrot_selftest_status` gauge (1=passed, 0=failed, 2=skipped), +//! - [`bitrot_selftest_passed`] for admin/health surfaces, +//! - `RUSTFS_BITROT_SELFTEST_STRICT=on` turns a failure into a startup error +//! (MinIO `bitrotSelfTest` Fatal parity); the default only degrades the +//! status so a bad build cannot brick an existing fleet on upgrade. + +use crate::storage_api::startup::background::{BitrotSelfTestError, bitrot_self_test}; +use metrics::gauge; +use std::future::Future; +use std::io; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::time::Instant; +use tracing::{debug, error, info}; + +const LOG_COMPONENT_MAIN: &str = "main"; +const LOG_SUBSYSTEM_STARTUP: &str = "startup"; +const EVENT_BITROT_SELFTEST: &str = "bitrot_selftest"; +const METRIC_BITROT_SELFTEST_STATUS: &str = "rustfs_bitrot_selftest_status"; + +/// Gauge values for [`METRIC_BITROT_SELFTEST_STATUS`]. +const STATUS_PASSED: f64 = 1.0; +const STATUS_FAILED: f64 = 0.0; +const STATUS_SKIPPED: f64 = 2.0; + +/// Internal cell values for [`BITROT_SELF_TEST_STATUS`]. +const STATUS_CELL_UNSET: u8 = 0; +const STATUS_CELL_PASSED: u8 = 1; +const STATUS_CELL_FAILED: u8 = 2; + +static BITROT_SELF_TEST_STATUS: AtomicU8 = AtomicU8::new(STATUS_CELL_UNSET); + +/// Last recorded self-test outcome: `None` before the first run, then +/// `Some(true)` on a passing check and `Some(false)` on a failed one (a +/// skipped check never publishes, so it cannot read as a pass). The cell is +/// last-writer-wins rather than set-once: production runs the self-test once, +/// and last-writer-wins keeps tests that exercise both outcomes +/// order-independent. +pub fn bitrot_selftest_passed() -> Option { + match BITROT_SELF_TEST_STATUS.load(Ordering::Acquire) { + STATUS_CELL_UNSET => None, + STATUS_CELL_PASSED => Some(true), + STATUS_CELL_FAILED => Some(false), + _ => None, + } +} + +/// Run the bitrot self-test and publish the outcome. In strict mode a failure +/// is returned as an error so the caller aborts startup. +pub(crate) async fn run_startup_bitrot_self_test(enabled: bool, strict: bool) -> io::Result<()> { + run_startup_bitrot_self_test_with(enabled, strict, bitrot_self_test).await +} + +async fn run_startup_bitrot_self_test_with(enabled: bool, strict: bool, run_check: F) -> io::Result<()> +where + F: FnOnce() -> Fut, + Fut: Future>, +{ + if !enabled { + gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_SKIPPED); + debug!( + target: "rustfs::main::run", + event = EVENT_BITROT_SELFTEST, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "skipped", + reason = "disabled", + "Bitrot self-test skipped" + ); + return Ok(()); + } + + let started = Instant::now(); + match run_check().await { + Ok(()) => { + BITROT_SELF_TEST_STATUS.store(STATUS_CELL_PASSED, Ordering::Release); + gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_PASSED); + info!( + target: "rustfs::main::run", + event = EVENT_BITROT_SELFTEST, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "passed", + duration_us = started.elapsed().as_micros() as u64, + "Bitrot self-test passed" + ); + } + Err(err) => { + BITROT_SELF_TEST_STATUS.store(STATUS_CELL_FAILED, Ordering::Release); + gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_FAILED); + error!( + target: "rustfs::main::run", + event = EVENT_BITROT_SELFTEST, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "failed", + duration_us = started.elapsed().as_micros() as u64, + error = %err, + "Bitrot self-test failed" + ); + if strict { + return Err(io::Error::other(format!("bitrot self-test failed: {err}"))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{BITROT_SELF_TEST_STATUS, STATUS_CELL_UNSET, bitrot_selftest_passed, run_startup_bitrot_self_test_with}; + use crate::storage_api::startup::background::BitrotSelfTestError; + use std::future::ready; + use std::sync::atomic::Ordering; + + fn failing_check() -> impl Future> { + ready(Err(BitrotSelfTestError::RoundtripReadback { + algorithm: "HighwayHash256S", + })) + } + + /// All scenarios run sequentially inside one test: the status cell is + /// process-global, so parallel per-scenario tests would race the reset and + /// read each other's outcomes (the exact order-dependent flake class this + /// module exists to avoid). + #[tokio::test] + async fn startup_self_test_publishes_outcome_and_strict_gates_abort() { + BITROT_SELF_TEST_STATUS.store(STATUS_CELL_UNSET, Ordering::Release); + + // Skipped: publishes nothing, never fails, never aborts. + run_startup_bitrot_self_test_with(false, true, || async { Ok(()) }) + .await + .expect("a disabled self-test must not fail even in strict mode"); + assert_eq!(bitrot_selftest_passed(), None, "a skipped run must leave the status unset"); + + // Passing: publishes Some(true), never fails. + run_startup_bitrot_self_test_with(true, false, || async { Ok(()) }) + .await + .expect("a passing check must never fail startup"); + assert_eq!(bitrot_selftest_passed(), Some(true), "a passing run must publish Some(true)"); + + // Failing, non-strict: publishes Some(false) but startup continues. + run_startup_bitrot_self_test_with(true, false, failing_check) + .await + .expect("a failed check must not abort startup in non-strict mode"); + assert_eq!(bitrot_selftest_passed(), Some(false), "a failing run must publish Some(false)"); + + // Failing, strict: startup error carries the failure and the published + // outcome stays a failure. + let err = run_startup_bitrot_self_test_with(true, true, failing_check) + .await + .expect_err("strict mode must turn a failed check into a startup error"); + assert!(err.to_string().contains("bitrot self-test failed")); + assert_eq!(bitrot_selftest_passed(), Some(false)); + } +} diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index f8e9d893d..b60a2bd20 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -76,6 +76,7 @@ pub mod allocator_reclaim; pub mod app; pub mod auth; pub mod auth_keystone; +pub(crate) mod bitrot_selftest; pub mod capacity; pub mod cluster_snapshot; pub mod config; diff --git a/rustfs/src/module_switches.rs b/rustfs/src/module_switches.rs index fcb0ffcef..ced58b0ec 100644 --- a/rustfs/src/module_switches.rs +++ b/rustfs/src/module_switches.rs @@ -33,6 +33,8 @@ pub(crate) const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED"; pub(crate) const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER"; pub(crate) const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED"; pub(crate) const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL"; +pub(crate) const ENV_BITROT_SELFTEST_ENABLE: &str = "RUSTFS_BITROT_SELFTEST_ENABLE"; +pub(crate) const ENV_BITROT_SELFTEST_STRICT: &str = "RUSTFS_BITROT_SELFTEST_STRICT"; static AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE); static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE); @@ -47,6 +49,18 @@ pub(crate) fn heal_enabled_from_env() -> bool { get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true) } +/// Whether the startup bitrot algorithm self-test runs, defaulting to on +/// (rustfs/backlog#1873). +pub(crate) fn bitrot_selftest_enabled_from_env() -> bool { + rustfs_utils::get_env_bool(ENV_BITROT_SELFTEST_ENABLE, true) +} + +/// Whether a failed bitrot self-test aborts startup instead of only logging +/// and exposing a failed status, defaulting to off. +pub(crate) fn bitrot_selftest_strict_from_env() -> bool { + rustfs_utils::get_env_bool(ENV_BITROT_SELFTEST_STRICT, false) +} + /// Last published audit-module state. pub fn is_audit_module_enabled() -> bool { AUDIT_MODULE_ENABLED.load(Ordering::Relaxed) diff --git a/rustfs/src/startup_background.rs b/rustfs/src/startup_background.rs index 8cee2c502..fd5366740 100644 --- a/rustfs/src/startup_background.rs +++ b/rustfs/src/startup_background.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::module_switches::{heal_enabled_from_env, scanner_enabled_from_env}; +use crate::bitrot_selftest::run_startup_bitrot_self_test; +use crate::module_switches::{ + bitrot_selftest_enabled_from_env, bitrot_selftest_strict_from_env, heal_enabled_from_env, scanner_enabled_from_env, +}; use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider}; use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; @@ -27,6 +30,12 @@ const LOG_SUBSYSTEM_STARTUP: &str = "startup"; const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured"; pub(crate) async fn init_background_service_runtime(store: Arc) -> Result { + // Pin the bitrot algorithms before anything can write or verify a shard: + // the check costs well under a millisecond, and in strict mode a drifted + // build must abort here rather than after it has touched data + // (rustfs/backlog#1873). + run_startup_bitrot_self_test(bitrot_selftest_enabled_from_env(), bitrot_selftest_strict_from_env()).await?; + let _ = create_ahm_services_cancel_token(); let enable_scanner = scanner_enabled_from_env(); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 5d217f18e..d1e7c0f3a 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -569,6 +569,10 @@ pub(crate) mod ecstore_erasure { pub(crate) use rustfs_ecstore::api::erasure::{BitrotReader, Erasure}; } +/// Startup bitrot algorithm self-test (rustfs/backlog#1873), re-exported for +/// the root facade's background-startup section. +pub(crate) use rustfs_ecstore::api::erasure::{BitrotSelfTestError, bitrot_self_test}; + pub(crate) mod ecstore_storage { #[cfg(test)] pub(crate) use rustfs_ecstore::api::storage::init_local_disks; diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index b9b8d3c64..e90c1f528 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -214,7 +214,9 @@ pub(crate) mod startup { } pub(crate) mod background { - pub(crate) use crate::storage::storage_api::{ECStore, set_workload_admission_snapshot_provider}; + pub(crate) use crate::storage::storage_api::{ + BitrotSelfTestError, ECStore, bitrot_self_test, set_workload_admission_snapshot_provider, + }; } pub(crate) mod bucket_metadata { From 23b17c2d5ac9d6e4e7ebdb44ee4efb0bad102f5c Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 15:04:49 +0800 Subject: [PATCH 64/71] feat(madmin): add a SigV4-signed admin client for heal and scanner APIs (HS-05) (#6166) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat(madmin): add a SigV4-signed admin client for heal and scanner APIs The madmin crate held only wire types; automation and mc-style tooling had no way to drive the heal/scanner admin surface without hand-rolled HTTP. Add `AdminClient`, which signs with the same rustfs-signer path the server authenticates (UNSIGNED-PAYLOAD marker, matching RustFS peer admin calls) and wraps: - heal_start / heal_status / heal_stop over POST /rustfs/admin/v3/heal/ (bucket/prefix path params percent-encoded per segment; stop models the server's two cancel branches: token-scoped task status vs path-scoped start-success receipt); - background_heal_status, scanner_status (freshness typed), plus ilm_expiry_status / replacement_recovery_status passthroughs; - a public get_json escape hatch for endpoints not wrapped yet. Wire types follow the madmin-go model (SDK-owned mirrors pinned by round-trip tests): HealOpts with serde defaults so partial settings objects decode, HealScanMode accepting both the numeric and name encodings, and status structs that type the fields operators branch on while flattening unknown nested payloads verbatim so server additions cannot break the client. Errors map to a closed AdminClientError enum (InvalidEndpoint / Transport / HttpStatus with body / Decode). Tests cover wire round-trips, path building, both stop branches, error mapping, and — via a dependency-free raw-TCP test server — that signed requests carry a SigV4 Authorization header, the right method/path/ query, and the expected JSON body. Closes rustfs/backlog#1869 (first increment; single-sourcing the wire structs server-side and an embedded-server e2e roundtrip are noted as follow-ups there). Co-authored-by: heihutu --- Cargo.lock | 5 + crates/madmin/Cargo.toml | 5 + crates/madmin/src/client.rs | 851 ++++++++++++++++++++++++++++++++++++ crates/madmin/src/lib.rs | 2 + 4 files changed, 863 insertions(+) create mode 100644 crates/madmin/src/client.rs diff --git a/Cargo.lock b/Cargo.lock index 38637d173..b521a731d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9825,14 +9825,19 @@ name = "rustfs-madmin" version = "1.0.0-rc.2" dependencies = [ "hotpath", + "http 1.5.0", "humantime", "hyper", "jiff", + "reqwest", "rmp-serde", + "rustfs-signer", + "s3s", "serde", "serde_json", "sysinfo", "time", + "tokio", ] [[package]] diff --git a/crates/madmin/Cargo.toml b/crates/madmin/Cargo.toml index 6a1fe0ff5..dfd346334 100644 --- a/crates/madmin/Cargo.toml +++ b/crates/madmin/Cargo.toml @@ -37,7 +37,11 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] [dependencies] hotpath.workspace = true humantime.workspace = true +http.workspace = true hyper = { workspace = true, features = ["http2", "http1", "server"] } +reqwest = { workspace = true, features = ["json"] } +rustfs-signer.workspace = true +s3s.workspace = true jiff = { workspace = true, features = ["serde"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } @@ -49,3 +53,4 @@ doctest = false [dev-dependencies] rmp-serde.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net"] } diff --git a/crates/madmin/src/client.rs b/crates/madmin/src/client.rs new file mode 100644 index 000000000..273f922a3 --- /dev/null +++ b/crates/madmin/src/client.rs @@ -0,0 +1,851 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Admin API HTTP client for heal and scanner management (rustfs/backlog#1869). +//! +//! [`AdminClient`] speaks the `/rustfs/admin/v3` surface with S3 SigV4 +//! request signing (the same scheme the server's admin router authenticates), +//! so `mc`-style tooling and automation can drive heal start/query/cancel and +//! read background-heal / scanner status without hand-rolling HTTP. +//! +//! Wire structs in this module mirror the server-side shapes +//! (`rustfs/src/admin/handlers/heal.rs`, `handlers/scanner.rs`, +//! `rustfs-common/src/heal_channel.rs`), following the madmin-go model where +//! the SDK owns its own copies and round-trip tests pin the encoding. Deeply +//! nested status payloads that the server composes from runtime types are +//! carried through as `serde_json::Value` and flattened maps rather than +//! duplicated field-for-field, so the client cannot silently drift on fields +//! it never interprets. + +use crate::heal_commands::HealResultItem; +use http::Method; +use serde::{Deserialize, Serialize, de}; +use std::time::Duration; + +/// Default admin API path prefix on a RustFS endpoint. +pub const DEFAULT_ADMIN_API_PREFIX: &str = "/rustfs/admin"; +/// Default SigV4 region when the server has no explicit region configured. +pub const DEFAULT_REGION: &str = "us-east-1"; + +/// Scan mode for a heal request, mirroring the server's numeric-or-name wire +/// encoding (`0` unknown/default, `1` normal, `2` deep). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum HealScanMode { + /// Server default; behaves as [`HealScanMode::Normal`]. + #[default] + Unknown, + /// Metadata-level checks only. + Normal, + /// Full bitrot verification while healing. + Deep, +} + +impl HealScanMode { + fn wire_number(self) -> u8 { + match self { + Self::Unknown => 0, + Self::Normal => 1, + Self::Deep => 2, + } + } + + fn from_wire_number(value: u8) -> Option { + match value { + 0 => Some(Self::Unknown), + 1 => Some(Self::Normal), + 2 => Some(Self::Deep), + _ => None, + } + } + + fn from_wire_name(value: &str) -> Option { + match value { + "unknown" => Some(Self::Unknown), + "normal" => Some(Self::Normal), + "deep" => Some(Self::Deep), + _ => None, + } + } +} + +impl Serialize for HealScanMode { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_u8(self.wire_number()) + } +} + +impl<'de> Deserialize<'de> for HealScanMode { + fn deserialize>(deserializer: D) -> Result { + struct HealScanModeVisitor; + + impl de::Visitor<'_> for HealScanModeVisitor { + type Value = HealScanMode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a heal scan mode number or name") + } + + fn visit_u64(self, value: u64) -> Result { + u8::try_from(value) + .ok() + .and_then(HealScanMode::from_wire_number) + .ok_or_else(|| E::custom(format!("unknown heal scan mode number: {value}"))) + } + + fn visit_str(self, value: &str) -> Result { + HealScanMode::from_wire_name(value).ok_or_else(|| E::custom(format!("unknown heal scan mode name: {value}"))) + } + } + + deserializer.deserialize_any(HealScanModeVisitor) + } +} + +/// Heal options for an admin heal request (mirror of the server body type). +/// Fields default on decode: a client should tolerate a server response whose +/// settings object omits fields it never set. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct HealOpts { + #[serde(default)] + pub recursive: bool, + #[serde(rename = "dryRun", default)] + pub dry_run: bool, + #[serde(default)] + pub remove: bool, + #[serde(default)] + pub recreate: bool, + #[serde(rename = "scanMode", default)] + pub scan_mode: HealScanMode, + #[serde(rename = "updateParity", default)] + pub update_parity: bool, + #[serde(rename = "nolock", default)] + pub no_lock: bool, + #[serde(rename = "pool", default)] + pub pool: Option, + #[serde(rename = "set", default)] + pub set: Option, +} + +/// Successful heal start / path-scoped cancel response. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HealStartSuccess { + pub client_token: String, + pub client_address: String, + #[serde(default)] + pub start_time: String, +} + +/// Heal task status response (query, cancel-with-token, start-then-poll). +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HealTaskStatus { + /// `running` | `finished` | `stopped` | `notFound`. + pub summary: String, + /// Failure detail for stopped tasks; empty otherwise. + #[serde(rename = "detail", default)] + pub failure_detail: String, + #[serde(default)] + pub start_time: String, + #[serde(default)] + pub settings: HealOpts, + #[serde(default)] + pub items: Vec, + #[serde(default)] + pub truncated: bool, + /// Live progress snapshot; the exact shape is owned by the heal runtime. + #[serde(default)] + pub progress: Option, +} + +/// `POST /v3/background-heal/status` response. Known top-level fields are +/// typed; the flattened heal info and operations matrix pass through verbatim. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BackgroundHealStatus { + /// `disabled` | `uninitialized` | `idle` | `active` | `degraded`. + pub state: String, + #[serde(default)] + pub heal_queue_length: u64, + #[serde(default)] + pub heal_active_tasks: u64, + #[serde(default)] + pub cluster_status_complete: bool, + #[serde(default)] + pub progress: Option, + /// Remaining wire fields (flattened `BackgroundHealInfo` plus the + /// priority-by-source operations matrix), carried verbatim. + #[serde(flatten)] + pub extra: serde_json::Map, +} + +/// `GET /v3/scanner/status` response, typed at the fields operators branch +/// on; everything else passes through verbatim. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScannerStatus { + pub enabled: bool, + /// `fresh` | `stale` | `unknown`; absent when the scanner never completed + /// a cycle. + #[serde(default)] + pub freshness: Option, + #[serde(flatten)] + pub extra: serde_json::Map, +} + +/// Freshness block of the scanner status response. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScannerFreshness { + /// `fresh` | `stale` | `unknown`. + pub state: String, +} + +impl ScannerStatus { + /// Convenience accessor for the freshness state string. + pub fn freshness(&self) -> &str { + self.freshness + .as_ref() + .map(|freshness| freshness.state.as_str()) + .unwrap_or("unknown") + } +} + +/// Everything that can go wrong in an admin client call. +#[derive(Debug)] +pub enum AdminClientError { + /// The endpoint URL could not be parsed. + InvalidEndpoint(String), + /// Request build/send failed (DNS, connect, timeout, body read). + Transport(reqwest::Error), + /// The server answered a non-2xx status. + HttpStatus { status: u16, body: String }, + /// The response body did not decode into the expected shape. + Decode { message: String }, +} + +impl std::fmt::Display for AdminClientError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidEndpoint(message) => write!(f, "invalid admin endpoint: {message}"), + Self::Transport(err) => write!(f, "admin request transport failure: {err}"), + Self::HttpStatus { status, body } => write!(f, "admin request failed with HTTP {status}: {body}"), + Self::Decode { message } => write!(f, "admin response decode failure: {message}"), + } + } +} + +impl std::error::Error for AdminClientError {} + +impl From for AdminClientError { + fn from(err: reqwest::Error) -> Self { + Self::Transport(err) + } +} + +/// A signed client for a RustFS admin API. +#[derive(Debug, Clone)] +pub struct AdminClient { + endpoint: reqwest::Url, + access_key: String, + secret_key: String, + session_token: String, + region: String, + api_prefix: String, + http: reqwest::Client, +} + +impl AdminClient { + /// Build a client for `endpoint` (e.g. `http://127.0.0.1:9000`) using root + /// or admin credentials. Requests are SigV4-signed with the same scheme + /// the server's admin router authenticates. + pub fn new(endpoint: &str, access_key: &str, secret_key: &str) -> Result { + let url = reqwest::Url::parse(endpoint).map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?; + if url.host_str().is_none() { + return Err(AdminClientError::InvalidEndpoint("endpoint has no host".to_string())); + } + let http = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .build() + .map_err(AdminClientError::Transport)?; + Ok(Self { + endpoint: url, + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + session_token: String::new(), + region: DEFAULT_REGION.to_string(), + api_prefix: DEFAULT_ADMIN_API_PREFIX.to_string(), + http, + }) + } + + /// Attach an STS session token (signed as `x-amz-security-token`). + pub fn with_session_token(mut self, session_token: impl Into) -> Self { + self.session_token = session_token.into(); + self + } + + /// Override the SigV4 region (defaults to `us-east-1`, matching a + /// region-less RustFS deployment). + pub fn with_region(mut self, region: impl Into) -> Self { + self.region = region.into(); + self + } + + /// Override the admin API path prefix (defaults to `/rustfs/admin`). + pub fn with_api_prefix(mut self, prefix: impl Into) -> Self { + self.api_prefix = prefix.into(); + self + } + + /// Start a heal. `bucket` empty and `prefix` empty heals the whole + /// deployment (requires `recursive` or a `pool`/`set` pair in `opts`, + /// enforced server-side); a bucket alone heals the bucket (the server + /// forces `recursive` for bucket heals). + pub async fn heal_start( + &self, + bucket: Option<&str>, + prefix: Option<&str>, + opts: &HealOpts, + force_start: bool, + ) -> Result { + let body = serde_json::to_vec(opts).map_err(|err| AdminClientError::Decode { + message: err.to_string(), + })?; + let mut query = Vec::new(); + if force_start { + query.push(("forceStart", "true".to_string())); + } + self.post_json(&heal_path(bucket, prefix), &query, body).await + } + + /// Query the status of the heal identified by `client_token` (the token + /// returned by [`Self::heal_start`]) at the path it was started on. + pub async fn heal_status( + &self, + bucket: Option<&str>, + prefix: Option<&str>, + client_token: &str, + ) -> Result { + self.post_json(&heal_path(bucket, prefix), &[("clientToken", client_token.to_string())], Vec::new()) + .await + } + + /// Stop a heal: with a `client_token` only that task is cancelled and its + /// final status returned; without one, every heal task at the path is + /// cancelled (the server answers with a start-success-shaped receipt). + pub async fn heal_stop( + &self, + bucket: Option<&str>, + prefix: Option<&str>, + client_token: Option<&str>, + ) -> Result { + let mut query = vec![("forceStop", "true".to_string())]; + if let Some(token) = client_token { + query.push(("clientToken", token.to_string())); + } + match client_token { + Some(_) => { + let status: HealTaskStatus = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?; + Ok(HealStopOutcome::Stopped(status)) + } + None => { + let success: HealStartSuccess = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?; + Ok(HealStopOutcome::PathStopped(success)) + } + } + } + + /// Cluster-aggregated background heal status. + pub async fn background_heal_status(&self) -> Result { + self.get_json("/v3/background-heal/status").await + } + + /// Data scanner status (enabled state, freshness, runtime config). + pub async fn scanner_status(&self) -> Result { + self.get_json("/v3/scanner/status").await + } + + /// ILM expiry worker status. The payload is owned by the expiry + /// subsystem and still evolving; returned verbatim. + pub async fn ilm_expiry_status(&self) -> Result { + self.get_json("/v3/ilm/expiry/status").await + } + + /// Durable replacement-recovery status (admin v4). The payload is owned + /// by the heal runtime; returned verbatim. + pub async fn replacement_recovery_status(&self) -> Result { + self.get_json("/v4/heal/replacement-recovery").await + } + + /// Signed GET returning a decoded JSON body; escape hatch for endpoints + /// this client does not wrap yet. + pub async fn get_json Deserialize<'de>>(&self, path: &str) -> Result { + let url = self.url_for(path, &[])?; + let request = self.sign_and_build(Method::GET, url, Vec::new(), None).await?; + self.execute(request).await + } + + /// Signed POST returning a decoded JSON body. + async fn post_json Deserialize<'de>>( + &self, + path: &str, + query: &[(&str, String)], + body: Vec, + ) -> Result { + let content_type = if body.is_empty() { None } else { Some("application/json") }; + let url = self.url_for(path, query)?; + let request = self.sign_and_build(Method::POST, url, body, content_type).await?; + self.execute(request).await + } + + fn url_for(&self, path: &str, query: &[(&str, String)]) -> Result { + let mut url = self + .endpoint + .join(&format!("{}{}", self.api_prefix.trim_end_matches('/'), path)) + .map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?; + if !query.is_empty() { + let mut pairs = url.query_pairs_mut(); + for (key, value) in query { + pairs.append_pair(key, value); + } + } + Ok(url) + } + + /// Build a SigV4-signed request via the same signer the server trusts, + /// then hand the signed headers to the HTTP client. The signature covers + /// method, path, query, and an unsigned-payload marker — the same shape + /// RustFS itself sends for peer admin calls. + async fn sign_and_build( + &self, + method: Method, + url: reqwest::Url, + body: Vec, + content_type: Option<&str>, + ) -> Result { + let authority = match (url.host_str(), url.port_or_known_default()) { + (Some(host), Some(port)) => format!("{host}:{port}"), + _ => return Err(AdminClientError::InvalidEndpoint("endpoint has no authority".to_string())), + }; + let mut builder = http::Request::builder() + .method(method.clone()) + .uri(url.as_str()) + .header(http::header::HOST, &authority) + .header("x-amz-content-sha256", rustfs_signer::constants::UNSIGNED_PAYLOAD); + if let Some(content_type) = content_type { + builder = builder.header(http::header::CONTENT_TYPE, content_type); + } + let unsigned = builder + .body(s3s::Body::empty()) + .map_err(|err| AdminClientError::InvalidEndpoint(format!("build request failed: {err}")))?; + let signed = rustfs_signer::sign_v4( + unsigned, + body.len() as i64, + &self.access_key, + &self.secret_key, + &self.session_token, + &self.region, + ); + + let mut request = self + .http + .request(method, url) + .body(body) + .build() + .map_err(AdminClientError::Transport)?; + let headers = request.headers_mut(); + for (name, value) in signed.headers().iter() { + // HOST is owned by the HTTP client; the signed value above was + // built from the same URL authority, so they always agree. + if name == http::header::HOST { + continue; + } + headers.insert(name, value.clone()); + } + Ok(request) + } + + async fn execute Deserialize<'de>>(&self, request: reqwest::Request) -> Result { + let response = self.http.execute(request).await?; + let status = response.status(); + let bytes = response.bytes().await?; + if !status.is_success() { + return Err(AdminClientError::HttpStatus { + status: status.as_u16(), + body: String::from_utf8_lossy(&bytes).into_owned(), + }); + } + serde_json::from_slice(&bytes).map_err(|err| AdminClientError::Decode { + message: err.to_string(), + }) + } +} + +/// Response of [`AdminClient::heal_stop`]: cancelling a single tokened task +/// answers with that task's status, cancelling a whole path answers with a +/// start-success-shaped receipt. +#[derive(Debug, Clone)] +pub enum HealStopOutcome { + Stopped(HealTaskStatus), + PathStopped(HealStartSuccess), +} + +fn heal_path(bucket: Option<&str>, prefix: Option<&str>) -> String { + match (bucket, prefix) { + (Some(bucket), Some(prefix)) if !bucket.is_empty() && !prefix.is_empty() => { + format!("/v3/heal/{}/{}", percent_encode_path_segment(bucket), percent_encode_path_segment(prefix)) + } + (Some(bucket), Some(_)) | (Some(bucket), None) if !bucket.is_empty() => { + format!("/v3/heal/{}", percent_encode_path_segment(bucket)) + } + _ => "/v3/heal/".to_string(), + } +} + +/// Encode a single path segment (slashes are content, not separators, inside +/// bucket/prefix path params). +fn percent_encode_path_segment(segment: &str) -> String { + let mut out = String::with_capacity(segment.len()); + for byte in segment.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(byte as char), + _ => out.push_str(&format!("%{byte:02X}")), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::{ + AdminClient, AdminClientError, BackgroundHealStatus, HealOpts, HealScanMode, HealStartSuccess, HealTaskStatus, + ScannerStatus, heal_path, percent_encode_path_segment, + }; + use serde_json::json; + use std::sync::{Arc, Mutex}; + + #[test] + fn heal_paths_cover_root_bucket_and_prefix() { + assert_eq!(heal_path(None, None), "/v3/heal/"); + assert_eq!(heal_path(Some(""), Some("")), "/v3/heal/"); + assert_eq!(heal_path(Some("bucket"), None), "/v3/heal/bucket"); + assert_eq!(heal_path(Some("bucket"), Some("pre/fix")), "/v3/heal/bucket/pre%2Ffix"); + } + + #[test] + fn path_segments_percent_encode_reserved_characters() { + assert_eq!(percent_encode_path_segment("a b"), "a%20b"); + assert_eq!(percent_encode_path_segment("a/b"), "a%2Fb"); + assert_eq!(percent_encode_path_segment("ü"), "%C3%BC"); + } + + #[test] + fn heal_opts_round_trip_through_the_server_wire_shape() { + let opts = HealOpts { + recursive: true, + dry_run: false, + remove: true, + recreate: false, + scan_mode: HealScanMode::Deep, + update_parity: true, + no_lock: false, + pool: Some(1), + set: Some(2), + }; + let wire = serde_json::to_value(&opts).unwrap(); + assert_eq!(wire["scanMode"], json!(2), "the server body decodes scanMode as a number"); + let back: HealOpts = serde_json::from_value(wire).unwrap(); + assert_eq!(back.scan_mode, HealScanMode::Deep); + assert_eq!(back.pool, Some(1)); + } + + #[test] + fn heal_scan_mode_accepts_both_wire_encodings() { + assert_eq!(serde_json::from_value::(json!(1)).unwrap(), HealScanMode::Normal); + assert_eq!(serde_json::from_value::(json!("deep")).unwrap(), HealScanMode::Deep); + assert!(serde_json::from_value::(json!(9)).is_err()); + assert!(serde_json::from_value::(json!("sideways")).is_err()); + } + + #[test] + fn heal_task_status_decodes_the_server_response_shape() { + let raw = json!({ + "summary": "finished", + "detail": "", + "startTime": "2026-08-17T00:00:00Z", + "settings": {"recursive": false, "scanMode": 1}, + "items": [{ + "resultId": 1, "type": "object", "bucket": "b", "object": "o", "versionId": "", "detail": "", + "parityBlocks": 2, "dataBlocks": 2, "diskCount": 4, "setCount": 1, + "before": {"drives": []}, "after": {"drives": []}, "objectSize": 128 + }], + "truncated": false + }); + let status: HealTaskStatus = serde_json::from_value(raw).unwrap(); + assert_eq!(status.summary, "finished"); + assert_eq!(status.items.len(), 1); + assert_eq!(status.settings.scan_mode, HealScanMode::Normal); + assert!(status.progress.is_none()); + } + + #[test] + fn background_heal_status_types_known_fields_and_passes_the_rest_through() { + let raw = json!({ + "state": "active", + "bitrotStartTime": "t", + "healQueueLength": 3, + "healActiveTasks": 1, + "healOperations": {"queueLength": 3}, + "clusterStatusComplete": true + }); + let status: BackgroundHealStatus = serde_json::from_value(raw).unwrap(); + assert_eq!(status.state, "active"); + assert_eq!(status.heal_queue_length, 3); + assert!(status.cluster_status_complete); + assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through"); + } + + #[test] + fn scanner_status_defaults_freshness_to_unknown() { + let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}}); + let status: ScannerStatus = serde_json::from_value(raw).unwrap(); + assert_eq!(status.freshness(), "stale"); + let bare: ScannerStatus = serde_json::from_value(json!({"enabled": false})).unwrap(); + assert_eq!(bare.freshness(), "unknown"); + } + + #[test] + fn invalid_endpoint_is_rejected_without_io() { + let err = AdminClient::new("not a url", "ak", "sk").unwrap_err(); + assert!(matches!(err, AdminClientError::InvalidEndpoint(_))); + } + + #[tokio::test] + async fn signed_requests_carry_sigv4_authorization_and_correct_target() { + let server = TestServer::spawn(r#"{"clientToken":"token-1","clientAddress":"127.0.0.1:9","startTime":"t"}"#, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "minioadmin", "minioadmin") + .expect("client builds against the test server"); + + let start: HealStartSuccess = client + .heal_start( + Some("bucket"), + None, + &HealOpts { + recursive: true, + ..Default::default() + }, + false, + ) + .await + .expect("signed heal start decodes"); + + assert_eq!(start.client_token, "token-1"); + let request = server.recorded(); + assert_eq!(request.method, "POST"); + assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket"); + assert!(!request.query.contains("forceStart"), "absent flags must not be sent"); + let auth = request.header("authorization").expect("request must be signed"); + assert!(auth.starts_with("AWS4-HMAC-SHA256"), "SigV4 scheme, got: {auth}"); + assert!(auth.contains("Credential=minioadmin/"), "credentials must be in the Authorization header"); + assert_eq!( + request.header("x-amz-content-sha256").as_deref(), + Some("UNSIGNED-PAYLOAD"), + "the client signs the same payload marker RustFS peer calls use" + ); + assert_eq!(request.header("content-type").as_deref(), Some("application/json")); + assert!(request.body.contains("\"recursive\":true")); + } + + #[tokio::test] + async fn query_sends_client_token_on_the_same_path() { + let body = r#"{"summary":"running","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#; + let server = TestServer::spawn(body, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + + let status = client + .heal_status(Some("bucket"), None, "token-1") + .await + .expect("status decodes"); + assert_eq!(status.summary, "running"); + let request = server.recorded(); + assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket"); + assert!(request.query.contains("clientToken=token-1")); + assert!(!request.query.contains("forceStop")); + } + + #[tokio::test] + async fn stop_without_token_takes_the_path_cancel_branch() { + let server = TestServer::spawn(r#"{"clientToken":"path","clientAddress":"c","startTime":"t"}"#, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + + let outcome = client.heal_stop(Some("bucket"), None, None).await.expect("path stop decodes"); + assert!(matches!(outcome, super::HealStopOutcome::PathStopped(_))); + let request = server.recorded(); + assert!(request.query.contains("forceStop=true")); + assert!(!request.query.contains("clientToken")); + } + + #[tokio::test] + async fn http_error_status_maps_to_a_typed_error_with_body() { + let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + let err = client.scanner_status().await.unwrap_err(); + match err { + AdminClientError::HttpStatus { status, body } => { + assert_eq!(status, 403); + assert!(body.contains("AccessDenied")); + } + other => panic!("expected HttpStatus, got {other:?}"), + } + } + + #[tokio::test] + async fn malformed_success_body_maps_to_a_decode_error() { + let server = TestServer::spawn("not json", 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + assert!(matches!(client.scanner_status().await.unwrap_err(), AdminClientError::Decode { .. })); + } + + /// One recorded request, parsed off the wire with the minimum needed for + /// assertions: method, path, query, headers, body. + #[derive(Debug, Clone)] + struct RecordedRequest { + method: String, + path: String, + query: String, + headers: Vec<(String, String)>, + body: String, + } + + impl RecordedRequest { + fn header(&self, name: &str) -> Option { + self.headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.clone()) + } + } + + /// Minimal HTTP/1.1 server: one canned response per connection, every + /// request recorded behind an `Arc`. Deliberately dependency-free — + /// the assertions only need the raw request bytes. + struct TestServer { + addr: std::net::SocketAddr, + requests: Arc>>, + } + + impl TestServer { + async fn spawn(response_body: &'static str, status: u16) -> Self { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("local addr"); + let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); + + let recorded = requests.clone(); + tokio::spawn(async move { + let reason = if status == 200 { "OK" } else { "Forbidden" }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{response_body}", + response_body.len() + ); + // Each request is a fresh connection (connection: close); a + // bounded loop serves every call a test makes while letting + // the task exit instead of lingering for the whole process. + for _ in 0..16 { + let Ok((mut stream, _)) = listener.accept().await else { + break; + }; + let mut buffer = Vec::with_capacity(2048); + let mut chunk = [0u8; 2048]; + // Read headers plus content-length body, or stop on close. + loop { + if let Some(end) = find_header_end(&buffer) { + let content_length = extract_content_length(&buffer[..end]); + if buffer.len() >= end + content_length { + break; + } + } + let n = match stream.read(&mut chunk).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + buffer.extend_from_slice(&chunk[..n]); + if buffer.len() > 64 * 1024 { + break; + } + } + if let Some(request) = parse_request(&buffer) { + recorded.lock().expect("recorded lock").push(request); + } + let _ = stream.write_all(response.as_bytes()).await; + let _ = stream.shutdown().await; + } + }); + + Self { addr, requests } + } + + fn recorded(&self) -> RecordedRequest { + self.requests + .lock() + .expect("recorded lock") + .last() + .cloned() + .expect("the client call must have produced one recorded request") + } + } + + fn find_header_end(buffer: &[u8]) -> Option { + buffer.windows(4).position(|window| window == b"\r\n\r\n").map(|pos| pos + 4) + } + + fn extract_content_length(headers: &[u8]) -> usize { + let text = String::from_utf8_lossy(headers).to_ascii_lowercase(); + text.lines() + .find_map(|line| line.strip_prefix("content-length:")) + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(0) + } + + fn parse_request(raw: &[u8]) -> Option { + let end = find_header_end(raw)?; + let head = String::from_utf8_lossy(&raw[..end]); + let body = String::from_utf8_lossy(&raw[end..]).into_owned(); + let mut lines = head.lines(); + let request_line = lines.next()?; + let mut parts = request_line.split_whitespace(); + let method = parts.next()?.to_string(); + let target = parts.next()?.to_string(); + let (path, query) = match target.split_once('?') { + Some((path, query)) => (path.to_string(), query.to_string()), + None => (target, String::new()), + }; + let headers = lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.trim().to_string(), value.trim().to_string())) + .collect(); + Some(RecordedRequest { + method, + path, + query, + headers, + body, + }) + } +} diff --git a/crates/madmin/src/lib.rs b/crates/madmin/src/lib.rs index 154663b11..a9d4bd8b9 100644 --- a/crates/madmin/src/lib.rs +++ b/crates/madmin/src/lib.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod client; pub mod group; pub mod heal_commands; pub mod health; @@ -25,6 +26,7 @@ pub mod trace; pub mod user; pub mod utils; +pub use client::*; pub use group::*; pub use info_commands::*; pub use policy::*; From 984c7057132936a4772d29c1e209903cca3db85a Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 16:24:54 +0800 Subject: [PATCH 65/71] docs(ecstore): fix bitrot comment typo (#6168) Co-authored-by: heihutu --- _typos.toml | 1 + crates/ecstore/src/erasure/coding/bitrot.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/_typos.toml b/_typos.toml index 12d50d60e..7ff843137 100644 --- a/_typos.toml +++ b/_typos.toml @@ -40,6 +40,7 @@ mak = "mak" gae = "gae" GAE = "GAE" thr = "thr" +mis = "mis" # s3-tests original test names (cannot be changed) nonexisted = "nonexisted" consts = "consts" diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index 698b1b02c..947bbb68e 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -835,7 +835,7 @@ pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096; /// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S` /// (the production default). Pinned so any platform or build where the -/// implementation drifts fails startup instead of mis-hashing shards. +/// implementation drifts fails startup instead of miss-hashing shards. const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [ 0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea, 0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06, From e0b87b0e7e48d5db66358dde292a80498df6e6ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Mon, 17 Aug 2026 17:47:36 +0800 Subject: [PATCH 66/71] fix(site-replication): admit only verifiable peer-edit fences (#6123) --- rustfs/src/admin/handlers/site_replication.rs | 219 +++++++++++++++++- 1 file changed, 214 insertions(+), 5 deletions(-) diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index fba311ba9..18eef001b 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -6023,7 +6023,10 @@ fn edit_generation_wall_clock() -> u64 { /// node's clock behind the clock that fed the previous lifetime) mints /// below the stale mark and the origin stays fenced — but only until real /// time passes the previous lifetime's last allocation, because every later -/// allocation takes the wall-clock floor again. Bounded by the skew, +/// allocation takes the wall-clock floor again (and never longer than +/// [`PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS`]: a regression past the window +/// leaves the mark implausibly distant and the origin runs unfenced +/// immediately). Bounded by the skew, /// self-healing, and no rollback window beyond the plain counter's: a /// delivery applies only at or above the receiver's mark, so the one /// cross-lifetime interleaving that can apply stale content — a @@ -6063,6 +6066,52 @@ fn peer_edit_fence(queries: &HashMap) -> Option<(String, u64)> { Some((origin.clone(), generation)) } +/// How far below the recorded high-water mark a delivery may sit and still +/// be fenced as stale. The distance a GENUINE superseded delivery can trail +/// its origin's mark is small: retransmissions re-run the sender flow and +/// mint a fresh generation (the retry queue keys on the bare path and never +/// replays a fenced URL), so only an in-flight straggler of the losing +/// fan-out race trails the mark, by delivery latency — minutes at the +/// outside. A mark further above than this window cannot be explained by +/// any genuine race, only by a forged fence (the shared service account +/// lets any peer stamp any origin) or by a persisted clock excursion the +/// origin has since left behind — and fencing on it would silently drop the +/// origin's real edits, so the stale check ignores it instead. +const PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS: u64 = 24 * 60 * 60 * 1_000_000_000; + +/// Whether an incoming fence may be honoured, as far as this site can vouch +/// for it. The sender's identity is unverifiable (shared service account), +/// so the check runs over what the receiving state knows: the claimed origin +/// must be a site this state currently replicates with — the same membership +/// rule the load-time mark pruning applies, so every mark recorded behind +/// this check is one a reload would keep — and not this site itself, which +/// never delivers edits to itself. The caller IGNORES an inadmissible fence +/// rather than failing the request: the delivery applies exactly as an +/// unstamped (pre-fence) delivery would, no high-water mark is read or +/// written, and the worst a forged fence achieves is forfeiting an ordering +/// guarantee its sender was never owed. The generation itself is NOT +/// bounded here: a genuine origin whose hybrid clock persisted a wall-clock +/// excursion allocates arbitrarily far in the future, and refusing to +/// record its marks would strip the ordering fence from exactly the +/// deliveries that still race — the staleness window on the read side is +/// what defuses forged marks instead. +fn peer_edit_fence_is_admissible(state: &SiteReplicationState, local_deployment_id: &str, fence: &(String, u64)) -> bool { + let (origin, generation) = fence; + if origin != local_deployment_id && state.peers.contains_key(origin) { + return true; + } + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "fence_origin_not_a_remote_peer", + origin = %origin, + generation = *generation, + "ignoring inadmissible peer-edit fence" + ); + false +} + /// True when a strictly newer edit from the same origin site already landed /// here. No lock on the sending side can order deliveries issued by two /// nodes of that site, so ordering is decided here, on the generation the @@ -6070,11 +6119,42 @@ fn peer_edit_fence(queries: &HashMap) -> Option<(String, u64)> { /// stale: one edit legitimately fans out several deliveries under a single /// generation (the ILM-expiry edit sends every peer's record), and a replay of /// an applied delivery re-applies the same edit idempotently. +/// +/// A mark more than [`PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS`] above the +/// delivery is implausible and does NOT fence: the shared service account +/// means any peer can stamp any origin, so a forged `u64::MAX`-scale mark +/// would otherwise silently swallow the origin's genuine edits for good. +/// Bounding the fence by distance instead of by an absolute ceiling keeps +/// ordering intact wherever the origin's clock actually operates — two +/// racing deliveries trail each other by seconds whether the hybrid clock +/// tracks wall time or persists a long-gone excursion far ahead of it — +/// while a mark no genuine race can explain merely downgrades the origin to +/// unfenced (pre-fence) delivery instead of dropping its edits. (One genuine +/// shape does land out here: a plain-counter straggler arriving after its +/// origin's first hybrid-clock edit. It gets the same downgrade — applied +/// unfenced — once, at upgrade time; fencing it instead would silence the +/// mirror case, a hybrid-clock origin downgraded back to the plain counter.) fn peer_edit_delivery_is_stale(state: &SiteReplicationState, origin: &str, generation: u64) -> bool { - state - .applied_edit_generations - .get(origin) - .is_some_and(|applied| *applied > generation) + let Some(applied) = state.applied_edit_generations.get(origin) else { + return false; + }; + if *applied <= generation { + return false; + } + if *applied - generation > PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "fence_mark_beyond_staleness_window", + origin, + generation, + applied_mark = *applied, + "ignoring implausibly distant peer-edit high-water mark" + ); + return false; + } + true } fn record_applied_peer_edit_generation(state: &mut SiteReplicationState, origin: &str, generation: u64) { @@ -10698,6 +10778,11 @@ impl Operation for SRPeerEditHandler { let outcome = update_site_replication_state_when_changed(move |state| { let mut incoming = incoming; let local_peer = local_peer_at_endpoint(commit_endpoint, state); + // The fence is self-reported — the shared service account means + // the sender cannot be identified — so it is honoured only after + // the admissibility check, against the same state it will gate. + let commit_fence = + commit_fence.filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence)); // Ordering fence: the sending site allocates the generation under // its state-object lock, so a delivery that lost the race carries // a generation this site has already passed. Applying it would @@ -13393,6 +13478,15 @@ mod tests { handler_block.contains("record_applied_peer_edit_generation(state, origin, *generation);"), "SRPeerEditHandler must record the applied generation so later stale deliveries are recognised" ); + // Fence hardening: origin and generation are self-reported by a + // caller the shared service account cannot identify, so the handler + // must pass the fence through the admissibility check — against the + // same state the fence gates, i.e. inside the transaction — before + // reading or raising any high-water mark. + assert!( + handler_block.contains(".filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence))"), + "SRPeerEditHandler must admit a fence only through peer_edit_fence_is_admissible inside the state transaction" + ); // P1-15 PR2: both halves of the fence and the edit they fence share // ONE transaction. Checking the fence against a state read outside the // lock would let the check pass on one snapshot and the write land on @@ -14769,6 +14863,121 @@ mod tests { assert!(peer_edit_delivery_is_stale(&state, origin, generation - 1)); } + /// A fence is self-reported: every site authenticates peer traffic with + /// the same site-replicator credential, so a compromised peer can stamp + /// ANY origin with ANY generation. An origin the receiver does not + /// replicate with — or the receiver itself — is ignored and plants no + /// mark; a mark a compromised peer plants for a CURRENT origin cannot + /// silence that origin, because the staleness window refuses to fence on + /// a mark implausibly far above the genuine deliveries. + #[test] + fn forged_peer_edit_fences_cannot_poison_the_high_water_marks() { + let mut state = SiteReplicationState { + peers: BTreeMap::from([ + ( + "site-local".to_string(), + PeerInfo { + deployment_id: "site-local".to_string(), + ..peer("local", "https://local.example:9000") + }, + ), + ( + "site-victim".to_string(), + PeerInfo { + deployment_id: "site-victim".to_string(), + ..peer("victim", "https://victim.example:9000") + }, + ), + ]), + ..Default::default() + }; + // An origin outside the current membership is refused outright... + let unknown = ("site-unknown".to_string(), 4u64); + assert!(!peer_edit_fence_is_admissible(&state, "site-local", &unknown)); + + // No site delivers edits to itself: a fence claiming the receiver as + // its origin is forged by construction, current peer or not. + let own = ("site-local".to_string(), 4u64); + assert!(!peer_edit_fence_is_admissible(&state, "site-local", &own)); + + // A current remote peer's fence is admitted and works end to end. + let genuine = ("site-victim".to_string(), 1u64); + assert!(peer_edit_fence_is_admissible(&state, "site-local", &genuine)); + assert!(!peer_edit_delivery_is_stale(&state, &genuine.0, genuine.1)); + record_applied_peer_edit_generation(&mut state, &genuine.0, genuine.1); + assert_eq!(state.applied_edit_generations.get("site-victim"), Some(&1)); + + // A forged u64::MAX-scale mark CAN be recorded — the shared service + // account means the receiver cannot tell the stamp was forged — but + // it is inert: the victim's genuine hybrid-clock deliveries sit far + // more than the staleness window below it, so they keep applying + // instead of being silently acked-and-dropped. + record_applied_peer_edit_generation(&mut state, "site-victim", u64::MAX); + assert!(!peer_edit_delivery_is_stale(&state, "site-victim", edit_generation_wall_clock())); + } + + /// The staleness window bounds the fence by DISTANCE from the mark, not + /// by an absolute clock ceiling, so ordering must hold wherever the + /// origin's hybrid clock actually operates. The regression that matters: + /// a temporary wall-clock excursion far in the future is persisted by + /// `next_peer_edit_generation` (`max(now, prev + 1)` never comes back + /// down), and two later edits g+1 then g can arrive in reverse order — + /// g must still be fenced, even though both generations dwarf the + /// receiver's clock. Conversely a mark further above a delivery than any + /// genuine race can explain must not fence it. + #[test] + fn peer_edit_fence_orders_a_persisted_future_clock_and_defuses_distant_marks() { + let mut state = SiteReplicationState { + peers: BTreeMap::from([( + "site-origin".to_string(), + PeerInfo { + deployment_id: "site-origin".to_string(), + ..peer("origin", "https://origin.example:9000") + }, + )]), + ..Default::default() + }; + + // The origin's clock once jumped ten years ahead; the hybrid clock + // keeps allocating from there long after the clock was corrected. + let excursion = edit_generation_wall_clock() + 10 * 365 * 24 * 60 * 60 * 1_000_000_000; + let fence = ("site-origin".to_string(), excursion + 1); + assert!(peer_edit_fence_is_admissible(&state, "site-local", &fence)); + record_applied_peer_edit_generation(&mut state, &fence.0, fence.1); + + // The reverse delivery of the race: g arrives after g+1 landed. + // Without the fence it would commit last and roll g+1 back. + assert!(peer_edit_delivery_is_stale(&state, "site-origin", excursion)); + // Equal generation (same edit's fan-out or a replay) still applies, + // as does the next edit. + assert!(!peer_edit_delivery_is_stale(&state, "site-origin", excursion + 1)); + assert!(!peer_edit_delivery_is_stale(&state, "site-origin", excursion + 2)); + + // The window's exact boundary: a delivery trailing the mark by the + // full window is still fenced; one nanosecond further is not — that + // distance is no longer explicable by a genuine race, only by a + // forged mark or an excursion the origin has left behind. + let mark = fence.1; + // A straggler trailing by a concrete hour must still be fenced — + // pins the window's real magnitude, not just its symbolic boundary. + assert!(peer_edit_delivery_is_stale(&state, "site-origin", mark - 60 * 60 * 1_000_000_000)); + assert!(peer_edit_delivery_is_stale( + &state, + "site-origin", + mark - PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS + )); + assert!(!peer_edit_delivery_is_stale( + &state, + "site-origin", + mark - PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS - 1 + )); + + // A pre-hybrid plain-counter origin trails such a mark by eons: it + // is not fenced (the rc.2-era downgrade case), it just runs + // unfenced until its counter regime catches up. + assert!(!peer_edit_delivery_is_stale(&state, "site-origin", 3)); + } + /// P1-15 review follow-up: a site that leaves the mesh drops below two /// peers, which clears its state object and restarts its generation /// counter at zero. A mark left over from its previous membership would From 59b7d13095e780fccbff829a19770a713f593884 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 19:40:56 +0800 Subject: [PATCH 67/71] feat(scanner): expose prefix-level bucket usage via admin API (HS-08) (#6171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat(scanner): expose prefix-level bucket usage via admin API The scanner's per-bucket, per-set usage caches already hold a path-keyed prefix tree, but dui() flattened it only to bucket names — consoles and operators had no way to ask "what does this prefix hold" without an S3 listing sweep (rustfs/backlog#1872, MinIO loadPrefixUsageFromBackend parity). Add: - data-usage: prefix_usage_in_cache — a shared aggregation over the entry map (arbitrary prefix, full counters, one-level sub-prefix breakdown with names recovered from the literal-path cache keys), hardened like the scanner's checked flatten: cycles, dangling child links, over-deep trees, and overflowing counters yield None rather than unbounded recursion or wrapped totals. - ecstore: ECStore::all_set_disks — iterate every erasure set so a query can read each set's own cache copy; the hash-routed store path would always land on one set. - scanner: bucket_prefix_usage — per-set loads (5s budget each, a slow set degrades to not-reporting instead of stalling the caller), merged across sets with partial/compacted/truncated flags, served from a bounded 30s cache (128 entries, hard-capped) that bucket writes invalidate through the dirty-usage hook. - admin: GET /rustfs/admin/v3/usage/{bucket}?prefix=&max-entries= behind the same any-of gate as datausageinfo (DataUsageInfoAdminAction OR ListBucketAction), rejecting unknown query parameters and clamping max-entries to 1..=10000. Route registered in the policy table (deferred MultipleActions, matching datausageinfo) and the route matrix test. Closes rustfs/backlog#1872. Co-authored-by: heihutu --- crates/data-usage/src/data_usage.rs | 286 ++++++++++++++++ crates/ecstore/src/store/mod.rs | 10 + crates/scanner/src/data_usage_define.rs | 10 +- crates/scanner/src/lib.rs | 2 + crates/scanner/src/prefix_usage.rs | 349 ++++++++++++++++++++ crates/scanner/src/scanner_io.rs | 4 + rustfs/src/admin/handlers/mod.rs | 1 + rustfs/src/admin/handlers/system.rs | 8 +- rustfs/src/admin/handlers/usage_prefix.rs | 142 ++++++++ rustfs/src/admin/mod.rs | 4 +- rustfs/src/admin/route_policy.rs | 5 + rustfs/src/admin/route_registration_test.rs | 1 + 12 files changed, 816 insertions(+), 6 deletions(-) create mode 100644 crates/scanner/src/prefix_usage.rs create mode 100644 rustfs/src/admin/handlers/usage_prefix.rs diff --git a/crates/data-usage/src/data_usage.rs b/crates/data-usage/src/data_usage.rs index ebbd03261..b08281d0f 100644 --- a/crates/data-usage/src/data_usage.rs +++ b/crates/data-usage/src/data_usage.rs @@ -870,6 +870,157 @@ pub struct DataUsageCacheInfo { pub snapshot_complete: bool, } +/// Prefix-level usage over a raw entry map — the shared core behind +/// [`DataUsageCache::prefix_usage`], usable by any cache-shaped reader (the +/// scanner's writer-side cache has the same map type). +/// +/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix +/// names come straight off the child keys — no reverse mapping exists or is +/// needed. A compacted prefix carries its aggregate but no children, which +/// the `compacted` flag reports so callers can say why the breakdown is +/// empty. `truncated` is set when the breakdown exceeded `max_entries` and +/// was cut (largest first). +pub fn prefix_usage_in_cache( + cache: &HashMap, + bucket: &str, + prefix: &str, + max_entries: usize, +) -> Option { + let prefix = prefix.trim_matches('/'); + let root = if prefix.is_empty() { + bucket.to_string() + } else { + format!("{bucket}/{prefix}") + }; + let entry = cache.get(&hash_path(&root).key())?.clone(); + + let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?); + + let child_prefix = format!("{root}/"); + let mut sub_prefixes: Vec = entry + .children + .iter() + .filter_map(|child_key| { + let child = cache.get(child_key)?; + let child_flat = flatten_entry(cache, child, 1)?; + // Child keys are literal `bucket/pre/name` paths; a trailing + // slash marks a directory object and is display-only here. + let name = child_key + .strip_prefix(child_prefix.as_str()) + .unwrap_or(child_key.as_str()) + .trim_end_matches('/') + .to_string(); + Some(PrefixUsageEntry { + prefix: name, + usage: PrefixUsageSummary::from_entry(&child_flat), + }) + }) + .collect(); + sub_prefixes.sort_by(|left, right| { + right + .usage + .size + .cmp(&left.usage.size) + .then_with(|| left.prefix.cmp(&right.prefix)) + }); + let truncated = sub_prefixes.len() > max_entries; + sub_prefixes.truncate(max_entries); + + Some(PrefixUsageQuery { + usage, + compacted: entry.compacted, + truncated, + sub_prefixes, + }) +} + +/// Maximum subtree depth [`flatten_entry`] will walk before declaring the +/// cache corrupt — the same bound the scanner's checked flatten uses. +const PREFIX_USAGE_MAX_DEPTH: usize = 1024; + +/// Flatten one entry's subtree into an aggregate: the free-function twin of +/// [`DataUsageCache::flatten`], carrying the scanner checked-flatten +/// hardening so a corrupt cache (cycles, over-deep trees, overflowing +/// counters) yields `None` instead of unbounded recursion or wrapped totals. +fn flatten_entry(cache: &HashMap, root: &DataUsageEntry, depth: usize) -> Option { + if depth > PREFIX_USAGE_MAX_DEPTH { + return None; + } + let mut flattened = DataUsageEntry::default(); + if !flattened.checked_merge(root) { + return None; + } + flattened.compacted = root.compacted; + // The root itself is not pre-seeded: it is merged above, and a corrupt + // child edge pointing back at the root's own key is still terminated by + // the visited set on first encounter. + let mut visited: HashSet<&str> = HashSet::new(); + let mut pending: Vec<(&String, usize)> = root.children.iter().map(|child| (child, depth + 1)).collect(); + while let Some((key, child_depth)) = pending.pop() { + if child_depth > PREFIX_USAGE_MAX_DEPTH || !visited.insert(key.as_str()) { + return None; + } + let entry = cache.get(key)?; + if !flattened.checked_merge(entry) { + return None; + } + pending.extend(entry.children.iter().map(|child| (child, child_depth + 1))); + } + flattened.children.clear(); + Some(flattened) +} + +/// Flattened counters of one prefix subtree, as returned by +/// [`DataUsageCache::prefix_usage`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PrefixUsageSummary { + pub size: u64, + pub objects: u64, + pub versions: u64, + pub delete_markers: u64, +} + +impl PrefixUsageSummary { + fn from_entry(entry: &DataUsageEntry) -> Self { + Self { + size: entry.size as u64, + objects: entry.objects as u64, + versions: entry.versions as u64, + delete_markers: entry.delete_markers as u64, + } + } + + /// Add another set's counters into this one (entries are partitioned by + /// set, so per-set results sum). + pub fn merge(&mut self, other: &Self) { + self.size = self.size.saturating_add(other.size); + self.objects = self.objects.saturating_add(other.objects); + self.versions = self.versions.saturating_add(other.versions); + self.delete_markers = self.delete_markers.saturating_add(other.delete_markers); + } +} + +/// One first-level sub-prefix row of a [`PrefixUsageQuery`]. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub struct PrefixUsageEntry { + pub prefix: String, + pub usage: PrefixUsageSummary, +} + +/// Result of [`DataUsageCache::prefix_usage`]. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PrefixUsageQuery { + pub usage: PrefixUsageSummary, + /// The prefix entry was compacted by the scanner: its aggregate is valid + /// but no sub-prefix breakdown exists on disk. + pub compacted: bool, + /// The breakdown had more entries than `max_entries`; the largest remain. + pub truncated: bool, + pub sub_prefixes: Vec, +} + /// Read-only projection of a scanner-written `.usage-cache.bin` file. /// /// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`) @@ -997,6 +1148,21 @@ impl DataUsageCache { } } + /// Prefix-level usage for one bucket subtree, plus the one-level + /// breakdown below it (rustfs/backlog#1872, MinIO + /// `loadPrefixUsageFromBackend` parity and beyond: arbitrary prefixes and + /// full counters instead of first-level sizes only). + /// + /// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix + /// names come straight off the child keys — no reverse mapping exists or + /// is needed. A compacted prefix carries its aggregate but no children, + /// which the `compacted` flag reports so callers can say why the + /// breakdown is empty. `truncated` is set when the breakdown exceeded + /// `max_entries` and was cut (largest first). + pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option { + prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries) + } + pub fn force_compact(&mut self, limit: usize) { if self.cache.len() < limit { return; @@ -1898,6 +2064,126 @@ mod tests { ); } + /// Build a cache shaped like `bucket/{a,b/{c,d}},bucket/loose` with + /// distinct counters so aggregation is observable. + fn prefix_usage_fixture_cache() -> DataUsageCache { + let mut cache = DataUsageCache::default(); + let mut insert = |path: &str, parent: &str, size: usize, objects: usize, versions: usize, delete_markers: usize| { + cache.replace( + path, + parent, + DataUsageEntry { + size, + objects, + versions, + delete_markers, + ..Default::default() + }, + ); + }; + insert("bucket", "", 0, 0, 0, 0); + insert("bucket/a", "bucket", 100, 1, 1, 0); + insert("bucket/b", "bucket", 0, 0, 0, 0); + insert("bucket/b/c", "bucket/b", 200, 2, 2, 1); + insert("bucket/b/d", "bucket/b", 40, 1, 3, 0); + insert("bucket/loose", "bucket", 10, 1, 1, 1); + cache + } + + #[test] + fn prefix_usage_aggregates_bucket_root_and_one_level_below() { + let cache = prefix_usage_fixture_cache(); + + let root = cache + .prefix_usage("bucket", "", 100) + .expect("root query must find the bucket entry"); + assert_eq!(root.usage.size, 350, "root aggregate flattens the whole subtree"); + assert_eq!(root.usage.objects, 5); + assert_eq!(root.usage.versions, 7); + assert_eq!(root.usage.delete_markers, 2); + assert!(!root.compacted); + assert!(!root.truncated); + // Breakdown is one level: b (240) before a (100) before loose (10), + // each flattened to its own subtree total. + let names: Vec<(&str, u64)> = root + .sub_prefixes + .iter() + .map(|entry| (entry.prefix.as_str(), entry.usage.size)) + .collect(); + assert_eq!(names, vec![("b", 240), ("a", 100), ("loose", 10)]); + } + + #[test] + fn prefix_usage_drills_into_arbitrary_prefixes() { + let cache = prefix_usage_fixture_cache(); + + let b = cache.prefix_usage("bucket", "b", 100).expect("nested prefix must resolve"); + assert_eq!(b.usage.size, 240); + assert_eq!(b.usage.versions, 5); + let names: Vec<&str> = b.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect(); + assert_eq!(names, vec!["c", "d"]); + + // Prefix slashes are normalized away. + let slashed = cache.prefix_usage("bucket", "/b/", 100).expect("slash-insensitive lookup"); + assert_eq!(slashed.usage.size, 240); + + assert!(cache.prefix_usage("bucket", "absent", 100).is_none(), "unknown prefix must be a miss"); + assert!(cache.prefix_usage("other", "", 100).is_none(), "unknown bucket must be a miss"); + } + + #[test] + fn prefix_usage_reports_and_respects_truncation() { + let cache = prefix_usage_fixture_cache(); + let capped = cache.prefix_usage("bucket", "", 2).expect("root query"); + assert!(capped.truncated, "three children capped to two must flag truncation"); + let names: Vec<&str> = capped.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect(); + assert_eq!(names, vec!["b", "a"], "largest prefixes survive the cut"); + } + + #[test] + fn prefix_usage_marks_compacted_entries() { + let mut cache = DataUsageCache::default(); + cache.replace( + "bucket", + "", + DataUsageEntry { + size: 999, + objects: 9, + compacted: true, + ..Default::default() + }, + ); + + let compacted = cache.prefix_usage("bucket", "", 100).expect("compacted root resolves"); + assert!(compacted.compacted, "compaction must be visible to callers"); + assert_eq!(compacted.usage.size, 999); + assert!(compacted.sub_prefixes.is_empty(), "a compacted entry carries no children"); + } + + #[test] + fn prefix_usage_rejects_cyclic_and_dangling_caches() { + // A self-referencing child (corrupt cache) must yield a miss for the + // whole query, not unbounded recursion. + let mut cache = prefix_usage_fixture_cache(); + if let Some(entry) = cache.cache.get_mut("bucket/b") { + entry.children.insert("bucket/b".to_string()); + } + assert!(cache.prefix_usage("bucket", "b", 100).is_none(), "a cyclic subtree must be rejected"); + // The unaffected sibling still answers. + assert!(cache.prefix_usage("bucket", "a", 100).is_some()); + + // A child key with no entry (dangling link) is rejected rather than + // silently dropped: half a tree would under-report usage. + let mut dangling = prefix_usage_fixture_cache(); + if let Some(entry) = dangling.cache.get_mut("bucket/b") { + entry.children.insert("bucket/b/ghost".to_string()); + } + assert!( + dangling.prefix_usage("bucket", "b", 100).is_none(), + "a dangling child link must be rejected" + ); + } + #[test] fn hash_path_uses_portable_slash_semantics() { for (input, expected) in [ diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index f51fa6df5..8d02ef441 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -216,6 +216,16 @@ impl std::fmt::Debug for ECStore { /// These delegate to the process-global statics. No local state — the globals /// remain the single source of truth until the migration is complete. impl ECStore { + /// Every erasure set across all pools, pool-major order. + /// + /// Read-only queries that must consult each set's own copy of a + /// per-bucket object (e.g. the scanner's `.usage-cache.bin`) iterate + /// this instead of the hash-routed store path, which would always land + /// on one set (rustfs/backlog#1872). + pub fn all_set_disks(&self) -> Vec> { + self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect() + } + /// Get server configuration (delegates to global) pub fn get_server_config(&self) -> Option { runtime_sources::server_config() diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 67a199661..8ac714314 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -28,7 +28,8 @@ use rustfs_common::heal_channel::HealScanMode; use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS; pub use rustfs_data_usage::{ AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, - DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, TierStats, hash_path, + DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry, + PrefixUsageQuery, PrefixUsageSummary, TierStats, hash_path, prefix_usage_in_cache, }; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use tokio::time::{Duration, Instant, sleep, timeout}; @@ -430,6 +431,13 @@ pub(crate) enum DataUsageCachePrepareOutcome { } impl DataUsageCache { + /// Prefix-level usage query over this (writer-side) cache; see + /// [`prefix_usage_in_cache`] for the semantics + /// (rustfs/backlog#1872). + pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option { + prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries) + } + pub(crate) fn prepare_for_scan( &mut self, name: &str, diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 4a5cc7543..36cc6b817 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -53,6 +53,7 @@ use tokio_util::sync::CancellationToken; pub mod data_usage_define; pub mod error; +pub mod prefix_usage; mod remote_scanner; pub mod runtime_config; pub mod scanner; @@ -64,6 +65,7 @@ pub(crate) mod storage_api; pub use data_usage_define::*; pub use error::ScannerError; +pub use prefix_usage::{BucketPrefixUsageResponse, bucket_prefix_usage, invalidate_prefix_usage_cache}; pub use remote_scanner::{ NS_SCANNER_MAX_REQUEST_BODY_SIZE, RemoteScannerAdmission, RemoteScannerRequest, admit_remote_scanner_request, claim_remote_scanner_request, decode_remote_scanner_request, preflight_remote_scanner_request, diff --git a/crates/scanner/src/prefix_usage.rs b/crates/scanner/src/prefix_usage.rs new file mode 100644 index 000000000..9f94957de --- /dev/null +++ b/crates/scanner/src/prefix_usage.rs @@ -0,0 +1,349 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Prefix-level bucket usage for admin/console consumers (rustfs/backlog#1872, +//! MinIO `loadPrefixUsageFromBackend` parity). +//! +//! The per-bucket, per-set `.usage-cache.bin` objects already hold a +//! path-keyed prefix tree; this module reads every set's copy through that +//! set's own object layer (the hash-routed store path would always land on +//! one set), aggregates the overlapping trees, and serves the result from a +//! bounded 30-second cache. Bucket writes poke the cache through the +//! dirty-usage hook so a fresh scan is visible immediately. + +use crate::data_usage_define::{DATA_USAGE_CACHE_NAME, DataUsageCache}; +use crate::error::ScannerError; +use crate::storage_api::owner::{ + EcstoreSetDisks, EcstoreStore, ecstore_is_reserved_or_invalid_bucket, ecstore_resolve_object_store_handle, +}; +use futures::future::join_all; +use rustfs_data_usage::{PrefixUsageEntry, PrefixUsageSummary}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime}; +use tracing::{debug, warn}; + +const LOG_COMPONENT_SCANNER: &str = "scanner"; +const LOG_SUBSYSTEM_PREFIX_USAGE: &str = "prefix_usage"; +const EVENT_PREFIX_USAGE_CACHE_STATE: &str = "prefix_usage_cache_state"; + +/// How long a computed breakdown stays fresh. MinIO uses the same 30s for +/// its prefix-usage cache; bucket writes additionally invalidate on the spot. +const CACHE_TTL: Duration = Duration::from_secs(30); +/// Hard entry cap for the result cache; exceeded, expired entries go first +/// and the map clears rather than growing past the bound. +const CACHE_MAX_ENTRIES: usize = 128; +/// Per-set cache read budget. The underlying loader retries for up to a +/// minute per attempt on backend errors — far too long for an admin GET, so +/// a slow set degrades to "not reporting" instead of stalling the caller. +const PER_SET_LOAD_TIMEOUT: Duration = Duration::from_secs(5); + +/// Aggregated prefix-usage answer across every erasure set. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BucketPrefixUsageResponse { + pub bucket: String, + pub prefix: String, + pub usage: PrefixUsageSummary, + /// Every reporting set's prefix entry was compacted: the aggregate is + /// valid, the sub-prefix breakdown is empty on disk. + pub compacted: bool, + /// The sub-prefix breakdown is incomplete: at least one reporting set + /// had the prefix compacted (or absent while others found it), so its + /// objects cannot be attributed to a sub-prefix. + pub sub_prefixes_partial: bool, + /// The breakdown exceeded the caller's entry limit; largest remain. + pub truncated: bool, + pub sub_prefixes: Vec, + /// Sets whose cache held this bucket and prefix. + pub sets_reporting: usize, + pub sets_total: usize, + /// Newest `last_update` across reporting sets, unix seconds. + pub last_update_unix_secs: Option, +} + +#[derive(Clone)] +struct CachedResponse { + computed_at: std::time::Instant, + response: Arc, +} + +/// Cache key: (lowercased bucket, normalized prefix, max entries). +type PrefixUsageCacheKey = (String, String, usize); +type PrefixUsageCacheMap = Option>; + +static PREFIX_USAGE_CACHE: Mutex = Mutex::new(None); + +/// Drop cached results for `bucket` (empty string clears everything). Wired +/// into the dirty-usage recording path so a write makes the next prefix +/// query recompute instead of serving up to `CACHE_TTL` seconds of stale +/// numbers. +pub fn invalidate_prefix_usage_cache(bucket: &str) { + let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let Some(map) = guard.as_mut() else { + return; + }; + if bucket.is_empty() { + map.clear(); + return; + } + map.retain(|(cached_bucket, ..), _| !cached_bucket.eq_ignore_ascii_case(bucket)); +} + +/// Query prefix usage for `bucket` (arbitrary `prefix`, empty = whole +/// bucket), merging every erasure set's own cache copy. `max_entries` bounds +/// the sub-prefix rows (largest first). +pub async fn bucket_prefix_usage( + bucket: &str, + prefix: &str, + max_entries: usize, +) -> Result { + if ecstore_is_reserved_or_invalid_bucket(bucket, true) { + return Err(ScannerError::Other(format!("invalid bucket name: {bucket}"))); + } + let normalized_prefix = prefix.trim_matches('/').to_string(); + let cache_key = (bucket.to_ascii_lowercase(), normalized_prefix.clone(), max_entries); + if let Some(response) = lookup_cached(&cache_key) { + return Ok((*response).clone()); + } + + let store = ecstore_resolve_object_store_handle() + .ok_or_else(|| ScannerError::Other("object store is not initialized".to_string()))?; + let response = Arc::new(compute_prefix_usage(store, bucket, &normalized_prefix, max_entries).await); + store_cached(cache_key, response.clone()); + Ok((*response).clone()) +} + +async fn compute_prefix_usage( + store: Arc, + bucket: &str, + prefix: &str, + max_entries: usize, +) -> BucketPrefixUsageResponse { + let sets: Vec> = store.all_set_disks(); + let sets_total = sets.len(); + let cache_name = format!("{bucket}/{DATA_USAGE_CACHE_NAME}"); + + let per_set = join_all(sets.into_iter().map(|set| { + let cache_name = cache_name.clone(); + async move { + let mut cache = DataUsageCache::default(); + // A set that has never scanned this bucket (or cannot be read + // within the budget) reports nothing — the remaining sets still + // produce a usable, flagged answer. + let loaded = match tokio::time::timeout(PER_SET_LOAD_TIMEOUT, cache.load(set, &cache_name)).await { + Ok(Ok(())) => cache, + Ok(Err(err)) => { + debug!( + target: "rustfs::scanner::prefix_usage", + event = EVENT_PREFIX_USAGE_CACHE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_PREFIX_USAGE, + bucket = %bucket, + state = "set_load_failed", + error = %err, + "Prefix usage set cache load failed" + ); + return None; + } + Err(_) => { + warn!( + target: "rustfs::scanner::prefix_usage", + event = EVENT_PREFIX_USAGE_CACHE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_PREFIX_USAGE, + bucket = %bucket, + state = "set_load_timeout", + "Prefix usage set cache load timed out" + ); + return None; + } + }; + if loaded.info.name != bucket { + // Empty or stale-scoped cache: this set has no data for the bucket. + return None; + } + let last_update = loaded.info.last_update; + let query = loaded.prefix_usage(bucket, prefix, max_entries); + Some((query, last_update)) + } + })) + .await; + + let mut usage = PrefixUsageSummary::default(); + let mut sub_prefix_map: HashMap = HashMap::new(); + let mut sets_reporting = 0usize; + let mut reporting_but_absent = 0usize; + let mut any_compacted = false; + let mut all_compacted = true; + let mut truncated = false; + let mut last_update: Option = None; + + for (query, set_last_update) in per_set.into_iter().flatten() { + // last_update counts every set that has scanned the bucket, even + // when the prefix itself is absent on that set. + if let Some(set_last_update) = set_last_update + && last_update.map(|current| set_last_update > current).unwrap_or(true) + { + last_update = Some(set_last_update); + } + let Some(query) = query else { + // The set knows the bucket but not this prefix: legitimate when + // the prefix's objects all hash to other sets, but it means the + // breakdown below cannot attribute that set's (zero) objects. + reporting_but_absent += 1; + continue; + }; + sets_reporting += 1; + usage.merge(&query.usage); + if query.compacted { + any_compacted = true; + } else { + all_compacted = false; + } + truncated |= query.truncated; + for entry in query.sub_prefixes { + sub_prefix_map.entry(entry.prefix).or_default().merge(&entry.usage); + } + } + + let mut sub_prefixes: Vec = sub_prefix_map + .into_iter() + .map(|(prefix, usage)| PrefixUsageEntry { prefix, usage }) + .collect(); + sub_prefixes.sort_by(|left, right| { + right + .usage + .size + .cmp(&left.usage.size) + .then_with(|| left.prefix.cmp(&right.prefix)) + }); + // Merged rows can exceed max_entries only when per-set truncation + // already flagged; enforce the caller bound on the merged view too. + if sub_prefixes.len() > max_entries { + truncated = true; + sub_prefixes.truncate(max_entries); + } + + let found = sets_reporting > 0; + BucketPrefixUsageResponse { + bucket: bucket.to_string(), + prefix: prefix.to_string(), + usage, + compacted: found && all_compacted, + sub_prefixes_partial: any_compacted || reporting_but_absent > 0, + truncated, + sub_prefixes, + sets_reporting, + sets_total, + last_update_unix_secs: last_update + .and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|dur| dur.as_secs()), + } +} + +fn lookup_cached(key: &(String, String, usize)) -> Option> { + let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let map = guard.as_mut()?; + let cached = map.get(key)?; + if cached.computed_at.elapsed() > CACHE_TTL { + map.remove(key); + return None; + } + Some(cached.response.clone()) +} + +fn store_cached(key: (String, String, usize), response: Arc) { + let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let map = guard.get_or_insert_with(HashMap::new); + // Bound the cache: drop expired entries first, and if the cap is still + // exceeded clear wholesale — the next queries recompute in milliseconds. + if map.len() >= CACHE_MAX_ENTRIES { + map.retain(|_, cached| cached.computed_at.elapsed() <= CACHE_TTL); + if map.len() >= CACHE_MAX_ENTRIES { + map.clear(); + } + } + map.insert( + key, + CachedResponse { + computed_at: std::time::Instant::now(), + response, + }, + ); +} + +#[cfg(test)] +mod tests { + use super::{CACHE_MAX_ENTRIES, PREFIX_USAGE_CACHE, invalidate_prefix_usage_cache, store_cached}; + use rustfs_data_usage::PrefixUsageSummary; + + fn response(bucket: &str) -> super::BucketPrefixUsageResponse { + super::BucketPrefixUsageResponse { + bucket: bucket.to_string(), + prefix: String::new(), + usage: PrefixUsageSummary::default(), + compacted: false, + sub_prefixes_partial: false, + truncated: false, + sub_prefixes: Vec::new(), + sets_reporting: 1, + sets_total: 1, + last_update_unix_secs: None, + } + } + + fn seed(bucket: &str, prefix: &str) { + store_cached( + (bucket.to_ascii_lowercase(), prefix.to_string(), 10), + std::sync::Arc::new(response(bucket)), + ); + } + + fn contains(bucket: &str, prefix: &str) -> bool { + PREFIX_USAGE_CACHE + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .as_ref() + .is_some_and(|map| map.contains_key(&(bucket.to_ascii_lowercase(), prefix.to_string(), 10))) + } + + /// All cache tests run inside one test to keep the process-global map + /// free of cross-test ordering (the flake class this module avoids). + #[test] + fn invalidation_scopes_to_bucket_and_cache_stays_bounded() { + invalidate_prefix_usage_cache(""); + seed("alpha", "x"); + seed("beta", "y"); + + // Case-insensitive bucket scoping. + invalidate_prefix_usage_cache("ALPHA"); + assert!(!contains("alpha", "x")); + assert!(contains("beta", "y")); + + // Wholesale clear. + invalidate_prefix_usage_cache(""); + assert!(!contains("beta", "y")); + + // Hard cap: overflow clears rather than grows. + for index in 0..=(CACHE_MAX_ENTRIES / 2) { + let bucket = format!("cap-bucket-{index}"); + seed(&bucket, "a"); + seed(&bucket, "b"); + } + let guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let map = guard.as_ref().expect("seeded"); + assert!(map.len() <= CACHE_MAX_ENTRIES, "cache must stay bounded, got {}", map.len()); + } +} diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 15ee9cca0..f722ff186 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -231,6 +231,10 @@ pub fn record_dirty_usage_bucket(bucket: &str) { dirty_buckets.len() }; global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets)); + // A write invalidates this bucket's prefix-usage answers on the spot so + // admin/console consumers never ride the full TTL after a change + // (rustfs/backlog#1872). + crate::prefix_usage::invalidate_prefix_usage_cache(bucket); DIRTY_USAGE_BUCKET_NOTIFY.notify_one(); } diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 8c837eb05..f0a32f402 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -64,6 +64,7 @@ mod target_descriptor; pub mod tier; pub mod tls_debug; pub mod trace; +pub mod usage_prefix; pub mod user; pub mod user_iam; pub mod user_lifecycle; diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index c7ad93b54..0909568e5 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -1158,10 +1158,10 @@ impl Operation for RuntimeCapabilitiesHandler { } } -/// Authorization gate for GET datausageinfo: any-of the dedicated admin action -/// OR the bucket listing action. Pinned by a unit test so the gate cannot -/// silently narrow or widen (rustfs/backlog#1306). -fn data_usage_info_gate_actions() -> Vec { +/// Authorization gate for GET datausageinfo (and prefix usage): any-of the +/// dedicated admin action OR the bucket listing action. Pinned by a unit test +/// so the gate cannot silently narrow or widen (rustfs/backlog#1306). +pub(crate) fn data_usage_info_gate_actions() -> Vec { vec![ Action::AdminAction(AdminAction::DataUsageInfoAdminAction), Action::S3Action(S3Action::ListBucketAction), diff --git a/rustfs/src/admin/handlers/usage_prefix.rs b/rustfs/src/admin/handlers/usage_prefix.rs new file mode 100644 index 000000000..10cce50d5 --- /dev/null +++ b/rustfs/src/admin/handlers/usage_prefix.rs @@ -0,0 +1,142 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Prefix-level bucket usage admin handler (rustfs/backlog#1872). +//! +//! `GET /rustfs/admin/v3/usage/{bucket}?prefix=&max-entries=` answers +//! "what does this bucket / this prefix hold" from the scanner's per-set +//! usage caches, with a one-level sub-prefix breakdown — the data console +//! buckets view MinIO serves from `loadPrefixUsageFromBackend`. + +use crate::admin::auth::validate_admin_request; +use crate::admin::handlers::system::data_usage_info_gate_actions; +use crate::admin::router::{AdminOperation, Operation, S3Router}; +use crate::auth::{check_key_valid, get_session_token}; +use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use http::{HeaderMap, HeaderValue, StatusCode}; +use hyper::Method; +use matchit::Params; +use s3s::header::CONTENT_TYPE; +use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; + +const JSON_CONTENT_TYPE: &str = "application/json"; +const DEFAULT_MAX_ENTRIES: usize = 1000; +const MAX_ENTRIES_LIMIT: usize = 10_000; + +pub struct BucketPrefixUsageHandler {} + +pub fn register_usage_prefix_route(r: &mut S3Router) -> std::io::Result<()> { + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/usage/{bucket}").as_str(), + AdminOperation(&BucketPrefixUsageHandler {}), + )?; + Ok(()) +} + +/// Parse `prefix` and `max-entries` from the query string. Unknown keys are +/// rejected so a typo'd parameter cannot silently change the answer's shape. +fn parse_usage_prefix_query(query: Option<&str>) -> S3Result<(String, usize)> { + let mut prefix: Option = None; + let mut max_entries: Option = None; + for (key, value) in url::form_urlencoded::parse(query.unwrap_or_default().as_bytes()) { + match key.as_ref() { + "prefix" => prefix = Some(value.into_owned()), + "max-entries" => { + max_entries = Some( + value + .parse::() + .map_err(|_| s3_error!(InvalidArgument, "max-entries must be a positive integer"))?, + ); + } + other => return Err(s3_error!(InvalidArgument, "unknown query parameter: {other}")), + } + } + let max_entries = max_entries.unwrap_or(DEFAULT_MAX_ENTRIES).clamp(1, MAX_ENTRIES_LIMIT); + Ok((prefix.unwrap_or_default(), max_entries)) +} + +#[async_trait::async_trait] +impl Operation for BucketPrefixUsageHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let remote_addr = req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)); + validate_admin_request(&req.headers, &cred, owner, false, data_usage_info_gate_actions(), remote_addr).await?; + + let bucket = params.get("bucket").unwrap_or_default().to_string(); + if bucket.is_empty() { + return Err(s3_error!(InvalidRequest, "bucket path parameter is required")); + } + let (prefix, max_entries) = parse_usage_prefix_query(req.uri.query())?; + + // Authorization is bucket-scoped by the same any-of gate as the + // datausageinfo route; the bucket name itself is validated by the + // scanner layer, which rejects reserved/invalid names. + let response = rustfs_scanner::bucket_prefix_usage(&bucket, &prefix, max_entries) + .await + .map_err(|err| s3_error!(InvalidArgument, "{}", err))?; + + let data = serde_json::to_vec(&response) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "parse prefix usage failed"))?; + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, HeaderValue::from_static(JSON_CONTENT_TYPE)); + + Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) + } +} + +#[cfg(test)] +mod tests { + use super::{DEFAULT_MAX_ENTRIES, MAX_ENTRIES_LIMIT, parse_usage_prefix_query}; + use s3s::S3Error; + + fn query(raw: &str) -> Result<(String, usize), S3Error> { + parse_usage_prefix_query(Some(raw)) + } + + #[test] + fn defaults_apply_when_no_query_is_given() { + assert_eq!(parse_usage_prefix_query(None).unwrap(), (String::new(), DEFAULT_MAX_ENTRIES)); + assert_eq!(query("").unwrap(), (String::new(), DEFAULT_MAX_ENTRIES)); + } + + #[test] + fn prefix_round_trips_url_encoded_characters() { + let (prefix, _) = query("prefix=pre%2Ffix%20name").unwrap(); + assert_eq!(prefix, "pre/fix name"); + } + + #[test] + fn max_entries_parses_and_clamps_to_documented_bounds() { + assert_eq!(query("max-entries=5").unwrap().1, 5); + assert_eq!(query("max-entries=0").unwrap().1, 1, "zero must clamp up, not mean unlimited"); + assert_eq!(query("max-entries=99999999").unwrap().1, MAX_ENTRIES_LIMIT); + assert!(query("max-entries=-3").is_err()); + assert!(query("max-entries=abc").is_err()); + } + + #[test] + fn unknown_parameters_are_rejected_not_ignored() { + assert!( + query("prefixes=x").is_err(), + "a typo'd parameter must fail the request, not widen the query" + ); + } +} diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index d61abf932..fe07d4056 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -40,7 +40,8 @@ use handlers::{ audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, durability as durability_handler, extensions, heal, health, idp_compat, ilm_transition, inspect_archive, kms, module_switch, object_data_cache, object_zip_download, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler, rebalance, - replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, user, + replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, usage_prefix, + user, }; use router::{AdminOperation, S3Router}; use s3s::route::S3Route; @@ -80,6 +81,7 @@ fn register_admin_routes(r: &mut S3Router) -> std::io::Result<() bucket_meta::register_bucket_meta_route(r)?; config_admin::register_config_route(r)?; scanner::register_scanner_route(r)?; + usage_prefix::register_usage_prefix_route(r)?; ilm_transition::register_ilm_transition_route(r)?; object_data_cache::register_object_data_cache_route(r)?; audit::register_audit_target_route(r)?; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 474d3ad0f..2c1ad5e1b 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -1558,6 +1558,11 @@ pub const DEFERRED_ADMIN_ROUTE_POLICIES: &[DeferredAdminRoutePolicy] = &[ "/rustfs/admin/v3/datausageinfo", DeferredRoutePolicyReason::MultipleActions, ), + deferred( + HttpMethod::Get, + "/rustfs/admin/v3/usage/{bucket}", + DeferredRoutePolicyReason::MultipleActions, + ), deferred( HttpMethod::Post, "/rustfs/admin/v3/object-zip-downloads", diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index f829b8c94..e81f13d17 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -172,6 +172,7 @@ fn expected_admin_route_matrix() -> Vec { admin_route(Method::POST, "/v4/inspect/archive"), admin_route(Method::GET, "/v3/storageinfo"), admin_route(Method::GET, "/v3/datausageinfo"), + admin_route_sample(Method::GET, "/v3/usage/{bucket}", "/v3/usage/test-bucket"), admin_route(Method::GET, "/v3/metrics"), admin_route(Method::GET, "/v3/object-data-cache/stats"), admin_route(Method::POST, "/v3/object-data-cache/flush"), From beb6e1383e9982dac6a1b06f8e7a2af735336094 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 18 Aug 2026 01:21:15 +0800 Subject: [PATCH 68/71] feat(helm): add TLSRoute passthrough support for gateway api (#6169) Add an optional TLS passthrough listener to the Gateway API support. When gatewayApi.listeners.tls.enabled is true, the Gateway gets a TLS listener with tls.mode: Passthrough and a TLSRoute is rendered to the RustFS service so TLS terminates at the backend (end-to-end encryption). Refs rustfs/rustfs#3862. --- helm/README.md | 6 +++++ helm/rustfs/templates/gateway-api/gateway.yml | 10 ++++++++ .../rustfs/templates/gateway-api/tlsroute.yml | 25 +++++++++++++++++++ helm/rustfs/values.yaml | 6 +++++ 4 files changed, 47 insertions(+) create mode 100644 helm/rustfs/templates/gateway-api/tlsroute.yml diff --git a/helm/README.md b/helm/README.md index 8bdd8b531..79b7c6fd6 100644 --- a/helm/README.md +++ b/helm/README.md @@ -273,6 +273,10 @@ uer. `ClusterIssuer` or `Issuer`. | | gatewayApi.listeners.http.port| int | `8000` | Gateway API http listener port. | | gatewayApi.listeners.https.name | string | `websecure` | Gateway API https listener name. | | gatewayApi.listeners.https.port| int | `8443` | Gateway API https listener port. | +| gatewayApi.listeners.tls.enabled | bool | `false` | Enable a TLS passthrough listener and generate a TLSRoute. | +| gatewayApi.listeners.tls.name | string | `tls` | Gateway API TLS passthrough listener name. | +| gatewayApi.listeners.tls.port | int | `443` | Gateway API TLS passthrough listener port. | +| gatewayApi.listeners.tls.backendPort | int | `null` | Backend service port that terminates TLS; defaults to the console port. | | gatewayApi.hostname | string | Hostname to access RustFS via gateway api. | | gatewayApi.secretName | string | Secret tls to via RustFS using HTTPS. | | gatewayApi.existingGateway.name | string | `""` | The existing gateway name, instead of creating a new one. | @@ -447,6 +451,8 @@ rustfs-route ["example.rustfs.com"] 172m Then, via RustFS instance via `https://example.rustfs.com` or `http://example.rustfs.com`. +For end-to-end encryption, set `gatewayApi.listeners.tls.enabled` to `true`. The chart then adds a `TLS` listener with `tls.mode: Passthrough` to the `Gateway` and generates a `TLSRoute` that forwards the encrypted stream to the RustFS service, where TLS is terminated on the backend side. Note that backend TLS termination must be configured on RustFS itself (for example `RUSTFS_TLS_PATH` pointing to server certificates), and the installed Gateway API CRDs must include `TLSRoute`. + # Uninstall Uninstalling the rustfs installation with command, diff --git a/helm/rustfs/templates/gateway-api/gateway.yml b/helm/rustfs/templates/gateway-api/gateway.yml index 207637b3e..8de988beb 100644 --- a/helm/rustfs/templates/gateway-api/gateway.yml +++ b/helm/rustfs/templates/gateway-api/gateway.yml @@ -26,5 +26,15 @@ spec: - name: {{ include "rustfs.fullname" $ }}-tls kind: Secret {{- end }} + {{- if .tls.enabled }} + - name: {{ .tls.name }} + port: {{ .tls.port }} + protocol: TLS + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: Same + {{- end }} {{- end }} {{- end }} diff --git a/helm/rustfs/templates/gateway-api/tlsroute.yml b/helm/rustfs/templates/gateway-api/tlsroute.yml new file mode 100644 index 000000000..e35f86c06 --- /dev/null +++ b/helm/rustfs/templates/gateway-api/tlsroute.yml @@ -0,0 +1,25 @@ +{{- if and .Values.gatewayApi.enabled .Values.gatewayApi.listeners.tls.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: TLSRoute +metadata: + name: {{ include "rustfs.fullname" . }}-tlsroute + namespace: {{ .Release.Namespace }} +spec: + parentRefs: + {{- if .Values.gatewayApi.existingGateway.name }} + - name: {{ .Values.gatewayApi.existingGateway.name }} + {{- if .Values.gatewayApi.existingGateway.namespace }} + namespace: {{ .Values.gatewayApi.existingGateway.namespace }} + {{- end }} + sectionName: {{ .Values.gatewayApi.listeners.tls.name }} + {{- else }} + - name: {{ include "rustfs.fullname" $ }}-gateway + sectionName: {{ .Values.gatewayApi.listeners.tls.name }} + {{- end }} + hostnames: + - {{ .Values.gatewayApi.hostname }} + rules: + - backendRefs: + - name: {{ include "rustfs.fullname" . }}-svc + port: {{ .Values.gatewayApi.listeners.tls.backendPort | default .Values.service.console.port }} +{{- end }} diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index 19826df96..d57317243 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -369,6 +369,12 @@ gatewayApi: https: name: websecure port: 8443 + tls: # Optional TLS passthrough listener; renders a TLSRoute so TLS terminates at the RustFS backend. + enabled: false + name: tls + port: 443 + # Service port that terminates TLS on the backend; defaults to the console port. + backendPort: null hostname: example.rustfs.com httpToHttpsRedirect: true existingGateway: From 7cb91a019084be368bbcaa750306628c1a127dce Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 07:51:56 +0800 Subject: [PATCH 69/71] chore(ecstore): adjudicate 32 bare dead_code allows (#6173) Replace every bare `#[allow(dead_code)]` in ecstore with either a deletion or a per-item allow carrying a `reason`. Blanket allows at module, struct, and impl level silence the lint for future members too, so each is narrowed to the members that are actually dead. Delete the dead cluster in `config/heal.rs` (`Config`, its three methods, `RUSTFS_BITROT_CYCLE_IN_MONTHS`, `parse_bitrot_config`) rather than annotate it: it has no callers and is unreachable outside the crate, and `parse_bitrot_config` would panic on its disabled path via `Duration::from_secs_f64(-1.0)`. `DEFAULT_KVS` stays, since the config registry uses it. Correct two `reason` strings on `Checksum::new` and `PutObjReader::md5_current_hex_string`, which are methods but carried a field-only rationale. Refs backlog#1823 Co-authored-by: houseme --- .../ecstore/src/bucket/bucket_target_sys.rs | 4 +- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 9 ++- .../bucket/lifecycle/tier_last_day_stats.rs | 5 +- .../src/bucket/lifecycle/tier_sweeper.rs | 10 +++- crates/ecstore/src/bucket/quota/mod.rs | 2 - crates/ecstore/src/client/api_get_object.rs | 4 +- crates/ecstore/src/client/api_get_options.rs | 1 - crates/ecstore/src/client/api_list.rs | 1 - crates/ecstore/src/client/api_put_object.rs | 4 +- crates/ecstore/src/client/api_remove.rs | 5 +- crates/ecstore/src/client/checksum.rs | 6 +- crates/ecstore/src/client/object_api_utils.rs | 3 +- crates/ecstore/src/config/audit.rs | 3 - crates/ecstore/src/config/heal.rs | 59 ------------------- crates/ecstore/src/config/mod.rs | 1 - crates/ecstore/src/core/pools.rs | 12 +++- crates/ecstore/src/disk/local.rs | 2 +- .../ecstore/src/services/rebalance/types.rs | 1 - .../ecstore/src/services/tier/tier_config.rs | 17 +++++- crates/ecstore/src/services/tier/tier_gen.rs | 1 - 20 files changed, 59 insertions(+), 91 deletions(-) diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 0a9b0f41c..00cf124f1 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -1549,8 +1549,8 @@ impl Default for PutObjectOptions { } } -#[allow(dead_code)] impl PutObjectOptions { + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag(&mut self, etag: &str) { if etag == "*" { self.custom_header @@ -1561,6 +1561,7 @@ impl PutObjectOptions { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag_except(&mut self, etag: &str) { if etag == "*" { self.custom_header @@ -1696,6 +1697,7 @@ impl PutObjectOptions { header } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn validate(&self, _c: Arc) -> Result<(), std::io::Error> { //if self.checksum.is_set() { /*if !self.trailing_header_support { diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 0366551e0..ec5aaf74e 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -456,16 +456,23 @@ impl<'a> LifecycleExpiryTrace<'a> { } } -#[allow(dead_code)] impl ExpiryStats { pub fn missed_tasks(&self) -> i64 { self.missed_expiry_tasks.load(Ordering::SeqCst) } + #[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" + )] fn missed_free_vers_tasks(&self) -> i64 { self.missed_freevers_tasks.load(Ordering::SeqCst) } + #[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" + )] fn missed_tier_journal_tasks(&self) -> i64 { self.missed_tier_journal_tasks.load(Ordering::SeqCst) } diff --git a/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs index b32fb32f5..2d8d6c2ea 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs @@ -80,7 +80,10 @@ impl LastDayTierStats { } } - #[allow(dead_code)] + #[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" + )] fn merge(&self, m: LastDayTierStats) -> LastDayTierStats { let mut cl = self.clone(); let mut cm = m; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index 2ce68fd03..0bde1a23c 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -177,9 +177,10 @@ fn should_record_remote_delete_failure(err: &std::io::Error) -> bool { } #[derive(Default)] -#[allow(dead_code)] struct ObjSweeper { + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] object: String, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] bucket: String, version_id: Option, versioned: bool, @@ -191,9 +192,9 @@ struct ObjSweeper { remote_object: String, } -#[allow(dead_code)] impl ObjSweeper { #[allow(clippy::new_ret_no_self)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub async fn new(bucket: &str, object: &str) -> Result { Ok(Self { object: object.into(), @@ -202,17 +203,20 @@ impl ObjSweeper { }) } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn with_version(&mut self, vid: Option) -> &Self { self.version_id = vid.clone(); self } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self { self.versioned = versioned; self.suspended = suspended; self } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn get_opts(&self) -> lifecycle::ObjectOpts { let mut opts = ObjectOpts { version_id: self.version_id.clone(), @@ -226,6 +230,7 @@ impl ObjSweeper { opts } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn set_transition_state(&mut self, info: TransitionedObject) { self.transition_tier = info.tier; self.transition_status = info.status; @@ -266,6 +271,7 @@ impl ObjSweeper { None } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub async fn sweep(&self, api: Arc) { let Some(je) = self.should_remove_remote_object() else { return; diff --git a/crates/ecstore/src/bucket/quota/mod.rs b/crates/ecstore/src/bucket/quota/mod.rs index 0ce5a3a52..3fed86728 100644 --- a/crates/ecstore/src/bucket/quota/mod.rs +++ b/crates/ecstore/src/bucket/quota/mod.rs @@ -312,9 +312,7 @@ mod tests { } #[derive(Deserialize)] struct LegacyBucketQuota { - #[allow(dead_code)] quota: Option, - #[allow(dead_code)] quota_type: LegacyQuotaType, } let legacy = serde_json::from_slice::(&json) diff --git a/crates/ecstore/src/client/api_get_object.rs b/crates/ecstore/src/client/api_get_object.rs index a9c69aad0..1eb6122ca 100644 --- a/crates/ecstore/src/client/api_get_object.rs +++ b/crates/ecstore/src/client/api_get_object.rs @@ -95,7 +95,6 @@ impl TransitionClient { } #[derive(Default)] -#[allow(dead_code)] pub struct GetRequest { pub buffer: Vec, pub offset: i64, @@ -107,11 +106,12 @@ pub struct GetRequest { pub setting_object_info: bool, } -#[allow(dead_code)] pub struct GetResponse { pub size: i64, //pub error: error, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] pub did_read: bool, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] pub object_info: ObjectInfo, } diff --git a/crates/ecstore/src/client/api_get_options.rs b/crates/ecstore/src/client/api_get_options.rs index 622c5a4c2..503b44f3a 100644 --- a/crates/ecstore/src/client/api_get_options.rs +++ b/crates/ecstore/src/client/api_get_options.rs @@ -27,7 +27,6 @@ use tracing::warn; use crate::client::api_error_response::err_invalid_argument; #[derive(Default)] -#[allow(dead_code)] pub struct AdvancedGetOptions { pub replication_delete_marker: bool, pub is_replication_ready_for_delete_marker: bool, diff --git a/crates/ecstore/src/client/api_list.rs b/crates/ecstore/src/client/api_list.rs index 6bd8591c3..a7f894a00 100644 --- a/crates/ecstore/src/client/api_list.rs +++ b/crates/ecstore/src/client/api_list.rs @@ -360,7 +360,6 @@ impl TransitionClient { } #[derive(Default)] -#[allow(dead_code)] pub struct ListObjectsOptions { reverse_versions: bool, with_versions: bool, diff --git a/crates/ecstore/src/client/api_put_object.rs b/crates/ecstore/src/client/api_put_object.rs index 5fbc3fd2c..bf9efe4a2 100644 --- a/crates/ecstore/src/client/api_put_object.rs +++ b/crates/ecstore/src/client/api_put_object.rs @@ -137,8 +137,8 @@ impl Default for PutObjectOptions { } } -#[allow(dead_code)] impl PutObjectOptions { + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag(&mut self, etag: &str) { if etag == "*" { self.custom_header.insert("If-Match", HeaderValue::from_static("*")); @@ -149,6 +149,7 @@ impl PutObjectOptions { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag_except(&mut self, etag: &str) { if etag == "*" { self.custom_header.insert("If-None-Match", HeaderValue::from_static("*")); @@ -259,6 +260,7 @@ impl PutObjectOptions { header } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> { //if self.checksum.is_set() { /*if !self.trailing_header_support { diff --git a/crates/ecstore/src/client/api_remove.rs b/crates/ecstore/src/client/api_remove.rs index 25c71d80a..573a2eed1 100644 --- a/crates/ecstore/src/client/api_remove.rs +++ b/crates/ecstore/src/client/api_remove.rs @@ -55,7 +55,6 @@ pub struct RemoveBucketOptions { const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024; #[derive(Debug)] -#[allow(dead_code)] pub struct AdvancedRemoveOptions { pub replication_delete_marker: bool, pub replication_status: ReplicationStatus, @@ -465,10 +464,10 @@ impl TransitionClient { } #[derive(Debug, Default)] -#[allow(dead_code)] pub struct RemoveObjectError { + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] object_name: String, - #[allow(dead_code)] + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] version_id: String, err: Option, } diff --git a/crates/ecstore/src/client/checksum.rs b/crates/ecstore/src/client/checksum.rs index c71394210..7bb96a6a3 100644 --- a/crates/ecstore/src/client/checksum.rs +++ b/crates/ecstore/src/client/checksum.rs @@ -372,8 +372,8 @@ pub struct Checksum { computed: bool, } -#[allow(dead_code)] impl Checksum { + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn new(t: ChecksumMode, b: &[u8]) -> Checksum { if t.is_set() && b.len() == t.raw_byte_len() { return Checksum { @@ -385,7 +385,7 @@ impl Checksum { Checksum::default() } - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn new_checksum_string(t: ChecksumMode, s: &str) -> Result { let b = match base64_decode(s.as_bytes()) { Ok(b) => b, @@ -412,7 +412,7 @@ impl Checksum { base64_encode(&self.r) } - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn raw(&self) -> Option> { if !self.is_set() { return None; diff --git a/crates/ecstore/src/client/object_api_utils.rs b/crates/ecstore/src/client/object_api_utils.rs index 484233fad..b4bd3e0b5 100644 --- a/crates/ecstore/src/client/object_api_utils.rs +++ b/crates/ecstore/src/client/object_api_utils.rs @@ -37,16 +37,17 @@ pub struct PutObjReader { //pub sealMD5Fn: SealMD5CurrFn, } -#[allow(dead_code)] impl PutObjReader { pub fn new(reader: HashReader) -> Self { Self { reader } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn md5_current_hex_string(&self) -> String { self.reader.checksum().map(|v| v.encoded).unwrap_or_default() } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> { self.reader = enc_reader; diff --git a/crates/ecstore/src/config/audit.rs b/crates/ecstore/src/config/audit.rs index b7ed83323..63c63b700 100644 --- a/crates/ecstore/src/config/audit.rs +++ b/crates/ecstore/src/config/audit.rs @@ -39,7 +39,6 @@ use rustfs_config::{ }; use std::sync::LazyLock; -#[allow(dead_code)] #[allow(clippy::declare_interior_mutable_const)] /// Default KVS for audit webhook settings. pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock = LazyLock::new(|| { @@ -117,7 +116,6 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock = LazyLock::new(|| { ]) }); -#[allow(dead_code)] #[allow(clippy::declare_interior_mutable_const)] /// Default KVS for audit MQTT settings. pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock = LazyLock::new(|| { @@ -375,7 +373,6 @@ pub static DEFAULT_AUDIT_NATS_KVS: LazyLock = LazyLock::new(|| { ]) }); -#[allow(dead_code)] pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock = LazyLock::new(|| { KVS(vec![ KV { diff --git a/crates/ecstore/src/config/heal.rs b/crates/ecstore/src/config/heal.rs index 4505d3ce0..8ef41efa1 100644 --- a/crates/ecstore/src/config/heal.rs +++ b/crates/ecstore/src/config/heal.rs @@ -12,12 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::error::{Error, Result}; use rustfs_config::server_config::{KV, KVS}; use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE}; -use rustfs_utils::string::parse_bool; use std::sync::LazyLock; -use std::time::Duration; pub static DEFAULT_KVS: LazyLock = LazyLock::new(|| { KVS(vec![KV { @@ -26,59 +23,3 @@ pub static DEFAULT_KVS: LazyLock = LazyLock::new(|| { hidden_if_empty: false, }]) }); - -#[derive(Debug, Default)] -pub struct Config { - pub bitrot: String, - pub sleep: Duration, - pub io_count: usize, - pub drive_workers: usize, - pub cache: Duration, -} - -impl Config { - pub fn bitrot_scan_cycle(&self) -> Duration { - self.cache - } - - pub fn get_workers(&self) -> usize { - self.drive_workers - } - - pub fn update(&mut self, nopts: &Config) { - self.bitrot = nopts.bitrot.clone(); - self.io_count = nopts.io_count; - self.sleep = nopts.sleep; - self.drive_workers = nopts.drive_workers; - } -} - -const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1; - -fn parse_bitrot_config(s: &str) -> Result { - match parse_bool(s) { - Ok(enabled) => { - if enabled { - Ok(Duration::from_secs_f64(0.0)) - } else { - Ok(Duration::from_secs_f64(-1.0)) - } - } - Err(_) => { - if !s.ends_with("m") { - return Err(Error::other("unknown format")); - } - - match s.trim_end_matches('m').parse::() { - Ok(months) => { - if months < RUSTFS_BITROT_CYCLE_IN_MONTHS { - return Err(Error::other(format!("minimum bitrot cycle is {RUSTFS_BITROT_CYCLE_IN_MONTHS} month(s)"))); - } - - Ok(Duration::from_secs(months * 30 * 24 * 60)) - } - Err(err) => Err(Error::other(err)), - } - } - } -} diff --git a/crates/ecstore/src/config/mod.rs b/crates/ecstore/src/config/mod.rs index 5e6e3a3e3..ff97d03e0 100644 --- a/crates/ecstore/src/config/mod.rs +++ b/crates/ecstore/src/config/mod.rs @@ -16,7 +16,6 @@ mod audit; pub mod com; -#[allow(dead_code)] pub mod heal; mod notify; mod oidc; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index f35017760..ec9da3708 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -1996,11 +1996,11 @@ impl PoolMeta { Ok(false) } - #[allow(dead_code)] pub fn validate(&self, pools: Vec>) -> Result { struct PoolInfo { position: usize, completed: bool, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] decom_started: bool, } @@ -4958,13 +4958,19 @@ fn is_disk_online_state(state: &str) -> bool { } #[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")] -#[allow(dead_code)] +#[allow( + dead_code, + reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)" +)] fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize { fallback_total_capacity_dedup(disks) } #[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")] -#[allow(dead_code)] +#[allow( + dead_code, + reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)" +)] fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize { fallback_free_capacity_dedup(disks) } diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 2cab189aa..1932744e8 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -6562,7 +6562,7 @@ impl LocalDisk { Ok(f) } - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn get_metrics(&self) -> DiskMetrics { DiskMetrics::default() } diff --git a/crates/ecstore/src/services/rebalance/types.rs b/crates/ecstore/src/services/rebalance/types.rs index b43e075b8..5f79275dc 100644 --- a/crates/ecstore/src/services/rebalance/types.rs +++ b/crates/ecstore/src/services/rebalance/types.rs @@ -132,7 +132,6 @@ impl RebalanceStopPropagationRecord { } } -#[allow(dead_code)] #[derive(Debug, Clone, Default)] pub struct DiskStat { pub total_space: u64, diff --git a/crates/ecstore/src/services/tier/tier_config.rs b/crates/ecstore/src/services/tier/tier_config.rs index a5be98866..149f96ef6 100644 --- a/crates/ecstore/src/services/tier/tier_config.rs +++ b/crates/ecstore/src/services/tier/tier_config.rs @@ -16,8 +16,16 @@ use serde::{Deserialize, Serialize}; use std::{fmt::Display, io}; use tracing::info; +#[allow( + dead_code, + reason = "tier config wire version stamped by the parity constructors below (backlog#1823)" +)] const C_TIER_CONFIG_VER: &str = "v1"; +#[allow( + dead_code, + reason = "tier-name validation message reached only from the parity constructors below (backlog#1823)" +)] const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty"; const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com"; const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[ @@ -264,7 +272,6 @@ impl Clone for TierConfig { } } -#[allow(dead_code)] impl TierConfig { pub(crate) fn clone_with_credentials(&self) -> Self { Self { @@ -284,6 +291,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn endpoint(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(), @@ -303,6 +311,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn bucket(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(), @@ -322,6 +331,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn prefix(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(), @@ -341,6 +351,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn region(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(), @@ -457,7 +468,7 @@ impl TierWasabi { } impl TierS3 { - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn create( name: &str, access_key: &str, @@ -528,7 +539,7 @@ pub struct TierMinIO { } impl TierMinIO { - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn create( name: &str, endpoint: &str, diff --git a/crates/ecstore/src/services/tier/tier_gen.rs b/crates/ecstore/src/services/tier/tier_gen.rs index 400466b47..63a692b85 100644 --- a/crates/ecstore/src/services/tier/tier_gen.rs +++ b/crates/ecstore/src/services/tier/tier_gen.rs @@ -14,7 +14,6 @@ use crate::services::tier::tier::TierConfigMgr; -#[allow(dead_code)] impl TierConfigMgr { pub fn msg_size(&self) -> usize { 100 From 360bceafced5bd63a39f013f7fc30aef162c50f2 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 08:29:29 +0800 Subject: [PATCH 70/71] feat(heal): add progress and trace observability (#6179) * feat(heal): track erasure set progress baseline Record erasure-set heal byte progress from per-object results and seed progress totals from complete usage-cache snapshots when available. Keep usage-cache failures observational so heal execution continues without a baseline. Co-Authored-By: heihutu * feat(heal): skip filtered erasure set versions Skip erasure-set versions written after the durable heal start time, and queue lifecycle-expired versions for expiry before skipping them. Track new-version and ILM-expired skips separately so progress can explain completed baseline work without treating these skips as retry-blocking failures. Co-Authored-By: heihutu * feat(heal): wire abandoned data-dir cleanup check Connect check_abandoned_parts through ECStore, pool, and set layers so heal can invoke the existing orphan data-dir reclaim path instead of returning NotImplemented. Add dry-run support to the reclaim scan and cover dry-run plus scoped set behavior with regression tests. Co-Authored-By: heihutu * feat(obs): add heal scanner trace bus Introduce an in-process broadcast trace bus with typed heal and scanner events, lazy event construction, and bounded lagged-subscriber behavior. Cover zero-subscriber publishing, subscription delivery, drop accounting, and lagged receivers with focused common-crate tests. Co-Authored-By: heihutu * feat(obs): stream heal trace events from admin API Wire the admin trace endpoint to the common trace bus for heal/scanner events, including kind, regex, and threshold filtering. Co-Authored-By: heihutu * feat(obs): emit heal trace events Publish heal task lifecycle and abandoned-parts cleanup events through the common trace bus so the admin trace stream has live heal diagnostics. Co-Authored-By: heihutu * feat(obs): emit scanner trace events Publish scanner folder, lifecycle action, and heal-candidate events through the common trace bus for live admin scanner diagnostics. Co-Authored-By: heihutu * fix(heal): route data usage loader through storage api Keep ECStore data-usage facade access behind the heal storage_api boundary so architecture migration guards can validate the heal progress path. Co-Authored-By: heihutu * perf(heal): avoid lifecycle snapshots on ordinary heal pages Only request lifecycle object snapshots when the heal pass has lifecycle expiry context. This keeps ordinary listing and disk-walk pages from cloning FileInfo/ObjectInfo payloads while preserving the skip path that queues expired versions. Co-Authored-By: heihutu * test(heal): update bug-fix mocks for lifecycle snapshots Carry the lifecycle snapshot opt-in argument through the remaining heal bug-fix test mocks so all-targets clippy covers the updated storage trait. Co-Authored-By: heihutu * test(rustfs): sync heal storage mock signature Update the rustfs storage RPC test mock for the lifecycle snapshot opt-in argument and cover it with rustfs all-targets clippy. Co-Authored-By: heihutu * test(e2e): allocate smoke ports across nextest processes Serialize E2E port selection with a small /tmp allocator so nextest workers do not reuse the same just-released ephemeral port before RustFS binds it. Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- Cargo.lock | 1 + crates/common/Cargo.toml | 1 + crates/common/src/lib.rs | 1 + crates/common/src/trace_bus.rs | 333 ++++++++++++++ crates/e2e_test/src/common.rs | 83 +++- crates/ecstore/src/api/mod.rs | 1 + crates/ecstore/src/bucket/lifecycle/mod.rs | 2 +- crates/ecstore/src/core/pools.rs | 91 ++++ crates/ecstore/src/core/sets.rs | 29 +- .../src/set_disk/core/io_primitives.rs | 22 + crates/ecstore/src/set_disk/mod.rs | 109 ++++- crates/ecstore/src/set_disk/ops/heal.rs | 61 ++- crates/ecstore/src/set_disk/ops/heal_walk.rs | 50 ++- crates/ecstore/src/store/heal.rs | 42 +- crates/ecstore/src/store/heal_walk.rs | 3 +- crates/heal/src/heal/channel.rs | 1 + crates/heal/src/heal/erasure_healer.rs | 344 ++++++++++++-- crates/heal/src/heal/manager.rs | 30 ++ crates/heal/src/heal/progress.rs | 166 ++++++- crates/heal/src/heal/storage.rs | 176 +++++++- crates/heal/src/heal/storage_api.rs | 13 +- crates/heal/src/heal/task.rs | 238 +++++++++- crates/heal/src/lib.rs | 1 + .../heal_b5_versioned_regression_test.rs | 2 +- .../tests/heal_b920_subquorum_union_test.rs | 6 +- crates/heal/tests/heal_bug_fixes_test.rs | 2 + crates/madmin/src/service_commands.rs | 10 +- crates/scanner/src/scanner_folder.rs | 254 +++++++++++ rustfs/src/admin/handlers/profile_admin.rs | 422 ++++++++++++++++-- rustfs/src/storage/rpc/node_service.rs | 1 + 30 files changed, 2383 insertions(+), 112 deletions(-) create mode 100644 crates/common/src/trace_bus.rs diff --git a/Cargo.lock b/Cargo.lock index b521a731d..85c91d523 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9280,6 +9280,7 @@ dependencies = [ "s3s", "serde", "serde_json", + "smallvec", "tokio", "tonic", "tracing", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 5c9094d61..ddc02cf2e 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -42,6 +42,7 @@ chrono = { workspace = true, features = ["serde"] } jiff = { workspace = true, features = ["serde"] } metrics = { workspace = true } serde = { workspace = true, features = ["derive"] } +smallvec = { workspace = true } rmp-serde = { workspace = true } s3s = { workspace = true, features = ["minio"] } tracing = { workspace = true } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 09240e25b..1ae200f24 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -19,6 +19,7 @@ pub mod last_minute; pub mod metrics; mod readiness; pub mod table_catalog; +pub mod trace_bus; pub use globals::*; pub use readiness::{GlobalReadiness, SystemStage}; diff --git a/crates/common/src/trace_bus.rs b/crates/common/src/trace_bus.rs new file mode 100644 index 000000000..5e6a90d60 --- /dev/null +++ b/crates/common/src/trace_bus.rs @@ -0,0 +1,333 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use smallvec::SmallVec; +use std::{ + sync::{ + Arc, OnceLock, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, SystemTime}, +}; +use tokio::sync::broadcast; + +const DEFAULT_TRACE_BUS_CAPACITY: usize = 1024; +const TRACE_ATTR_INLINE_CAPACITY: usize = 8; + +static GLOBAL_TRACE_BUS: OnceLock = OnceLock::new(); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TraceKind { + Heal, + Scanner, +} + +impl TraceKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Heal => "heal", + Self::Scanner => "scanner", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TraceFunc { + HealTask, + HealBucket, + HealObject, + HealCheckAbandonedParts, + HealErasureSetPage, + ScannerFolder, + ScannerIlmAction, + ScannerHealCandidate, + Dropped, +} + +impl TraceFunc { + pub const fn as_str(self) -> &'static str { + match self { + Self::HealTask => "heal.Task", + Self::HealBucket => "heal.Bucket", + Self::HealObject => "heal.Object", + Self::HealCheckAbandonedParts => "heal.CheckAbandonedParts", + Self::HealErasureSetPage => "heal.ErasureSetPage", + Self::ScannerFolder => "scanner.Folder", + Self::ScannerIlmAction => "scanner.IlmAction", + Self::ScannerHealCandidate => "scanner.HealCandidate", + Self::Dropped => "trace.Dropped", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TraceVal { + Bool(bool), + U64(u64), + I64(i64), + Str(Arc), +} + +impl From for TraceVal { + fn from(value: bool) -> Self { + Self::Bool(value) + } +} + +impl From for TraceVal { + fn from(value: u64) -> Self { + Self::U64(value) + } +} + +impl From for TraceVal { + fn from(value: i64) -> Self { + Self::I64(value) + } +} + +impl From<&str> for TraceVal { + fn from(value: &str) -> Self { + Self::Str(Arc::from(value)) + } +} + +impl From for TraceVal { + fn from(value: String) -> Self { + Self::Str(Arc::from(value)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TraceAttr { + pub key: &'static str, + pub value: TraceVal, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TraceEvent { + pub kind: TraceKind, + pub func: TraceFunc, + pub time: SystemTime, + pub bucket: Option>, + pub object: Option>, + pub duration: Duration, + pub bytes: u64, + pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>, +} + +impl TraceEvent { + pub fn new(kind: TraceKind, func: TraceFunc) -> Self { + Self { + kind, + func, + time: SystemTime::now(), + bucket: None, + object: None, + duration: Duration::ZERO, + bytes: 0, + attrs: SmallVec::new(), + } + } + + pub fn with_bucket(mut self, bucket: impl Into>) -> Self { + self.bucket = Some(bucket.into()); + self + } + + pub fn with_object(mut self, object: impl Into>) -> Self { + self.object = Some(object.into()); + self + } + + pub fn with_duration(mut self, duration: Duration) -> Self { + self.duration = duration; + self + } + + pub fn with_bytes(mut self, bytes: u64) -> Self { + self.bytes = bytes; + self + } + + pub fn with_attr(mut self, key: &'static str, value: impl Into) -> Self { + self.attrs.push(TraceAttr { + key, + value: value.into(), + }); + self + } +} + +#[derive(Debug)] +pub struct TraceBus { + sender: broadcast::Sender>, + subscriber_count: Arc, +} + +impl TraceBus { + pub fn new(capacity: usize) -> Self { + let capacity = capacity.max(1); + let (sender, _receiver) = broadcast::channel(capacity); + Self { + sender, + subscriber_count: Arc::new(AtomicUsize::new(0)), + } + } + + pub fn subscriber_count(&self) -> usize { + self.subscriber_count.load(Ordering::Acquire) + } + + pub fn subscribe(&self) -> TraceSubscription { + let receiver = self.sender.subscribe(); + self.subscriber_count.fetch_add(1, Ordering::AcqRel); + TraceSubscription { + receiver, + subscriber_count: Arc::clone(&self.subscriber_count), + } + } + + pub fn emit(&self, build: impl FnOnce() -> TraceEvent) -> bool { + if self.subscriber_count() == 0 { + return false; + } + + self.sender.send(Arc::new(build())).is_ok() + } +} + +impl Default for TraceBus { + fn default() -> Self { + Self::new(DEFAULT_TRACE_BUS_CAPACITY) + } +} + +#[derive(Debug)] +pub struct TraceSubscription { + receiver: broadcast::Receiver>, + subscriber_count: Arc, +} + +impl TraceSubscription { + pub async fn recv(&mut self) -> Result, broadcast::error::RecvError> { + self.receiver.recv().await + } + + pub fn try_recv(&mut self) -> Result, broadcast::error::TryRecvError> { + self.receiver.try_recv() + } +} + +impl Drop for TraceSubscription { + fn drop(&mut self) { + self.subscriber_count.fetch_sub(1, Ordering::AcqRel); + } +} + +pub fn global_trace_bus() -> &'static TraceBus { + GLOBAL_TRACE_BUS.get_or_init(TraceBus::default) +} + +pub fn subscribe_trace_events() -> TraceSubscription { + global_trace_bus().subscribe() +} + +pub fn trace_emit(build: impl FnOnce() -> TraceEvent) -> bool { + global_trace_bus().emit(build) +} + +pub fn trace_subscriber_count() -> usize { + global_trace_bus().subscriber_count() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicUsize; + + #[test] + fn trace_emit_skips_builder_without_subscribers() { + let bus = TraceBus::new(4); + let built = AtomicUsize::new(0); + + let sent = bus.emit(|| { + built.fetch_add(1, Ordering::Relaxed); + TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask) + }); + + assert!(!sent); + assert_eq!(built.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn trace_subscriber_receives_event() { + let bus = TraceBus::new(4); + let mut subscription = bus.subscribe(); + + assert!(bus.emit(|| { + TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject) + .with_bucket("bucket") + .with_object("object") + .with_duration(Duration::from_millis(7)) + .with_bytes(11) + .with_attr("dry", true) + })); + + let event = subscription + .recv() + .await + .expect("subscriber should receive emitted trace event"); + + assert_eq!(event.kind, TraceKind::Heal); + assert_eq!(event.func, TraceFunc::HealObject); + assert_eq!(event.bucket.as_deref(), Some("bucket")); + assert_eq!(event.object.as_deref(), Some("object")); + assert_eq!(event.duration, Duration::from_millis(7)); + assert_eq!(event.bytes, 11); + assert_eq!( + event.attrs.as_slice(), + &[TraceAttr { + key: "dry", + value: TraceVal::Bool(true) + }] + ); + } + + #[test] + fn trace_subscription_drop_decrements_count() { + let bus = TraceBus::new(4); + let subscription = bus.subscribe(); + + assert_eq!(bus.subscriber_count(), 1); + drop(subscription); + assert_eq!(bus.subscriber_count(), 0); + } + + #[tokio::test] + async fn lagged_subscriber_drops_events_without_blocking_publishers() { + let bus = TraceBus::new(2); + let mut subscription = bus.subscribe(); + + for index in 0_u64..4 { + assert!(bus.emit(|| { TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder).with_attr("index", index) })); + } + + let err = subscription + .recv() + .await + .expect_err("receiver should observe lag instead of blocking publishers"); + assert!(matches!(err, broadcast::error::RecvError::Lagged(_))); + } +} diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index f1fcaa20a..460c764ed 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -32,6 +32,7 @@ use rustfs_signer::sign_v4; use s3s::Body; use std::ffi::OsStr; use std::fs as stdfs; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::sync::Once; @@ -51,6 +52,11 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] = &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")]; pub const TEST_BUCKET: &str = "e2e-test-bucket"; const RUSTFS_FULL_FEATURE: &str = "full"; +const TEST_PORT_MIN: u16 = 20_000; +const TEST_PORT_RANGE: u16 = 40_000; +const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port"; +const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock"; +const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30); fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option { let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy(); @@ -67,6 +73,64 @@ fn configured_capture_log_path(temp_dir: &str) -> Option { capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned()) } +struct PortAllocatorGuard; + +impl PortAllocatorGuard { + async fn acquire() -> Result> { + loop { + match stdfs::create_dir(TEST_PORT_LOCK_DIR) { + Ok(()) => return Ok(Self), + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + remove_stale_port_allocator_lock(); + sleep(Duration::from_millis(10)).await; + } + Err(err) => return Err(err.into()), + } + } + } +} + +impl Drop for PortAllocatorGuard { + fn drop(&mut self) { + let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR); + } +} + +fn advance_test_port(port: u16) -> u16 { + let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE; + TEST_PORT_MIN + offset +} + +fn seeded_test_port() -> u16 { + let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16; + TEST_PORT_MIN + offset +} + +fn read_next_test_port() -> u16 { + stdfs::read_to_string(TEST_PORT_COUNTER_PATH) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port)) + .unwrap_or_else(seeded_test_port) +} + +fn remove_stale_port_allocator_lock() { + let Ok(metadata) = stdfs::metadata(TEST_PORT_LOCK_DIR) else { + return; + }; + let Ok(modified) = metadata.modified() else { + return; + }; + if modified.elapsed().is_ok_and(|elapsed| elapsed > TEST_PORT_LOCK_STALE_AFTER) { + let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR); + } +} + +fn write_next_test_port(port: u16) -> Result<(), Box> { + stdfs::write(TEST_PORT_COUNTER_PATH, port.to_string())?; + Ok(()) +} + pub(crate) fn capture_command_logs( command: &mut Command, log_path: Option<&str>, @@ -508,10 +572,21 @@ impl RustFSTestEnvironment { /// Find an available port for the test pub async fn find_available_port() -> Result> { use std::net::TcpListener; - let listener = TcpListener::bind("127.0.0.1:0")?; - let port = listener.local_addr()?.port(); - drop(listener); - Ok(port) + let _guard = PortAllocatorGuard::acquire().await?; + let mut next_port = read_next_test_port(); + + for _ in 0..TEST_PORT_RANGE { + let port = next_port; + next_port = advance_test_port(next_port); + write_next_test_port(next_port)?; + + if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) { + drop(listener); + return Ok(port); + } + } + + Err("no available E2E test port found".into()) } /// Kill any existing RustFS processes diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 5526b7d00..4b8bc3249 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -483,6 +483,7 @@ pub mod store_list { } pub mod storage { + pub use crate::core::pools::HealLifecycleExpiryContext; pub use crate::store::HealWalkVersion; pub use crate::store::{ ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks, diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index 823263072..6d8e64f1b 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -19,7 +19,7 @@ pub mod core; pub mod evaluator; pub mod manual_transition_job; mod metadata_boundary; -pub(crate) use metadata_boundary::get_expiry_configs; +pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs}; mod object_lock_boundary; pub use self::core as lifecycle; mod replication_sink; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index ec9da3708..02fe70f29 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -16,6 +16,7 @@ use crate::bucket::replication::replication_state_from_filemeta; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::bucket::{ lifecycle::{ + LifecycleExpiryConfigs, bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::{ LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle, @@ -2335,6 +2336,10 @@ fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool { ) } +fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool { + action.delete() +} + fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result { if !apply_actions || applied { return Ok(true); @@ -2385,7 +2390,80 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement( } } +pub struct HealLifecycleExpiryContext { + configs: LifecycleExpiryConfigs, +} + impl ECStore { + pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result> { + if bucket == RUSTFS_META_BUCKET { + return Ok(None); + } + + let configs = get_expiry_configs(self, bucket).await?; + if configs.lifecycle.is_none() { + return Ok(None); + } + + Ok(Some(HealLifecycleExpiryContext { configs })) + } + + pub async fn enqueue_heal_lifecycle_expiry( + self: &Arc, + context: &HealLifecycleExpiryContext, + bucket: &str, + object: &str, + version_id: Option<&str>, + object_info: Option<&crate::object_api::ObjectInfo>, + ) -> Result { + let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else { + return Ok(false); + }; + + let object_info = if let Some(object_info) = object_info { + if object_info.bucket != bucket || object_info.name != object { + return Ok(false); + } + let snapshot_version_id = object_info + .version_id + .filter(|version_id| !version_id.is_nil()) + .map(|version_id| version_id.to_string()); + if snapshot_version_id.as_deref() != version_id { + return Ok(false); + } + object_info.clone() + } else { + match self + .get_object_info( + bucket, + object, + &ObjectOptions { + version_id: version_id.map(str::to_string), + versioned: version_id.is_some(), + expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id), + ..Default::default() + }, + ) + .await + { + Ok(object_info) => object_info, + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false), + Err(err) => return Err(err), + } + }; + + let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await; + if !lifecycle_action_skips_heal_version(event.action) { + return Ok(false); + } + + if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? { + return Ok(false); + } + + Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await) + } + async fn save_current_pool_meta(&self) -> Result<()> { let _save_guard = self.pool_meta_save_gate.lock().await; let snapshot = { @@ -4287,6 +4365,19 @@ mod tests { )); } + #[test] + fn lifecycle_action_skips_heal_version_for_every_delete_action() { + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction)); + assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction)); + assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction)); + assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction)); + } + #[test] fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() { let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false) diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index acb8b53b9..1d1bcedeb 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -1140,11 +1140,11 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets { Err(Error::DiskNotFound) } - #[tracing::instrument(skip(self))] - async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { - // Multipart orphan reconciliation is intentionally retained above the pool/set layers - // until there is a concrete caller and a stable lower-level contract to implement. - Err(StorageError::NotImplemented) + #[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))] + async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { + self.get_disks_for_heal_object(object, opts)? + .check_abandoned_parts(bucket, object, opts) + .await } } @@ -1996,7 +1996,7 @@ mod tests { } #[tokio::test] - async fn sets_check_abandoned_parts_returns_typed_not_implemented_error() { + async fn sets_check_abandoned_parts_rejects_invalid_set_scope() { let format = FormatV3::new(1, 1); let sets = Sets { id: format.id, @@ -2021,10 +2021,21 @@ mod tests { }; let err = sets - .check_abandoned_parts("bucket", "object", &HealOpts::default()) + .check_abandoned_parts( + "bucket", + "object", + &HealOpts { + set: Some(1), + ..Default::default() + }, + ) .await - .expect_err("abandoned-parts ownership should stay above the pool/set storage layers"); - assert!(matches!(err, StorageError::NotImplemented)); + .expect_err("out-of-range abandoned-parts set scope must fail closed"); + assert!( + matches!(err, StorageError::InvalidArgument(_, ref field, ref reason) + if field == "set" && reason.contains("invalid heal set index 1")), + "unexpected invalid set error: {err:?}" + ); } // Builds a single-set `Sets` over `SET_DRIVE_COUNT` local temp-dir disks, diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 41bd6dac7..7af25a64c 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -4860,6 +4860,14 @@ impl SetDisks { /// is best-effort maintenance: individual delete failures are logged and /// skipped rather than propagated. pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result { + self.reclaim_orphan_data_dirs_inner(bucket, object, false).await + } + + pub(crate) async fn dry_run_reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result { + self.reclaim_orphan_data_dirs_inner(bucket, object, true).await + } + + async fn reclaim_orphan_data_dirs_inner(&self, bucket: &str, object: &str, dry_run: bool) -> disk::error::Result { let disks = self.get_disks_internal().await; // Phase 1 (read-only): build the referenced-data-dir union and record the @@ -4967,6 +4975,20 @@ impl SetDisks { continue; } let stray = format!("{object}/{dir}"); + if dry_run { + removed += 1; + debug!( + target: "rustfs_ecstore::set_disk", + event = "heal_abandoned_parts", + component = "ecstore", + subsystem = "heal", + state = "dry_run_matched", + result = "matched", + bucket, object, data_dir = %dir, + "Heal abandoned parts dry-run matched orphaned data directory" + ); + continue; + } match disk .delete( bucket, diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index a04c5db82..5a56ea36e 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -6998,6 +6998,100 @@ mod tests { assert!(object_dir.join(STORAGE_FORMAT_FILE).exists(), "metadata must be preserved"); } + async fn recv_abandoned_parts_trace( + trace: &mut rustfs_common::trace_bus::TraceSubscription, + bucket: &str, + object: &str, + state: &str, + ) -> rustfs_common::trace_bus::TraceEvent { + for _ in 0..32 { + let event = tokio::time::timeout(std::time::Duration::from_secs(1), trace.recv()) + .await + .expect("abandoned-parts trace event should arrive") + .expect("trace bus should stay open"); + if event.kind == rustfs_common::trace_bus::TraceKind::Heal + && event.func == rustfs_common::trace_bus::TraceFunc::HealCheckAbandonedParts + && event.bucket.as_deref() == Some(bucket) + && event.object.as_deref() == Some(object) + && trace_attr_string(&event, "state").as_deref() == Some(state) + { + return (*event).clone(); + } + } + + panic!("expected abandoned-parts trace state {state} for {bucket}/{object}"); + } + + fn trace_attr_string(event: &rustfs_common::trace_bus::TraceEvent, key: &str) -> Option { + event.attrs.iter().find_map(|attr| { + if attr.key != key { + return None; + } + Some(match &attr.value { + rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(), + }) + }) + } + + #[tokio::test] + async fn check_abandoned_parts_dry_run_counts_without_deleting() { + let mut trace = rustfs_common::trace_bus::subscribe_trace_events(); + let (dir, disk) = make_single_local_disk().await; + let live = Uuid::new_v4(); + let orphan = Uuid::new_v4(); + + let object_dir = dir.path().join("bucket").join("obj"); + write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await; + fs::create_dir_all(object_dir.join(live.to_string())) + .await + .expect("live data dir should be created"); + fs::create_dir_all(object_dir.join(orphan.to_string())) + .await + .expect("orphan data dir should be created"); + + let set = make_set_disks_with(vec![Some(disk)]).await; + set.check_abandoned_parts( + "bucket", + "obj", + &HealOpts { + dry_run: true, + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("dry-run abandoned-parts check should succeed"); + let dry_run_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "dry_run_matched").await; + assert_eq!(trace_attr_string(&dry_run_trace, "dry_run").as_deref(), Some("true")); + assert_eq!(trace_attr_string(&dry_run_trace, "data_dirs").as_deref(), Some("1")); + + assert!(object_dir.join(live.to_string()).exists(), "referenced data dir must be preserved"); + assert!(object_dir.join(orphan.to_string()).exists(), "dry-run must not remove orphaned data dir"); + + set.check_abandoned_parts( + "bucket", + "obj", + &HealOpts { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("abandoned-parts check should reclaim stale data dir"); + let reclaim_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "reclaimed").await; + assert_eq!(trace_attr_string(&reclaim_trace, "dry_run").as_deref(), Some("false")); + assert_eq!(trace_attr_string(&reclaim_trace, "data_dirs").as_deref(), Some("1")); + + assert!( + object_dir.join(live.to_string()).exists(), + "referenced data dir must remain after reclaim" + ); + assert!(!object_dir.join(orphan.to_string()).exists(), "orphaned data dir must be removed"); + } + #[tokio::test] async fn reclaim_orphan_data_dirs_recovers_deferred_cleanup_after_restart() { let (dir, disk) = make_single_local_disk().await; @@ -12233,11 +12327,18 @@ mod tests { .expect_err("unsupported copy_object_part should return a typed error"); assert!(matches!(copy_part_err, StorageError::NotImplemented)); - let abandoned_err = set_disks - .check_abandoned_parts("bucket", "object", &HealOpts::default()) + set_disks + .check_abandoned_parts( + "bucket", + "object", + &HealOpts { + dry_run: true, + no_lock: true, + ..Default::default() + }, + ) .await - .expect_err("abandoned-parts check should stay in the upper reconciliation layer"); - assert!(matches!(abandoned_err, StorageError::NotImplemented)); + .expect("abandoned-parts check should be callable on empty disk sets"); } #[tokio::test] diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 274b33f43..de688fbdb 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -16,6 +16,7 @@ use super::super::*; use crate::disk::disk_store::DiskStoreRenameDataExt; use crate::io_support::bitrot::object_mmap_read_enabled; use crate::storage_api_contracts::namespace::NamespaceLocking as _; +use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit}; use tracing::trace; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; @@ -2057,11 +2058,61 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { Err(Error::DiskNotFound) } - #[tracing::instrument(skip(self))] - async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { - // Multipart orphan reconciliation is intentionally retained above the set layer - // until there is a concrete caller and a stable lower-level contract to implement. - Err(StorageError::NotImplemented) + #[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))] + async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { + let started_at = std::time::Instant::now(); + let _write_lock_guard = if !opts.no_lock { + let ns_lock = self.new_ns_lock(bucket, object).await?; + Some( + ns_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?, + ) + } else { + None + }; + + let removed = if opts.dry_run { + self.dry_run_reclaim_orphan_data_dirs(bucket, object).await? + } else { + self.reclaim_orphan_data_dirs(bucket, object).await? + }; + let state = if opts.dry_run && removed > 0 { + "dry_run_matched" + } else if removed > 0 { + "reclaimed" + } else { + "checked" + }; + let data_dirs = u64::try_from(removed).unwrap_or(u64::MAX); + + trace_emit(|| { + TraceEvent::new(TraceKind::Heal, TraceFunc::HealCheckAbandonedParts) + .with_bucket(bucket) + .with_object(object) + .with_duration(started_at.elapsed()) + .with_attr("state", state) + .with_attr("dry_run", opts.dry_run) + .with_attr("data_dirs", data_dirs) + }); + + if removed > 0 { + trace!( + event = "heal_abandoned_parts", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, + state = if opts.dry_run { "dry_run_matched" } else { "reclaimed" }, + result = "ok", + bucket, + object, + dry_run = opts.dry_run, + data_dirs = removed, + "Heal abandoned parts checked object data directories" + ); + } + + Ok(()) } } diff --git a/crates/ecstore/src/set_disk/ops/heal_walk.rs b/crates/ecstore/src/set_disk/ops/heal_walk.rs index a39a9abe7..ea1ea66cf 100644 --- a/crates/ecstore/src/set_disk/ops/heal_walk.rs +++ b/crates/ecstore/src/set_disk/ops/heal_walk.rs @@ -23,6 +23,7 @@ //! per-version `SetDisks::heal_object`. use super::super::*; +use crate::object_api::ObjectInfo; use std::collections::HashSet; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -39,12 +40,16 @@ const BACKGROUND_WALKDIR_STALL_TIMEOUT: Duration = Duration::from_secs(60); /// it must not gate healing logic — the delete-marker vs data path is chosen /// inside `ops/heal.rs` from the resolved latest metadata. `version_id` is /// normalized (nil/absent UUID => `None`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct HealWalkVersion { /// object key pub name: String, /// normalized version id (`None` when the version is nil/absent) pub version_id: Option, + /// version modification time as Unix nanoseconds + pub mod_time_unix_nanos: Option, + /// object snapshot for lifecycle evaluation + pub lifecycle_object_info: Option, /// whether this version is a delete marker (observability only) pub is_delete_marker: bool, } @@ -63,6 +68,7 @@ struct HealWalkCollector { bucket: String, batch_objects: usize, version_budget: usize, + include_lifecycle_object_info: bool, objects: Mutex>, decode_error: Mutex>, version_total: AtomicUsize, @@ -116,10 +122,25 @@ impl HealWalkCollector { let mut versions = Vec::with_capacity(fiv.versions.len() + fiv.free_versions.len()); for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) { + let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil()); + let lifecycle_object_info = if self.include_lifecycle_object_info { + let mut lifecycle_fi = fi.clone(); + lifecycle_fi.version_id = version_uuid; + Some(ObjectInfo::from_file_info( + &lifecycle_fi, + &self.bucket, + &entry.name, + version_uuid.is_some(), + )) + } else { + None + }; versions.push(HealWalkVersion { name: entry.name.clone(), // Normalize: nil/absent version id => None. - version_id: fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()), + version_id: version_uuid.map(|u| u.to_string()), + mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()), + lifecycle_object_info, is_delete_marker: fi.deleted, }); } @@ -173,11 +194,26 @@ impl HealWalkCollector { } }; for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) { - let vid = fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()); + let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil()); + let vid = version_uuid.map(|u| u.to_string()); if seen.insert(vid.clone()) { + let lifecycle_object_info = if self.include_lifecycle_object_info { + let mut lifecycle_fi = fi.clone(); + lifecycle_fi.version_id = version_uuid; + Some(ObjectInfo::from_file_info( + &lifecycle_fi, + &self.bucket, + &entry.name, + version_uuid.is_some(), + )) + } else { + None + }; versions.push(HealWalkVersion { name: entry.name.clone(), version_id: vid, + mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()), + lifecycle_object_info, is_delete_marker: fi.deleted, }); } @@ -255,6 +291,7 @@ impl SetDisks { forward_to: Option<&str>, batch_objects: usize, version_budget: usize, + include_lifecycle_object_info: bool, ) -> disk::error::Result<(Vec, Option, bool)> { assert!(batch_objects >= 2, "heal_walk_versions_page requires batch_objects >= 2"); @@ -264,6 +301,7 @@ impl SetDisks { bucket: bucket.to_string(), batch_objects, version_budget: version_budget.max(1), + include_lifecycle_object_info, objects: Mutex::new(Vec::new()), decode_error: Mutex::new(None), version_total: AtomicUsize::new(0), @@ -347,6 +385,7 @@ mod tests { bucket: "bucket".to_string(), batch_objects: 2, version_budget: 2, + include_lifecycle_object_info: false, objects: Mutex::new(Vec::new()), decode_error: Mutex::new(None), version_total: AtomicUsize::new(0), @@ -388,6 +427,8 @@ mod tests { HealWalkVersion { name: name.to_string(), version_id: Some(id.to_string()), + mod_time_unix_nanos: None, + lifecycle_object_info: None, is_delete_marker: dm, } } @@ -491,6 +532,7 @@ mod tests { bucket: "bucket".to_string(), batch_objects: 1000, version_budget: 10_000, + include_lifecycle_object_info: false, objects: Mutex::new(Vec::new()), version_total: AtomicUsize::new(0), decode_error: Mutex::new(None), @@ -567,7 +609,7 @@ mod tests { .expect("corrupt test metadata should be written"); let error = set_disks - .heal_walk_versions_page(bucket, "", None, 2, 2) + .heal_walk_versions_page(bucket, "", None, 2, 2, false) .await .expect_err("semantic metadata corruption must fail the heal disk walk"); diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 3efa4ed5f..d10abe740 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -18,6 +18,7 @@ use tracing::trace; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_HEAL: &str = "heal"; +const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts"; const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed"; const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started"; @@ -256,13 +257,40 @@ impl ECStore { #[instrument(skip(self))] pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { - let _ = (bucket, object, opts); - // Stale multipart reconciliation is already owned by the lifecycle-driven - // background cleanup path in `bucket_lifecycle_ops.rs`. There is currently - // no stable object-heal contract that should fan this request out through - // pool/set storage layers, so keep the placeholder explicit at the ECStore - // boundary instead of dispatching into lower layers. - Err(StorageError::NotImplemented) + let object = encode_dir_object(object); + let pools = self.get_pools_for_heal_object(opts)?; + + let mut futures = Vec::with_capacity(pools.len()); + for pool in pools.iter() { + futures.push(pool.check_abandoned_parts(bucket, &object, opts)); + } + + let mut first_error = None; + for result in join_all(futures).await { + if let Err(err) = result + && first_error.is_none() + { + first_error = Some(err); + } + } + + if let Some(err) = first_error { + return Err(err); + } + + trace!( + event = EVENT_HEAL_ABANDONED_PARTS, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, + state = "completed", + result = "ok", + bucket, + object, + dry_run = opts.dry_run, + "Heal abandoned parts completed" + ); + + Ok(()) } } diff --git a/crates/ecstore/src/store/heal_walk.rs b/crates/ecstore/src/store/heal_walk.rs index 39d50007d..191e87e54 100644 --- a/crates/ecstore/src/store/heal_walk.rs +++ b/crates/ecstore/src/store/heal_walk.rs @@ -34,6 +34,7 @@ impl ECStore { forward_to: Option<&str>, batch_objects: usize, version_budget: usize, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { if pool_idx >= self.pools.len() || set_idx >= self.pools[pool_idx].disk_set.len() { return Err(Error::other(format!( @@ -43,7 +44,7 @@ impl ECStore { } self.pools[pool_idx].disk_set[set_idx] - .heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget) + .heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget, include_lifecycle_object_info) .await .map_err(Error::from) } diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index d2be2545d..23cf2f168 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -767,6 +767,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> crate::Result<(Vec, Option, bool)> { Ok((vec![], None, false)) } diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 8f1810bb6..04075837c 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -23,13 +23,14 @@ use crate::heal::{ }; use crate::{Error, Result}; use futures::{StreamExt, stream::FuturesUnordered}; -use metrics::gauge; +use metrics::{counter, gauge}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode}; use rustfs_madmin::heal_commands::HealResultItem; use std::sync::{ Arc, atomic::{AtomicUsize, Ordering}, }; +use std::time::{Duration, UNIX_EPOCH}; use tokio::sync::{RwLock, Semaphore}; use tracing::{debug, error, warn}; @@ -47,6 +48,21 @@ enum HealObjectOutcome { Failed, } +fn result_object_size_u64(result: &HealResultItem) -> u64 { + u64::try_from(result.object_size).unwrap_or(u64::MAX) +} + +const NEW_VERSION_SKIP_GRACE_SECS: u64 = 60; +const NANOS_PER_SECOND: i128 = 1_000_000_000; + +fn should_skip_new_version(mod_time_unix_nanos: Option, started_at_secs: u64) -> bool { + let Some(mod_time_unix_nanos) = mod_time_unix_nanos else { + return false; + }; + let cutoff_secs = started_at_secs.saturating_add(NEW_VERSION_SKIP_GRACE_SECS); + mod_time_unix_nanos > i128::from(cutoff_secs).saturating_mul(NANOS_PER_SECOND) +} + struct PageConcurrencyGuard { in_flight: Arc, set_label: String, @@ -492,6 +508,7 @@ impl ErasureSetHealer { &mut skipped_objects, resume_manager, checkpoint_manager, + state.start_time, ) .await; @@ -658,6 +675,7 @@ impl ErasureSetHealer { skipped_objects: &mut u64, resume_manager: &ResumeManager, checkpoint_manager: &CheckpointManager, + started_at_secs: u64, ) -> Result<()> { debug!( target: "rustfs::heal::erasure_healer", @@ -710,6 +728,7 @@ impl ErasureSetHealer { // The end-of-pass summary reports the full failed/skipped counts. let mut transient_skip_samples_logged = 0_u64; let mut failure_samples_logged = 0_u64; + let mut bytes_processed = self.progress.read().await.bytes_processed; // backlog#920: select the per-erasure-set DISK-WALK union enumerator when // the scan is Deep OR the request came from AutoHeal — these are the paths @@ -718,17 +737,25 @@ impl ErasureSetHealer { // which stays the default. let use_disk_walk = matches!(self.heal_opts.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal); + let lifecycle_expiry_context = self.storage.load_heal_lifecycle_expiry_context(bucket).await?; + let include_lifecycle_object_info = lifecycle_expiry_context.is_some(); loop { self.verify_replacement_identity_fence("page scan").await?; // Get one page of object versions let (objects, next_token, is_truncated) = if use_disk_walk { self.storage - .list_versions_for_heal_page_disk_walk(set_disk_id, bucket, "", continuation_token.as_deref()) + .list_versions_for_heal_page_disk_walk( + set_disk_id, + bucket, + "", + continuation_token.as_deref(), + include_lifecycle_object_info, + ) .await? } else { self.storage - .list_objects_for_heal_page(bucket, "", continuation_token.as_deref()) + .list_objects_for_heal_page(bucket, "", continuation_token.as_deref(), include_lifecycle_object_info) .await? }; let page_is_empty = objects.is_empty(); @@ -736,6 +763,7 @@ impl ErasureSetHealer { let page_resume_index = *current_object_index; let semaphore = Arc::new(Semaphore::new(page_concurrency_limit)); let mut page_tasks = FuturesUnordered::new(); + let mut completed_in_page = 0usize; // Capture the last version identity of this page for the anti-loop guard. let page_last = objects.last().map(|item| (item.name.clone(), item.version_id.clone())); @@ -751,6 +779,75 @@ impl ErasureSetHealer { continue; } + if should_skip_new_version(item.mod_time_unix_nanos, started_at_secs) { + checkpoint_manager.add_processed_object(key).await?; + *processed_objects = processed_objects.saturating_add(1); + completed_in_page = completed_in_page.saturating_add(1); + counter!("rustfs_heal_skipped_new_versions_total").increment(1); + { + let mut progress = self.progress.write().await; + progress.record_skipped_new_version(); + progress.set_current_object(Some(format!("skipped_new: {bucket}/{}", item.name))); + progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + } + debug!( + target: "rustfs::heal::erasure_healer", + event = EVENT_HEAL_ERASURE_OBJECT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, + set_disk_id, + bucket, + object = %item.name, + version_id = ?item.version_id, + state = "skipped_new_version", + "Erasure set object version skipped because it was written after heal started" + ); + if completed_in_page.is_multiple_of(100) { + checkpoint_manager.update_position(bucket_index, page_resume_index).await?; + } + continue; + } + + if let Some(context) = lifecycle_expiry_context.as_ref() + && self + .storage + .enqueue_heal_lifecycle_expiry( + context, + bucket, + &item.name, + item.version_id.as_deref(), + item.lifecycle_object_info.as_ref(), + ) + .await? + { + checkpoint_manager.add_processed_object(key).await?; + *processed_objects = processed_objects.saturating_add(1); + completed_in_page = completed_in_page.saturating_add(1); + counter!("rustfs_heal_skipped_ilm_expired_total").increment(1); + { + let mut progress = self.progress.write().await; + progress.record_skipped_ilm_expired(); + progress.set_current_object(Some(format!("skipped_ilm: {bucket}/{}", item.name))); + progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + } + debug!( + target: "rustfs::heal::erasure_healer", + event = EVENT_HEAL_ERASURE_OBJECT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, + set_disk_id, + bucket, + object = %item.name, + version_id = ?item.version_id, + state = "skipped_ilm_expired", + "Erasure set object version skipped because lifecycle expiry was queued" + ); + if completed_in_page.is_multiple_of(100) { + checkpoint_manager.update_position(bucket_index, page_resume_index).await?; + } + continue; + } + resume_manager .set_current_item(Some(bucket.to_string()), Some(item.name.clone())) .await?; @@ -777,7 +874,7 @@ impl ErasureSetHealer { let _permit = match permit { Ok(permit) => permit, - Err(err) => return (dedup_key, object_name, version_id, Err(err)), + Err(err) => return (dedup_key, object_name, version_id, (0, Err(err))), }; let _in_flight_guard = PageConcurrencyGuard::new(in_flight, set_label); @@ -788,7 +885,7 @@ impl ErasureSetHealer { // recorded as skipped-ok rather than failed. The delete-marker // vs data path is chosen internally in ops/heal.rs. let result = if cancel_token.is_cancelled() { - Err(Error::TaskCancelled) + (0, Err(Error::TaskCancelled)) } else { match storage .heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts) @@ -797,8 +894,9 @@ impl ErasureSetHealer { Ok((result, None)) if target_outcomes_complete(&result, &target_endpoints) => { + let object_size = result_object_size_u64(&result); if !replacement_commit_evidence_required { - Ok(true) + (object_size, Ok(true)) } else { match storage .replacement_targets_have_version( @@ -810,27 +908,42 @@ impl ErasureSetHealer { ) .await { - Ok(true) => Ok(true), - Ok(false) => Err(Error::transient_skip(format!( + Ok(true) => (object_size, Ok(true)), + Ok(false) => (object_size, Err(Error::transient_skip(format!( "Skipped heal for {bucket_name}/{object_name} because replacement target readback did not confirm the committed version" - ))), - Err(err) => Err(Error::transient_skip(format!( + )))), + Err(err) => (object_size, Err(Error::transient_skip(format!( "Skipped heal for {bucket_name}/{object_name} because replacement target readback failed: {err}" - ))), + )))), } } - } - Ok((_result, None)) if !target_endpoints.is_empty() => Err(Error::transient_skip(format!( - "Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed" - ))), - Ok((_result, None)) => Ok(true), - Ok((_, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => Ok(false), - Ok((_, Some(err))) | Err(err) => match Self::classify_heal_object_error(&err) { - HealObjectOutcome::Absent => Ok(false), - HealObjectOutcome::Transient => Err(Error::transient_skip(format!( - "Skipped heal for {bucket_name}/{object_name} due to transient error: {err}" + }, + Ok((result, None)) if !target_endpoints.is_empty() => ( + result_object_size_u64(&result), + Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed" ))), - HealObjectOutcome::Failed => Err(err), + ), + Ok((result, None)) => (result_object_size_u64(&result), Ok(true)), + Ok((result, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => { + (result_object_size_u64(&result), Ok(false)) + } + Ok((result, Some(err))) => { + let object_size = result_object_size_u64(&result); + match Self::classify_heal_object_error(&err) { + HealObjectOutcome::Absent => (object_size, Ok(false)), + HealObjectOutcome::Transient => (object_size, Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} due to transient error: {err}" + )))), + HealObjectOutcome::Failed => (object_size, Err(err)), + } + } + Err(err) => match Self::classify_heal_object_error(&err) { + HealObjectOutcome::Absent => (0, Ok(false)), + HealObjectOutcome::Transient => (0, Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} due to transient error: {err}" + )))), + HealObjectOutcome::Failed => (0, Err(err)), }, } }; @@ -839,11 +952,12 @@ impl ErasureSetHealer { }); } - let mut completed_in_page = 0usize; while let Some((key, object, version_id, result)) = page_tasks.next().await { + let (object_size, result) = result; match result { Ok(true) => { *successful_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); checkpoint_manager.add_processed_object(key).await?; debug!( target: "rustfs::heal::erasure_healer", @@ -861,6 +975,7 @@ impl ErasureSetHealer { Ok(false) => { checkpoint_manager.add_processed_object(key).await?; *successful_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -877,6 +992,7 @@ impl ErasureSetHealer { Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err), Err(Error::TransientSkip { message }) => { *skipped_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); checkpoint_manager.add_skipped_object(key).await?; demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", { event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -893,6 +1009,7 @@ impl ErasureSetHealer { } Err(err) => { *failed_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); checkpoint_manager.add_failed_object(key).await?; demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", { event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -911,6 +1028,11 @@ impl ErasureSetHealer { *processed_objects += 1; completed_in_page += 1; + { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(format!("{bucket}/{object}"))); + progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + } if completed_in_page.is_multiple_of(100) { checkpoint_manager.update_position(bucket_index, page_resume_index).await?; @@ -964,7 +1086,9 @@ impl ErasureSetHealer { progress.objects_scanned = state.total_objects; progress.objects_healed = state.successful_objects; progress.objects_failed = state.failed_objects; - progress.bytes_processed = 0; // set to 0 for now, can be extended later + progress.bytes_processed = 0; // Resume state tracks object counts, not byte counters. + progress.start_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.start_time)); + progress.last_update_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.last_update)); progress.set_current_object(state.current_object.clone()); } } @@ -1135,13 +1259,15 @@ mod resume_loop_tests { //! that emits programmable multi-version pages. These exercise the real loop //! logic (cursor seeding, per-version dedup, anti-loop guard, absence //! handling) — not merely a mock's own output. - use super::{ErasureSetHealer, target_outcomes_complete}; + use super::{ + ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete, + }; use crate::heal::progress::HealProgress; use crate::heal::resume::{ CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, compose_key, }; - use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo, HealStorageAPI}; + use crate::heal::storage::{DiskStatus, HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI}; use crate::heal::storage_api::status::BucketInfo; use crate::heal::{ BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk, @@ -1149,7 +1275,7 @@ mod resume_loop_tests { use crate::{Error, Result}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; - use std::collections::{HashMap, VecDeque}; + use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -1160,10 +1286,37 @@ mod resume_loop_tests { HealListItem { name: name.to_string(), version_id: version.map(str::to_string), + mod_time_unix_nanos: None, + lifecycle_object_info: None, is_delete_marker: delete_marker, } } + fn item_with_mod_time(name: &str, version: Option<&str>, mod_time_secs: u64) -> HealListItem { + HealListItem { + name: name.to_string(), + version_id: version.map(str::to_string), + mod_time_unix_nanos: Some(i128::from(mod_time_secs).saturating_mul(NANOS_PER_SECOND)), + lifecycle_object_info: None, + is_delete_marker: false, + } + } + + #[test] + fn new_version_filter_respects_grace_boundary() { + let started_at = 1_700_000_000; + + assert!(!should_skip_new_version(None, started_at)); + assert!(!should_skip_new_version( + Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS).saturating_mul(NANOS_PER_SECOND)), + started_at, + )); + assert!(should_skip_new_version( + Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS + 1).saturating_mul(NANOS_PER_SECOND)), + started_at, + )); + } + #[test] fn target_outcomes_require_each_requested_endpoint_once_and_ok() { let result = HealResultItem { @@ -1246,8 +1399,10 @@ mod resume_loop_tests { /// Target-specific physical readback evidence per `compose_key`; the /// fake models a healthy backend unless a test explicitly revokes it. replacement_commit_evidence: Mutex>, + lifecycle_expired: Mutex>, /// every heal_object call recorded as (name, version_id) heal_calls: Mutex)>>, + list_include_lifecycle_object_info: Mutex>, replacement_target_identity_sequences: Mutex>>, fail_listing: AtomicBool, } @@ -1274,9 +1429,15 @@ mod resume_loop_tests { .unwrap() .insert(compose_key(name, version), ReplacementCommitEvidence::Error(message.to_string())); } + fn set_lifecycle_expired(&self, name: &str, version: Option<&str>) { + self.lifecycle_expired.lock().unwrap().insert(compose_key(name, version)); + } fn calls(&self) -> Vec<(String, Option)> { self.heal_calls.lock().unwrap().clone() } + fn list_include_lifecycle_object_info_calls(&self) -> Vec { + self.list_include_lifecycle_object_info.lock().unwrap().clone() + } fn fail_listing(&self) { self.fail_listing.store(true, Ordering::SeqCst); } @@ -1330,6 +1491,23 @@ mod resume_loop_tests { async fn get_object_checksum(&self, _b: &str, _o: &str) -> Result> { Ok(None) } + async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result> { + Ok((!self.lifecycle_expired.lock().unwrap().is_empty()).then(HealLifecycleExpiryContext::test)) + } + async fn enqueue_heal_lifecycle_expiry( + &self, + _context: &HealLifecycleExpiryContext, + _bucket: &str, + object: &str, + version_id: Option<&str>, + _object_info: Option<&HealObjectInfo>, + ) -> Result { + Ok(self + .lifecycle_expired + .lock() + .unwrap() + .contains(&compose_key(object, version_id))) + } async fn heal_object( &self, _bucket: &str, @@ -1386,7 +1564,12 @@ mod resume_loop_tests { _bucket: &str, _prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { + self.list_include_lifecycle_object_info + .lock() + .unwrap() + .push(include_lifecycle_object_info); if self.fail_listing.load(Ordering::SeqCst) { return Err(Error::other("injected listing failure")); } @@ -1476,6 +1659,7 @@ mod resume_loop_tests { /// Drive one bucket heal pass; returns (processed, successful, failed, skipped, result). async fn run(env: &Env) -> (u64, u64, u64, u64, Result<()>) { + let state = env.resume.get_state().await; let mut current_object_index = 0usize; let mut processed = 0u64; let mut successful = 0u64; @@ -1494,6 +1678,7 @@ mod resume_loop_tests { &mut skipped, &env.resume, &env.checkpoint, + state.start_time, ) .await; (processed, successful, failed, skipped, result) @@ -1559,6 +1744,7 @@ mod resume_loop_tests { let mut successful = 0; let mut failed = 0; let mut skipped = 0; + let started_at = env.resume.get_state().await.start_time; let error = healer .heal_bucket_with_resume( @@ -1572,6 +1758,7 @@ mod resume_loop_tests { &mut skipped, &env.resume, &env.checkpoint, + started_at, ) .await .expect_err("a remounted target must not begin a new page scan"); @@ -1641,6 +1828,109 @@ mod resume_loop_tests { assert_eq!(skipped, 0); } + #[tokio::test] + async fn erasure_set_progress_accumulates_healed_object_bytes() { + let env = make_env().await; + env.storage.set_page( + None, + Page { + items: vec![item("first", Some("v1"), false), item("second", Some("v2"), false)], + next: None, + truncated: false, + }, + ); + env.storage.set_result( + "first", + Some("v1"), + HealResultItem { + object_size: 1024, + ..Default::default() + }, + ); + env.storage.set_result( + "second", + Some("v2"), + HealResultItem { + object_size: 2048, + ..Default::default() + }, + ); + + let (processed, successful, failed, skipped, result) = run(&env).await; + + result.expect("page heal should succeed"); + assert_eq!(processed, 2); + assert_eq!(successful, 2); + assert_eq!(failed, 0); + assert_eq!(skipped, 0); + let progress = env.healer.progress.read().await; + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 2); + assert_eq!(progress.objects_failed, 0); + assert_eq!(progress.bytes_processed, 3072); + assert!(matches!(progress.current_object.as_deref(), Some("b/first" | "b/second"))); + } + + #[tokio::test] + async fn erasure_set_skips_versions_written_after_heal_started() { + let env = make_env().await; + let started_at = env.resume.get_state().await.start_time; + env.storage.set_page( + None, + Page { + items: vec![ + item_with_mod_time("old", Some("v1"), started_at + NEW_VERSION_SKIP_GRACE_SECS), + item_with_mod_time("new", Some("v2"), started_at + NEW_VERSION_SKIP_GRACE_SECS + 1), + ], + next: None, + truncated: false, + }, + ); + + let (processed, successful, failed, skipped, result) = run(&env).await; + + result.expect("page heal should succeed"); + assert_eq!(processed, 2); + assert_eq!(successful, 1); + assert_eq!(failed, 0); + assert_eq!(skipped, 0); + assert_eq!(env.storage.calls(), vec![("old".to_string(), Some("v1".to_string()))]); + let progress = env.healer.progress.read().await; + assert_eq!(progress.skipped_new_versions, 1); + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 1); + assert_eq!(progress.objects_failed, 0); + } + + #[tokio::test] + async fn erasure_set_skips_versions_queued_for_lifecycle_expiry() { + let env = make_env().await; + env.storage.set_page( + None, + Page { + items: vec![item("expired", Some("v1"), false), item("kept", Some("v2"), false)], + next: None, + truncated: false, + }, + ); + env.storage.set_lifecycle_expired("expired", Some("v1")); + + let (processed, successful, failed, skipped, result) = run(&env).await; + + result.expect("page heal should succeed"); + assert_eq!(processed, 2); + assert_eq!(successful, 1); + assert_eq!(failed, 0); + assert_eq!(skipped, 0); + assert_eq!(env.storage.calls(), vec![("kept".to_string(), Some("v2".to_string()))]); + assert_eq!(env.storage.list_include_lifecycle_object_info_calls(), vec![true]); + let progress = env.healer.progress.read().await; + assert_eq!(progress.skipped_ilm_expired, 1); + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 1); + assert_eq!(progress.objects_failed, 0); + } + #[tokio::test] async fn bucket_listing_failure_does_not_mark_set_completed() { let env = make_env().await; diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 79ba6b169..66b8f637f 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -2385,8 +2385,27 @@ impl HealManager { snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned); snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed); snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed); + snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions); + snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired); + snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count); + snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size); snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed); + snapshot.start_time = match (snapshot.start_time, progress.start_time) { + (Some(current), Some(next)) => Some(current.min(next)), + (None, next) => next, + (current, None) => current, + }; + snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) { + (Some(current), Some(next)) => Some(current.max(next)), + (None, next) => next, + (current, None) => current, + }; + if progress.current_object.is_some() { + snapshot.current_object = progress.current_object; + } } + snapshot.refresh_progress_percentage(); + snapshot.refresh_estimated_completion_time(); Some(snapshot) } @@ -3208,6 +3227,7 @@ impl HealManager { } else { completed_task.get_status().await }; + let completed_progress = completed_task.get_progress().await; let completed_status_entry = CompletedHealStatus { heal_type: completed_task.heal_type.clone(), status: completed_status.clone(), @@ -3223,6 +3243,7 @@ impl HealManager { match completed_status { HealTaskStatus::Completed => { stats.update_task_completion(true); + stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed); } HealTaskStatus::Retrying { .. } => {} _ => { @@ -3749,6 +3770,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { Ok((Vec::new(), None, false)) } @@ -5396,6 +5418,8 @@ mod tests { )); { let mut progress = first.progress.write().await; + progress.start_time = Some(SystemTime::now() - Duration::from_secs(20)); + progress.set_total_baseline(12, 8192); progress.update_progress(7, 3, 1, 4096); } @@ -5405,6 +5429,8 @@ mod tests { )); { let mut progress = second.progress.write().await; + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + progress.set_total_baseline(8, 4096); progress.update_progress(11, 5, 2, 2048); } @@ -5419,7 +5445,11 @@ mod tests { assert_eq!(progress.objects_scanned, 18); assert_eq!(progress.objects_healed, 8); assert_eq!(progress.objects_failed, 3); + assert_eq!(progress.objects_total_count, 20); + assert_eq!(progress.objects_total_size, 12288); assert_eq!(progress.bytes_processed, 6144); + assert!((progress.progress_percentage - 50.0).abs() < 0.001); + assert!(progress.estimated_completion_time.is_some()); } #[tokio::test] diff --git a/crates/heal/src/heal/progress.rs b/crates/heal/src/heal/progress.rs index cb602b5de..981aa01fe 100644 --- a/crates/heal/src/heal/progress.rs +++ b/crates/heal/src/heal/progress.rs @@ -13,7 +13,7 @@ // limitations under the License. use serde::{Deserialize, Serialize}; -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; #[derive(Debug, Default, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -24,6 +24,14 @@ pub struct HealProgress { pub objects_healed: u64, /// Objects failed pub objects_failed: u64, + /// Versions skipped because they were written after this heal started + pub skipped_new_versions: u64, + /// Versions skipped because lifecycle already selected them for expiry + pub skipped_ilm_expired: u64, + /// Baseline object count from the latest complete usage snapshot + pub objects_total_count: u64, + /// Baseline object bytes from the latest complete usage snapshot + pub objects_total_size: u64, /// Bytes processed pub bytes_processed: u64, /// Current object @@ -54,10 +62,56 @@ impl HealProgress { self.bytes_processed = bytes; self.last_update_time = Some(SystemTime::now()); - // calculate progress percentage - let total = scanned + healed + failed; + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + pub fn set_total_baseline(&mut self, objects_total_count: u64, objects_total_size: u64) { + self.objects_total_count = objects_total_count; + self.objects_total_size = objects_total_size; + self.last_update_time = Some(SystemTime::now()); + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + pub fn record_skipped_new_version(&mut self) { + self.skipped_new_versions = self.skipped_new_versions.saturating_add(1); + self.last_update_time = Some(SystemTime::now()); + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + pub fn record_skipped_ilm_expired(&mut self) { + self.skipped_ilm_expired = self.skipped_ilm_expired.saturating_add(1); + self.last_update_time = Some(SystemTime::now()); + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + fn completed_for_baseline(&self) -> u64 { + self.objects_healed + .saturating_add(self.objects_failed) + .saturating_add(self.skipped_new_versions) + .saturating_add(self.skipped_ilm_expired) + } + + pub(crate) fn refresh_progress_percentage(&mut self) { + if self.objects_total_size > 0 { + self.progress_percentage = ((self.bytes_processed as f64 / self.objects_total_size as f64) * 100.0).min(100.0); + return; + } + if self.objects_total_count > 0 { + let completed = self.completed_for_baseline(); + self.progress_percentage = ((completed as f64 / self.objects_total_count as f64) * 100.0).min(100.0); + return; + } + + let total = self + .objects_scanned + .saturating_add(self.objects_healed) + .saturating_add(self.objects_failed); if total > 0 { - self.progress_percentage = (healed as f64 / total as f64) * 100.0; + self.progress_percentage = (self.objects_healed as f64 / total as f64) * 100.0; } } @@ -66,9 +120,36 @@ impl HealProgress { self.last_update_time = Some(SystemTime::now()); } + pub fn refresh_estimated_completion_time(&mut self) { + let Some(start_time) = self.start_time else { + self.estimated_completion_time = None; + return; + }; + if self.is_completed() || !(0.0..100.0).contains(&self.progress_percentage) || self.bytes_processed == 0 { + self.estimated_completion_time = None; + return; + } + + let elapsed = match SystemTime::now().duration_since(start_time) { + Ok(elapsed) if !elapsed.is_zero() => elapsed, + _ => { + self.estimated_completion_time = None; + return; + } + }; + let estimated_total_secs = elapsed.as_secs_f64() * 100.0 / self.progress_percentage; + self.estimated_completion_time = start_time.checked_add(Duration::from_secs_f64(estimated_total_secs)); + } + pub fn is_completed(&self) -> bool { - self.progress_percentage >= 100.0 - || self.objects_scanned > 0 && self.objects_healed + self.objects_failed >= self.objects_scanned + if self.progress_percentage >= 100.0 { + return true; + } + if self.objects_total_count > 0 || self.objects_total_size > 0 { + return false; + } + + self.objects_scanned > 0 && self.objects_healed.saturating_add(self.objects_failed) >= self.objects_scanned } pub fn get_success_rate(&self) -> f64 { @@ -158,6 +239,10 @@ mod tests { assert_eq!(progress.objects_scanned, 0); assert_eq!(progress.objects_healed, 0); assert_eq!(progress.objects_failed, 0); + assert_eq!(progress.skipped_new_versions, 0); + assert_eq!(progress.skipped_ilm_expired, 0); + assert_eq!(progress.objects_total_count, 0); + assert_eq!(progress.objects_total_size, 0); assert_eq!(progress.bytes_processed, 0); assert_eq!(progress.progress_percentage, 0.0); assert!(progress.start_time.is_some()); @@ -181,6 +266,73 @@ mod tests { assert!(progress.last_update_time.is_some()); } + #[test] + fn test_heal_progress_estimates_completion_time_from_progress() { + let mut progress = HealProgress::new(); + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + + progress.update_progress(100, 25, 0, 4096); + + let eta = progress + .estimated_completion_time + .expect("partial byte progress should estimate completion"); + assert!(eta > SystemTime::now()); + } + + #[test] + fn test_heal_progress_uses_byte_baseline_for_percentage() { + let mut progress = HealProgress::new(); + progress.set_total_baseline(10, 8192); + + progress.update_progress(100, 25, 0, 4096); + + assert!((progress.progress_percentage - 50.0).abs() < 0.001); + } + + #[test] + fn test_heal_progress_uses_object_baseline_when_bytes_unknown() { + let mut progress = HealProgress::new(); + progress.set_total_baseline(10, 0); + + progress.update_progress(100, 3, 2, 0); + + assert!((progress.progress_percentage - 50.0).abs() < 0.001); + } + + #[test] + fn test_heal_progress_counts_skipped_versions_for_object_baseline() { + let mut progress = HealProgress::new(); + progress.set_total_baseline(10, 0); + + progress.update_progress(100, 3, 2, 0); + progress.record_skipped_new_version(); + + assert_eq!(progress.skipped_new_versions, 1); + assert!((progress.progress_percentage - 60.0).abs() < 0.001); + } + + #[test] + fn test_heal_progress_does_not_estimate_completion_without_bytes() { + let mut progress = HealProgress::new(); + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + + progress.update_progress(100, 25, 0, 0); + + assert!(progress.estimated_completion_time.is_none()); + } + + #[test] + fn test_heal_progress_with_baseline_is_not_completed_by_processed_count() { + let mut progress = HealProgress::new(); + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + progress.set_total_baseline(10, 8192); + + progress.update_progress(1, 1, 0, 1024); + + assert!(!progress.is_completed()); + assert!(progress.estimated_completion_time.is_some()); + } + #[test] fn test_heal_progress_update_progress_zero_total() { let mut progress = HealProgress::new(); @@ -251,6 +403,8 @@ mod tests { assert_eq!(json["objectsScanned"], 10); assert_eq!(json["objectsHealed"], 8); assert_eq!(json["objectsFailed"], 2); + assert_eq!(json["skippedNewVersions"], 0); + assert_eq!(json["skippedIlmExpired"], 0); assert_eq!(json["bytesProcessed"], 1024); assert_eq!(json["currentObject"], "test-bucket/test-object"); assert!(json["progressPercentage"].is_number()); diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index f5101ac1c..fbe51f050 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use tracing::{debug, error, warn}; +use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached}; use super::storage_api::storage::{ BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _, ObjectOperations as _, StorageAdminApi, @@ -29,6 +30,37 @@ use super::storage_api::storage::{ use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity}; pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader}; +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct HealBucketUsageBaseline { + pub objects_count: u64, + pub bytes: u64, +} + +pub struct HealLifecycleExpiryContext { + inner: HealLifecycleExpiryContextInner, +} + +enum HealLifecycleExpiryContextInner { + Ecstore(EcstoreHealLifecycleExpiryContext), + #[allow(dead_code)] + Test, +} + +impl HealLifecycleExpiryContext { + fn ecstore(inner: EcstoreHealLifecycleExpiryContext) -> Self { + Self { + inner: HealLifecycleExpiryContextInner::Ecstore(inner), + } + } + + #[cfg(test)] + pub(crate) fn test() -> Self { + Self { + inner: HealLifecycleExpiryContextInner::Test, + } + } +} + const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_STORAGE: &str = "storage"; const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io"; @@ -272,6 +304,10 @@ pub struct HealListItem { pub name: String, /// normalized version id (`None` when the version is nil/absent) pub version_id: Option, + /// version modification time as Unix nanoseconds + pub mod_time_unix_nanos: Option, + /// object snapshot for lifecycle evaluation + pub lifecycle_object_info: Option, /// whether this version is a delete marker (observability only) pub is_delete_marker: bool, } @@ -329,6 +365,28 @@ pub trait HealStorageAPI: Send + Sync { /// Get bucket info async fn get_bucket_info(&self, bucket: &str) -> Result>; + /// Aggregate usage-cache baselines for the requested buckets. + async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result> { + Ok(None) + } + + /// Load per-bucket lifecycle expiry context for heal skips. + async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result> { + Ok(None) + } + + /// Queue lifecycle expiry for a version that heal can skip. + async fn enqueue_heal_lifecycle_expiry( + &self, + _context: &HealLifecycleExpiryContext, + _bucket: &str, + _object: &str, + _version_id: Option<&str>, + _object_info: Option<&HealObjectInfo>, + ) -> Result { + Ok(false) + } + /// Fix bucket metadata async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>; @@ -409,6 +467,7 @@ pub trait HealStorageAPI: Send + Sync { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)>; /// List versions for healing via a per-erasure-set DISK-WALK union enumerator @@ -427,8 +486,10 @@ pub trait HealStorageAPI: Send + Sync { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { - self.list_objects_for_heal_page(bucket, prefix, continuation_token).await + self.list_objects_for_heal_page(bucket, prefix, continuation_token, include_lifecycle_object_info) + .await } /// Get disk for resume functionality. @@ -1021,6 +1082,85 @@ impl HealStorageAPI for ECStoreHealStorage { } } + async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result> { + if buckets.is_empty() { + return Ok(None); + } + + let info = match ecstore_load_admin_data_usage_from_backend_cached(self.ecstore.clone()).await { + Ok(info) if info.is_complete_bucket_usage_snapshot() => info, + Ok(_) | Err(_) => return Ok(None), + }; + + let mut baseline = HealBucketUsageBaseline::default(); + for bucket in buckets { + if let Some(usage) = info.buckets_usage.get(bucket) { + baseline.objects_count = baseline.objects_count.saturating_add(usage.objects_count); + baseline.bytes = baseline.bytes.saturating_add(usage.size); + } + } + + Ok(Some(baseline)) + } + + async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result> { + match self.ecstore.load_heal_lifecycle_expiry_context(bucket).await { + Ok(Some(context)) => Ok(Some(HealLifecycleExpiryContext::ecstore(context))), + Ok(None) => Ok(None), + Err(err) => { + debug!( + target: "rustfs::heal::storage", + event = EVENT_HEAL_STORAGE_ADMIN_OP, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_STORAGE, + operation = "load_heal_lifecycle_expiry_context", + bucket, + result = "failed", + error = %err, + "Heal storage lifecycle expiry context load failed" + ); + Ok(None) + } + } + } + + async fn enqueue_heal_lifecycle_expiry( + &self, + context: &HealLifecycleExpiryContext, + bucket: &str, + object: &str, + version_id: Option<&str>, + object_info: Option<&HealObjectInfo>, + ) -> Result { + let context = match &context.inner { + HealLifecycleExpiryContextInner::Ecstore(context) => context, + HealLifecycleExpiryContextInner::Test => return Ok(false), + }; + match self + .ecstore + .enqueue_heal_lifecycle_expiry(context, bucket, object, version_id, object_info) + .await + { + Ok(queued) => Ok(queued), + Err(err) => { + debug!( + target: "rustfs::heal::storage", + event = EVENT_HEAL_STORAGE_ADMIN_OP, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_STORAGE, + operation = "enqueue_heal_lifecycle_expiry", + bucket, + object, + version_id = ?version_id, + result = "failed", + error = %err, + "Heal storage lifecycle expiry check failed" + ); + Ok(false) + } + } + } + async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> { debug!( target: "rustfs::heal::storage", @@ -1436,7 +1576,7 @@ impl HealStorageAPI for ECStoreHealStorage { loop { let (page_objects, next_token, is_truncated) = self - .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref()) + .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false) .await?; all_objects.extend(page_objects); @@ -1471,6 +1611,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { debug!( target: "rustfs::heal::storage", @@ -1522,10 +1663,19 @@ impl HealStorageAPI for ECStoreHealStorage { let page_objects: Vec = list_info .objects .into_iter() - .map(|obj| HealListItem { - name: obj.name, - version_id: obj.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()), - is_delete_marker: obj.delete_marker, + .map(|mut obj| { + obj.version_id = obj.version_id.filter(|u| !u.is_nil()); + let version_id = obj.version_id.map(|u| u.to_string()); + let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()); + let is_delete_marker = obj.delete_marker; + let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone()); + HealListItem { + name: obj.name, + version_id, + mod_time_unix_nanos, + lifecycle_object_info, + is_delete_marker, + } }) .collect(); let page_count = page_objects.len(); @@ -1562,6 +1712,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { // Per-page bounds for the disk-walk union enumerator. Objects are atomic // (never split across pages), so version_budget only bounds how many @@ -1590,7 +1741,16 @@ impl HealStorageAPI for ECStoreHealStorage { let (versions, next_forward, is_truncated) = self .ecstore - .heal_walk_versions_page(pool_idx, set_idx, bucket, prefix, forward_to.as_deref(), BATCH_OBJECTS, VERSION_BUDGET) + .heal_walk_versions_page( + pool_idx, + set_idx, + bucket, + prefix, + forward_to.as_deref(), + BATCH_OBJECTS, + VERSION_BUDGET, + include_lifecycle_object_info, + ) .await .map_err(|e| { error!( @@ -1614,6 +1774,8 @@ impl HealStorageAPI for ECStoreHealStorage { .map(|v| HealListItem { name: v.name, version_id: v.version_id, + mod_time_unix_nanos: v.mod_time_unix_nanos, + lifecycle_object_info: v.lifecycle_object_info, is_delete_marker: v.is_delete_marker, }) .collect(); diff --git a/crates/heal/src/heal/storage_api.rs b/crates/heal/src/heal/storage_api.rs index 417998e39..ff722b418 100644 --- a/crates/heal/src/heal/storage_api.rs +++ b/crates/heal/src/heal/storage_api.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME; +pub(crate) use rustfs_ecstore::api::data_usage::{ + DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME, + load_admin_data_usage_from_backend_cached as ecstore_load_admin_data_usage_from_backend_cached, +}; pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint; pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult}; pub(crate) use rustfs_ecstore::api::disk::{ @@ -25,7 +28,9 @@ pub(crate) use rustfs_ecstore::api::disk::{ pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk}; pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError}; pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read; -pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore; +pub(crate) use rustfs_ecstore::api::storage::{ + ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext, +}; use rustfs_storage_api as storage_contracts; pub(crate) mod owner { @@ -34,8 +39,8 @@ pub(crate) mod owner { pub(crate) use super::{ ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, - EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, - ecstore_local_disk_map_read, + EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext, + EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read, }; #[cfg(test)] diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 62123c418..f6472eb65 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -19,11 +19,12 @@ use crate::heal::{ resume::{ CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match, }, - storage::{HealStorageAPI, next_heal_listing_token}, + storage::{HealBucketUsageBaseline, HealStorageAPI, next_heal_listing_token}, }; use crate::{Error, Result}; use metrics::{counter, histogram}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode}; +use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit}; use rustfs_madmin::heal_commands::HealResultItem; use rustfs_utils::path::SLASH_SEPARATOR; use serde::{Deserialize, Serialize}; @@ -178,6 +179,17 @@ pub enum HealPriority { Urgent = 3, } +impl HealPriority { + fn as_str(self) -> &'static str { + match self { + Self::Low => "low", + Self::Normal => "normal", + Self::High => "high", + Self::Urgent => "urgent", + } + } +} + /// Heal options #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HealOptions { @@ -498,6 +510,61 @@ impl HealTask { } } + fn emit_trace_task_state(&self, state: &'static str, duration: Duration, error: Option<&Error>) { + trace_emit(|| { + let mut event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask) + .with_duration(duration) + .with_attr("task_id", self.id.as_str()) + .with_attr("heal_type", self.heal_type.log_kind()) + .with_attr("state", state) + .with_attr("source", self.source.as_str()) + .with_attr("priority", self.priority.as_str()) + .with_attr("retry_attempts", u64::from(self.retry_attempts)) + .with_attr("dry_run", self.options.dry_run); + + event = match &self.heal_type { + HealType::Cluster => event, + HealType::Object { + bucket, + object, + version_id, + } => { + let event = event.with_bucket(bucket.as_str()).with_object(object.as_str()); + match version_id { + Some(version_id) => event.with_attr("version_id", version_id.as_str()), + None => event, + } + } + HealType::Bucket { bucket } => event.with_bucket(bucket.as_str()), + HealType::Prefix { bucket, prefix } => event.with_bucket(bucket.as_str()).with_object(prefix.as_str()), + HealType::ErasureSet { buckets, set_disk_id } => { + let bucket_count = u64::try_from(buckets.len()).unwrap_or(u64::MAX); + event + .with_attr("set_disk_id", set_disk_id.as_str()) + .with_attr("bucket_count", bucket_count) + } + HealType::Metadata { bucket, object } => event.with_bucket(bucket.as_str()).with_object(object.as_str()), + HealType::ECDecode { + bucket, + object, + version_id, + } => { + let event = event.with_bucket(bucket.as_str()).with_object(object.as_str()); + match version_id { + Some(version_id) => event.with_attr("version_id", version_id.as_str()), + None => event, + } + } + HealType::MRF { meta_path } => event.with_object(meta_path.as_str()), + }; + + match error { + Some(error) => event.with_attr("error", error.to_string()), + None => event, + } + }); + } + async fn remaining_timeout(&self) -> Result> { if let Some(total) = self.options.timeout { let start_instant = { *self.task_start_instant.read().await }; @@ -717,6 +784,7 @@ impl HealTask { queue_delay = ?queue_delay, "Heal task started" }); + self.emit_trace_task_state("started", Duration::ZERO, None); let result = match &self.heal_type { HealType::Cluster => self.heal_cluster().await, @@ -805,6 +873,14 @@ impl HealTask { } } + let terminal_state = match &result { + Ok(_) => "completed", + Err(Error::TaskCancelled) => "cancelled", + Err(Error::TaskTimeout) => "timed_out", + Err(_) => "failed", + }; + self.emit_trace_task_state(terminal_state, start_instant.elapsed(), result.as_ref().err()); + result } @@ -1535,7 +1611,7 @@ impl HealTask { let (objects, next_token, is_truncated) = self .await_with_control( self.storage - .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref()), + .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false), ) .await?; @@ -1697,6 +1773,23 @@ impl HealTask { Ok(()) } + async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> { + let baseline = match self + .await_with_control(self.storage.erasure_set_usage_baseline(buckets)) + .await + { + Ok(Some(baseline)) => baseline, + Ok(None) => return Ok(()), + Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err), + Err(_) => return Ok(()), + }; + + let HealBucketUsageBaseline { objects_count, bytes } = baseline; + let mut progress = self.progress.write().await; + progress.set_total_baseline(objects_count, bytes); + Ok(()) + } + async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> { debug!( target: "rustfs::heal::task", @@ -2298,6 +2391,8 @@ impl HealTask { None }; + self.apply_erasure_set_usage_baseline(&buckets).await?; + let healing_marker = format!("{set_disk_id}:{}", self.id); if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() { let state = resume_manager.get_state().await; @@ -2602,7 +2697,8 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(4, 4, 0, 0); + let bytes_processed = progress.bytes_processed; + progress.update_progress(4, 4, 0, bytes_processed); } match result { @@ -2658,6 +2754,7 @@ mod tests { use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk}; use super::*; use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo}; + use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; use std::collections::{HashMap, VecDeque}; use std::sync::Mutex; @@ -3203,6 +3300,8 @@ mod tests { block_heal_object: Mutex, resume_disk: Mutex>, replacement_resume_disk: Mutex>, + usage_baseline: Mutex>, + usage_baseline_error: Mutex, } #[test] @@ -3265,11 +3364,69 @@ mod tests { assert_eq!(samples_logged, MAX_BUCKET_FAILURE_LOG_SAMPLES); } + #[tokio::test] + async fn execute_emits_heal_trace_task_state() { + let mut trace = subscribe_trace_events(); + let storage = Arc::new(MockStorage::default()); + let task = HealTask::from_request( + HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())), + storage, + ); + + task.execute().await.expect("mock object heal should complete"); + + let started = recv_trace_task_state(&mut trace, &task.id, "started").await; + assert_eq!(started.kind, TraceKind::Heal); + assert_eq!(started.func, TraceFunc::HealTask); + assert_eq!(started.bucket.as_deref(), Some("bucket-a")); + assert_eq!(started.object.as_deref(), Some("object-a")); + assert_eq!(trace_attr_string(&started, "heal_type").as_deref(), Some("object")); + assert_eq!(trace_attr_string(&started, "source").as_deref(), Some("internal")); + assert_eq!(trace_attr_string(&started, "version_id").as_deref(), Some("version-a")); + + let completed = recv_trace_task_state(&mut trace, &task.id, "completed").await; + assert_eq!(completed.kind, TraceKind::Heal); + assert_eq!(completed.func, TraceFunc::HealTask); + assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed")); + } + + async fn recv_trace_task_state(trace: &mut TraceSubscription, task_id: &str, state: &str) -> TraceEvent { + for _ in 0..32 { + let event = tokio::time::timeout(Duration::from_secs(1), trace.recv()) + .await + .expect("trace event should arrive") + .expect("trace bus should stay open"); + if trace_attr_string(&event, "task_id").as_deref() == Some(task_id) + && trace_attr_string(&event, "state").as_deref() == Some(state) + { + return (*event).clone(); + } + } + + panic!("expected trace state {state} for task {task_id}"); + } + + fn trace_attr_string(event: &TraceEvent, key: &str) -> Option { + event.attrs.iter().find_map(|attr| { + if attr.key != key { + return None; + } + Some(match &attr.value { + TraceVal::Bool(value) => value.to_string(), + TraceVal::U64(value) => value.to_string(), + TraceVal::I64(value) => value.to_string(), + TraceVal::Str(value) => value.to_string(), + }) + }) + } + /// Build a latest, non-delete-marker heal list item with no version id. fn heal_item(name: &str) -> HealListItem { HealListItem { name: name.to_string(), version_id: None, + mod_time_unix_nanos: None, + lifecycle_object_info: None, is_delete_marker: false, } } @@ -3357,6 +3514,13 @@ mod tests { })) } + async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result> { + if *self.usage_baseline_error.lock().unwrap() { + return Err(Error::Other("usage baseline unavailable".to_string())); + } + Ok(*self.usage_baseline.lock().unwrap()) + } + async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> { Ok(()) } @@ -3540,6 +3704,7 @@ mod tests { bucket: &str, prefix: &str, continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { self.listed_prefixes.lock().unwrap().push(prefix.to_string()); if *self.truncate_without_token.lock().unwrap() { @@ -4654,6 +4819,73 @@ mod tests { assert!(storage.object_heal_opts.lock().unwrap().is_empty()); } + #[tokio::test] + async fn erasure_set_heal_applies_usage_baseline_to_progress() { + let temp = TempDir::new().expect("temporary directory should be created"); + let disk = make_resume_disk(&temp).await; + let storage = Arc::new(MockStorage { + resume_disk: Mutex::new(Some(disk)), + usage_baseline: Mutex::new(Some(HealBucketUsageBaseline { + objects_count: 10, + bytes: 8, + })), + ..Default::default() + }); + let request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage); + + task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string()) + .await + .expect("erasure set heal should complete"); + + let progress = task.get_progress().await; + assert_eq!(progress.objects_total_count, 10); + assert_eq!(progress.objects_total_size, 8); + assert_eq!(progress.bytes_processed, 2); + assert!((progress.progress_percentage - 25.0).abs() < 0.001); + } + + #[tokio::test] + async fn erasure_set_heal_ignores_usage_baseline_errors() { + let temp = TempDir::new().expect("temporary directory should be created"); + let disk = make_resume_disk(&temp).await; + let storage = Arc::new(MockStorage { + resume_disk: Mutex::new(Some(disk)), + usage_baseline_error: Mutex::new(true), + ..Default::default() + }); + let request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage); + + task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string()) + .await + .expect("usage baseline failures should not fail erasure set heal"); + + let progress = task.get_progress().await; + assert_eq!(progress.objects_total_count, 0); + assert_eq!(progress.objects_total_size, 0); + } + #[tokio::test] async fn resumable_erasure_set_execution_is_cancelled_while_object_heal_is_pending() { let temp = TempDir::new().expect("temporary directory should be created"); diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index 7c156304d..3dd29b064 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -445,6 +445,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool), Error> { Ok((Vec::new(), None, false)) } diff --git a/crates/heal/tests/heal_b5_versioned_regression_test.rs b/crates/heal/tests/heal_b5_versioned_regression_test.rs index 61a542955..95e0f0a90 100644 --- a/crates/heal/tests/heal_b5_versioned_regression_test.rs +++ b/crates/heal/tests/heal_b5_versioned_regression_test.rs @@ -176,7 +176,7 @@ async fn enumerate_all_versions(heal_storage: &Arc, bucket: let mut token: Option = None; loop { let (page, next, truncated) = heal_storage - .list_objects_for_heal_page(bucket, "", token.as_deref()) + .list_objects_for_heal_page(bucket, "", token.as_deref(), false) .await .expect("list_objects_for_heal_page failed"); items.extend(page); diff --git a/crates/heal/tests/heal_b920_subquorum_union_test.rs b/crates/heal/tests/heal_b920_subquorum_union_test.rs index 6d188b3f3..8d4ac65f2 100644 --- a/crates/heal/tests/heal_b920_subquorum_union_test.rs +++ b/crates/heal/tests/heal_b920_subquorum_union_test.rs @@ -166,7 +166,7 @@ async fn enumerate_b5(heal_storage: &Arc, bucket: &str) -> V let mut token: Option = None; loop { let (page, next, truncated) = heal_storage - .list_objects_for_heal_page(bucket, "", token.as_deref()) + .list_objects_for_heal_page(bucket, "", token.as_deref(), false) .await .expect("b5 list page failed"); items.extend(page); @@ -187,7 +187,7 @@ async fn enumerate_disk_walk(heal_storage: &Arc, bucket: &st let mut token: Option = None; loop { let (page, next, truncated) = heal_storage - .list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref()) + .list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref(), false) .await .expect("disk-walk list page failed"); items.extend(page); @@ -418,7 +418,7 @@ mod serial_tests { let mut pages = 0usize; loop { let (versions, next_forward, truncated) = ecstore - .heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000) + .heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000, false) .await .expect("heal_walk_versions_page failed"); pages += 1; diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index 7b8737f19..ba59142fd 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -242,6 +242,7 @@ fn test_heal_task_status_atomic_update() { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> rustfs_heal::Result<(Vec, Option, bool)> { Ok((vec![], None, false)) } @@ -385,6 +386,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> rustfs_heal::Result<(Vec, Option, bool)> { Ok((Vec::new(), None, false)) } diff --git a/crates/madmin/src/service_commands.rs b/crates/madmin/src/service_commands.rs index ddda978be..1b5d0b2a9 100644 --- a/crates/madmin/src/service_commands.rs +++ b/crates/madmin/src/service_commands.rs @@ -43,7 +43,7 @@ pub struct ServiceTraceOpts { #[allow(dead_code)] impl ServiceTraceOpts { - fn trace_types(&self) -> TraceType { + pub fn trace_types(&self) -> TraceType { let mut tt = TraceType::default(); tt.set_if(self.s3, &TraceType::S3); tt.set_if(self.internal, &TraceType::INTERNAL); @@ -72,6 +72,14 @@ impl ServiceTraceOpts { tt } + pub fn only_errors(&self) -> bool { + self.only_errors + } + + pub fn threshold(&self) -> Duration { + self.threshold + } + pub fn parse_params(&mut self, uri: &Uri) -> Result<(), String> { let query_pairs: HashMap<_, _> = uri .query() diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 25f1e758d..820cbb12f 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -41,6 +41,7 @@ use rustfs_common::metrics::{ CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn, current_path_updater, global_metrics, }; +use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count}; use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration}; @@ -430,6 +431,113 @@ fn non_negative_i64_to_u64(value: i64) -> u64 { value.max(0) as u64 } +fn trace_start_instant() -> Option { + (trace_subscriber_count() > 0).then(Instant::now) +} + +fn emit_scanner_folder_trace(root: &str, folder: &str, objects: u64, started_at: Option, state: &'static str) { + let Some(started_at) = started_at else { + return; + }; + + trace_emit(|| { + let (bucket, prefix) = path2_bucket_object_with_base_path(root, folder); + TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder) + .with_bucket(bucket) + .with_object(prefix) + .with_duration(started_at.elapsed()) + .with_attr("state", state) + .with_attr("objects", objects) + }); +} + +fn emit_scanner_ilm_action_trace( + bucket: &str, + object: &str, + action: IlmAction, + count: u64, + queued: bool, + started_at: Option, +) { + let Some(started_at) = started_at else { + return; + }; + + let state = if queued { "queued" } else { "not_queued" }; + trace_emit(|| { + TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction) + .with_bucket(bucket) + .with_object(object) + .with_duration(started_at.elapsed()) + .with_attr("state", state) + .with_attr("action", action.as_str()) + .with_attr("count", count) + .with_attr("queued", queued) + }); +} + +struct ScannerHealCandidateTraceContext { + bucket: String, + object: Option, + version_id: Option, + scan_mode: Option, + started_at: Instant, +} + +fn scanner_heal_candidate_trace_context(request: &HealChannelRequest) -> Option { + let started_at = trace_start_instant()?; + Some(ScannerHealCandidateTraceContext { + bucket: request.bucket.clone(), + object: request.object_prefix.clone(), + version_id: request.object_version_id.clone(), + scan_mode: request.scan_mode, + started_at, + }) +} + +struct ScannerHealCandidateTrace<'a> { + candidate_type: &'static str, + bucket: &'a str, + object: Option<&'a str>, + version_id: Option<&'a str>, + priority: HealChannelPriority, + scan_mode: Option, + result: Result, + started_at: Instant, +} + +fn emit_scanner_heal_candidate_trace(trace: ScannerHealCandidateTrace<'_>) { + trace_emit(|| { + let (state, admission, error) = match trace.result { + Ok(result) if result.is_admitted() => ("admitted", describe_heal_admission(result), None), + Ok(result) => ("not_admitted", describe_heal_admission(result), None), + Err(error) => ("submit_failed", "channel_error".to_string(), Some(error)), + }; + let mut event = TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerHealCandidate) + .with_bucket(trace.bucket) + .with_duration(trace.started_at.elapsed()) + .with_attr("state", state) + .with_attr("candidate_type", trace.candidate_type) + .with_attr("priority", heal_priority_label(trace.priority)) + .with_attr("admission", admission); + + if let Some(object) = trace.object { + event = event.with_object(object); + } + if let Some(version_id) = trace.version_id { + event = event.with_attr("version_id", version_id); + } + if let Some(scan_mode) = trace.scan_mode { + event = event.with_attr("scan_mode", scan_mode.as_str()); + } + if let Some(error) = error { + event = event.with_attr("error", error); + } + + event + }); +} + fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary) { into.size = into.size.saturating_add(summary.total_size); into.versions = into.versions.saturating_add(summary.versions); @@ -677,9 +785,22 @@ async fn send_scanner_heal_request( request: HealChannelRequest, ) -> Result { let priority = request.priority; + let trace_context = scanner_heal_candidate_trace_context(&request); match send_heal_request_with_admission(request).await { Ok(result) => { record_heal_candidate_admission(candidate_type, priority, result); + if let Some(trace_context) = trace_context.as_ref() { + emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace { + candidate_type, + bucket: &trace_context.bucket, + object: trace_context.object.as_deref(), + version_id: trace_context.version_id.as_deref(), + priority, + scan_mode: trace_context.scan_mode, + result: Ok(result), + started_at: trace_context.started_at, + }); + } Ok(result) } Err(err) => { @@ -690,6 +811,18 @@ async fn send_scanner_heal_request( "result" => "channel_error".to_string() ) .increment(1); + if let Some(trace_context) = trace_context.as_ref() { + emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace { + candidate_type, + bucket: &trace_context.bucket, + object: trace_context.object.as_deref(), + version_id: trace_context.version_id.as_deref(), + priority, + scan_mode: trace_context.scan_mode, + result: Err(err.as_str()), + started_at: trace_context.started_at, + }); + } Err(ScannerError::Other(err)) } } @@ -905,7 +1038,9 @@ impl ScannerItem { "Scanner lifecycle action dispatched" ); let done_ilm = Metrics::time_ilm(event.action); + let trace_started_at = trace_start_instant(); let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { done_ilm(1)(); remaining_versions = 0; @@ -957,7 +1092,9 @@ impl ScannerItem { "Scanner lifecycle action dispatched" ); let done_ilm = Metrics::time_ilm(event.action); + let trace_started_at = trace_start_instant(); let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { done_ilm(1)(); if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction { @@ -995,7 +1132,9 @@ impl ScannerItem { "Scanner lifecycle action dispatched" ); let done_ilm = Metrics::time_ilm(event.action); + let trace_started_at = trace_start_instant(); let queued = apply_transition_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { done_ilm(1)(); } @@ -1019,7 +1158,21 @@ impl ScannerItem { let action = event.action; let count = u64::try_from(to_delete_objs.len()).unwrap_or(u64::MAX); let done_ilm = Metrics::time_ilm(action); + let trace_started_at = trace_start_instant(); let queued = enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner).await; + if let Some(trace_started_at) = trace_started_at { + let state = if queued { "queued" } else { "not_queued" }; + trace_emit(|| { + TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction) + .with_bucket(self.bucket.as_str()) + .with_object(self.object_path()) + .with_duration(trace_started_at.elapsed()) + .with_attr("state", state) + .with_attr("action", action.as_str()) + .with_attr("count", count) + .with_attr("queued", queued) + }); + } if record_scanner_ilm_action_if_queued(global_metrics(), action, count, queued) { done_ilm(count)(); remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len()); @@ -1830,6 +1983,7 @@ impl FolderScanner { into: &mut DataUsageEntry, ) -> Result<(), ScannerError> { let done_folder = Metrics::time(Metric::ScanFolder); + let trace_started_at = trace_start_instant(); if ctx.is_cancelled() { return Err(ScannerError::Other("Operation cancelled".to_string())); @@ -2895,6 +3049,8 @@ impl FolderScanner { } done_folder(); + let scanned_objects = u64::try_from(into.objects).unwrap_or(u64::MAX); + emit_scanner_folder_trace(&self.root, &folder.name, scanned_objects, trace_started_at, "completed"); Ok(()) } @@ -4400,6 +4556,104 @@ mod tests { ); } + #[tokio::test] + async fn scanner_trace_helpers_emit_expected_events() { + let mut trace = rustfs_common::trace_bus::subscribe_trace_events(); + + emit_scanner_folder_trace( + "/tmp/rustfs-scanner-trace", + "/tmp/rustfs-scanner-trace/bucket-a/folder-a", + 7, + Some(Instant::now()), + "completed", + ); + let folder = recv_scanner_trace_event( + &mut trace, + TraceFunc::ScannerFolder, + Some("bucket-a"), + Some("folder-a"), + Some("completed"), + ) + .await; + assert_eq!(trace_attr_string(&folder, "objects").as_deref(), Some("7")); + + emit_scanner_ilm_action_trace("bucket-a", "object-a", IlmAction::DeleteAction, 2, true, Some(Instant::now())); + let ilm = recv_scanner_trace_event( + &mut trace, + TraceFunc::ScannerIlmAction, + Some("bucket-a"), + Some("object-a"), + Some("queued"), + ) + .await; + assert_eq!(trace_attr_string(&ilm, "action").as_deref(), Some("delete")); + assert_eq!(trace_attr_string(&ilm, "count").as_deref(), Some("2")); + assert_eq!(trace_attr_string(&ilm, "queued").as_deref(), Some("true")); + + emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace { + candidate_type: "object", + bucket: "bucket-a", + object: Some("object-a"), + version_id: Some("version-a"), + priority: HealChannelPriority::High, + scan_mode: Some(HealScanMode::Deep), + result: Ok(HealAdmissionResult::Merged), + started_at: Instant::now(), + }); + let heal_candidate = recv_scanner_trace_event( + &mut trace, + TraceFunc::ScannerHealCandidate, + Some("bucket-a"), + Some("object-a"), + Some("admitted"), + ) + .await; + assert_eq!(trace_attr_string(&heal_candidate, "candidate_type").as_deref(), Some("object")); + assert_eq!(trace_attr_string(&heal_candidate, "priority").as_deref(), Some("high")); + assert_eq!(trace_attr_string(&heal_candidate, "scan_mode").as_deref(), Some("deep")); + assert_eq!(trace_attr_string(&heal_candidate, "version_id").as_deref(), Some("version-a")); + assert_eq!(trace_attr_string(&heal_candidate, "admission").as_deref(), Some("merged")); + } + + async fn recv_scanner_trace_event( + trace: &mut rustfs_common::trace_bus::TraceSubscription, + func: TraceFunc, + bucket: Option<&str>, + object: Option<&str>, + state: Option<&str>, + ) -> TraceEvent { + for _ in 0..32 { + let event = tokio::time::timeout(Duration::from_secs(1), trace.recv()) + .await + .expect("scanner trace event should arrive") + .expect("trace bus should stay open"); + if event.kind == TraceKind::Scanner + && event.func == func + && event.bucket.as_deref() == bucket + && event.object.as_deref() == object + && state.is_none_or(|state| trace_attr_string(&event, "state").as_deref() == Some(state)) + { + return (*event).clone(); + } + } + + panic!("expected scanner trace event {func:?} for bucket {bucket:?} object {object:?}"); + } + + fn trace_attr_string(event: &TraceEvent, key: &str) -> Option { + event.attrs.iter().find_map(|attr| { + if attr.key != key { + return None; + } + Some(match &attr.value { + rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(), + }) + }) + } + #[test] fn test_build_high_priority_heal_admission_error_contains_context() { let err = build_high_priority_heal_admission_error( diff --git a/rustfs/src/admin/handlers/profile_admin.rs b/rustfs/src/admin/handlers/profile_admin.rs index d4a85360a..d14af9f3b 100644 --- a/rustfs/src/admin/handlers/profile_admin.rs +++ b/rustfs/src/admin/handlers/profile_admin.rs @@ -23,18 +23,24 @@ use futures::{Stream, StreamExt}; use http::{HeaderMap, HeaderValue}; use hyper::{Method, StatusCode}; use matchit::Params; +use regex::Regex; +use rustfs_common::trace_bus::{TraceEvent, TraceKind, TraceVal, subscribe_trace_events}; use rustfs_madmin::service_commands::ServiceTraceOpts; +use rustfs_madmin::trace::TraceType; use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::header::CONTENT_TYPE; use s3s::stream::{ByteStream, DynByteStream}; use s3s::{Body, S3Request, S3Response, S3Result, StdError, s3_error}; use serde::Serialize; +use std::collections::HashMap; use std::pin::Pin; use std::task::{Context, Poll}; -use std::time::Duration; +use std::time::{Duration, SystemTime}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tracing::error; +use url::form_urlencoded; #[derive(Serialize)] struct ProfileStatus { @@ -206,16 +212,164 @@ impl Stream for TraceStream { impl ByteStream for TraceStream {} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct TraceKindFilter { + heal: bool, + scanner: bool, +} + +impl TraceKindFilter { + const ALL_SUPPORTED: Self = Self { + heal: true, + scanner: true, + }; + + fn from_request(uri: &hyper::Uri, trace_types: TraceType) -> S3Result { + let mut has_kind = false; + let mut filter = Self { + heal: false, + scanner: false, + }; + + for (key, value) in trace_query_pairs(uri) { + if key != "kind" { + continue; + } + has_kind = true; + for item in value.split(',') { + match item.trim().to_ascii_lowercase().as_str() { + "heal" | "healing" => filter.heal = true, + "scanner" => filter.scanner = true, + "all" => return Ok(Self::ALL_SUPPORTED), + _ => return Err(s3_error!(InvalidRequest, "invalid trace kind")), + } + } + } + + if has_kind { + return Ok(filter); + } + + if trace_types.mask() == 0 || trace_query_flag(uri, "all") { + return Ok(Self::ALL_SUPPORTED); + } + + Ok(Self { + heal: trace_types.overlaps(&TraceType::HEALING), + scanner: trace_types.overlaps(&TraceType::SCANNER), + }) + } + + const fn matches(self, kind: TraceKind) -> bool { + match kind { + TraceKind::Heal => self.heal, + TraceKind::Scanner => self.scanner, + } + } +} + +#[derive(Debug)] +struct TraceStreamFilter { + kinds: TraceKindFilter, + regex: Option, + threshold: Duration, +} + +impl TraceStreamFilter { + fn from_request(uri: &hyper::Uri, opts: &ServiceTraceOpts) -> S3Result { + if opts.only_errors() { + return Err(s3_error!( + InvalidRequest, + "trace error-only filter is not supported for heal/scanner trace" + )); + } + + Ok(Self { + kinds: TraceKindFilter::from_request(uri, opts.trace_types())?, + regex: trace_regex_filter(uri)?, + threshold: opts.threshold(), + }) + } + + fn matches_kind(&self, kind: TraceKind) -> bool { + self.kinds.matches(kind) + } + + fn matches_record(&self, record: &TraceWireRecord) -> bool { + record.duration >= self.threshold && self.regex.as_ref().is_none_or(|regex| record.matches_regex(regex)) + } +} + +#[derive(Serialize)] +struct TraceWireRecord { + #[serde(rename = "type")] + trace_type: u64, + #[serde(rename = "nodename")] + node_name: String, + #[serde(rename = "funcname")] + func_name: String, + #[serde(rename = "time")] + time: String, + #[serde(rename = "path")] + path: String, + #[serde(rename = "dur")] + duration: Duration, + #[serde(rename = "bytes", skip_serializing_if = "Option::is_none")] + bytes: Option, + #[serde(rename = "msg", skip_serializing_if = "Option::is_none")] + message: Option, + #[serde(rename = "custom", skip_serializing_if = "Option::is_none")] + custom: Option>, +} + +impl TraceWireRecord { + fn from_event(node_name: &str, event: &TraceEvent) -> Self { + Self { + trace_type: trace_type_mask(event.kind), + node_name: node_name.to_owned(), + func_name: event.func.as_str().to_owned(), + time: trace_time_string(event.time), + path: trace_path(event), + duration: event.duration, + bytes: trace_bytes(event.bytes), + message: None, + custom: trace_custom_attrs(event), + } + } + + fn dropped(node_name: &str, dropped: u64) -> Self { + let mut custom = HashMap::new(); + custom.insert("dropped_events".to_string(), dropped.to_string()); + + Self { + trace_type: 0, + node_name: node_name.to_owned(), + func_name: "trace.Dropped".to_string(), + time: trace_time_string(SystemTime::now()), + path: String::new(), + duration: Duration::ZERO, + bytes: None, + message: Some("trace subscriber lagged".to_string()), + custom: Some(custom), + } + } + + fn matches_regex(&self, regex: &Regex) -> bool { + regex.is_match(&self.func_name) + || regex.is_match(&self.path) + || self.message.as_ref().is_some_and(|message| regex.is_match(message)) + || self + .custom + .as_ref() + .is_some_and(|custom| custom.iter().any(|(key, value)| regex.is_match(key) || regex.is_match(value))) + } +} + /// `GET /v3/trace` — stream real-time server trace events. /// -/// RustFS emits diagnostics through the `tracing` pipeline but does not expose -/// an in-process subscriber that can fan trace events out to an admin client -/// (there is no request-trace broadcast channel). Rather than return an opaque -/// `501` — which would make `mc admin trace` fail to connect — this honors the -/// streaming NDJSON contract: it validates the requested trace filters, opens -/// the stream, emits a single capability record explaining that live tracing is -/// not wired, then holds the connection open with keep-alives. No fabricated -/// trace records are ever sent. +/// RustFS currently publishes heal and scanner diagnostics through the common +/// trace bus. The admin endpoint exposes those events as MinIO-shaped NDJSON +/// records while keeping unsupported trace classes filtered out. pub struct TraceHandler {} #[async_trait::async_trait] @@ -228,24 +382,13 @@ impl Operation for TraceHandler { let mut opts = ServiceTraceOpts::default(); opts.parse_params(&req.uri) .map_err(|_| s3_error!(InvalidRequest, "invalid trace parameters"))?; + let filter = TraceStreamFilter::from_request(&req.uri, &opts)?; let node_name = sysinfo::System::host_name().unwrap_or_else(|| "rustfs".to_string()); - let (tx, rx) = mpsc::channel::>(8); + let mut subscription = subscribe_trace_events(); + let (tx, rx) = mpsc::channel::>(64); spawn_traced(async move { - let notice = serde_json::json!({ - "nodename": node_name, - "funcname": "admin.Trace", - "msg": "RustFS does not expose an in-process trace-event subscriber; live tracing is not yet available", - "err": "trace_streaming_unsupported", - }); - if let Ok(mut encoded) = serde_json::to_vec(¬ice) { - encoded.push(b'\n'); - if tx.send(Ok(Bytes::from(encoded))).await.is_err() { - return; - } - } - let mut ticker = tokio::time::interval(Duration::from_secs(15)); ticker.tick().await; loop { @@ -256,6 +399,26 @@ impl Operation for TraceHandler { break; } } + received = subscription.recv() => { + match received { + Ok(event) => { + if !filter.matches_kind(event.kind) { + continue; + } + let record = TraceWireRecord::from_event(&node_name, &event); + if filter.matches_record(&record) && send_trace_record(&tx, &record).await.is_err() { + break; + } + } + Err(tokio::sync::broadcast::error::RecvError::Lagged(dropped)) => { + let record = TraceWireRecord::dropped(&node_name, dropped); + if send_trace_record(&tx, &record).await.is_err() { + break; + } + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + } + } } } }); @@ -269,17 +432,115 @@ impl Operation for TraceHandler { } } +async fn send_trace_record(tx: &mpsc::Sender>, record: &TraceWireRecord) -> Result<(), ()> { + let Some(encoded) = encode_ndjson(record) else { + return Ok(()); + }; + tx.send(Ok(encoded)).await.map_err(|_| ()) +} + +fn encode_ndjson(value: &impl Serialize) -> Option { + let mut encoded = serde_json::to_vec(value).ok()?; + encoded.push(b'\n'); + Some(Bytes::from(encoded)) +} + +fn trace_query_pairs(uri: &hyper::Uri) -> impl Iterator + '_ { + uri.query() + .into_iter() + .flat_map(|query| form_urlencoded::parse(query.as_bytes())) + .map(|(key, value)| (key.into_owned(), value.into_owned())) +} + +fn trace_query_flag(uri: &hyper::Uri, flag: &str) -> bool { + trace_query_pairs(uri).any(|(key, value)| key == flag && value == "true") +} + +fn trace_regex_filter(uri: &hyper::Uri) -> S3Result> { + trace_query_pairs(uri) + .find_map(|(key, value)| { + if key == "filter" && !value.is_empty() { + Some(value) + } else { + None + } + }) + .map(|pattern| Regex::new(&pattern).map_err(|_| s3_error!(InvalidRequest, "invalid trace filter"))) + .transpose() +} + +fn trace_type_mask(kind: TraceKind) -> u64 { + match kind { + TraceKind::Heal => TraceType::HEALING.mask(), + TraceKind::Scanner => TraceType::SCANNER.mask(), + } +} + +fn trace_time_string(time: SystemTime) -> String { + match OffsetDateTime::from(time).format(&Rfc3339) { + Ok(value) => value, + Err(_) => "1970-01-01T00:00:00Z".to_string(), + } +} + +fn trace_path(event: &TraceEvent) -> String { + match (event.bucket.as_deref(), event.object.as_deref()) { + (Some(bucket), Some(object)) if !object.is_empty() => format!("{bucket}/{object}"), + (Some(bucket), _) => bucket.to_owned(), + (None, Some(object)) => object.to_owned(), + (None, None) => String::new(), + } +} + +fn trace_bytes(bytes: u64) -> Option { + if bytes == 0 { + return None; + } + + match i64::try_from(bytes) { + Ok(value) => Some(value), + Err(_) => Some(i64::MAX), + } +} + +fn trace_custom_attrs(event: &TraceEvent) -> Option> { + if event.attrs.is_empty() { + return None; + } + + Some( + event + .attrs + .iter() + .map(|attr| (attr.key.to_string(), trace_value_string(&attr.value))) + .collect(), + ) +} + +fn trace_value_string(value: &TraceVal) -> String { + match value { + TraceVal::Bool(value) => value.to_string(), + TraceVal::U64(value) => value.to_string(), + TraceVal::I64(value) => value.to_string(), + TraceVal::Str(value) => value.to_string(), + } +} + #[cfg(test)] mod tests { use super::{ ProfileControlHandler, ProfileHandler, ProfileStatusHandler, ProfilingDownloadHandler, ProfilingStartHandler, - TraceHandler, + TraceHandler, TraceKindFilter, TraceStreamFilter, TraceWireRecord, }; use crate::admin::router::Operation; use http::{Extensions, HeaderMap, Uri}; use hyper::Method; use matchit::Params; - use s3s::{Body, S3ErrorCode, S3Request}; + use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind}; + use rustfs_madmin::service_commands::ServiceTraceOpts; + use rustfs_madmin::trace::TraceType; + use s3s::{Body, S3ErrorCode, S3Request, S3Result}; + use std::time::{Duration, UNIX_EPOCH}; fn build_profile_request(uri: &'static str) -> S3Request { S3Request { @@ -295,6 +556,13 @@ mod tests { } } + fn build_trace_stream_filter(uri: &'static str) -> S3Result { + let uri = Uri::from_static(uri); + let mut opts = ServiceTraceOpts::default(); + opts.parse_params(&uri).expect("test trace params should parse"); + TraceStreamFilter::from_request(&uri, &opts) + } + #[tokio::test] async fn profile_handler_rejects_missing_credentials() { let result = ProfileHandler {} @@ -358,4 +626,108 @@ mod tests { .expect_err("trace must reject anonymous requests"); assert_eq!(err.code(), &S3ErrorCode::AccessDenied); } + + #[test] + fn trace_kind_filter_supports_kind_query() { + let uri = Uri::from_static("/rustfs/admin/v3/trace?kind=heal"); + let filter = TraceKindFilter::from_request(&uri, TraceType::default()).expect("kind filter should parse"); + + assert!(filter.matches(TraceKind::Heal)); + assert!(!filter.matches(TraceKind::Scanner)); + } + + #[test] + fn trace_kind_filter_defaults_to_supported_events_without_type_flags() { + let uri = Uri::from_static("/rustfs/admin/v3/trace"); + let filter = TraceKindFilter::from_request(&uri, TraceType::default()).expect("empty filter should parse"); + + assert!(filter.matches(TraceKind::Heal)); + assert!(filter.matches(TraceKind::Scanner)); + } + + #[test] + fn trace_kind_filter_rejects_unknown_kind() { + let uri = Uri::from_static("/rustfs/admin/v3/trace?kind=s3"); + let err = TraceKindFilter::from_request(&uri, TraceType::default()).expect_err("unknown kind should fail"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn trace_stream_filter_matches_regex_against_path_and_attrs() { + let filter = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&filter=data/.%2Bxl.meta") + .expect("regex filter should parse"); + let event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject) + .with_bucket("data") + .with_object("dir/xl.meta") + .with_attr("dry_run", true); + let record = TraceWireRecord::from_event("node-a", &event); + + assert!(filter.matches_kind(event.kind)); + assert!(filter.matches_record(&record)); + } + + #[test] + fn trace_stream_filter_rejects_invalid_regex() { + let err = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&filter=[").expect_err("invalid regex should fail"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn trace_stream_filter_applies_threshold() { + let filter = + build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&threshold=10ms").expect("threshold should parse"); + let short = TraceWireRecord::from_event( + "node-a", + &TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject).with_duration(Duration::from_millis(9)), + ); + let long = TraceWireRecord::from_event( + "node-a", + &TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject).with_duration(Duration::from_millis(10)), + ); + + assert!(!filter.matches_record(&short)); + assert!(filter.matches_record(&long)); + } + + #[test] + fn trace_stream_filter_rejects_error_only_filter() { + let err = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&err=true").expect_err("err filter should fail"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn trace_wire_record_contains_madmin_trace_fields() { + let event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject) + .with_bucket("bucket") + .with_object("object") + .with_duration(Duration::from_millis(3)) + .with_bytes(17) + .with_attr("dry", true); + let mut record = TraceWireRecord::from_event("node-a", &event); + record.time = "1970-01-01T00:00:00Z".to_string(); + + let value = serde_json::to_value(&record).expect("trace record should serialize"); + + assert_eq!(value["type"], TraceType::HEALING.mask()); + assert_eq!(value["nodename"], "node-a"); + assert_eq!(value["funcname"], "heal.Object"); + assert_eq!(value["time"], "1970-01-01T00:00:00Z"); + assert_eq!(value["path"], "bucket/object"); + assert_eq!(value["bytes"], 17); + assert_eq!(value["custom"]["dry"], "true"); + } + + #[test] + fn trace_wire_record_formats_epoch_time() { + let event = TraceEvent { + time: UNIX_EPOCH, + ..TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder) + }; + let record = TraceWireRecord::from_event("node-a", &event); + + assert_eq!(record.time, "1970-01-01T00:00:00Z"); + } } diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index d06e0892f..a39ca6c3a 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -2420,6 +2420,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> rustfs_heal::Result<(Vec, Option, bool)> { Ok((Vec::new(), None, false)) } From 35a30cd6144f5f37f3b03e2964b73655ed408274 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 08:46:32 +0800 Subject: [PATCH 71/71] feat(scanner): emit excess alerts as S3 notification events (HS-04) (#6176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(scanner): emit excess alerts as S3 notification events The excess-versions / excess-version-size / excess-folders alerts were metrics-and-logs only; consoles and external auditors had no way to hear them (rustfs/backlog#1868, HS-04). MinIO emits s3:ObjectManyVersions / s3:ObjectLargeVersions / s3:PrefixManyFolders for the same conditions — RustFS carries those as EventName::Scanner* with s3:Scanner:* wire names that already existed unpublished. The three alert sites now also dispatch through the standard event pipeline (send_event via the storage_api owner facade), carrying the actual values and thresholds in req_params and UserAgent "Scanner". Without a cooldown a single over-threshold object would re-emit on every ~60s scan cycle, so emissions are edge-held per (kind, bucket, object) for 24h (RUSTFS_SCANNER_ALERT_COOLDOWN_SECS, 0 = every cycle), backed by a process-global map with a 4096-key hard cap that clears rather than grows. Metrics and structured logs stay level-triggered every cycle; only the notification events are held back. A restart resets the cooldown deliberately: one re-emission per still-hot key buys back visibility after the restarts that accompany incident response. Tests pin the edge-hold semantics (first fires, immediate re-check held, independent keys, cooldown expiry re-fires, zero cooldown always emits, hard bound) in one sequential test for the process-global map, and pin the emitted wire names against EventName's canonical string forms so a subscribed bucket notification can never silently stop matching. docs/operations/scanner-excess-alerts.md documents the three events, the metric-vs-event cadence difference, and the HS-15 threshold deltas (alert_excess_folders 65538 vs MinIO 50000 is deliberate: Proxmox Backup Server chunk layout compatibility). Closes rustfs/backlog#1868. Co-Authored-By: heihutu * docs(operations): split scanner excess alerts into English and Chinese pages The page shipped Chinese-only; keep it as scanner-excess-alerts_zh.md and add a faithful English translation at the original path, cross-linked at the top of both. Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- Cargo.lock | 1 + crates/ecstore/src/api/mod.rs | 2 +- crates/scanner/Cargo.toml | 3 + crates/scanner/src/scanner_folder.rs | 229 +++++++++++++++++++- crates/scanner/src/storage_api.rs | 7 +- docs/operations/scanner-excess-alerts.md | 37 ++++ docs/operations/scanner-excess-alerts_zh.md | 37 ++++ 7 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 docs/operations/scanner-excess-alerts.md create mode 100644 docs/operations/scanner-excess-alerts_zh.md diff --git a/Cargo.lock b/Cargo.lock index 85c91d523..cda581ffb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10252,6 +10252,7 @@ dependencies = [ "rustfs-ecstore", "rustfs-filemeta", "rustfs-lock", + "rustfs-s3-types", "rustfs-storage-api", "rustfs-utils", "s3s", diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 4b8bc3249..3a031d60a 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -380,7 +380,7 @@ pub mod erasure { pub mod event { pub use crate::event::name::EventName; - pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook}; + pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook, send_event}; } pub mod global { diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 16ff34a8e..0e2d039c6 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -108,6 +108,9 @@ temp-env = { workspace = true } tempfile = { workspace = true } uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } +# Test-only: pins the emitted scanner alert wire names against the canonical +# EventName string forms subscribers configure (rustfs/backlog#1868). +rustfs-s3-types.workspace = true # Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via # the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6). rustfs-ecstore = { workspace = true, features = ["test-util"] } diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 820cbb12f..15a01507c 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fs::FileType; use std::io::ErrorKind; -use std::sync::{Arc, Once}; +use std::sync::{Arc, Mutex, Once}; use std::time::{Duration, Instant, SystemTime}; use crate::ReplTargetSizeSummary; @@ -32,6 +32,7 @@ use crate::scanner_io::{ SCANNER_SKIP_FILE_ERROR, ScannerIODisk as _, is_scanner_metadata_corrupt_error, is_scanner_metadata_transient_error, }; use crate::sleeper::DynamicSleeper; +use crate::storage_api::owner::{EcstoreEventArgs, ecstore_send_event}; use metrics::{counter, describe_counter}; use rustfs_common::heal_channel::{ HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest, @@ -98,6 +99,101 @@ const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders const METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL: &str = "rustfs_scanner_pending_heal_prune_total"; const METRIC_SCANNER_PENDING_HEAL_MALFORMED_TOTAL: &str = "rustfs_scanner_pending_heal_malformed_total"; const MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET: usize = 128; + +// --- scanner excess alerts as S3 notification events (rustfs/backlog#1868) -- +// +// The excess-versions / excess-version-size / excess-folders alerts were +// metrics-and-logs only; subscribers (consoles, external auditors) had no way +// to hear them. MinIO emits s3:ObjectManyVersions / s3:ObjectLargeVersions / +// s3:PrefixManyFolders for the same conditions — RustFS carries those as +// EventName::Scanner* with the wire names below. Without a cooldown a single +// over-threshold object would re-emit on every scan cycle (~a minute), so +// emissions are edge-held per (kind, bucket, object) for 24h. + +/// `s3:Scanner:ManyVersions` (MinIO `s3:ObjectManyVersions`). +pub const EVENT_SCANNER_MANY_VERSIONS: &str = "s3:Scanner:ManyVersions"; +/// `s3:Scanner:LargeVersions` (MinIO `s3:ObjectLargeVersions`). +pub const EVENT_SCANNER_LARGE_VERSIONS: &str = "s3:Scanner:LargeVersions"; +/// `s3:Scanner:BigPrefix` (MinIO `s3:PrefixManyFolders`). +pub const EVENT_SCANNER_BIG_PREFIX: &str = "s3:Scanner:BigPrefix"; +const ENV_SCANNER_ALERT_COOLDOWN_SECS: &str = "RUSTFS_SCANNER_ALERT_COOLDOWN_SECS"; +const DEFAULT_SCANNER_ALERT_COOLDOWN_SECS: u64 = 86_400; +/// Hard cap on distinct cooldown keys; a pathological number of over-threshold +/// objects clears the map wholesale instead of growing without bound (the +/// worst case is one re-emission per still-hot key per scan cycle). +const MAX_SCANNER_ALERT_COOLDOWN_KEYS: usize = 4096; + +/// Distinct alert kinds sharing one cooldown map. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum ScannerAlertKind { + ManyVersions, + LargeVersions, + BigPrefix, +} + +type ScannerAlertCooldownKey = (ScannerAlertKind, String, String); +type ScannerAlertCooldownMap = HashMap; + +static SCANNER_ALERT_EMISSION_COOLDOWN: Mutex> = Mutex::new(None); + +fn scanner_alert_cooldown() -> Duration { + let raw = std::env::var(ENV_SCANNER_ALERT_COOLDOWN_SECS) + .ok() + .and_then(|v| v.parse::().ok()); + Duration::from_secs(raw.unwrap_or(DEFAULT_SCANNER_ALERT_COOLDOWN_SECS)) +} + +/// Edge-held emission gate: returns `true` (and records the cooldown) only +/// when this (kind, bucket, object) last fired longer than the cooldown ago — +/// or never. Metrics and logs stay level-triggered every cycle; only the +/// notification events are held back. +fn scanner_alert_emission_allows(kind: ScannerAlertKind, bucket: &str, object: &str, cooldown: Duration) -> bool { + let key = (kind, bucket.to_string(), object.to_string()); + let mut guard = SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + let guard = guard.get_or_insert_with(ScannerAlertCooldownMap::new); + let now = Instant::now(); + // Expired entries leave first; the cap is still exceeded only when live + // keys alone overflow it, in which case a wholesale clear trades one + // extra emission per hot key for a hard memory bound. + if guard.len() >= MAX_SCANNER_ALERT_COOLDOWN_KEYS { + guard.retain(|_, fired_at| now.duration_since(*fired_at) < cooldown); + if guard.len() >= MAX_SCANNER_ALERT_COOLDOWN_KEYS { + guard.clear(); + } + } + match guard.get(&key) { + Some(fired_at) if now.duration_since(*fired_at) < cooldown => false, + _ => { + guard.insert(key, now); + true + } + } +} + +/// Emit a scanner alert as an S3 notification event through the standard +/// dispatch pipeline. Fire-and-forget: the notify layer owns delivery, +/// retry, and target filtering; the scanner never waits on it. +fn emit_scanner_alert_event(event_name: &str, bucket: &str, object: &str, size: i64, details: &[(&str, String)]) { + let mut req_params = HashMap::with_capacity(details.len()); + for (key, value) in details { + req_params.insert((*key).to_string(), value.clone()); + } + ecstore_send_event(EcstoreEventArgs { + event_name: event_name.to_string(), + bucket_name: bucket.to_string(), + object: crate::ScannerObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size, + ..Default::default() + }, + req_params, + user_agent: "Scanner".to_string(), + ..Default::default() + }); +} const MAX_PENDING_SCANNER_HEALS_PER_BUCKET: usize = 10_000; static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new(); @@ -1350,6 +1446,7 @@ impl ScannerItem { fn alert_excessive_versions(&self, remaining_versions: usize, cumulative_size: i64) { ensure_scanner_alert_metrics_registered(); let (too_many_versions, too_large_versions) = should_alert_excessive_versions(remaining_versions, cumulative_size); + let object_path = self.object_path(); if too_many_versions { global_metrics().record_scanner_source_executed(ScannerWorkSource::Alerts, 1); counter!( @@ -1357,13 +1454,26 @@ impl ScannerItem { "bucket" => self.bucket.clone() ) .increment(1); + if scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, &self.bucket, &object_path, scanner_alert_cooldown()) + { + emit_scanner_alert_event( + EVENT_SCANNER_MANY_VERSIONS, + &self.bucket, + &object_path, + cumulative_size, + &[ + ("versions", remaining_versions.to_string()), + ("threshold", scanner_excess_versions_threshold().to_string()), + ], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_FOLDER, bucket = %self.bucket, - object = %self.object_path(), + object = %object_path, versions = remaining_versions, threshold = scanner_excess_versions_threshold(), state = "excess_versions", @@ -1377,13 +1487,31 @@ impl ScannerItem { "bucket" => self.bucket.clone() ) .increment(1); + if scanner_alert_emission_allows( + ScannerAlertKind::LargeVersions, + &self.bucket, + &object_path, + scanner_alert_cooldown(), + ) { + emit_scanner_alert_event( + EVENT_SCANNER_LARGE_VERSIONS, + &self.bucket, + &object_path, + cumulative_size, + &[ + ("versions", remaining_versions.to_string()), + ("cumulativeSize", cumulative_size.to_string()), + ("threshold", scanner_excess_version_size_threshold().to_string()), + ], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_FOLDER, bucket = %self.bucket, - object = %self.object_path(), + object = %object_path, versions = remaining_versions, cumulative_size, threshold = scanner_excess_version_size_threshold(), @@ -1764,6 +1892,15 @@ impl FolderScanner { "root" => self.root.clone() ) .increment(1); + if scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, &self.root, folder, scanner_alert_cooldown()) { + emit_scanner_alert_event( + EVENT_SCANNER_BIG_PREFIX, + &self.root, + folder, + 0, + &[("folders", total_folders.to_string()), ("threshold", threshold.to_string())], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, @@ -3232,6 +3369,90 @@ mod tests { #[cfg(unix)] use std::os::unix::fs::{PermissionsExt, symlink}; use std::sync::Mutex; + + /// Reset the process-global alert cooldown map; test-only. + fn reset_alert_cooldowns() { + *SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(ScannerAlertCooldownMap::new()); + } + + /// The emitted event-name strings must be exactly what `EventName` + /// serializes, or a bucket notification subscribed to the documented name + /// would silently never match (rustfs/backlog#1868). + #[test] + fn scanner_alert_wire_names_match_canonical_event_names() { + use rustfs_s3_types::EventName; + assert_eq!(EVENT_SCANNER_MANY_VERSIONS, EventName::ScannerManyVersions.to_string()); + assert_eq!(EVENT_SCANNER_LARGE_VERSIONS, EventName::ScannerLargeVersions.to_string()); + assert_eq!(EVENT_SCANNER_BIG_PREFIX, EventName::ScannerBigPrefix.to_string()); + } + + fn cooldown_map_len() -> usize { + SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .as_ref() + .map(|map| map.len()) + .unwrap_or(0) + } + + /// Backdate every recorded cooldown so the next check fires again. + fn expire_all_alert_cooldowns(cooldown: Duration) { + let now = Instant::now(); + let mut guard = SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if let Some(map) = guard.as_mut() { + for fired_at in map.values_mut() { + if let Some(expired) = now.checked_sub(cooldown + Duration::from_secs(1)) { + *fired_at = expired; + } + } + } + } + + /// The emission gate is the only thing standing between an over-threshold + /// object and one S3 event per scan cycle, so its edge semantics get + /// pinned directly. All scenarios share one #[test] because the cooldown + /// map is process-global and parallel tests would read each other's + /// firings. + #[test] + fn scanner_alert_emission_is_edge_held_per_key_and_bounded() { + reset_alert_cooldowns(); + let cooldown = Duration::from_secs(3600); + + // First firing allows, an immediate re-check is held. + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + assert!(!scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + + // Different kind, object, and bucket are independent keys. + assert!(scanner_alert_emission_allows(ScannerAlertKind::LargeVersions, "bkt", "obj", cooldown)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "other", cooldown)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "other", "obj", cooldown)); + assert_eq!(cooldown_map_len(), 4); + + // After the cooldown elapses the same key fires again. + expire_all_alert_cooldowns(cooldown); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + + // A zero cooldown degenerates to always-emit (operators may want that). + assert!(scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", "dir", Duration::ZERO)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", "dir", Duration::ZERO)); + + // Hard bound: overflow the cap with zero-cooldown keys and confirm the + // map clears rather than growing past it. + reset_alert_cooldowns(); + for index in 0..=(MAX_SCANNER_ALERT_COOLDOWN_KEYS + 8) { + let _ = scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", &format!("dir-{index}"), Duration::ZERO); + } + assert!( + cooldown_map_len() <= MAX_SCANNER_ALERT_COOLDOWN_KEYS, + "cooldown map must stay bounded, got {}", + cooldown_map_len() + ); + } + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use temp_env::{with_var, with_var_unset}; use tracing_subscriber::fmt::MakeWriter; diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index e77033aa2..cbd7c6486 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -78,6 +78,7 @@ pub(crate) use rustfs_ecstore::api::disk::{ pub(crate) use rustfs_ecstore::api::error::{ Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError, }; +pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_event as ecstore_send_event}; #[cfg(test)] pub(crate) use rustfs_ecstore::api::layout::{ EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints, @@ -110,8 +111,8 @@ pub(crate) mod owner { ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS, ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, - EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc, - EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, + EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreEventArgs, + EcstoreLcEventSrc, EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi, ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, @@ -121,7 +122,7 @@ pub(crate) mod owner { ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, - ecstore_save_config, scanner_replication_config_for_lifecycle_eval, + ecstore_save_config, ecstore_send_event, scanner_replication_config_for_lifecycle_eval, }; #[cfg(test)] diff --git a/docs/operations/scanner-excess-alerts.md b/docs/operations/scanner-excess-alerts.md new file mode 100644 index 000000000..8079daba6 --- /dev/null +++ b/docs/operations/scanner-excess-alerts.md @@ -0,0 +1,37 @@ +# Scanner Excess Alerts: Metrics, S3 Events, and Thresholds + +> 中文版：[scanner-excess-alerts_zh.md](scanner-excess-alerts_zh.md) + +Date: 2026-08-18 (rustfs/backlog#1868 / HS-04; includes the HS-15 threshold-delta notes) + +The background scanner detects three classes of "excess" conditions while it walks buckets and surfaces them as alerts. This page documents each alert's trigger condition, the subscribable S3 event, the cooldown semantics, and the threshold differences versus MinIO — for operators debugging alerts and for event consumers wiring up subscriptions. + +## The three alerts + +| Alert | Trigger (per scan cycle) | Metric | S3 event (RustFS wire name) | MinIO event name | +|---|---|---|---|---| +| Excess versions | Retained versions of one object ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | +| Excess version size | Cumulative bytes of all versions of one object ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | +| Excess folders | Direct subfolders of one directory > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | + +Subscribe like any bucket notification: configure a notification on the target bucket with the RustFS wire name above (or the `s3:Scanner:*` wildcard). Events carry `UserAgent: Scanner` as their origin marker, and `req_params` holds the observed value and the threshold (`versions` / `cumulativeSize` / `folders` / `threshold`), so consumers can judge severity directly. + +## Metrics and events fire on different cadences + +- **Metrics and structured logs are level-triggered**: as long as the object stays over the threshold, every scan cycle counts and logs it (default cycle ≈ 60s; see `scanner:speed`). +- **S3 events are edge-triggered with a cooldown**: the same (alert kind, bucket, object) emits at most once per cooldown window — 24 hours by default (`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`; set it to 0 to emit every cycle). When the window lapses and the object is still over the threshold, the event fires again. The cooldown table lives in process memory with a 4096-entry hard cap; on overflow it is cleared and rebuilt (worst case: one extra emission per still-hot key). +- A process restart resets the cooldown (every still-over-threshold object emits once more after a restart) — deliberately: restarts usually accompany incident response, and the re-emission buys visibility. + +## Threshold defaults and the MinIO deltas (HS-15) + +| Config key | ENV | RustFS default | MinIO default | Notes | +|---|---|---|---|---| +| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | Identical | +| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | Same order of magnitude; different unit basis (TiB vs TB) | +| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **Deliberate divergence**: 65538 tolerates the Proxmox Backup Server chunk layout (65536 chunks per directory plus the directory's own entries); MinIO's 50000 would fire continuously for PBS users. Set it to 50000 explicitly to match MinIO behavior | + +All three keys accept both env and admin config (`PUT /rustfs/admin/v3/config`, `scanner` subsystem); hot updates take effect immediately. + +## Why the event names are mapped + +RustFS's event enum (`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`) keeps the repo's established `s3:Scanner:*` wire names (literally different from MinIO's `s3:ObjectManyVersions`; the enum comments preserve the mapping). Subscribers should use the RustFS wire names in this page. If you need MinIO-literal compatibility, map the names on the console/consumer side — do not change the published wire names. diff --git a/docs/operations/scanner-excess-alerts_zh.md b/docs/operations/scanner-excess-alerts_zh.md new file mode 100644 index 000000000..d4f4995ac --- /dev/null +++ b/docs/operations/scanner-excess-alerts_zh.md @@ -0,0 +1,37 @@ +# Scanner 超限告警：指标、S3 事件与阈值 + +> English version: [scanner-excess-alerts.md](scanner-excess-alerts.md) + +日期：2026-08-18（rustfs/backlog#1868 / HS-04，含 HS-15 阈值差异说明） + +后台 scanner 在扫描过程中检测三类"超限"状态并对外告警。本文说明每类告警的触发条件、可订阅的 S3 事件、冷却语义，以及与 MinIO 的阈值差异，供运维排障与事件消费方对接。 + +## 三类告警 + +| 告警 | 触发条件（任一扫描周期） | 指标 | S3 事件（RustFS wire 名） | MinIO 对应事件名 | +|---|---|---|---|---| +| 版本数超限 | 单对象保留版本数 ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | +| 版本总大小超限 | 单对象全部版本累计字节 ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | +| 子目录数超限 | 单目录直接子目录数 > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | + +订阅方式与普通桶通知一致：对目标桶配置 notification，事件名填上表 RustFS wire 名（或通配 `s3:Scanner:*`）。事件以 `UserAgent: Scanner` 标记来源，`req_params` 携带实际值与阈值（`versions` / `cumulativeSize` / `folders` / `threshold`），便于消费方直接判断严重程度。 + +## 指标与事件的触发节奏不同 + +- **指标与结构化日志是电平触发**：只要对象仍在阈值之上，每个扫描周期都会计数/打日志（默认周期约 60s，见 `scanner:speed`）。 +- **S3 事件是边沿触发 + 冷却**：同一 (告警类型， 桶, 对象) 在冷却窗口内只发一次，默认 24 小时（`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`，设 0 表示每周期都发）。窗口过后对象仍超限会再次发出。冷却表在进程内有 4096 条硬顶，超限清空重建（最坏情况是每个仍超限的 key 多发一次）。 +- 进程重启会重置冷却（重启后每个仍超限的对象会再发一次）——这是有意为之：重启常伴随排障，重发提供可见性。 + +## 阈值默认值与 MinIO 差异（HS-15） + +| 配置键 | ENV | RustFS 默认 | MinIO 默认 | 差异说明 | +|---|---|---|---|---| +| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | 一致 | +| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | 语义同量级，单位口径不同（TiB vs TB） | +| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **有意差异**：65538 兼容 Proxmox Backup Server 的 chunk 布局（每目录 65536 个 chunk + 目录自身条目），按 MinIO 的 50000 会对 PBS 用户持续误报。如需与 MinIO 行为一致可显式配置为 50000 | + +三个键均支持 env 与 admin config（`PUT /rustfs/admin/v3/config` 的 `scanner` 子系统）双通道，热更新即时生效。 + +## 事件名映射的由来 + +RustFS 的事件枚举（`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`）沿用仓库既有 wire 名 `s3:Scanner:*`（与 MinIO 的 `s3:ObjectManyVersions` 字面不同，枚举注释中保留了映射关系）。订阅方应以本文的 RustFS wire 名为准；如需 MinIO 字面兼容，请在 console/消费侧做名称映射，不要修改已发布的 wire 名。