diff --git a/.vscode/launch.json b/.vscode/launch.json index f113b1d5a..394c1e979 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -172,7 +172,7 @@ ], }, { - "name": "Debug executable target/debug/rustfs with sse", + "name": "Debug executable target/debug/rustfs with sse kms", "type": "lldb", "request": "launch", "program": "${workspaceFolder}/target/debug/rustfs", @@ -200,7 +200,7 @@ // 2. kms local backend test key // "RUSTFS_KMS_ENABLE": "true", // "RUSTFS_KMS_BACKEND": "local", - // "RUSTFS_KMS_KEY_DIR": "./target/kms-key-dir", + // "RUSTFS_KMS_KEY_DIR": "/tmp/kms-key-dir", // "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password // "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key", @@ -228,6 +228,29 @@ "rust" ], }, + { + "name": "Debug executable target/debug/rustfs with local sse", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/target/debug/rustfs", + "args": [], + "cwd": "${workspaceFolder}", + "env": { + "RUSTFS_ACCESS_KEY": "rustfsadmin", + "RUSTFS_SECRET_KEY": "rustfsadmin", + "RUSTFS_VOLUMES": "./target/volumes/test{1...4}", + "RUSTFS_ADDRESS": ":9000", + "RUSTFS_CONSOLE_ENABLE": "true", + "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001", + "RUSTFS_OBS_LOG_DIRECTORY": "./target/logs", + "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", + "RUSTFS_SSE_S3_MASTER_KEY": "xGb3aYSp825j2tPpg8JrUzghiXsIkfdOtmrsJ/iafiM=", + "RUST_LOG": "rustfs=debug,ecstore=debug,s3s=debug,iam=debug", + }, + "sourceLanguages": [ + "rust" + ], + }, { "type": "lldb", "request": "launch", diff --git a/Cargo.lock b/Cargo.lock index 6086ee54c..617f54b45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3941,7 +3941,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5361,7 +5361,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8097,7 +8097,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9477,6 +9477,8 @@ name = "rustfs-lifecycle" version = "1.0.0-beta.11" dependencies = [ "async-trait", + "metrics", + "metrics-util", "proptest", "rustfs-common", "rustfs-config", @@ -9714,6 +9716,7 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "ipnetwork", "libunftp", "md5", "percent-encoding", @@ -9731,6 +9734,7 @@ dependencies = [ "rustfs-rio", "rustfs-storage-api", "rustfs-tls-runtime", + "rustfs-trusted-proxies", "rustfs-utils", "rustls", "s3s", @@ -10221,7 +10225,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10294,7 +10298,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10351,7 +10355,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.14.1" -source = "git+https://github.com/s3s-project/s3s.git?rev=8136db4ac8253c0ccfd86f8216e00e0235747757#8136db4ac8253c0ccfd86f8216e00e0235747757" +source = "git+https://github.com/cxymds/s3s.git?rev=afa1796ec64cd1e1c78dfee46c4b4b72bbc7b39d#afa1796ec64cd1e1c78dfee46c4b4b72bbc7b39d" dependencies = [ "arc-swap", "arrayvec", @@ -11494,10 +11498,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -12598,7 +12602,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e6e66d836..3329a6e39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -288,7 +288,7 @@ redis = { version = "1.4.1" } rustix = { version = "1.1.4" } rust-embed = { version = "8.12.0" } rustc-hash = { version = "2.1.3" } -s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "8136db4ac8253c0ccfd86f8216e00e0235747757" } +s3s = { git = "https://github.com/cxymds/s3s.git", rev = "afa1796ec64cd1e1c78dfee46c4b4b72bbc7b39d" } serial_test = "4.0.1" shadow-rs = { default-features = false, version = "2.0.0" } siphasher = "1.0.3" diff --git a/crates/e2e_test/src/reliant/grpc_lock_server.rs b/crates/e2e_test/src/reliant/grpc_lock_server.rs index 2525be58b..301125188 100644 --- a/crates/e2e_test/src/reliant/grpc_lock_server.rs +++ b/crates/e2e_test/src/reliant/grpc_lock_server.rs @@ -400,6 +400,20 @@ impl NodeService for MinimalLockNodeService { Err(Status::unimplemented("lock-only test server")) } + async fn prepare_part_transaction( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("lock-only test server")) + } + + async fn settle_part_transaction( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("lock-only test server")) + } + async fn rename_file( &self, _request: Request, diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 61ef15198..fc042f97f 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -303,9 +303,9 @@ pub mod disk { pub use crate::disk::{ BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, - FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, RUSTFS_META_BUCKET, - ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, - WalkDirOptions, new_disk, validate_batch_read_version_item_count, + FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, + PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, + STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count, }; pub use bytes::Bytes; pub use endpoint::Endpoint; diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 0e04bca35..2d9598339 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -1182,8 +1182,12 @@ impl TransitionState { } fn new_with_capacity(capacity: usize) -> Arc { - let capacity = capacity.max(1); let queue_send_timeout = resolve_transition_queue_send_timeout(); + Self::new_with_capacity_and_timeout(capacity, queue_send_timeout) + } + + fn new_with_capacity_and_timeout(capacity: usize, queue_send_timeout: StdDuration) -> Arc { + let capacity = capacity.max(1); let (tx1, rx1) = bounded(capacity); Arc::new(Self { transition_tx: tx1, @@ -1250,13 +1254,12 @@ impl TransitionState { return false; } let bucket = bucket.to_string(); - let scheduled = Arc::clone(&self.compensation_buckets); let state = Arc::clone(self); tokio::spawn(async move { Self::inc_counter(&state.compensation_running_tasks); state.record_scanner_transition_state(); let Some(api) = runtime_sources::object_store_handle() else { - scheduled.lock().unwrap().remove(&bucket); + state.finish_bucket_compensation(&bucket); Self::add_counter(&state.compensation_running_tasks, -1); state.record_scanner_transition_state(); debug!( @@ -1292,13 +1295,25 @@ impl TransitionState { ); } - scheduled.lock().unwrap().remove(&bucket); + state.finish_bucket_compensation(&bucket); Self::add_counter(&state.compensation_running_tasks, -1); state.record_scanner_transition_state(); }); true } + fn finish_bucket_compensation(&self, bucket: &str) { + match self.compensation_buckets.lock() { + Ok(mut scheduled) => { + scheduled.remove(bucket); + } + Err(poisoned) => { + poisoned.into_inner().remove(bucket); + self.compensation_buckets.clear_poison(); + } + } + } + #[inline] fn inc_counter(counter: &AtomicI64) { Self::add_counter(counter, 1); @@ -1535,17 +1550,35 @@ impl TransitionState { let outcome = match self.transition_tx.try_send(Some(task)) { Ok(()) => TransitionEnqueueOutcome::Queued, - Err(async_channel::TrySendError::Full(_)) => { + Err(async_channel::TrySendError::Full(task)) => { Self::inc_counter(&self.queue_full_tasks); - debug!( - bucket = %oi.bucket, - object = %oi.name, - source = ?src, - "transition queue is full; deferring to scanner/backfill" - ); - TransitionEnqueueOutcome::QueueFull + let send_timeout = self.transition_queue_send_timeout; + match tokio::time::timeout(send_timeout, self.transition_tx.send(task)).await { + Ok(Ok(())) => TransitionEnqueueOutcome::Queued, + Ok(Err(_)) => { + self.schedule_bucket_compensation(&oi.bucket); + TransitionEnqueueOutcome::QueueClosed + } + Err(_) => { + Self::inc_counter(&self.queue_send_timeout_tasks); + self.schedule_bucket_compensation(&oi.bucket); + debug!( + bucket = %oi.bucket, + object = %oi.name, + source = ?src, + timeout_ms = send_timeout.as_millis() as u64, + event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + state = "queue_send_timed_out", + "Scanner transition enqueue timed out; scheduled bucket compensation" + ); + TransitionEnqueueOutcome::QueueFull + } + } } Err(async_channel::TrySendError::Closed(_)) => { + self.schedule_bucket_compensation(&oi.bucket); debug!( bucket = %oi.bucket, object = %oi.name, @@ -1730,7 +1763,7 @@ impl TransitionState { } pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) { - let mut tier_stats = self.last_day_stats.lock().unwrap(); + let mut tier_stats = self.lock_last_day_stats(); tier_stats .entry(tier.to_string()) .and_modify(|e| e.add_stats(ts)) @@ -1738,7 +1771,7 @@ impl TransitionState { } pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats { - let tier_stats = self.last_day_stats.lock().unwrap(); + let tier_stats = self.lock_last_day_stats(); let mut res = DailyAllTierStats::with_capacity(tier_stats.len()); for (tier, st) in tier_stats.iter() { res.insert(tier.clone(), st.clone()); @@ -1746,6 +1779,18 @@ impl TransitionState { res } + fn lock_last_day_stats(&self) -> std::sync::MutexGuard<'_, HashMap> { + match self.last_day_stats.lock() { + Ok(stats) => stats, + Err(poisoned) => { + let mut stats = poisoned.into_inner(); + stats.clear(); + self.last_day_stats.clear_poison(); + stats + } + } + } + pub async fn update_workers(api: Arc, n: i64) { Self::update_workers_inner(api, n).await; } @@ -1767,7 +1812,27 @@ impl TransitionState { fn resize_workers_to(api: Arc, n: i64, requested: i64, absolute_max: i64) { let target = n as usize; let transition_state = runtime_sources::transition_state_handle(); - let mut workers = transition_state.workers.lock().unwrap(); + let runtime = match tokio::runtime::Handle::try_current() { + Ok(runtime) => runtime, + Err(err) => { + warn!( + event = EVENT_LIFECYCLE_WORKER_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + error = %err, + state = "resize_failed", + "Lifecycle worker pool requires a Tokio runtime" + ); + return; + } + }; + // Runtime lookup happens before locking, and the guard is dropped before + // metrics/logging callbacks. Poison therefore means a Vec mutation may + // have unwound and worker tracking cannot be reconstructed safely. + let mut workers = transition_state + .workers + .lock() + .expect("transition worker tracking mutex poisoned"); let tracked_workers = workers.len(); workers.retain(|worker| !worker.handle.is_finished()); let pruned_finished_workers = tracked_workers.saturating_sub(workers.len()); @@ -1777,7 +1842,7 @@ impl TransitionState { let clone_api = api.clone(); let cancel = CancellationToken::new(); let worker_cancel = cancel.clone(); - let handle = tokio::spawn(async move { + let handle = runtime.spawn(async move { TransitionState::worker_with_cancel(clone_api, worker_cancel).await; }); workers.push(TransitionWorker { cancel, handle }); @@ -1791,6 +1856,7 @@ impl TransitionState { let current_workers = workers.len() as i64; transition_state.num_workers.store(current_workers, Ordering::SeqCst); + drop(workers); transition_state.record_scanner_transition_state(); debug!( @@ -4903,6 +4969,7 @@ mod tests { FreeVersionRecoveryStats, RecoveryWalkTestAction, list_tier_free_versions, recover_tier_free_versions_with_cancel, set_recovery_bucket_list_wait_hook, set_recovery_walk_test_hook, }; + use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats; use crate::bucket::lifecycle::tier_sweeper::Jentry; use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; use crate::bucket::metadata_sys; @@ -4936,6 +5003,7 @@ mod tests { use http::HeaderMap; use rustfs_common::metrics::{IlmAction, global_metrics}; use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX; + use rustfs_data_usage::TierStats; use rustfs_filemeta::{FileInfo, FileMeta}; use s3s::dto::{ BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry, OutputLocation, @@ -6546,6 +6614,96 @@ mod tests { assert_eq!(state.transition_rx.len(), 1); } + #[tokio::test] + #[serial] + async fn scanner_transition_enqueue_waits_for_saturated_queue_to_recover() { + let state = TransitionState::new_with_capacity(1); + let first_object = ObjectInfo { + bucket: "bucket".to_string(), + name: "first".to_string(), + ..Default::default() + }; + let deferred_object = ObjectInfo { + bucket: "bucket".to_string(), + name: "deferred".to_string(), + ..Default::default() + }; + let event = crate::bucket::lifecycle::lifecycle::Event { + action: IlmAction::TransitionAction, + ..Default::default() + }; + + assert!( + state.queue_transition_task(&first_object, &event, &LcEventSrc::Scanner).await, + "first scanner transition should fill the queue" + ); + + let deferred = state.queue_transition_task(&deferred_object, &event, &LcEventSrc::Scanner); + tokio::pin!(deferred); + assert!( + (&mut deferred).now_or_never().is_none(), + "a saturated scanner queue should apply bounded backpressure instead of dropping the task" + ); + + let first_task = state + .transition_rx + .recv() + .await + .expect("queue should remain open") + .expect("first queued transition task should be present"); + state.release_transition(&first_task.obj_info); + + assert!( + deferred.await, + "the deferred scanner transition should enqueue as soon as capacity recovers" + ); + let recovered_task = state + .transition_rx + .recv() + .await + .expect("queue should remain open") + .expect("deferred transition task should be present"); + assert_eq!(recovered_task.obj_info.name, deferred_object.name); + } + + #[tokio::test] + #[serial] + async fn scanner_transition_sustained_saturation_schedules_compensation() { + let state = TransitionState::new_with_capacity_and_timeout(1, StdDuration::ZERO); + let first_object = ObjectInfo { + bucket: "saturated-bucket".to_string(), + name: "first".to_string(), + ..Default::default() + }; + let missed_object = ObjectInfo { + bucket: "saturated-bucket".to_string(), + name: "missed".to_string(), + ..Default::default() + }; + let event = crate::bucket::lifecycle::lifecycle::Event { + action: IlmAction::TransitionAction, + ..Default::default() + }; + + assert!( + state.queue_transition_task(&first_object, &event, &LcEventSrc::Scanner).await, + "first scanner transition should fill the queue" + ); + assert!( + !state + .queue_transition_task(&missed_object, &event, &LcEventSrc::Scanner) + .await, + "a continuously saturated queue should report that the object was not admitted" + ); + assert_eq!(state.queue_full_tasks(), 1); + assert_eq!(state.queue_send_timeout_tasks(), 1); + assert_eq!( + state.compensation_scheduled_tasks(), + 1, + "timed-out scanner work must schedule a bounded bucket backfill" + ); + } + #[tokio::test] #[serial] async fn scanner_transition_enqueue_updates_transition_status() { @@ -7071,6 +7229,46 @@ mod tests { assert_eq!(state.compensation_pending_tasks(), 1); } + #[test] + fn poisoned_compensation_set_can_release_completed_bucket() { + let state = TransitionState::new_with_capacity(1); + state + .compensation_buckets + .lock() + .expect("fresh mutex should lock") + .insert("bucket-a".to_string()); + let poison_target = Arc::clone(&state.compensation_buckets); + let _ = std::thread::spawn(move || { + let _guard = poison_target.lock().expect("fresh mutex should lock"); + panic!("poison compensation set"); + }) + .join(); + + state.finish_bucket_compensation("bucket-a"); + assert_eq!(state.compensation_pending_tasks(), 0); + assert!( + state.compensation_buckets.lock().is_ok(), + "validated compensation state must clear poison" + ); + } + + #[test] + fn poisoned_tier_stats_are_reset_before_reuse() { + let state = TransitionState::new_with_capacity(1); + let poison_target = Arc::clone(&state.last_day_stats); + let _ = std::thread::spawn(move || { + let mut stats = poison_target.lock().expect("fresh mutex should lock"); + stats.insert("stale".to_string(), LastDayTierStats::default()); + panic!("poison tier stats"); + }) + .join(); + + state.add_lastday_stats("fresh", TierStats::default()); + let stats = state.get_daily_all_tier_stats(); + assert!(!stats.contains_key("stale"), "possibly partial statistics must be discarded"); + assert!(stats.contains_key("fresh"), "statistics must accept new samples after recovery"); + } + #[tokio::test(flavor = "current_thread")] async fn scanner_transition_state_reports_compensation_pending_buckets() { let state = TransitionState::new_with_capacity(1); @@ -7361,6 +7559,23 @@ mod tests { TransitionState::resize_workers_to(ecstore, original_workers, original_workers, absolute_max); } + #[tokio::test] + #[serial] + async fn transition_worker_resize_without_runtime_does_not_poison_tracking() { + let (_paths, ecstore) = setup_test_env().await; + let transition_state = runtime_sources::transition_state_handle(); + let resize = std::thread::spawn(move || { + TransitionState::resize_workers_to(ecstore, 1, 1, resolve_transition_workers_absolute_max()); + }) + .join(); + + assert!(resize.is_ok(), "missing Tokio runtime must not panic while worker tracking is locked"); + assert!( + transition_state.workers.lock().is_ok(), + "failed resize must leave worker tracking unpoisoned" + ); + } + #[test] fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() { let now = OffsetDateTime::now_utc(); @@ -9979,6 +10194,61 @@ mod tests { assert_eq!(backend.remove_count().await, 0); } + #[cfg(feature = "test-util")] + #[tokio::test] + async fn journal_replay_deletes_confirmed_exact_provider_token() { + let (_disk_paths, ecstore) = setup_test_env().await; + let (backend, _) = register_recovery_mock_tier(&ecstore).await; + let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM") + .await + .expect("mock tier lease should be available"); + let identity = lease.backend_identity(); + backend + .set_put_remote_version(Some("provider-version-token".to_string())) + .await; + lease + .put( + "remote/object", + crate::client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"candidate")), + 9, + ) + .await + .expect("confirmed remote candidate should be seeded"); + backend.set_remove_failure(true); + backend.set_reject_non_empty_remote_versions(true); + let je = Jentry { + obj_name: "remote/object".to_string(), + version_id: "provider-version-token".to_string(), + tier_name: "WARM".to_string(), + backend_identity: Some(identity), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + }; + + crate::set_disk::cleanup_rejected_transition_upload_durably( + &lease, + &je.obj_name, + &je.version_id, + true, + Some(ecstore.clone()), + ) + .await + .expect("failed immediate cleanup should remain durable in the journal"); + assert!(backend.contains(&je.obj_name).await); + + backend.set_remove_failure(false); + crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je) + .await + .expect("identity-bound exact journal must retry confirmed candidate cleanup"); + + assert!(!backend.contains(&je.obj_name).await); + assert_eq!(backend.exact_remove_count(), 2); + assert_eq!( + backend.remove_versions().await, + vec![("remote/object".to_string(), "provider-version-token".to_string())] + ); + } + async fn seed_recoverable_free_version( disk_paths: &[PathBuf], bucket: &str, diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index b7cb3cc32..a217d4978 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -20,7 +20,10 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, warn}; use crate::bucket::lifecycle::config_boundary; -use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity}; +use crate::bucket::lifecycle::tier_sweeper::{ + Jentry, delete_confirmed_transition_candidate_exact_with_manager_and_identity, + delete_object_from_remote_tier_idempotent_with_manager_and_identity, +}; use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; @@ -270,15 +273,26 @@ pub async fn process_tier_delete_journal_entry(api: Arc, je: &Jentry) - let backend_identity = je .backend_identity .ok_or_else(|| std::io::Error::other("legacy tier delete journal has no durable backend identity"))?; - delete_object_from_remote_tier_idempotent_with_manager_and_identity( - &je.obj_name, - &je.version_id, - &je.tier_name, - backend_identity, - &api.tier_config_mgr(), - je.version_id_exact, - ) - .await?; + if je.version_id_exact { + delete_confirmed_transition_candidate_exact_with_manager_and_identity( + &je.obj_name, + &je.version_id, + &je.tier_name, + backend_identity, + &api.tier_config_mgr(), + ) + .await?; + } else { + delete_object_from_remote_tier_idempotent_with_manager_and_identity( + &je.obj_name, + &je.version_id, + &je.tier_name, + backend_identity, + &api.tier_config_mgr(), + false, + ) + .await?; + } remove_tier_delete_journal_entry(api, je).await } diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index c5fbc29a9..c4ca3b809 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -252,7 +252,10 @@ impl ObjSweeper { version_id: self.transition_version_id.clone(), tier_name: self.transition_tier.clone(), backend_identity: None, - version_id_exact: false, + version_id_exact: matches!( + self.transition_version_state, + rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact + ), version_state: self.transition_version_state, }); } @@ -335,7 +338,7 @@ async fn delete_object_from_remote_tier_raw_with_manager( let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name) .await .map_err(std::io::Error::other)?; - delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false).await + delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await } async fn delete_object_from_remote_tier_raw_with_lease( @@ -343,8 +346,11 @@ async fn delete_object_from_remote_tier_raw_with_lease( rv_id: &str, lease: &TierOperationLease, version_id_exact: bool, + validate_remote_version_id: bool, ) -> Result<(), std::io::Error> { - lease.validate_remote_version_id(rv_id)?; + if validate_remote_version_id { + lease.validate_remote_version_id(rv_id)?; + } if remote_delete_breaker_is_open(Instant::now()).await { metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1); @@ -440,7 +446,53 @@ pub(crate) async fn delete_object_from_remote_tier_with_lease_idempotent( lease: &TierOperationLease, version_id_exact: bool, ) -> Result { - match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact).await { + delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, version_id_exact, true).await +} + +pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idempotent( + obj_name: &str, + rv_id: &str, + lease: &TierOperationLease, +) -> Result { + if rv_id.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "confirmed versioned transition candidate requires a non-empty remote version", + )); + } + #[cfg(test)] + if obj_name == "remote/empty-guard-probe" { + CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, true, false).await +} + +#[cfg(test)] +static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity( + obj_name: &str, + rv_id: &str, + tier_name: &str, + backend_identity: TierDestinationId, + tier_config_mgr: &Arc>, +) -> Result { + let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, tier_name, backend_identity) + .await + .map_err(std::io::Error::other)?; + delete_confirmed_transition_candidate_exact_with_lease_idempotent(obj_name, rv_id, &lease).await +} + +async fn delete_object_from_remote_tier_with_lease_idempotent_inner( + obj_name: &str, + rv_id: &str, + lease: &TierOperationLease, + version_id_exact: bool, + validate_remote_version_id: bool, +) -> Result { + match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact, validate_remote_version_id) + .await + { Ok(()) => Ok(RemoteTierDeleteOutcome::Deleted), Err(err) if is_remote_tier_not_found_error(&err) => Ok(RemoteTierDeleteOutcome::AlreadyRemoved), Err(err) => { @@ -465,6 +517,7 @@ pub fn transitioned_delete_journal_entry( versioned: bool, suspended: bool, transitioned: &TransitionedObject, + transition_version_state: rustfs_filemeta::TransitionVersionState, ) -> Option { let sweeper = ObjSweeper { version_id, @@ -473,6 +526,7 @@ pub fn transitioned_delete_journal_entry( transition_status: transitioned.status.clone(), transition_tier: transitioned.tier.clone(), transition_version_id: transitioned.version_id.clone(), + transition_version_state, remote_object: transitioned.name.clone(), ..Default::default() }; @@ -480,8 +534,13 @@ pub fn transitioned_delete_journal_entry( sweeper.should_remove_remote_object() } -pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject) -> Option { - if transitioned.status != lifecycle::TRANSITION_COMPLETE { +pub fn transitioned_force_delete_journal_entry( + transitioned: &TransitionedObject, + transition_version_state: rustfs_filemeta::TransitionVersionState, +) -> Option { + if transitioned.status != lifecycle::TRANSITION_COMPLETE + || transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown + { return None; } @@ -490,8 +549,11 @@ pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject version_id: transitioned.version_id.clone(), tier_name: transitioned.tier.clone(), backend_identity: None, - version_id_exact: false, - version_state: rustfs_filemeta::TransitionVersionState::Unknown, + version_id_exact: matches!( + transition_version_state, + rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact + ), + version_state: transition_version_state, }) } @@ -500,11 +562,14 @@ mod test { use crate::client::signer_error::invalid_utf8_header_error; use super::{ - ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, RemoteDeleteBreaker, RemoteTierDeleteOutcome, + CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, + RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity, - is_remote_tier_not_found_error, is_signer_header_error, set_remote_tier_delete_test_hook, - should_record_remote_delete_failure, + is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, + should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry, }; + use crate::storage_api_contracts::lifecycle::TransitionedObject; + use rustfs_filemeta::TransitionVersionState; use std::io::{Error, ErrorKind}; use std::time::{Duration, Instant}; @@ -548,6 +613,43 @@ mod test { assert!(should_record_remote_delete_failure(&Error::other("NoSuchVersion"))); } + #[test] + fn transitioned_delete_journal_preserves_remote_version_state() { + let cases = [ + (TransitionVersionState::Unknown, "legacy-version", None), + (TransitionVersionState::KnownDisabled, "", Some(false)), + (TransitionVersionState::SuspendedNull, "null", Some(true)), + (TransitionVersionState::Exact, "opaque-version", Some(true)), + ]; + + for (state, version_id, expected_exact) in cases { + let transitioned = TransitionedObject { + name: "remote/object".to_string(), + version_id: version_id.to_string(), + tier: "WARM".to_string(), + status: lifecycle::TRANSITION_COMPLETE.to_string(), + ..Default::default() + }; + let regular = transitioned_delete_journal_entry(None, false, false, &transitioned, state); + let forced = transitioned_force_delete_journal_entry(&transitioned, state); + + match expected_exact { + Some(expected_exact) => { + let regular = regular.expect("known version state should produce a regular delete journal entry"); + assert_eq!(regular.version_state, state); + assert_eq!(regular.version_id_exact, expected_exact); + let forced = forced.expect("known version state should produce a forced delete journal entry"); + assert_eq!(forced.version_state, state); + assert_eq!(forced.version_id_exact, expected_exact); + } + None => { + assert!(regular.is_none()); + assert!(forced.is_none()); + } + } + } + } + #[tokio::test] #[serial_test::serial] async fn idempotent_remote_delete_treats_hooked_nosuchversion_as_already_removed() { @@ -670,6 +772,55 @@ mod test { assert_eq!(backend.remove_versions().await, vec![("remote/object".to_string(), String::new())]); } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial] + async fn confirmed_transition_cleanup_deletes_exact_provider_token() { + CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.store(0, std::sync::atomic::Ordering::Relaxed); + let manager = crate::services::tier::tier::TierConfigMgr::new(); + let backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await; + let lease = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM") + .await + .expect("test tier lease should be available"); + let identity = lease.backend_identity(); + drop(lease); + backend.set_reject_non_empty_remote_versions(true); + + let outcome = delete_confirmed_transition_candidate_exact_with_manager_and_identity( + "remote/object", + "provider-version-token", + "WARM", + identity, + &manager, + ) + .await + .expect("confirmed upload compensation should delete the exact provider token"); + + assert_eq!(outcome, RemoteTierDeleteOutcome::Deleted); + assert_eq!(backend.exact_remove_count(), 1); + assert_eq!( + backend.remove_versions().await, + vec![("remote/object".to_string(), "provider-version-token".to_string())] + ); + + let err = delete_confirmed_transition_candidate_exact_with_manager_and_identity( + "remote/empty-guard-probe", + "", + "WARM", + identity, + &manager, + ) + .await + .expect_err("confirmed versioned cleanup must reject an empty token"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!(backend.remove_count().await, 1); + assert_eq!( + CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.load(std::sync::atomic::Ordering::Relaxed), + 0, + "empty remote versions must be rejected before exact cleanup dispatch" + ); + } + #[test] fn breaker_opens_at_threshold_and_recovers_after_window() { let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30)); diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index 823ae1d2a..e3f3036e8 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -22,7 +22,10 @@ use uuid::Uuid; use crate::bucket::lifecycle::config_boundary; use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; -use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idempotent_with_manager_and_identity; +use crate::bucket::lifecycle::tier_sweeper::{ + delete_confirmed_transition_candidate_exact_with_manager_and_identity, + delete_object_from_remote_tier_idempotent_with_manager_and_identity, +}; use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result as EcstoreResult}; use crate::object_api::ObjectOptions; @@ -708,6 +711,21 @@ async fn recover_unknown_upload_outcome( TransitionCandidateProbe::UnversionedPresent => { cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await } + TransitionCandidateProbe::VersionedPresent(version_id) + if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) => + { + delete_confirmed_transition_candidate_exact_with_manager_and_identity( + &transaction.remote_object, + &version_id, + &transaction.tier_name, + transaction.backend_fingerprint, + &api.tier_config_mgr(), + ) + .await + .map_err(Error::other)?; + delete_transition_transaction_record(api, transaction.transaction_id).await?; + Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) + } TransitionCandidateProbe::VersionedPresent(version_id) => { cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await } diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index f937e0eb2..97dbf9147 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -741,6 +741,8 @@ impl BucketMetadataSys { if let Some(config) = &bm.policy_config { Ok((config.clone(), bm.policy_config_updated_at)) + } else if !bm.policy_config_json.is_empty() { + Ok((serde_json::from_slice(&bm.policy_config_json)?, bm.policy_config_updated_at)) } else { Err(Error::ConfigNotFound) } @@ -1051,6 +1053,22 @@ mod tests { ); } + #[tokio::test] + async fn get_bucket_policy_rejects_malformed_cached_policy() { + let (_dirs, ecstore) = isolated_store_over_temp_disks().await; + let sys = BucketMetadataSys::new(ecstore); + let mut metadata = BucketMetadata::new("malformed-policy"); + metadata.policy_config_json = b"{".to_vec(); + sys.set("malformed-policy".to_string(), Arc::new(metadata)).await; + + let err = sys + .get_bucket_policy("malformed-policy") + .await + .expect_err("malformed persisted policy must not be treated as missing"); + + assert!(matches!(err, Error::Io(_)), "malformed persisted policy must surface its parse failure"); + } + fn target(bucket: &str, id: &str) -> BucketTarget { BucketTarget { source_bucket: bucket.to_string(), diff --git a/crates/ecstore/src/bucket/policy_sys.rs b/crates/ecstore/src/bucket/policy_sys.rs index 14e54252e..75a9a4473 100644 --- a/crates/ecstore/src/bucket/policy_sys.rs +++ b/crates/ecstore/src/bucket/policy_sys.rs @@ -15,23 +15,26 @@ use super::metadata_sys::get_bucket_metadata_sys; use crate::error::{Result, StorageError}; use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs}; -use tracing::info; pub struct PolicySys {} impl PolicySys { pub async fn is_allowed(args: &BucketPolicyArgs<'_>) -> bool { - match Self::get(args.bucket).await { - Ok(cfg) => return cfg.is_allowed(args).await, - Err(err) => { - if err != StorageError::ConfigNotFound { - info!("config get err {:?}", err); - } - } - } - - args.is_owner + matches!(Self::try_is_allowed(args).await, Ok(true)) } + + pub async fn try_is_allowed(args: &BucketPolicyArgs<'_>) -> Result { + Self::is_allowed_with_policy(args, Self::get(args.bucket).await).await + } + + async fn is_allowed_with_policy(args: &BucketPolicyArgs<'_>, policy: Result) -> Result { + match policy { + Ok(policy) => Ok(policy.is_allowed(args).await), + Err(StorageError::ConfigNotFound) => Ok(args.is_owner), + Err(err) => Err(err), + } + } + pub async fn get(bucket: &str) -> Result { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; @@ -41,3 +44,91 @@ impl PolicySys { Ok(cfg) } } + +#[cfg(test)] +mod tests { + use super::{PolicySys, StorageError}; + use rustfs_policy::policy::action::{Action, S3Action}; + use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs}; + use std::collections::HashMap; + + fn args<'a>( + is_owner: bool, + groups: &'a Option>, + conditions: &'a HashMap>, + ) -> BucketPolicyArgs<'a> { + BucketPolicyArgs { + bucket: "bucket", + action: Action::S3Action(S3Action::GetObjectAction), + is_owner, + account: "account", + groups, + conditions, + object: "object", + } + } + + #[tokio::test] + async fn missing_policy_preserves_owner_and_iam_fallback_semantics() { + let groups = None; + let conditions = HashMap::new(); + assert!( + PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Err(StorageError::ConfigNotFound),) + .await + .expect("missing policy should preserve owner access") + ); + assert!( + !PolicySys::is_allowed_with_policy(&args(false, &groups, &conditions), Err(StorageError::ConfigNotFound),) + .await + .expect("missing policy should defer non-owner access to IAM") + ); + } + + #[tokio::test] + async fn policy_load_failures_propagate() { + let groups = None; + let conditions = HashMap::new(); + for (failure, expected_message) in [ + (StorageError::Io(std::io::Error::other("policy read failed")), "policy read failed"), + ( + StorageError::other("bucket metadata sys not initialized for this instance"), + "bucket metadata sys not initialized for this instance", + ), + ] { + let result = PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Err(failure)).await; + + assert!( + matches!(result, Err(StorageError::Io(ref err)) if err.to_string().contains(expected_message)), + "policy I/O and uninitialized metadata failures must propagate instead of granting owner access" + ); + } + } + + #[tokio::test] + async fn explicit_bucket_deny_precedes_iam_allow() { + let groups = None; + let conditions = HashMap::new(); + let policy: BucketPolicy = serde_json::from_str( + r#"{ + "Version":"2012-10-17", + "Statement":[{ + "Effect":"Deny", + "Principal":{"AWS":"*"}, + "Action":["s3:GetObject"], + "Resource":["arn:aws:s3:::bucket/*"] + }] +}"#, + ) + .expect("deny policy should parse"); + + let bucket_allowed = PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Ok(policy)) + .await + .expect("loaded bucket policy should evaluate"); + let iam_allowed = true; + let request_allowed = bucket_allowed && iam_allowed; + + assert!(iam_allowed, "test precondition: IAM grants the action"); + assert!(!bucket_allowed, "test precondition: bucket policy explicitly denies the action"); + assert!(!request_allowed, "explicit bucket Deny must reject before IAM Allow fallback"); + } +} diff --git a/crates/ecstore/src/client/api_put_object_streaming.rs b/crates/ecstore/src/client/api_put_object_streaming.rs index 4bc5393a3..9d67f3ebb 100644 --- a/crates/ecstore/src/client/api_put_object_streaming.rs +++ b/crates/ecstore/src/client/api_put_object_streaming.rs @@ -22,7 +22,7 @@ use bytes::Bytes; use futures::future::join_all; use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; use std::io::Error; -use std::sync::RwLock; +use std::sync::{Mutex, MutexGuard, RwLock}; use std::{collections::HashMap, sync::Arc}; use time::{OffsetDateTime, format_description}; use tokio::io::AsyncReadExt; @@ -46,6 +46,14 @@ use crate::client::utils::base64_encode; use rustfs_utils::path::trim_etag; use s3s::header::X_AMZ_EXPIRATION; +fn lock_md5_hasher( + md5_hasher: &Mutex>, +) -> Result>, std::io::Error> { + md5_hasher + .lock() + .map_err(|_| std::io::Error::other("MD5 hasher state is unavailable")) +} + /// Read exactly `want` bytes for a single multipart part, or fewer if the reader /// reaches EOF first. Advances the reader so the next call returns the following /// part. Replaces the previous per-part `read_all()`/`to_vec()`, which drained @@ -177,7 +185,7 @@ impl TransitionClient { let length = buf.len(); if opts.send_content_md5 { - let mut md5_hasher = self.md5_hasher.lock().unwrap(); + let mut md5_hasher = lock_md5_hasher(&self.md5_hasher)?; let md5_hash = match md5_hasher.as_mut() { Some(hasher) => hasher, None => return Err(std::io::Error::other("MD5 hasher not initialized")), @@ -370,7 +378,7 @@ impl TransitionClient { let mut md5_base64: String = "".to_string(); if opts.send_content_md5 { - let mut md5_hasher = clone_self.md5_hasher.lock().unwrap(); + let mut md5_hasher = lock_md5_hasher(&clone_self.md5_hasher)?; let md5_hash = match md5_hasher.as_mut() { Some(hasher) => hasher, None => { @@ -418,6 +426,9 @@ impl TransitionClient { } let results = join_all(futures).await; + for result in results { + result?; + } select! { err = err_rx.recv() => { @@ -620,10 +631,12 @@ fn collect_complete_parts(parts_info: &HashMap, total_parts_cou #[cfg(test)] mod tests { - use super::{ObjectPart, ReaderImpl, collect_complete_parts, read_multipart_part}; + use super::{ObjectPart, ReaderImpl, collect_complete_parts, lock_md5_hasher, read_multipart_part}; use crate::object_api::GetObjectReader; use bytes::Bytes; + use rustfs_utils::hash::HashAlgorithm; use std::collections::HashMap; + use std::sync::{Arc, Mutex}; // Drive a reader through the same per-part loop the multipart stream uses and // collect the size of every part. Regression for rustfs/rustfs#4811: the old @@ -733,4 +746,18 @@ mod tests { "a gap in the parts map must be an error, not a panic" ); } + + #[test] + fn poisoned_md5_state_fails_closed() { + let hasher = Arc::new(Mutex::new(Some(HashAlgorithm::Md5))); + let poison_target = Arc::clone(&hasher); + let _ = std::thread::spawn(move || { + let _guard = poison_target.lock().expect("fresh mutex should lock"); + panic!("poison MD5 state"); + }) + .join(); + + let error = lock_md5_hasher(&hasher).expect_err("poisoned hash state must not be reused"); + assert_eq!(error.kind(), std::io::ErrorKind::Other); + } } diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index b1411a078..300f443d2 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -24,8 +24,8 @@ use crate::cluster::rpc::internode_data_transport::{ use crate::disk::error::{Error, Result}; use crate::disk::{ BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, - DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one, + DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions, + RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one, disk_store::{ DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE, get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout, @@ -48,9 +48,10 @@ use rustfs_protos::proto_gen::node_service::RenamePartRequest; use rustfs_protos::proto_gen::node_service::{ BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, - MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, - ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, StatVolumeRequest, - UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient, + MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, + ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, + RenameFileRequest, SettlePartTransactionRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, + WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient, }; use serde::{Serialize, de::DeserializeOwned}; use std::{ @@ -2481,6 +2482,71 @@ impl DiskAPI for RemoteDisk { .await } + #[tracing::instrument(level = "trace", skip_all)] + async fn prepare_part_transaction( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + meta: Bytes, + ) -> Result<()> { + self.execute_with_timeout( + || async { + let mut client = self + .get_client() + .await + .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; + let mut request = Request::new(PreparePartTransactionRequest { + disk: self.endpoint.to_string(), + src_volume: src_volume.to_string(), + src_path: src_path.to_string(), + dst_volume: dst_volume.to_string(), + dst_path: dst_path.to_string(), + meta, + }); + let canonical_body = rustfs_protos::canonical_prepare_part_transaction_request_body(request.get_ref()); + attach_mutation_body_digest(&mut request, canonical_body, "prepare_part_transaction")?; + + let response = client.prepare_part_transaction(request).await?.into_inner(); + if !response.success { + return Err(response.error.unwrap_or_default().into()); + } + Ok(()) + }, + get_max_timeout_duration(), + ) + .await + } + + #[tracing::instrument(level = "trace", skip_all)] + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> { + self.execute_with_timeout( + || async { + let mut client = self + .get_client() + .await + .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; + let mut request = Request::new(SettlePartTransactionRequest { + disk: self.endpoint.to_string(), + volume: volume.to_string(), + path: path.to_string(), + rollback: action == PartTransactionAction::Rollback, + }); + let canonical_body = rustfs_protos::canonical_settle_part_transaction_request_body(request.get_ref()); + attach_mutation_body_digest(&mut request, canonical_body, "settle_part_transaction")?; + + let response = client.settle_part_transaction(request).await?.into_inner(); + if !response.success { + return Err(response.error.unwrap_or_default().into()); + } + Ok(()) + }, + get_max_timeout_duration(), + ) + .await + } + #[tracing::instrument(level = "trace", skip_all)] async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { trace!( diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index 8920df6bc..4d8d9f4e7 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -16,6 +16,7 @@ use crate::disk::error_reduce::count_errs; use crate::error::{Error, Result}; use crate::layout::set_heal::{formats_to_drives_info, new_heal_format_sets}; +use crate::multipart_listing::paginate_multipart_listing; use crate::storage_api_contracts::{ bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, @@ -49,7 +50,10 @@ use rustfs_filemeta::FileInfo; use rustfs_lock::NamespaceLockWrapper; use rustfs_madmin::heal_commands::HealResultItem; use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash}; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; use tokio::sync::RwLock; use tokio::sync::broadcast::{Receiver, Sender}; use tokio::time::Duration; @@ -63,6 +67,8 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo; type ObjectInfoOrErr = StorageObjectInfoOrErr; type WalkOptions = StorageWalkOptions bool>; +const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4; + #[derive(Debug, Clone)] pub struct Sets { pub id: Uuid, @@ -799,9 +805,52 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets { delimiter: Option, max_uploads: usize, ) -> Result { - self.get_disks_by_key(prefix) - .list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, max_uploads) - .await + let per_set_limit = max_uploads.saturating_add(1); + let results = futures::stream::iter(self.disk_set.iter().cloned()) + .map(|set| { + let key_marker = key_marker.clone(); + let upload_id_marker = upload_id_marker.clone(); + let delimiter = delimiter.clone(); + async move { + set.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, per_set_limit) + .await + } + }) + .buffer_unordered(LIST_MULTIPART_SETS_CONCURRENCY) + .collect::>() + .await; + + let mut uploads = Vec::new(); + let mut common_prefixes = HashSet::new(); + let mut source_truncated = false; + for result in results { + let page = result?; + uploads.extend(page.uploads); + common_prefixes.extend(page.common_prefixes); + source_truncated |= page.is_truncated; + } + + let page = paginate_multipart_listing( + uploads, + common_prefixes.into_iter().collect(), + key_marker.as_deref(), + key_marker.as_ref().and(upload_id_marker.as_deref()), + max_uploads, + source_truncated, + ); + + Ok(ListMultipartsInfo { + key_marker, + upload_id_marker, + next_key_marker: page.next_key_marker, + next_upload_id_marker: page.next_upload_id_marker, + max_uploads, + is_truncated: page.is_truncated, + uploads: page.uploads, + common_prefixes: page.common_prefixes, + prefix: prefix.to_owned(), + delimiter, + }) } #[tracing::instrument(skip(self))] async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { @@ -1111,6 +1160,7 @@ mod tests { use crate::layout::endpoints::SetupType; use crate::storage_api_contracts::heal::HealOperations as _; use crate::storage_api_contracts::list::ListOperations as _; + use crate::storage_api_contracts::multipart::MultipartOperations as _; use rustfs_lock::client::local::LocalClient; use serial_test::serial; @@ -1248,6 +1298,194 @@ mod tests { assert_eq!(result, (Some(3), Some(1), Some(0))); } + async fn multipart_listing_test_sets() -> (Vec, Arc) { + let format = FormatV3::new(2, 2); + let mut temp_dirs = Vec::new(); + let mut all_endpoints = Vec::new(); + let mut disk_sets = Vec::new(); + + for set_index in 0..2 { + let mut endpoints = Vec::new(); + let mut disks = Vec::new(); + for disk_index in 0..2 { + let temp_dir = tempfile::tempdir().expect("tempdir should be created"); + let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8")) + .expect("endpoint should parse"); + endpoint.set_pool_index(0); + endpoint.set_set_index(set_index); + endpoint.set_disk_index(disk_index); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("disk should be created"); + let mut disk_format = format.clone(); + disk_format.erasure.this = format.erasure.sets[set_index][disk_index]; + save_format_file(&Some(disk.clone()), &Some(disk_format)) + .await + .expect("format should be saved"); + temp_dirs.push(temp_dir); + all_endpoints.push(endpoint.clone()); + endpoints.push(endpoint); + disks.push(Some(disk)); + } + disk_sets.push( + SetDisks::new( + "test-owner".to_string(), + Arc::new(RwLock::new(disks)), + 2, + 1, + 0, + set_index, + endpoints, + format.clone(), + vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())], + ) + .await, + ); + } + + let sets = Arc::new(Sets { + id: format.id, + disk_set: disk_sets, + pool_idx: 0, + endpoints: PoolEndpoints { + legacy: false, + set_count: 2, + drives_per_set: 2, + endpoints: Endpoints::from(all_endpoints), + cmd_line: String::new(), + platform: String::new(), + }, + format, + parity_count: 1, + set_count: 2, + set_drive_count: 2, + default_parity_count: 1, + distribution_algo: DistributionAlgoVersion::V1, + exit_signal: None, + ctx: bootstrap_ctx(), + }); + (temp_dirs, sets) + } + + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn list_multipart_uploads_merges_all_sets_without_pagination_loss() { + let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await; + let (_temp_dirs, sets) = multipart_listing_test_sets().await; + let bucket = format!("multipart-list-{}", Uuid::new_v4().simple()); + sets.make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + let mut keys_by_set = [Vec::new(), Vec::new()]; + for index in 0..100 { + let key = format!("logs/{index:03}.bin"); + let set_index = sets.get_hashed_set_index(&key); + if keys_by_set[set_index].len() < 2 { + keys_by_set[set_index].push(key); + } + if keys_by_set.iter().all(|keys| keys.len() == 2) { + break; + } + } + assert!(keys_by_set.iter().all(|keys| keys.len() == 2), "test keys must span both sets"); + + let repeated_key = keys_by_set[0][0].clone(); + let mut expected = Vec::new(); + for key in keys_by_set.iter().flatten() { + let upload = sets + .new_multipart_upload(&bucket, key, &ObjectOptions::default()) + .await + .expect("multipart upload should be created"); + expected.push((key.clone(), upload.upload_id)); + } + let second = sets + .new_multipart_upload(&bucket, &repeated_key, &ObjectOptions::default()) + .await + .expect("second upload for the same key should be created"); + expected.push((repeated_key, second.upload_id)); + expected.sort(); + + let mut actual = Vec::new(); + let mut key_marker = None; + let mut upload_id_marker = None; + for _ in 0..expected.len() + 1 { + let page = sets + .list_multipart_uploads(&bucket, "logs/", key_marker.clone(), upload_id_marker.clone(), None, 2) + .await + .expect("multipart page should list across every set"); + assert!(page.uploads.len() <= 2); + actual.extend( + page.uploads + .iter() + .map(|upload| (upload.object.clone(), upload.upload_id.clone())), + ); + if !page.is_truncated { + break; + } + key_marker = page.next_key_marker; + upload_id_marker = page.next_upload_id_marker; + } + + assert_eq!(actual, expected, "set-level merge must return every upload exactly once"); + let mut deduped = actual.clone(); + deduped.dedup(); + assert_eq!(deduped.len(), actual.len(), "set-level pagination must not duplicate uploads"); + + let mut nested_by_set = [None, None]; + for index in 0..100 { + let key = format!("nested/group-{index:03}/file.bin"); + let set_index = sets.get_hashed_set_index(&key); + nested_by_set[set_index].get_or_insert(key); + if nested_by_set.iter().all(Option::is_some) { + break; + } + } + for key in nested_by_set.iter().flatten() { + sets.new_multipart_upload(&bucket, key, &ObjectOptions::default()) + .await + .expect("nested multipart upload should be created"); + } + let mut expected_prefixes = nested_by_set + .iter() + .flatten() + .map(|key| { + key.rsplit_once('/') + .expect("nested key should contain a delimiter") + .0 + .to_string() + + "/" + }) + .collect::>(); + expected_prefixes.sort(); + + let first = sets + .list_multipart_uploads(&bucket, "nested/", None, None, Some("/".to_string()), 1) + .await + .expect("first delimiter page should list across every set"); + assert!(first.is_truncated); + assert_eq!(first.common_prefixes, expected_prefixes[..1]); + let second = sets + .list_multipart_uploads( + &bucket, + "nested/", + first.next_key_marker, + first.next_upload_id_marker, + Some("/".to_string()), + 1, + ) + .await + .expect("second delimiter page should list across every set"); + assert!(!second.is_truncated); + assert_eq!(second.common_prefixes, expected_prefixes[1..]); + } + #[tokio::test(flavor = "multi_thread")] #[serial] async fn sets_list_objects_v2_lists_objects_within_the_pool() { diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 2ae0ed8d1..01171b8f1 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -35,7 +35,7 @@ use std::{ Arc, atomic::{AtomicI64, AtomicU32, AtomicU64, Ordering}, }, - time::Duration, + time::{Duration, SystemTime, UNIX_EPOCH}, }; use tokio::{sync::RwLock, time}; use tokio_util::sync::CancellationToken; @@ -339,21 +339,19 @@ impl Drop for DiskHealthWaitingGuard<'_> { impl DiskHealthTracker { /// Create a new disk health tracker pub fn new() -> Self { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() as i64; + let now = current_unix_time(); + let now_nanos = unix_nanos(now); Self { - last_success: AtomicI64::new(now), - last_started: AtomicI64::new(now), + last_success: AtomicI64::new(now_nanos), + last_started: AtomicI64::new(now_nanos), status: AtomicU32::new(DISK_HEALTH_OK), waiting: AtomicU32::new(0), runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32), consecutive_failures: AtomicU32::new(0), consecutive_successes: AtomicU32::new(0), offline_since_unix_secs: AtomicI64::new(0), - last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000), + last_transition_unix_secs: AtomicI64::new(unix_secs_i64(now)), last_capacity_total: AtomicU64::new(0), last_capacity_used: AtomicU64::new(0), last_capacity_free: AtomicU64::new(0), @@ -363,11 +361,7 @@ impl DiskHealthTracker { /// Log a successful operation pub fn log_success(&self) { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() as i64; - self.last_success.store(now, Ordering::Relaxed); + self.last_success.store(current_unix_nanos(), Ordering::Relaxed); } pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) { @@ -429,11 +423,14 @@ impl DiskHealthTracker { } pub fn offline_duration(&self) -> Option { + self.offline_duration_at(current_unix_secs()) + } + + fn offline_duration_at(&self, now: u64) -> Option { let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire); if offline_since <= 0 { return None; } - let now = current_unix_secs(); Some(Duration::from_secs(now.saturating_sub(offline_since as u64))) } @@ -492,6 +489,12 @@ impl DiskHealthTracker { /// Remote disks are marked faulty on timeout/network errors; the init loop retries with the /// same [`DiskStore`] handles, which would otherwise fail immediately at `is_faulty()`. pub fn reset_for_store_init_retry(&self, endpoint: &Endpoint) { + self.reset_for_store_init_retry_at(endpoint, current_unix_time()); + } + + fn reset_for_store_init_retry_at(&self, endpoint: &Endpoint, now: Duration) { + let now_nanos = unix_nanos(now); + let now_secs = unix_secs_i64(now); self.status.store(DISK_HEALTH_OK, Ordering::Release); self.runtime_state .store(RuntimeDriveHealthState::Online as u32, Ordering::Release); @@ -499,11 +502,9 @@ impl DiskHealthTracker { self.consecutive_successes.store(0, Ordering::Release); self.offline_since_unix_secs.store(0, Ordering::Release); self.waiting.store(0, Ordering::Release); - let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap(); - let now_nanos = now.as_nanos() as i64; self.last_success.store(now_nanos, Ordering::Relaxed); self.last_started.store(now_nanos, Ordering::Relaxed); - self.last_transition_unix_secs.store(now.as_secs() as i64, Ordering::Release); + self.last_transition_unix_secs.store(now_secs, Ordering::Release); record_drive_runtime_state(endpoint, RuntimeDriveHealthState::Online); } @@ -612,10 +613,33 @@ impl DiskHealthTracker { } fn current_unix_secs() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs() + // Zero is reserved as "not recorded" by health timestamp atomics. + current_unix_time().as_secs().max(1) +} + +fn current_unix_nanos() -> i64 { + unix_nanos(current_unix_time()) +} + +fn current_unix_time() -> Duration { + unix_time_since_epoch(SystemTime::now()) +} + +fn unix_time_since_epoch(time: SystemTime) -> Duration { + time.duration_since(UNIX_EPOCH).unwrap_or(Duration::ZERO) +} + +fn unix_nanos(time: Duration) -> i64 { + i64::try_from(time.as_nanos()).unwrap_or(i64::MAX) +} + +fn unix_secs_i64(time: Duration) -> i64 { + i64::try_from(time.as_secs()).unwrap_or(i64::MAX) +} + +fn elapsed_since(last_nanos: i64, now_nanos: i64) -> Duration { + let elapsed_nanos = now_nanos.saturating_sub(last_nanos).max(0); + Duration::from_nanos(u64::try_from(elapsed_nanos).unwrap_or(u64::MAX)) } impl Default for DiskHealthTracker { @@ -635,11 +659,7 @@ struct HealthDiskCtxValue { impl HealthDiskCtxValue { fn log_success(&self) { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() as i64; - self.last_success.store(now, Ordering::Relaxed); + self.last_success.store(current_unix_nanos(), Ordering::Relaxed); } } @@ -774,12 +794,7 @@ impl LocalDiskWrapper { } let last_success_nanos = health.last_success.load(Ordering::Relaxed); - let elapsed = Duration::from_nanos( - (std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() as i64 - last_success_nanos) as u64 - ); + let elapsed = elapsed_since(last_success_nanos, current_unix_nanos()); if elapsed < SKIP_IF_SUCCESS_BEFORE { continue; @@ -1091,11 +1106,7 @@ impl LocalDiskWrapper { self.check_disk_stale().await?; // Record operation start - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() as i64; - self.health.last_started.store(now, Ordering::Relaxed); + self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed); let _waiting_guard = self.health.waiting_guard(); if timeout_duration == Duration::ZERO { @@ -1317,11 +1328,7 @@ impl DiskAPI for LocalDiskWrapper { } // Record operation start - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() as i64; - self.health.last_started.store(now, Ordering::Relaxed); + self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed); self.health.increment_waiting(); // Execute the operation @@ -1473,6 +1480,33 @@ impl DiskAPI for LocalDiskWrapper { .await } + async fn prepare_part_transaction( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + meta: Bytes, + ) -> Result<()> { + self.track_disk_health( + || async { + self.disk + .prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta) + .await + }, + get_max_timeout_duration(), + ) + .await + } + + async fn settle_part_transaction(&self, volume: &str, path: &str, action: crate::disk::PartTransactionAction) -> Result<()> { + self.track_disk_health( + || async { self.disk.settle_part_transaction(volume, path, action).await }, + get_max_timeout_duration(), + ) + .await + } + async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { self.track_disk_health(|| async { self.disk.delete(volume, path, opt).await }, get_max_timeout_duration()) .await @@ -2243,4 +2277,53 @@ mod tests { assert!(health.mark_offline(&endpoint, "again")); assert!(health.is_faulty()); } + + #[test] + fn unix_time_clamps_epoch_and_pre_epoch_to_zero() { + let before_epoch = UNIX_EPOCH + .checked_sub(Duration::from_nanos(1)) + .expect("one nanosecond before the Unix epoch should be representable"); + + assert_eq!(unix_time_since_epoch(UNIX_EPOCH), Duration::ZERO); + assert_eq!(unix_time_since_epoch(before_epoch), Duration::ZERO); + } + + #[test] + fn elapsed_and_offline_duration_saturate_on_clock_rollback() { + let health = DiskHealthTracker::new(); + health.offline_since_unix_secs.store(10, Ordering::Release); + + assert_eq!(elapsed_since(10, 12), Duration::from_nanos(2)); + assert_eq!(elapsed_since(10, 9), Duration::ZERO); + assert_eq!(health.offline_duration_at(9), Some(Duration::ZERO)); + } + + #[test] + fn pre_epoch_retry_reset_updates_the_complete_health_state() { + let endpoint = Endpoint::try_from("/tmp/reset-store-init-retry-pre-epoch").expect("endpoint should parse"); + let health = DiskHealthTracker::new(); + health.status.store(DISK_HEALTH_FAULTY, Ordering::Release); + health + .runtime_state + .store(RuntimeDriveHealthState::Offline as u32, Ordering::Release); + health.consecutive_failures.store(3, Ordering::Release); + health.consecutive_successes.store(2, Ordering::Release); + health.offline_since_unix_secs.store(11, Ordering::Release); + health.waiting.store(4, Ordering::Release); + health.last_success.store(12, Ordering::Release); + health.last_started.store(13, Ordering::Release); + health.last_transition_unix_secs.store(14, Ordering::Release); + + health.reset_for_store_init_retry_at(&endpoint, unix_time_since_epoch(UNIX_EPOCH - Duration::from_nanos(1))); + + assert_eq!(health.status.load(Ordering::Acquire), DISK_HEALTH_OK); + assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online); + assert_eq!(health.consecutive_failures.load(Ordering::Acquire), 0); + assert_eq!(health.consecutive_successes.load(Ordering::Acquire), 0); + assert_eq!(health.offline_since_unix_secs.load(Ordering::Acquire), 0); + assert_eq!(health.waiting.load(Ordering::Acquire), 0); + assert_eq!(health.last_success.load(Ordering::Acquire), 0); + assert_eq!(health.last_started.load(Ordering::Acquire), 0); + assert_eq!(health.last_transition_unix_secs.load(Ordering::Acquire), 0); + } } diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index d6afb4c32..c24e38385 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -19,7 +19,8 @@ use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_r use crate::disk::{ BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, - FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, + FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, PART_TRANSACTION_NEW_META, + PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int, endpoint::Endpoint, @@ -80,6 +81,10 @@ const ENV_BITROT_SIZE_MISMATCH_RETRY_COUNT: &str = "RUSTFS_BITROT_SIZE_MISMATCH_ const ENV_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS: &str = "RUSTFS_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS"; const DEFAULT_BITROT_SIZE_MISMATCH_RETRY_COUNT: u64 = 2; const DEFAULT_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS: u64 = 100; +const PART_TRANSACTION_OLD_DATA: &str = "old.data"; +const PART_TRANSACTION_OLD_DATA_ABSENT: &str = "old.data.absent"; +const PART_TRANSACTION_OLD_META_ABSENT: &str = "old.meta.absent"; +const PART_TRANSACTION_PUBLISH_META: &str = "publish.meta"; enum ReadAllError { Open(std::io::Error), Disk(DiskError), @@ -141,6 +146,28 @@ fn remove_dir_all_if_exists(path: &Path) -> std::io::Result<()> { } } +fn snapshot_part_transaction_file(src: &Path, backup: &Path, absent: &Path) -> std::io::Result<()> { + match std::fs::symlink_metadata(src) { + Ok(metadata) if metadata.is_file() => std::fs::hard_link(src, backup), + Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction source is not a file")), + Err(err) if err.kind() == ErrorKind::NotFound => std::fs::write(absent, []), + Err(err) => Err(err), + } +} + +fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, restore: &Path) -> std::io::Result<()> { + match std::fs::symlink_metadata(backup) { + Ok(metadata) if metadata.is_file() => { + remove_file_if_exists(restore)?; + std::fs::hard_link(backup, restore)?; + std::fs::rename(restore, current) + } + Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction backup is not a file")), + Err(err) if err.kind() == ErrorKind::NotFound && absent.is_file() => remove_file_if_exists(current), + Err(err) => Err(err), + } +} + fn rollback_committed_rename_std( dst_file_path: &Path, new_data_path: Option<&Path>, @@ -6447,6 +6474,151 @@ impl DiskAPI for LocalDisk { Ok(resp) } + #[tracing::instrument(level = "trace", skip_all)] + async fn prepare_part_transaction( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + meta: Bytes, + ) -> Result<()> { + let src_volume_dir = self.get_bucket_path(src_volume)?; + let dst_volume_dir = self.get_bucket_path(dst_volume)?; + if !skip_access_checks(src_volume) { + super::fs::access_std(&src_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?; + } + if !skip_access_checks(dst_volume) { + super::fs::access_std(&dst_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?; + } + + let src_file_path = self.get_object_path(src_volume, src_path)?; + let dst_file_path = self.get_object_path(dst_volume, dst_path)?; + let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?; + let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?; + for path in [&src_file_path, &dst_file_path, &dst_meta_path, &transaction_path] { + check_path_length(path.to_string_lossy().as_ref())?; + } + + let durability = effective_durability(dst_volume); + tokio::task::spawn_blocking(move || { + let source = std::fs::symlink_metadata(&src_file_path).map_err(to_file_error)?; + if !source.is_file() { + return Err(DiskError::FileAccessDenied); + } + if transaction_path.exists() { + return Err(DiskError::FileAccessDenied); + } + + let Some(transaction_parent) = transaction_path.parent() else { + return Err(DiskError::InvalidPath); + }; + std::fs::create_dir_all(transaction_parent).map_err(to_file_error)?; + let staging_path = transaction_parent.join(format!(".part-txn-{}", Uuid::new_v4())); + std::fs::create_dir(&staging_path).map_err(to_file_error)?; + + let prepare_result = (|| -> std::io::Result<()> { + snapshot_part_transaction_file( + &dst_file_path, + &staging_path.join(PART_TRANSACTION_OLD_DATA), + &staging_path.join(PART_TRANSACTION_OLD_DATA_ABSENT), + )?; + snapshot_part_transaction_file( + &dst_meta_path, + &staging_path.join(PART_TRANSACTION_OLD_META), + &staging_path.join(PART_TRANSACTION_OLD_META_ABSENT), + )?; + + let mut new_meta = std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(staging_path.join(PART_TRANSACTION_NEW_META))?; + std::io::Write::write_all(&mut new_meta, &meta)?; + if durability.syncs_commit_metadata() { + new_meta.sync_data()?; + os::fsync_dir_std(&staging_path)?; + } + std::fs::rename(&staging_path, &transaction_path)?; + if durability.syncs_commit_metadata() { + os::fsync_dir_std(transaction_parent)?; + } + Ok(()) + })(); + + if let Err(err) = prepare_result { + let _ = remove_dir_all_if_exists(&staging_path); + return Err(to_file_error(err).into()); + } + + Ok(()) + }) + .await + .map_err(DiskError::from)? + } + + #[tracing::instrument(level = "trace", skip_all)] + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> { + self.get_bucket_path(volume)?; + let current_data_path = self.get_object_path(volume, path)?; + let current_meta_path = self.get_object_path(volume, &format!("{path}.meta"))?; + let transaction_path = self.get_object_path(volume, &crate::disk::part_transaction_path(path))?; + for candidate in [¤t_data_path, ¤t_meta_path, &transaction_path] { + check_path_length(candidate.to_string_lossy().as_ref())?; + } + let durability = effective_durability(volume); + + tokio::task::spawn_blocking(move || { + match std::fs::symlink_metadata(&transaction_path) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(to_file_error(err).into()), + } + + if action == PartTransactionAction::Rollback { + std::fs::write(transaction_path.join(PART_TRANSACTION_ROLLBACK), []).map_err(to_file_error)?; + if durability.syncs_commit_metadata() { + os::fsync_dir_std(&transaction_path).map_err(to_file_error)?; + } + restore_part_transaction_file( + ¤t_data_path, + &transaction_path.join(PART_TRANSACTION_OLD_DATA), + &transaction_path.join(PART_TRANSACTION_OLD_DATA_ABSENT), + &transaction_path.join("restore.data"), + ) + .map_err(to_file_error)?; + restore_part_transaction_file( + ¤t_meta_path, + &transaction_path.join(PART_TRANSACTION_OLD_META), + &transaction_path.join(PART_TRANSACTION_OLD_META_ABSENT), + &transaction_path.join("restore.meta"), + ) + .map_err(to_file_error)?; + if durability.syncs_commit_metadata() + && let Some(parent) = current_data_path.parent() + { + os::fsync_dir_std(parent).map_err(to_file_error)?; + } + } + + let Some(parent) = transaction_path.parent() else { + return Err(DiskError::InvalidPath); + }; + let cleanup_path = parent.join(format!(".part-txn-settled-{}", Uuid::new_v4())); + std::fs::rename(&transaction_path, &cleanup_path).map_err(to_file_error)?; + if durability.syncs_commit_metadata() { + os::fsync_dir_std(parent).map_err(to_file_error)?; + } + remove_dir_all_if_exists(&cleanup_path).map_err(to_file_error)?; + Ok(()) + }) + .await + .map_err(DiskError::from)??; + + self.io_backend.invalidate_cached_fd(volume, path).await; + Ok(()) + } + #[tracing::instrument(level = "trace", skip_all)] async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> { let src_volume_dir = self.get_bucket_path(src_volume)?; @@ -6528,6 +6700,42 @@ impl DiskAPI for LocalDisk { } } + let transaction_publish_meta = if src_is_dir { + None + } else { + let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?; + let transaction_meta_path = transaction_path.join(PART_TRANSACTION_NEW_META); + match fs::read(&transaction_meta_path).await { + Ok(expected_meta) => { + if expected_meta.as_slice() != meta.as_ref() { + return Err(DiskError::FileCorrupt); + } + + let publish_meta_path = transaction_path.join(PART_TRANSACTION_PUBLISH_META); + let source_meta_path = transaction_meta_path.clone(); + let publish_path = publish_meta_path.clone(); + tokio::task::spawn_blocking(move || { + remove_file_if_exists(&publish_path)?; + std::fs::hard_link(source_meta_path, &publish_path) + }) + .await + .map_err(DiskError::from)? + .map_err(to_file_error)?; + Some(publish_meta_path) + } + // Old peers know only RenamePart. The new coordinator never + // reaches rename unless prepare succeeded, so an absent + // transaction directory identifies the rolling-upgrade legacy + // path. A present directory without new.meta is corruption. + Err(err) if err.kind() == ErrorKind::NotFound => match fs::metadata(&transaction_path).await { + Ok(_) => return Err(DiskError::FileCorrupt), + Err(meta_err) if meta_err.kind() == ErrorKind::NotFound => None, + Err(meta_err) => return Err(to_file_error(meta_err).into()), + }, + Err(err) => return Err(to_file_error(err).into()), + } + }; + // UploadPart is acknowledged once this rename lands, so the part data and // its directory entry must be durable before we return. Relaxed keeps the // part payload fdatasync but leaves the directory entry to the page cache. @@ -6561,7 +6769,17 @@ impl DiskAPI for LocalDisk { return Err(DiskError::FileAccessDenied); } - self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?; + if let Some(transaction_publish_meta) = transaction_publish_meta { + let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?; + rename_all(&transaction_publish_meta, &dst_meta_path, &dst_volume_dir).await?; + if durability.syncs_commit_metadata() + && let Some(parent) = dst_meta_path.parent() + { + os::fsync_dir(parent).await.map_err(to_file_error)?; + } + } else { + self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?; + } if let Some(parent) = src_file_path.parent() { self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await?; @@ -8282,6 +8500,12 @@ mod test { file_info.data_dir = data_dir; file_info.data = data; file_info.size = size; + file_info.parts = vec![ObjectPartInfo { + number: 1, + size: usize::try_from(size).expect("test object size should fit usize"), + actual_size: size, + ..Default::default() + }]; file_info.mod_time = Some(OffsetDateTime::now_utc()); file_info } @@ -9370,9 +9594,15 @@ mod test { .await .expect("source part should be written"); + disk.prepare_part_transaction(tmp_volume, "upload/part.1", bucket, "object/part.1", meta.clone()) + .await + .expect("part transaction should be prepared"); disk.rename_part(tmp_volume, "upload/part.1", bucket, "object/part.1", meta.clone()) .await .expect("rename_part should commit part"); + disk.settle_part_transaction(bucket, "object/part.1", PartTransactionAction::Commit) + .await + .expect("part transaction should be committed"); assert_eq!( disk.read_all(bucket, "object/part.1") @@ -9390,6 +9620,71 @@ mod test { matches!(disk.read_all(tmp_volume, "upload/part.1").await, Err(DiskError::FileNotFound)), "source part must be removed after a successful commit" ); + + let legacy_payload = Bytes::from_static(b"legacy peer payload"); + let legacy_meta = Bytes::from_static(b"legacy peer metadata"); + disk.write_all(tmp_volume, "legacy/part.1", legacy_payload.clone()) + .await + .expect("legacy source part should be written"); + disk.rename_part(tmp_volume, "legacy/part.1", bucket, "object/part.1", legacy_meta.clone()) + .await + .expect("pre-transaction peer RenamePart should remain supported"); + assert_eq!( + disk.read_all(bucket, "object/part.1") + .await + .expect("legacy destination part should be readable"), + legacy_payload + ); + assert_eq!( + disk.read_all(bucket, "object/part.1.meta") + .await + .expect("legacy destination metadata should be readable"), + legacy_meta + ); + } + + #[tokio::test] + async fn test_part_transaction_rolls_back_data_published_before_metadata() { + use tempfile::tempdir; + + let dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + ensure_test_volume(&disk, "tmp").await; + ensure_test_volume(&disk, "bucket").await; + + disk.write_all("tmp", "upload/part.1", Bytes::from_static(b"new data")) + .await + .expect("new part should be staged"); + disk.write_all("bucket", "object/part.1", Bytes::from_static(b"old data")) + .await + .expect("old part data should be staged"); + disk.write_all("bucket", "object/part.1.meta", Bytes::from_static(b"old metadata")) + .await + .expect("old part metadata should be staged"); + + disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", Bytes::from_static(b"new metadata")) + .await + .expect("part transaction should be prepared"); + disk.rename_file("tmp", "upload/part.1", "bucket", "object/part.1") + .await + .expect("data publication should succeed"); + disk.settle_part_transaction("bucket", "object/part.1", PartTransactionAction::Rollback) + .await + .expect("part transaction should roll back"); + + assert_eq!( + disk.read_all("bucket", "object/part.1") + .await + .expect("old part data should be restored"), + Bytes::from_static(b"old data") + ); + assert_eq!( + disk.read_all("bucket", "object/part.1.meta") + .await + .expect("old part metadata should be restored"), + Bytes::from_static(b"old metadata") + ); } struct BlockingScanWriter { diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index d683e860f..fc17998e8 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -38,6 +38,16 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json"; pub const HEALING_MARKER_PATH: &str = "healing.bin"; pub const STORAGE_FORMAT_FILE: &str = "xl.meta"; pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp"; +pub const PART_TRANSACTION_NEW_META: &str = "new.meta"; +pub const PART_TRANSACTION_OLD_META: &str = "old.meta"; +pub const PART_TRANSACTION_ROLLBACK: &str = "rollback"; + +pub fn part_transaction_path(part_path: &str) -> String { + match part_path.rsplit_once('/') { + Some((parent, name)) => format!("{parent}/.{name}.rustfs-txn"), + None => format!(".{part_path}.rustfs-txn"), + } +} use crate::cluster::rpc::RemoteDisk; use crate::cluster::rpc::build_internode_data_transport_from_env; @@ -62,6 +72,12 @@ pub type DiskStore = Arc; pub type FileReader = Box; pub type FileWriter = Box; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PartTransactionAction { + Commit, + Rollback, +} + #[derive(Clone, Copy, Debug)] pub struct MmapCopyStageMetrics { pub(crate) path: &'static str, @@ -381,6 +397,35 @@ impl DiskAPI for Disk { } } + async fn prepare_part_transaction( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + meta: Bytes, + ) -> Result<()> { + match self { + Disk::Local(local_disk) => { + local_disk + .prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta) + .await + } + Disk::Remote(remote_disk) => { + remote_disk + .prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta) + .await + } + } + } + + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> { + match self { + Disk::Local(local_disk) => local_disk.settle_part_transaction(volume, path, action).await, + Disk::Remote(remote_disk) => remote_disk.settle_part_transaction(volume, path, action).await, + } + } + #[tracing::instrument(level = "trace", skip_all)] async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { match self { @@ -659,6 +704,19 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { // ReadFileStream async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>; async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()>; + async fn prepare_part_transaction( + &self, + _src_volume: &str, + _src_path: &str, + _dst_volume: &str, + _dst_path: &str, + _meta: Bytes, + ) -> Result<()> { + Err(DiskError::MethodNotAllowed) + } + async fn settle_part_transaction(&self, _volume: &str, _path: &str, _action: PartTransactionAction) -> Result<()> { + Err(DiskError::MethodNotAllowed) + } async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; // VerifyFile async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result; diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index 2c658e4e4..41705912b 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -443,7 +443,9 @@ pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorith size.div_ceil(shard_size) * algo.size() + size } -/// Verify an interleaved per-block bitrot shard file. +/// Verify an interleaved per-block bitrot shard file and consume the reader +/// through EOF. Bytes beyond the encoded length are corruption, even when every +/// expected block has a valid hash. /// /// The read loop below assumes every block on disk is `[hash][data]` (streaming /// bitrot). It is therefore only valid for the streaming Highway variants, whose @@ -487,6 +489,11 @@ pub async fn bitrot_verify( left -= read; } + let mut trailing = [0u8; 1]; + if r.read(&mut trailing).await? != 0 { + return Err(std::io::Error::other("bitrot shard file has trailing data")); + } + Ok(()) } @@ -860,12 +867,19 @@ mod tests { .await .expect("valid bitrot shard file should verify"); + let mut truncated = written.clone(); + truncated.pop(); + let err = bitrot_verify(Cursor::new(truncated), written.len(), data.len(), algo.clone(), shard_size) + .await + .expect_err("one-byte-short shard file must be rejected while reading"); + assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof); + let err = bitrot_verify(Cursor::new(written.clone()), written.len() - 1, data.len(), algo.clone(), shard_size) .await .expect_err("wrong file size must be rejected before reading data"); assert!(err.to_string().contains("size mismatch")); - let mut corrupt = written; + let mut corrupt = written.clone(); let last = corrupt.len() - 1; corrupt[last] ^= 0x80; let err = bitrot_verify( @@ -878,6 +892,21 @@ mod tests { .await .expect_err("hash mismatch must reject corrupted data"); assert!(err.to_string().contains("hash mismatch")); + + for trailing in [vec![0xa5], vec![0xa5; 17]] { + let mut oversized = written.clone(); + oversized.extend_from_slice(&trailing); + let err = bitrot_verify( + Cursor::new(oversized), + written.len(), + data.len(), + HashAlgorithm::HighwayHash256S, + shard_size, + ) + .await + .expect_err("trailing bytes after a valid encoded shard must be rejected"); + assert!(err.to_string().contains("trailing data")); + } } #[tokio::test] @@ -909,6 +938,22 @@ mod tests { assert!(err.to_string().contains("hash mismatch")); } + #[tokio::test] + async fn bitrot_verify_accepts_exact_legacy_streaming_layout() { + let data = b"legacy streaming bitrot"; + let shard_size = 8; + let algo = HashAlgorithm::HighwayHash256SLegacy; + let mut writer = BitrotWriter::new(Cursor::new(Vec::new()), shard_size, algo.clone()); + for chunk in data.chunks(shard_size) { + writer.write(chunk).await.expect("legacy streaming shard should encode"); + } + let written = writer.into_inner().into_inner(); + + bitrot_verify(Cursor::new(written.clone()), written.len(), data.len(), algo, shard_size) + .await + .expect("exact legacy streaming shard should remain valid"); + } + #[tokio::test] async fn write_all_vectored_retries_partial_hash_and_data_writes_and_rejects_zero_write() { let mut writer = LimitedVectoredWriter { diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index c249f64b3..1330cc10b 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -28,7 +28,6 @@ use md5::{Digest, Md5}; use rustfs_kms::{KmsUnavailableError, is_data_key_envelope, types::ObjectEncryptionContext}; use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; use rustfs_utils::path::path_join_buf; -#[cfg(feature = "rio-v2")] use serde::Deserialize; #[cfg(feature = "rio-v2")] use sha2::Sha256; @@ -44,6 +43,7 @@ const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv"; const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size"; const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size"; const DEFAULT_SSE_ALGORITHM: &str = "AES256"; +const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1; #[cfg(feature = "rio-v2")] const DARE_PAYLOAD_SIZE: i64 = 64 * 1024; #[cfg(feature = "rio-v2")] @@ -1716,16 +1716,39 @@ fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str, object_context fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> { let encrypted_dek = std::str::from_utf8(encrypted_dek).map_err(|_| Error::other("managed DEK is not valid UTF-8"))?; - let parts: Vec<&str> = encrypted_dek.split(':').collect(); - if parts.len() != 2 { - return Err(Error::other("invalid managed DEK format")); + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct LocalSseDekEnvelope<'a> { + version: u8, + nonce: &'a str, + ciphertext: &'a str, } + let (nonce, ciphertext) = match serde_json::from_str::>(encrypted_dek) { + Ok(envelope) => { + if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION { + return Err(Error::other(format!("unsupported managed DEK format version: {}", envelope.version))); + } + (envelope.nonce, envelope.ciphertext) + } + Err(_) => { + // DEPRECATED: read-only compatibility for persisted colon-delimited DEKs. + // RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs. + let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else { + return Err(Error::other("invalid managed DEK format")); + }; + if ciphertext.contains(':') { + return Err(Error::other("invalid managed DEK format")); + } + (nonce, ciphertext) + } + }; + let nonce_vec = BASE64_STANDARD - .decode(parts[0]) + .decode(nonce) .map_err(|_| Error::other("invalid managed DEK nonce"))?; let ciphertext = BASE64_STANDARD - .decode(parts[1]) + .decode(ciphertext) .map_err(|_| Error::other("invalid managed DEK ciphertext"))?; let nonce_array: [u8; 12] = nonce_vec @@ -2324,9 +2347,36 @@ mod tests { let cipher = Aes256Gcm::new(&key); let nonce = Nonce::from([0u8; 12]); let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt managed dek"); + serde_json::json!({ + "version": LOCAL_SSE_DEK_FORMAT_VERSION, + "nonce": BASE64_STANDARD.encode(nonce), + "ciphertext": BASE64_STANDARD.encode(ciphertext), + }) + .to_string() + } + + fn encrypt_legacy_managed_dek_for_test(dek: [u8; 32], master_key: [u8; 32]) -> String { + let key = Key::::from(master_key); + let cipher = Aes256Gcm::new(&key); + let nonce = Nonce::from([0u8; 12]); + let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt legacy managed dek"); format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext)) } + #[test] + fn decrypt_rustfs_local_sse_dek_rejects_unknown_json_version() { + let envelope = serde_json::json!({ + "version": LOCAL_SSE_DEK_FORMAT_VERSION + 1, + "nonce": BASE64_STANDARD.encode([0u8; 12]), + "ciphertext": BASE64_STANDARD.encode([0u8; 48]), + }) + .to_string(); + + let error = + decrypt_rustfs_local_sse_dek(envelope.as_bytes()).expect_err("unknown local SSE DEK versions must fail closed"); + assert!(error.to_string().contains("unsupported managed DEK format version")); + } + #[cfg(feature = "rio-v2")] fn seal_managed_s3_object_key_for_test( bucket: &str, @@ -2433,7 +2483,7 @@ mod tests { async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([7u8; 32])))], async { let data_key = [0x24; 32]; let base_nonce = [0x14; 12]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [7u8; 32]); + let encrypted_dek = encrypt_legacy_managed_dek_for_test(data_key, [7u8; 32]); let metadata = HashMap::from([ ( INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index c10020177..9440ebfe2 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -46,7 +46,10 @@ use crate::diagnostics::get::{ GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, }; -use crate::disk::OldCurrentSize; +use crate::disk::{ + OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, + part_transaction_path, +}; use crate::erasure::coding::BitrotReader; use crate::io_support::bitrot::ShardReader; use crate::io_support::bitrot::{ @@ -3176,6 +3179,155 @@ impl SetDisks { } } + async fn recover_part_transaction(&self, dst_object: &str, write_quorum: usize) -> disk::error::Result { + let disks = self.get_disks_internal().await; + let transaction_path = part_transaction_path(dst_object); + let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}"); + let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}"); + let current_meta_path = format!("{dst_object}.meta"); + + let reads = disks.iter().map(|disk| { + let disk = disk.clone(); + let transaction_meta_path = transaction_meta_path.clone(); + let rollback_path = rollback_path.clone(); + let current_meta_path = current_meta_path.clone(); + async move { + let Some(disk) = disk else { + return Ok((None, None, false)); + }; + let transaction_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await { + Ok(meta) => Some(meta), + Err(DiskError::FileNotFound) => None, + Err(err) => return Err(err), + }; + let rollback = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await { + Ok(_) => true, + Err(DiskError::FileNotFound) => false, + Err(err) => return Err(err), + }; + let current_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, ¤t_meta_path).await { + Ok(meta) => Some(meta), + Err(DiskError::FileNotFound | DiskError::DiskNotFound) => None, + Err(_) => None, + }; + Ok((transaction_meta, current_meta, rollback)) + } + }); + let observations = join_all(reads).await.into_iter().collect::>>()?; + if observations.iter().all(|(transaction, _, _)| transaction.is_none()) { + return Ok(false); + } + + let mut current_counts: HashMap = HashMap::new(); + for (_, current, _) in &observations { + if let Some(current) = current { + *current_counts.entry(current.clone()).or_default() += 1; + } + } + let current_quorum = current_counts + .into_iter() + .find_map(|(meta, count)| (count >= write_quorum).then_some(meta)); + + let old_meta_path = format!("{transaction_path}/{PART_TRANSACTION_OLD_META}"); + let old_meta_absent_path = format!("{transaction_path}/old.meta.absent"); + let decisions = observations + .iter() + .enumerate() + .filter_map(|(index, (transaction_meta, _, rollback))| { + transaction_meta.as_ref().map(|meta| (index, meta.clone(), *rollback)) + }) + .map(|(index, transaction_meta, rollback)| { + let disk = disks[index].clone(); + let old_meta_path = old_meta_path.clone(); + let old_meta_absent_path = old_meta_absent_path.clone(); + let current_quorum = current_quorum.clone(); + async move { + let Some(disk) = disk else { + return Err(DiskError::DiskNotFound); + }; + let action = if rollback { + PartTransactionAction::Rollback + } else if current_quorum.as_ref() == Some(&transaction_meta) { + PartTransactionAction::Commit + } else if let Some(current_quorum) = current_quorum { + match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_path).await { + Ok(old_meta) if old_meta == current_quorum => PartTransactionAction::Rollback, + Ok(_) => PartTransactionAction::Commit, + Err(DiskError::FileNotFound) => { + match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_absent_path).await { + Ok(_) => PartTransactionAction::Commit, + Err(_) => return Err(DiskError::FileCorrupt), + } + } + Err(err) => return Err(err), + } + } else { + PartTransactionAction::Rollback + }; + disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_object, action) + .await?; + Ok(action == PartTransactionAction::Commit) + } + }); + + let results = join_all(decisions).await; + if let Some(err) = results.iter().find_map(|result| result.as_ref().err()) { + return Err(err.clone()); + } + Ok(results.iter().any(|result| matches!(result, Ok(true)))) + } + + pub(in crate::set_disk) async fn recover_part_transactions( + &self, + part_path: &str, + read_quorum: usize, + write_quorum: usize, + ) -> disk::error::Result<()> { + let disks = self.get_disks_internal().await; + let listings = join_all(disks.iter().map(|disk| { + let disk = disk.clone(); + async move { + let Some(disk) = disk else { + return Err(DiskError::DiskNotFound); + }; + disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1) + .await + } + })) + .await; + + let mut transaction_parts = HashSet::new(); + let mut errs = Vec::with_capacity(listings.len()); + for listing in listings { + match listing { + Ok(entries) => { + errs.push(None); + for entry in entries { + let name = entry.trim_end_matches('/'); + let Some(part_number) = name + .strip_prefix(".part.") + .and_then(|name| name.strip_suffix(".rustfs-txn")) + .and_then(|number| number.parse::().ok()) + else { + continue; + }; + transaction_parts.insert(part_number); + } + } + Err(err) => errs.push(Some(err)), + } + } + if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { + return Err(err); + } + + for part_number in transaction_parts { + self.recover_part_transaction(&format!("{part_path}part.{part_number}"), write_quorum) + .await?; + } + Ok(()) + } + #[tracing::instrument(skip(disks, meta))] #[allow(clippy::too_many_arguments)] pub(in crate::set_disk) async fn rename_part( @@ -3189,23 +3341,46 @@ impl SetDisks { write_quorum: usize, quorum_context: Option>, ) -> disk::error::Result>> { + self.recover_part_transaction(dst_object, write_quorum).await?; + let src_bucket = Arc::new(src_bucket.to_string()); let src_object = Arc::new(src_object.to_string()); let dst_bucket = Arc::new(dst_bucket.to_string()); let dst_object = Arc::new(dst_object.to_string()); - // Do NOT pre-delete the destination part before renaming: the per-disk - // `rename_part` replaces `part.N` atomically (std::fs::rename) and rewrites - // `part.N.meta`, so the pre-delete is redundant — and destructive. It - // opened a window where an already-committed (ACKed) part was removed on - // every disk before the new rename landed, so a re-upload that then failed - // quorum destroyed the committed part outright (backlog#853 / #799 B4). - // The atomic rename overwrites in place; on quorum failure below we roll - // the destination back. + let prepare_tasks = disks.iter().map(|disk| { + let disk = disk.clone(); + let src_bucket = src_bucket.clone(); + let src_object = src_object.clone(); + let dst_bucket = dst_bucket.clone(); + let dst_object = dst_object.clone(); + let meta = meta.clone(); + async move { + let disk = disk?; + Some( + disk.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta) + .await, + ) + } + }); + let prepare_results = join_all(prepare_tasks).await; + let prepare_errs = prepare_results + .into_iter() + .map(|result| match result { + Some(Ok(())) => None, + Some(Err(err)) => Some(err), + None => Some(DiskError::DiskNotFound), + }) + .collect::>(); + let prepared_disks = Self::eval_disks(disks, &prepare_errs); + if reduce_write_quorum_errs(&prepare_errs, OBJECT_OP_IGNORED_ERRS, write_quorum).is_some() { + self.recover_part_transaction(&dst_object, write_quorum).await?; + return Err(DiskError::ErasureWriteQuorum); + } let mut errs = Vec::with_capacity(disks.len()); - let futures = disks.iter().map(|disk| { + let futures = prepared_disks.iter().map(|disk| { let disk = disk.clone(); let meta = meta.clone(); let src_bucket = src_bucket.clone(); @@ -3255,19 +3430,42 @@ impl SetDisks { ); } - if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + let reduced_err = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum); + if let Some(err) = reduced_err { + let rollbacks = prepared_disks.iter().filter_map(|disk| { + disk.clone().map(|disk| { + let dst_object = dst_object.clone(); + async move { + disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Rollback) + .await + } + }) + }); + let rollback_results = join_all(rollbacks).await; + self.recover_part_transaction(&dst_object, write_quorum).await?; + if let Some(rollback_err) = rollback_results.iter().find_map(|result| result.as_ref().err()) { + warn!(error = %rollback_err, "rename_part rollback did not settle on every prepared disk"); + } if let Some(context) = quorum_context { log_multipart_write_quorum_failure(context, &errs, write_quorum, &err); } else { warn!("rename_part errs {:?}", &errs); } - self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")]) - .await; return Err(err); } - let disks = Self::eval_disks(disks, &errs); - Ok(disks) + let committed = self.recover_part_transaction(&dst_object, write_quorum).await?; + if !committed { + let err = DiskError::ErasureWriteQuorum; + if let Some(context) = quorum_context { + log_multipart_write_quorum_failure(context, &errs, write_quorum, &err); + } else { + warn!("rename_part errs {:?}", &errs); + } + return Err(err); + } + + Ok(Self::eval_disks(&prepared_disks, &errs)) } pub(in crate::set_disk) fn eval_disks(disks: &[Option], errs: &[Option]) -> Vec> { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index bdd66c945..8d6c73948 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -690,6 +690,8 @@ mod ops; #[cfg(feature = "test-util")] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; pub(crate) use ops::object::body_cache_plaintext_len; +#[cfg(test)] +pub(crate) use ops::object::cleanup_rejected_transition_upload_durably; mod read; mod replication; pub(crate) mod shard_source; diff --git a/crates/ecstore/src/set_disk/ops/bitrot_self_verify.rs b/crates/ecstore/src/set_disk/ops/bitrot_self_verify.rs index 694d78a7d..93c038e6a 100644 --- a/crates/ecstore/src/set_disk/ops/bitrot_self_verify.rs +++ b/crates/ecstore/src/set_disk/ops/bitrot_self_verify.rs @@ -199,6 +199,32 @@ mod tests { .expect("healthy temp shard should verify"); assert_eq!(verified, 1); + for trailing in [vec![0xa5], vec![0xa5; 17]] { + let mut oversized = encoded.to_vec(); + oversized.extend_from_slice(&trailing); + disk.write_all(RUSTFS_META_TMP_BUCKET, path, Bytes::from(oversized)) + .await + .expect("oversized temp shard should be staged"); + let err = verify_written_bitrot_shards( + &[Some(disk.clone())], + None, + BitrotSelfVerifyTarget { + operation: "put_object", + bucket: "bucket", + object: "object", + part_number: None, + volume: RUSTFS_META_TMP_BUCKET, + path, + logical_shard_size, + shard_size, + write_quorum: 1, + }, + ) + .await + .expect_err("disk shard with trailing bytes must not be committed"); + assert!(err.to_string().contains("trailing data")); + } + let mut corrupt = encoded.to_vec(); let last = corrupt.len() - 1; corrupt[last] ^= 0x01; @@ -225,4 +251,40 @@ mod tests { .expect_err("corrupt no-parity temp shard must not be committed"); assert!(err.to_string().contains("bitrot self-verify failed")); } + + #[tokio::test] + async fn no_parity_inline_self_verify_rejects_trailing_bytes() { + let (_temp_dirs, disks, _set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await; + let shard_size = 16usize; + let payload = b"inline bitrot payload"; + let encoded = encode_streaming_shard(payload, shard_size).await; + let disks = disks.into_iter().map(Some).collect::>(); + + for trailing in [vec![0xa5], vec![0xa5; 17]] { + let mut oversized = encoded.to_vec(); + oversized.extend_from_slice(&trailing); + let parts = [FileInfo { + data: Some(Bytes::from(oversized)), + ..Default::default() + }]; + let err = verify_written_bitrot_shards( + &disks, + Some(&parts), + BitrotSelfVerifyTarget { + operation: "put_object", + bucket: "bucket", + object: "inline-object", + part_number: None, + volume: RUSTFS_META_TMP_BUCKET, + path: "unused-for-inline", + logical_shard_size: payload.len(), + shard_size, + write_quorum: 1, + }, + ) + .await + .expect_err("inline shard with trailing bytes must not be committed"); + assert!(err.to_string().contains("trailing data")); + } + } } diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index ee866fb7e..672af4c66 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -19,6 +19,71 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_HEAL: &str = "heal"; const EVENT_HEAL_OBJECT_RENAME: &str = "heal_object_rename"; +const HEAL_RENAME_INCOMPLETE: &str = "heal rename incomplete"; + +#[cfg(test)] +static HEAL_RENAME_FAILURES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + +#[cfg(test)] +struct HealRenameFailureScope { + bucket: String, + object: String, +} + +#[cfg(test)] +impl HealRenameFailureScope { + fn install(bucket: &str, object: &str, disk_indexes: &[usize]) -> Self { + let mut failures = HEAL_RENAME_FAILURES + .lock() + .expect("heal rename failure registry should not poison"); + assert!( + !failures + .iter() + .any(|(registered_bucket, registered_object, _)| { registered_bucket == bucket && registered_object == object }), + "heal rename failures must be installed once per object" + ); + failures.extend( + disk_indexes + .iter() + .map(|index| (bucket.to_string(), object.to_string(), *index)), + ); + Self { + bucket: bucket.to_string(), + object: object.to_string(), + } + } +} + +#[cfg(test)] +impl Drop for HealRenameFailureScope { + fn drop(&mut self) { + HEAL_RENAME_FAILURES + .lock() + .expect("heal rename failure registry should not poison") + .retain(|(bucket, object, _)| bucket != &self.bucket || object != &self.object); + } +} + +#[cfg(test)] +fn should_fail_heal_rename(bucket: &str, object: &str, disk_index: usize) -> bool { + let mut failures = HEAL_RENAME_FAILURES + .lock() + .expect("heal rename failure registry should not poison"); + if let Some(position) = failures + .iter() + .position(|entry| entry == &(bucket.to_string(), object.to_string(), disk_index)) + { + failures.swap_remove(position); + true + } else { + false + } +} + +#[cfg(not(test))] +fn should_fail_heal_rename(_bucket: &str, _object: &str, _disk_index: usize) -> bool { + false +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct PartFailureSummary { @@ -659,8 +724,12 @@ impl SetDisks { } } // Rename from tmp location to the actual location. + // MinIO stops on the first RenameData error. RustFS intentionally + // continues per target, but reports any residue after all attempts + // so successful repairs survive and failed targets remain retryable. let mut rename_attempts = 0usize; let mut rename_successes = 0usize; + let mut healed_disks = vec![None; out_dated_disks.len()]; for (index, outdated_disk) in out_dated_disks.iter().enumerate() { if let Some(disk) = outdated_disk { rename_attempts += 1; @@ -669,9 +738,18 @@ impl SetDisks { // Attempt a rename now from healed data to final location. parts_metadata[index].set_healing(); - let rename_result = disk - .rename_data(RUSTFS_META_TMP_BUCKET, &tmp_id, parts_metadata[index].clone(), bucket, object) - .await; + let rename_result = if should_fail_heal_rename(bucket, object, index) { + Err(DiskError::Unexpected) + } else { + disk.rename_data( + RUSTFS_META_TMP_BUCKET, + &tmp_id, + parts_metadata[index].clone(), + bucket, + object, + ) + .await + }; if let Err(err) = &rename_result { warn!( @@ -690,6 +768,7 @@ impl SetDisks { ); } else { rename_successes += 1; + healed_disks[index] = Some(disk.clone()); if parts_metadata[index].is_remote() { let rm_data_dir = parts_metadata[index].data_dir.expect("operation should succeed").to_string(); @@ -734,15 +813,18 @@ impl SetDisks { .await .map_err(DiskError::other)?; - if rename_attempts > 0 && rename_successes == 0 { + self.record_healed_capacity_scope(&healed_disks); + + if rename_successes < rename_attempts { return Ok(( result, - Some(DiskError::other(format!("all healed data rename attempts failed for {bucket}/{object}"))), + Some(DiskError::other(format!( + "{HEAL_RENAME_INCOMPLETE}: {rename_successes} of {rename_attempts} targets committed for \ + {bucket}/{object}" + ))), )); } - self.record_healed_capacity_scope(&out_dated_disks); - // The object is healthy here; sweep any data dirs left behind // by pre-#3510 unversioned overwrites, which the dangling paths // above never touch (issues #3231, #3191). Best effort — a @@ -1312,11 +1394,13 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { #[cfg(test)] mod heal_result_report_tests { use super::SetDisks; + use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope}; use crate::disk::endpoint::Endpoint; use crate::disk::error::DiskError; use crate::disk::format::FormatV3; - use crate::disk::{DiskOption, DiskStore, new_disk}; + use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk}; use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated; use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use crate::{config::storageclass, store::init_format::save_format_file}; @@ -1406,6 +1490,171 @@ mod heal_result_report_tests { (dir, set) } + async fn non_trash_tmp_entries(temp_dirs: &[TempDir]) -> Vec { + let mut entries = Vec::new(); + for dir in temp_dirs { + let tmp = dir.path().join(RUSTFS_META_TMP_BUCKET); + let mut read_dir = match tokio::fs::read_dir(&tmp).await { + Ok(read_dir) => read_dir, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, + Err(err) => panic!("tmp directory should be readable: {err}"), + }; + while let Some(entry) = read_dir.next_entry().await.expect("tmp entry should be readable") { + let name = entry.file_name().to_string_lossy().into_owned(); + if name != ".trash" { + entries.push(name); + } + } + } + entries + } + + #[tokio::test] + #[serial_test::serial] + async fn heal_rename_outcome_matrix_reports_partial_and_retries_failed_targets() { + for (case, failed_attempts, expect_error) in [ + ("ok-ok", Vec::new(), false), + ("ok-err", vec![1], true), + ("err-ok", vec![0], true), + ("err-err", vec![0, 1], true), + ] { + let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = format!("heal-rename-{case}"); + let object = "object.bin"; + for disk in &disks { + disk.make_volume(&bucket).await.expect("bucket volume should be created"); + } + + let payload = vec![0x5a; 1024 * 1024]; + let mut reader = PutObjReader::from_vec(payload); + set.put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + let source = disks[2] + .read_version("", &bucket, object, "", &ReadOptions::default()) + .await + .expect("source metadata should be readable"); + let data_dir = source.data_dir.expect("non-inline source should have a data directory"); + let tmp_entries_before_heal = non_trash_tmp_entries(&temp_dirs).await; + let target_slots = { + let mut slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1]; + slots.sort_unstable(); + slots + }; + let failed_slots = failed_attempts + .iter() + .map(|attempt| target_slots[*attempt]) + .collect::>(); + let failed_physical_indexes = [0, 1] + .into_iter() + .filter(|index| failed_slots.contains(&(source.erasure.distribution[*index] - 1))) + .collect::>(); + + for index in [0, 1] { + tokio::fs::remove_file( + temp_dirs[index] + .path() + .join(&bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"), + ) + .await + .expect("target shard should be removed before heal"); + } + + let failure_scope = HealRenameFailureScope::install(&bucket, object, &failed_slots); + let (first_result, first_error) = set + .heal_object( + &bucket, + object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("heal should report its per-target rename outcome"); + drop(failure_scope); + + assert_eq!(first_error.is_some(), expect_error, "{case}: aggregate status must match target outcomes"); + if let Some(error) = first_error { + let error = error.to_string(); + assert!( + error.contains(HEAL_RENAME_INCOMPLETE), + "{case}: partial/all failure must have an explicit retryable status: {error}" + ); + assert!( + error.contains(&format!("{} of 2 targets committed", 2 - failed_slots.len())), + "{case}: aggregate status must distinguish partial from all-target failure: {error}" + ); + } + for index in [0, 1] { + let expected = if failed_physical_indexes.contains(&index) { + DriveState::Missing + } else { + DriveState::Ok + }; + assert_eq!( + first_result.after.drives[index].state, + expected.to_string(), + "{case}: after.drives must reflect the actual rename outcome at index {index}" + ); + assert_eq!( + temp_dirs[index] + .path() + .join(&bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1") + .exists(), + !failed_physical_indexes.contains(&index), + "{case}: tmp cleanup must neither delete committed shards nor expose failed targets" + ); + } + let tmp_entries_after_heal = non_trash_tmp_entries(&temp_dirs).await; + assert!( + tmp_entries_after_heal + .iter() + .all(|entry| tmp_entries_before_heal.contains(entry)), + "{case}: first heal must not leave a new temporary shard: {tmp_entries_after_heal:?}" + ); + + if !failed_slots.is_empty() { + let (retry_result, retry_error) = set + .heal_object( + &bucket, + object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("second heal should retry failed targets"); + assert!(retry_error.is_none(), "{case}: second heal should complete remaining targets"); + for index in [0, 1] { + assert_eq!( + retry_result.after.drives[index].state, + DriveState::Ok.to_string(), + "{case}: second heal must converge target {index}" + ); + } + let tmp_entries_after_retry = non_trash_tmp_entries(&temp_dirs).await; + assert!( + tmp_entries_after_retry + .iter() + .all(|entry| tmp_entries_before_heal.contains(entry)), + "{case}: retry must not leave a new temporary shard: {tmp_entries_after_retry:?}" + ); + } + } + } + // Regression for #955: an offline disk must contribute exactly one drive // record. Before the fix the offline branch fell through and pushed a second // (Corrupt) record for the same disk, so `before/after.drives` grew to diff --git a/crates/ecstore/src/set_disk/ops/heal_walk.rs b/crates/ecstore/src/set_disk/ops/heal_walk.rs index 70291b039..3a87a0898 100644 --- a/crates/ecstore/src/set_disk/ops/heal_walk.rs +++ b/crates/ecstore/src/set_disk/ops/heal_walk.rs @@ -69,6 +69,13 @@ struct HealWalkCollector { } impl HealWalkCollector { + fn lock_objects(&self) -> disk::error::Result>> { + self.objects.lock().map_err(|_| { + self.cancel.cancel(); + DiskError::FileCorrupt + }) + } + /// Expand one resolved entry into its versions and record it. Cancels the /// walk once EITHER page bound (distinct object names OR expanded versions) /// is met — always at a sorted object-key boundary so a heavily-versioned @@ -105,7 +112,9 @@ impl HealWalkCollector { let added = versions.len(); let (objs_len, ver_total) = { - let mut objects = self.objects.lock().unwrap(); + let Ok(mut objects) = self.lock_objects() else { + return; + }; objects.push(HealWalkObject { name: entry.name, versions, @@ -239,7 +248,10 @@ impl SetDisks { Err(err) => return Err(err), } - let objects = std::mem::take(&mut *collector.objects.lock().unwrap()); + let objects = { + let mut objects = collector.lock_objects()?; + std::mem::take(&mut *objects) + }; let truncated = collector.truncated.load(Ordering::SeqCst); Ok(finalize_heal_walk_page(objects, forward_to, truncated)) } @@ -310,4 +322,28 @@ mod tests { assert_eq!(next_forward, None, "a complete final page must not carry a resume cursor"); assert!(!truncated); } + + #[test] + fn poisoned_collector_state_cancels_the_walk() { + let collector = Arc::new(HealWalkCollector { + bucket: "bucket".to_string(), + batch_objects: 2, + version_budget: 2, + objects: Mutex::new(Vec::new()), + version_total: AtomicUsize::new(0), + truncated: AtomicBool::new(false), + cancel: CancellationToken::new(), + }); + let poison_target = Arc::clone(&collector); + let _ = std::thread::spawn(move || { + let _guard = poison_target.objects.lock().expect("fresh mutex should lock"); + panic!("poison heal collector"); + }) + .join(); + + let error = collector.lock_objects().expect_err("poisoned collection must fail closed"); + assert_eq!(error, DiskError::FileCorrupt); + assert!(collector.cancel.is_cancelled(), "a poisoned page collector must cancel its walk"); + assert!(collector.objects.lock().is_err(), "poisoned state must remain fail-closed"); + } } diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index a3228e079..fd3416fa8 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -1289,8 +1289,18 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } } + if !opts.no_lock && object_lock_guard.is_none() { + object_lock_guard = Some( + self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object) + .await?, + ); + } + let upload_guard = self + .acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts) + .await?; + let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?; - let (mut fi, mut files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?; + let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?; let has_layout_candidate = range_seek_rollout_enabled && fi .data_dir @@ -1303,22 +1313,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { &token, ) }); - let upload_guard = if has_layout_candidate { - if object_lock_guard.is_none() { - object_lock_guard = Some( - self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object) - .await?, - ); - } - let guard = self - .acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts) - .await?; - (fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?; - guard - } else { - None - }; - let quorum_validated_layout_token = upload_guard.as_ref().and_then(|_| { + let quorum_validated_layout_token = if has_layout_candidate { fi.data_dir .filter(|data_dir| !data_dir.is_nil()) .map(|data_dir| data_dir.to_string()) @@ -1329,7 +1324,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { token, ) }) - }); + } else { + None + }; rustfs_utils::http::metadata_compat::remove_str( &mut fi.metadata, crate::object_api::ENCRYPTED_PART_LAYOUT_CANDIDATE_SUFFIX, @@ -1356,6 +1353,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { // let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution); let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil())); + self.recover_part_transactions(&part_path, read_quorum, write_quorum) + .await + .map_err(|err| to_object_err(err.into(), vec![bucket, object]))?; let part_meta_paths = uploaded_parts .iter() @@ -1713,12 +1713,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } } - if !opts.no_lock && object_lock_guard.is_none() { - object_lock_guard = Some( - self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object) - .await?, - ); - } // Phase 2 (backlog#899): fence the commit on lock loss before any destructive // step. If the refresh heartbeat has observed a refresh-quorum loss, another // writer may have re-acquired this object's lock; proceeding would race a @@ -2183,6 +2177,201 @@ mod tests { ) } + async fn assert_quorum_minus_one_retry_preserves_completable_part( + disk_count: usize, + parity: usize, + success_indices: &[usize], + ) { + use tokio::io::AsyncReadExt as _; + + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(disk_count, 0, parity).await; + let bucket = format!("multipart-retry-{disk_count}-{}", success_indices[0]); + let object = "object"; + make_bucket_on_all(&disk_stores, &bucket).await; + + let payload = vec![0x41; 1 << 20]; + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, &bucket, object, &payload, &ObjectOptions::default()).await; + let (upload_meta, _) = set_disks + .check_upload_id_exists(&bucket, object, &upload_id, false) + .await + .expect("staged upload metadata should be readable"); + let upload_path = SetDisks::get_upload_id_dir(&bucket, object, &upload_id); + let write_quorum = upload_meta.write_quorum(set_disks.default_write_quorum()); + assert_eq!(success_indices.len() + 1, write_quorum); + let part_path = format!( + "{}/{}/part.1", + upload_path, + upload_meta.data_dir.expect("multipart upload should have a data directory") + ); + let acknowledged_meta = disk_stores[0] + .read_all(RUSTFS_META_MULTIPART_BUCKET, &format!("{part_path}.meta")) + .await + .expect("acknowledged part metadata should be readable"); + let retry_path = format!("{}/part.1", Uuid::new_v4()); + + let mut retry_disks = vec![None; disk_stores.len()]; + for &index in success_indices { + disk_stores[index] + .write_all(RUSTFS_META_TMP_BUCKET, &retry_path, Bytes::from_static(b"retry shard")) + .await + .expect("retry shard should be staged"); + retry_disks[index] = Some(disk_stores[index].clone()); + } + + let err = set_disks + .rename_part( + &retry_disks, + RUSTFS_META_TMP_BUCKET, + &retry_path, + RUSTFS_META_MULTIPART_BUCKET, + &part_path, + acknowledged_meta, + write_quorum, + None, + ) + .await + .expect_err("quorum-minus-one renamed shards must remain below write quorum"); + assert_eq!(err, DiskError::ErasureWriteQuorum); + + for (index, disk) in disk_stores.iter().enumerate() { + assert!( + disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path).await.is_ok(), + "old acknowledged shard on disk {index} must survive" + ); + } + + set_disks + .clone() + .complete_multipart_upload(&bucket, object, &upload_id, parts, &ObjectOptions::default()) + .await + .expect("the old acknowledged part should remain completable"); + let mut reader = set_disks + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("completed object should be readable"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("completed object should stream fully"); + assert_eq!(restored, payload); + } + + #[tokio::test] + async fn upload_part_retry_quorum_failure_preserves_old_part_across_ec_geometries() { + assert_quorum_minus_one_retry_preserves_completable_part(4, 2, &[0, 1]).await; + assert_quorum_minus_one_retry_preserves_completable_part(4, 2, &[2, 3]).await; + assert_quorum_minus_one_retry_preserves_completable_part(6, 2, &[0, 1, 2]).await; + assert_quorum_minus_one_retry_preserves_completable_part(6, 2, &[3, 4, 5]).await; + } + + #[tokio::test] + async fn complete_multipart_upload_recovers_interrupted_part_retry() { + use tokio::io::AsyncReadExt as _; + + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(6, 0, 2).await; + let bucket = "multipart-retry-recovery"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + let payload = vec![0x42; 1 << 20]; + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await; + let (upload_meta, _) = set_disks + .check_upload_id_exists(bucket, object, &upload_id, false) + .await + .expect("staged upload metadata should be readable"); + let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id); + let part_path = format!( + "{}/{}/part.1", + upload_path, + upload_meta.data_dir.expect("multipart upload should have a data directory") + ); + let retry_meta = Bytes::from_static(b"interrupted retry metadata"); + + for (index, disk) in disk_stores.iter().enumerate().take(3) { + let retry_path = format!("{}/part.1", Uuid::new_v4()); + disk.write_all(RUSTFS_META_TMP_BUCKET, &retry_path, Bytes::from_static(b"interrupted retry shard")) + .await + .expect("retry shard should be staged"); + disk.prepare_part_transaction( + RUSTFS_META_TMP_BUCKET, + &retry_path, + RUSTFS_META_MULTIPART_BUCKET, + &part_path, + retry_meta.clone(), + ) + .await + .expect("part transaction should be prepared"); + disk.rename_part( + RUSTFS_META_TMP_BUCKET, + &retry_path, + RUSTFS_META_MULTIPART_BUCKET, + &part_path, + retry_meta.clone(), + ) + .await + .unwrap_or_else(|err| panic!("retry shard {index} should be published: {err}")); + } + + set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload_id, parts, &ObjectOptions::default()) + .await + .expect("completion should roll back the interrupted quorum-minus-one retry"); + + let mut reader = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("completed object should be readable"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("completed object should stream fully"); + assert_eq!(restored, payload); + } + + #[tokio::test] + async fn rename_part_quorum_failure_without_old_part_removes_new_shards() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await; + let src_path = format!("{}/part.1", Uuid::new_v4()); + let dst_path = format!("{}/part.1", Uuid::new_v4()); + + let mut retry_disks = vec![None; disk_stores.len()]; + for index in [0, 1] { + disk_stores[index] + .write_all(RUSTFS_META_TMP_BUCKET, &src_path, Bytes::from_static(b"new shard")) + .await + .expect("new shard should be staged"); + retry_disks[index] = Some(disk_stores[index].clone()); + } + + let err = set_disks + .rename_part( + &retry_disks, + RUSTFS_META_TMP_BUCKET, + &src_path, + RUSTFS_META_MULTIPART_BUCKET, + &dst_path, + Bytes::from_static(b"retry metadata"), + 3, + None, + ) + .await + .expect_err("two renamed shards must remain below write quorum"); + assert_eq!(err, DiskError::ErasureWriteQuorum); + for (index, disk) in disk_stores.iter().enumerate() { + assert!( + matches!(disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &dst_path).await, Err(DiskError::FileNotFound)), + "failed first upload must not leave a destination on disk {index}" + ); + } + } + async fn make_multipart_lock_test_set_disks() -> Arc { let endpoints = vec![ Endpoint::try_from("http://127.0.0.1:9000/data").expect("first endpoint should parse"), diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index a5021e728..8d97cfad1 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -25,7 +25,10 @@ use crate::set_disk::read::GetObjectDownstreamWriter; use crate::bucket::lifecycle::{ tier_delete_journal::{persist_tier_delete_journal_entry, remove_tier_delete_journal_entry}, - tier_sweeper::{Jentry, RemoteTierDeleteOutcome, delete_object_from_remote_tier_with_lease_idempotent}, + tier_sweeper::{ + Jentry, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_lease_idempotent, + delete_object_from_remote_tier_with_lease_idempotent, + }, transition_transaction::{ TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record, @@ -34,6 +37,7 @@ use crate::bucket::lifecycle::{ }; use crate::diagnostics::get::GetObjectFailureReason; use crate::disk::OldCurrentSize; +use crate::error::is_err_invalid_upload_id; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; use crate::services::tier::tier::{TierConfigMgr, TierOperationLease}; use crate::store::ECStore; @@ -98,6 +102,126 @@ fn full_object_plaintext_len(range: &Option, opts: &ObjectOptions Some(object_info.size) } +const RESTORE_MULTIPART_ABORT_FAILURES_TOTAL: &str = "rustfs_restore_multipart_abort_failures_total"; + +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RestoreMultipartFailurePoint { + InvalidPartSize, + RangeOverflow, + TierGet, + HashReader, + PutPart, + SizeMismatch, + Complete, +} + +#[cfg(test)] +static RESTORE_MULTIPART_FAILURE_POINT: std::sync::Mutex> = std::sync::Mutex::new(None); +#[cfg(test)] +static RESTORE_MULTIPART_UPLOAD_ID: std::sync::Mutex> = std::sync::Mutex::new(None); +#[cfg(test)] +static RESTORE_MULTIPART_ABORT_ATTEMPTS: AtomicU64 = AtomicU64::new(0); + +#[cfg(test)] +fn restore_multipart_failure_is(point: RestoreMultipartFailurePoint) -> bool { + *RESTORE_MULTIPART_FAILURE_POINT + .lock() + .expect("restore multipart failure-point lock must not be poisoned") + == Some(point) +} + +#[cfg(test)] +fn fail_restore_multipart_at(point: RestoreMultipartFailurePoint) -> Result<()> { + if restore_multipart_failure_is(point) { + return Err(StorageError::Unexpected); + } + Ok(()) +} + +struct RestoreMultipartUploadCleanup { + store: Arc, + bucket: String, + object: String, + upload_id: String, + armed: bool, +} + +impl RestoreMultipartUploadCleanup { + fn new(store: Arc, bucket: &str, object: &str, upload_id: &str) -> Self { + Self { + store, + bucket: bucket.to_string(), + object: object.to_string(), + upload_id: upload_id.to_string(), + armed: true, + } + } + + async fn abort(&mut self) { + if !self.armed { + return; + } + self.armed = false; + #[cfg(test)] + RESTORE_MULTIPART_ABORT_ATTEMPTS.fetch_add(1, Ordering::Relaxed); + if let Err(err) = self + .store + .abort_multipart_upload(&self.bucket, &self.object, &self.upload_id, &ObjectOptions::default()) + .await + && !is_err_invalid_upload_id(&err) + { + metrics::counter!(RESTORE_MULTIPART_ABORT_FAILURES_TOTAL).increment(1); + warn!( + bucket = self.bucket, + object = self.object, + upload_id = self.upload_id, + error = ?err, + "failed to abort incomplete multipart restore" + ); + } + } + + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for RestoreMultipartUploadCleanup { + fn drop(&mut self) { + if !self.armed { + return; + } + let store = Arc::clone(&self.store); + let bucket = self.bucket.clone(); + let object = self.object.clone(); + let upload_id = self.upload_id.clone(); + #[cfg(test)] + RESTORE_MULTIPART_ABORT_ATTEMPTS.fetch_add(1, Ordering::Relaxed); + // Cancellation while the restore runtime is alive is cleaned + // asynchronously. Runtime teardown itself is only best-effort because + // Drop cannot await storage IO; normal error exits use abort() above. + if let Ok(handle) = tokio::runtime::Handle::try_current() { + handle.spawn(async move { + if let Err(err) = store + .abort_multipart_upload(&bucket, &object, &upload_id, &ObjectOptions::default()) + .await + && !is_err_invalid_upload_id(&err) + { + metrics::counter!(RESTORE_MULTIPART_ABORT_FAILURES_TOTAL).increment(1); + warn!( + bucket, + object, + upload_id, + error = ?err, + "failed to abort cancelled multipart restore" + ); + } + }); + } + } +} + pub(crate) fn body_cache_plaintext_len( range: &Option, opts: &ObjectOptions, @@ -1542,7 +1666,11 @@ pub(crate) async fn cleanup_uncommitted_transition_upload( cleanup_version: &str, version_id_exact: bool, ) -> std::io::Result { - delete_object_from_remote_tier_with_lease_idempotent(object, cleanup_version, lease, version_id_exact).await + if version_id_exact { + delete_confirmed_transition_candidate_exact_with_lease_idempotent(object, cleanup_version, lease).await + } else { + delete_object_from_remote_tier_with_lease_idempotent(object, cleanup_version, lease, false).await + } } fn log_transition_upload_cleanup_failure(lease: &TierOperationLease, object: &str, cleanup_version: &str, err: &std::io::Error) { @@ -1679,7 +1807,7 @@ impl Drop for TransitionUploadCleanup { } } -async fn cleanup_rejected_transition_upload_durably( +pub(crate) async fn cleanup_rejected_transition_upload_durably( lease: &TierOperationLease, object: &str, cleanup_version: &str, @@ -1858,12 +1986,83 @@ async fn advance_and_save_transition_transaction( next: TransitionTransactionState, remote_version: Option, ) -> Result<()> { + #[cfg(test)] + record_transition_uploaded_save_attempt(transaction, next); transaction .advance(transaction.fence(), next, remote_version) .map_err(Error::other)?; save_transition_transaction_if_available(api, transaction).await } +#[cfg(test)] +struct TransitionUploadedSaveProbeState { + bucket: String, + object: String, + attempts: std::sync::atomic::AtomicUsize, +} + +#[cfg(test)] +struct TransitionUploadedSaveProbe { + state: Arc, +} + +#[cfg(test)] +static TRANSITION_UPLOADED_SAVE_PROBE: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +impl TransitionUploadedSaveProbe { + fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(TransitionUploadedSaveProbeState { + bucket: bucket.to_string(), + object: object.to_string(), + attempts: std::sync::atomic::AtomicUsize::new(0), + }); + let mut slot = TRANSITION_UPLOADED_SAVE_PROBE + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("transition uploaded-save probe mutex should not poison"); + assert!(slot.is_none(), "transition uploaded-save probe must be installed by one test at a time"); + *slot = Some(Arc::clone(&state)); + drop(slot); + Self { state } + } + + fn attempts(&self) -> usize { + self.state.attempts.load(std::sync::atomic::Ordering::Acquire) + } +} + +#[cfg(test)] +impl Drop for TransitionUploadedSaveProbe { + fn drop(&mut self) { + let mut slot = TRANSITION_UPLOADED_SAVE_PROBE + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("transition uploaded-save probe mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction, next: TransitionTransactionState) { + if next != TransitionTransactionState::Uploaded { + return; + } + let state = TRANSITION_UPLOADED_SAVE_PROBE + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("transition uploaded-save probe mutex should not poison") + .as_ref() + .filter(|state| state.bucket == transaction.source.bucket && state.object == transaction.source.object) + .cloned(); + if let Some(state) = state { + state.attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + } +} + async fn delete_transition_transaction_if_available(api: Option<&Arc>, transaction_id: Uuid) -> Result<()> { if let Some(api) = api { return delete_transition_transaction_record(api.clone(), transaction_id).await; @@ -2142,10 +2341,7 @@ fn persisted_transition_version_with_gate( remote_version_state_writer_enabled: bool, ) -> std::io::Result<(Option, rustfs_filemeta::TransitionVersionState)> { if remote_version.is_empty() { - return Err(std::io::Error::new( - std::io::ErrorKind::Unsupported, - "a missing remote tier object version does not prove that bucket versioning is disabled", - )); + return Ok((None, rustfs_filemeta::TransitionVersionState::KnownDisabled)); } match Uuid::parse_str(remote_version) { @@ -2165,6 +2361,82 @@ fn persisted_transition_version_with_gate( } } +#[cfg(test)] +#[derive(Default)] +struct ObjectTaggingCommitBarrierState { + bucket: String, + object: String, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +struct ObjectTaggingCommitBarrier { + state: Arc, +} + +#[cfg(test)] +static OBJECT_TAGGING_COMMIT_BARRIER: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +impl ObjectTaggingCommitBarrier { + fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(ObjectTaggingCommitBarrierState { + bucket: bucket.to_string(), + object: object.to_string(), + ..Default::default() + }); + let mut slot = OBJECT_TAGGING_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("object tagging commit barrier mutex should not poison"); + assert!(slot.is_none(), "object tagging commit barrier must be installed by one test at a time"); + *slot = Some(Arc::clone(&state)); + drop(slot); + Self { state } + } + + async fn wait_until_paused(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("object tagging should reach the deterministic commit barrier"); + } + + fn release(&self) { + self.state.release.notify_one(); + } +} + +#[cfg(test)] +impl Drop for ObjectTaggingCommitBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut slot = OBJECT_TAGGING_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("object tagging commit barrier mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +async fn pause_object_tagging_commit(bucket: &str, object: &str) { + let barrier = OBJECT_TAGGING_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("object tagging commit barrier mutex should not poison") + .as_ref() + .filter(|barrier| barrier.bucket == bucket && barrier.object == object) + .cloned(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } +} + #[cfg(test)] mod transition_upload_completion_tests { use super::*; @@ -2333,7 +2605,10 @@ mod transition_version_id_tests { #[test] fn normalizes_persisted_unversioned_ids_and_preserves_put_constraints() { - assert!(persisted_transition_version("").is_err()); + assert_eq!( + persisted_transition_version("").expect("empty remote version identifies an unversioned tier"), + (None, TransitionVersionState::KnownDisabled) + ); assert!(persisted_transition_version(&Uuid::nil().to_string()).is_err()); let nil_put_response = Uuid::nil().to_string(); let nil_candidate = TransitionUploadCandidate::from_put_response(nil_put_response.clone()); @@ -2396,13 +2671,54 @@ mod transition_version_id_tests { expected ); } - assert!( - persisted_transition_version_with_gate("", true).is_err(), - "an empty PUT response must remain unknown without authoritative bucket state" + assert_eq!( + persisted_transition_version_with_gate("", true).expect("empty remote version identifies an unversioned tier"), + (None, TransitionVersionState::KnownDisabled) ); } } +impl SetDisks { + async fn update_object_tags_locked( + &self, + operation: &'static str, + bucket: &str, + object: &str, + tags: &str, + opts: &ObjectOptions, + ) -> Result { + let object_lock_guard = if opts.no_lock { + None + } else { + Some(self.acquire_write_lock_diag(operation, bucket, object).await?) + }; + // Force the full quorum fanout (allow_early_stop=false): `disks` is the + // write target below, and an early-stop subset would only carry read + // quorum, failing write quorum on update_object_meta (backlog#872). + let (mut fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?; + + fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned()); + + #[cfg(test)] + pause_object_tagging_commit(bucket, object).await; + // Fence the read-modify-write before any disk can merge metadata derived + // from this read after another writer has reacquired the same key. + if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: operation, + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + + self.update_object_meta(bucket, object, fi.clone(), disks.as_slice()).await?; + + Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended)) + } +} + #[async_trait::async_trait] impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { type Error = Error; @@ -3628,6 +3944,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await; return Err(err.into()); } + let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) { + Ok(version) => version, + Err(err) => { + let cleanup_api = transition_cleanup_store(&self.ctx).await; + if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await { + return Err(StorageError::Io(std::io::Error::other(format!( + "{err}; rejected remote upload cleanup failed: {cleanup_err}" + )))); + } + delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object) + .await; + return Err(err.into()); + } + }; if let Err(err) = advance_and_save_transition_transaction( transaction_api.as_ref(), &mut transaction, @@ -3645,20 +3975,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await; return Err(err); } - let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) { - Ok(version) => version, - Err(err) => { - let cleanup_api = transition_cleanup_store(&self.ctx).await; - if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await { - return Err(StorageError::Io(std::io::Error::other(format!( - "{err}; rejected remote upload cleanup failed: {cleanup_err}" - )))); - } - delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object) - .await; - return Err(err.into()); - } - }; let mut commit_opts = opts.clone(); commit_opts.no_lock = true; @@ -3952,116 +4268,124 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } let res = self_.clone().new_multipart_upload(bucket, object, &ropts).await?; - //if err != nil { - // return set_restore_header_fn(&mut oi, err).await; - //} - - let mut uploaded_parts: Vec = vec![]; - let parts = Arc::clone(&oi.parts); - let mut part_offset: i64 = 0; - for part_info in parts.iter() { - let mut part_opts = opts.clone(); - part_opts.part_number = Some(part_info.number); - if part_info.actual_size <= 0 { - return set_restore_header_fn( - &mut oi, - Some(Error::other(format!("invalid multipart restore part size {}", part_info.actual_size))), + #[cfg(test)] + { + *RESTORE_MULTIPART_UPLOAD_ID + .lock() + .expect("restore multipart upload-id lock must not be poisoned") = Some(res.upload_id.clone()); + } + let mut upload_cleanup = RestoreMultipartUploadCleanup::new(self_.clone(), bucket, object, &res.upload_id); + let restore_result: Result = async { + let mut uploaded_parts: Vec = vec![]; + let parts = Arc::clone(&oi.parts); + let mut part_offset: i64 = 0; + for part_info in parts.iter() { + let mut part_opts = opts.clone(); + part_opts.part_number = Some(part_info.number); + #[cfg(test)] + fail_restore_multipart_at(RestoreMultipartFailurePoint::InvalidPartSize)?; + if part_info.actual_size <= 0 { + return Err(Error::other(format!("invalid multipart restore part size {}", part_info.actual_size))); + } + #[cfg(test)] + fail_restore_multipart_at(RestoreMultipartFailurePoint::RangeOverflow)?; + let part_end = part_offset + .checked_add(part_info.actual_size - 1) + .ok_or_else(|| Error::other("multipart restore part range overflow".to_string()))?; + let rs = Some(HTTPRangeSpec { + is_suffix_length: false, + start: part_offset, + end: part_end, + }); + part_offset = part_end + .checked_add(1) + .ok_or_else(|| Error::other("multipart restore part offset overflow".to_string()))?; + #[cfg(test)] + fail_restore_multipart_at(RestoreMultipartFailurePoint::TierGet)?; + let gr = get_transitioned_object_reader_with_tier_manager( + bucket, + object, + &rs, + &HeaderMap::new(), + &oi, + &part_opts, + &self_.ctx.tier_config_mgr(), ) - .await; - } - let part_end = match part_offset.checked_add(part_info.actual_size - 1) { - Some(end) => end, - None => { - return set_restore_header_fn( - &mut oi, - Some(Error::other("multipart restore part range overflow".to_string())), - ) - .await; - } - }; - let rs = Some(HTTPRangeSpec { - is_suffix_length: false, - start: part_offset, - end: part_end, - }); - part_offset = match part_end.checked_add(1) { - Some(next) => next, - None => { - return set_restore_header_fn( - &mut oi, - Some(Error::other("multipart restore part offset overflow".to_string())), - ) - .await; - } - }; - let gr = match get_transitioned_object_reader_with_tier_manager( - bucket, - object, - &rs, - &HeaderMap::new(), - &oi, - &part_opts, - &self_.ctx.tier_config_mgr(), - ) - .await - { - Ok(reader) => reader, - Err(err) => { - return set_restore_header_fn(&mut oi, Some(StorageError::Io(err))).await; - } - }; - let reader = BufReader::new(gr.stream); - let hash_reader = HashReader::from_stream(reader, part_info.actual_size, part_info.actual_size, None, None, false)?; - let mut p_reader = PutObjReader::new(hash_reader); - let p_info = self_ - .clone() - .put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default()) - .await?; - //if let Err(err) = p_info { - // return set_restore_header_fn(&mut oi, err).await; - //} - if p_info.size as i64 != part_info.actual_size { - return set_restore_header_fn( - &mut oi, - Some(Error::other(ObjectApiError::InvalidObjectState(GenericError { + .await + .map_err(StorageError::Io)?; + let reader = BufReader::new(gr.stream); + #[cfg(test)] + fail_restore_multipart_at(RestoreMultipartFailurePoint::HashReader)?; + let hash_reader = + HashReader::from_stream(reader, part_info.actual_size, part_info.actual_size, None, None, false)?; + let mut p_reader = PutObjReader::new(hash_reader); + #[cfg(test)] + fail_restore_multipart_at(RestoreMultipartFailurePoint::PutPart)?; + let p_info = self_ + .clone() + .put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default()) + .await?; + #[cfg(test)] + let p_info = if restore_multipart_failure_is(RestoreMultipartFailurePoint::SizeMismatch) { + let mut injected = p_info; + injected.size = 0; + injected + } else { + p_info + }; + if p_info.size as i64 != part_info.actual_size { + return Err(Error::other(ObjectApiError::InvalidObjectState(GenericError { bucket: bucket.to_string(), object: object.to_string(), ..Default::default() - }))), - ) - .await; + }))); + } + uploaded_parts.push(CompletePart { + part_num: p_info.part_num, + etag: p_info.etag, + checksum_crc32: None, + checksum_crc32c: None, + checksum_sha1: None, + checksum_sha256: None, + checksum_crc64nvme: None, + }); } - uploaded_parts.push(CompletePart { - part_num: p_info.part_num, - etag: p_info.etag, - checksum_crc32: None, - checksum_crc32c: None, - checksum_sha1: None, - checksum_sha256: None, - checksum_crc64nvme: None, - }); + #[cfg(test)] + if restore_multipart_failure_is(RestoreMultipartFailurePoint::Complete) { + uploaded_parts + .first_mut() + .expect("multipart restore must contain at least one uploaded part") + .etag = Some("injected-invalid-complete-etag".to_string()); + } + self_ + .clone() + .complete_multipart_upload( + bucket, + object, + &res.upload_id, + uploaded_parts, + &ObjectOptions { + mod_time: oi.mod_time, + version_id: oi.version_id.map(|version| version.to_string()), + user_defined: restore_commit_metadata, + // Inherit the restore write lock (see ropts.no_lock above): + // the commit phase re-acquires this object's write lock. + no_lock: opts.no_lock, + ..Default::default() + }, + ) + .await } - let restored_info = match self_ - .clone() - .complete_multipart_upload( - bucket, - object, - &res.upload_id, - uploaded_parts, - &ObjectOptions { - mod_time: oi.mod_time, - version_id: oi.version_id.map(|version| version.to_string()), - user_defined: restore_commit_metadata, - // Inherit the restore write lock (see ropts.no_lock above): - // the commit phase re-acquires this object's write lock. - no_lock: opts.no_lock, - ..Default::default() - }, - ) - .await - { - Ok(info) => info, - Err(err) => return set_restore_header_fn(&mut oi, Some(err)).await, + .await; + let restored_info = match restore_result { + Ok(info) => { + upload_cleanup.disarm(); + info + } + Err(err) => { + upload_cleanup.abort().await; + return set_restore_header_fn(&mut oi, Some(err)).await; + } }; send_event(EventArgs { event_name: EventName::ObjectRestoreCompleted.as_str().to_string(), @@ -4076,32 +4400,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { #[tracing::instrument(level = "debug", skip(self))] async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result { - // Acquire write-lock for tag update (metadata write) - // if !opts.no_lock { - // let guard_opt = self - // .namespace_lock - // .lock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10)) - // .await?; - // if guard_opt.is_none() { - // return Err(Error::other("can not get lock. please retry".to_string())); - // } - // _lock_guard = guard_opt; - // } - // Force the full quorum fanout (allow_early_stop=false): `disks` is the - // write target below, and an early-stop subset would only carry read - // quorum, failing write quorum on update_object_meta (backlog#872). - let (mut fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?; - - fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned()); - - self.update_object_meta(bucket, object, fi.clone(), disks.as_slice()).await?; - - Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended)) + self.update_object_tags_locked("put_object_tags", bucket, object, tags, opts) + .await } #[tracing::instrument(skip(self))] async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { - self.put_object_tags(bucket, object, "", opts).await + self.update_object_tags_locked("delete_object_tags", bucket, object, "", opts) + .await } #[tracing::instrument(skip(self))] @@ -4515,8 +4821,10 @@ mod transition_commit_failure_tests { use crate::disk::DiskAPI as _; use crate::services::tier::test_util::{MockWarmBackend, register_mock_tier}; use crate::services::tier::tier::TierConfigMgr; + use crate::storage_api_contracts::multipart::MultipartOperations as _; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use http::HeaderMap; + use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status}; use s3s::dto::RestoreRequest; use tokio::io::AsyncReadExt; @@ -4538,7 +4846,7 @@ mod transition_commit_failure_tests { #[tokio::test] async fn rejected_unsupported_remote_versions_are_cleaned_up() { - for remote_version in ["", "null", "opaque-version-token"] { + for remote_version in ["null", "opaque-version-token"] { let manager = TierConfigMgr::new(); let backend = register_mock_tier(&manager, "WARM").await; let lease = TierConfigMgr::acquire_operation_lease(&manager, "WARM") @@ -4564,6 +4872,150 @@ mod transition_commit_failure_tests { } } + #[serial_test::serial(restore_multipart_failure_point)] + async fn multipart_restore_aborts_every_post_create_failure() { + let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + RESTORE_MULTIPART_ABORT_ATTEMPTS.store(0, Ordering::Relaxed); + let bucket = "restore-multipart-failure-cleanup-bucket"; + let object = "object.bin"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let source_upload = set_disks + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("source multipart upload should be created"); + let mut first_reader = PutObjReader::from_vec(vec![b'a'; 5 * 1024 * 1024]); + let first = set_disks + .put_object_part(bucket, object, &source_upload.upload_id, 1, &mut first_reader, &ObjectOptions::default()) + .await + .expect("first source part should be staged"); + let mut second_reader = PutObjReader::from_vec(vec![b'b'; 1024 * 1024]); + let second = set_disks + .put_object_part(bucket, object, &source_upload.upload_id, 2, &mut second_reader, &ObjectOptions::default()) + .await + .expect("second source part should be staged"); + let original = set_disks + .clone() + .complete_multipart_upload( + bucket, + object, + &source_upload.upload_id, + vec![ + CompletePart { + part_num: first.part_num, + etag: first.etag, + ..Default::default() + }, + CompletePart { + part_num: second.part_num, + etag: second.etag, + ..Default::default() + }, + ], + &ObjectOptions::default(), + ) + .await + .expect("source multipart upload should complete"); + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + set_disks + .transition_object( + bucket, + object, + &ObjectOptions { + no_lock: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name, + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }, + ) + .await + .expect("multipart source should transition before restore"); + + for point in [ + RestoreMultipartFailurePoint::InvalidPartSize, + RestoreMultipartFailurePoint::RangeOverflow, + RestoreMultipartFailurePoint::TierGet, + RestoreMultipartFailurePoint::HashReader, + RestoreMultipartFailurePoint::PutPart, + RestoreMultipartFailurePoint::SizeMismatch, + RestoreMultipartFailurePoint::Complete, + ] { + *RESTORE_MULTIPART_FAILURE_POINT + .lock() + .expect("restore multipart failure-point lock must not be poisoned") = Some(point); + *RESTORE_MULTIPART_UPLOAD_ID + .lock() + .expect("restore multipart upload-id lock must not be poisoned") = None; + let mut opts = ObjectOptions::default(); + opts.transition.restore_request.days = Some(1); + set_disks + .clone() + .restore_transitioned_object(bucket, object, &opts) + .await + .expect_err("injected post-create restore failure must surface"); + let upload_id = RESTORE_MULTIPART_UPLOAD_ID + .lock() + .expect("restore multipart upload-id lock must not be poisoned") + .clone() + .expect("restore must create an upload before the injected failure"); + let err = set_disks + .get_multipart_info(bucket, object, &upload_id, &ObjectOptions::default()) + .await + .expect_err("failed restore upload must immediately disappear"); + assert!(is_err_invalid_upload_id(&err), "{point:?}: unexpected upload lookup error: {err:?}"); + let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id); + for temp_dir in &temp_dirs { + assert!( + !temp_dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path).exists(), + "{point:?}: failed restore must remove staged multipart data" + ); + } + } + assert_eq!( + RESTORE_MULTIPART_ABORT_ATTEMPTS.load(Ordering::Relaxed), + 7, + "every injected post-create failure must attempt an abort" + ); + *RESTORE_MULTIPART_FAILURE_POINT + .lock() + .expect("restore multipart failure-point lock must not be poisoned") = None; + let mut opts = ObjectOptions::default(); + opts.transition.restore_request.days = Some(1); + set_disks + .clone() + .restore_transitioned_object(bucket, object, &opts) + .await + .expect("multipart restore should complete after failure injection is cleared"); + assert_eq!( + RESTORE_MULTIPART_ABORT_ATTEMPTS.load(Ordering::Relaxed), + 7, + "successful multipart completion must disarm cleanup without aborting" + ); + let restored = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("successful multipart restore must leave the committed object intact"); + let restore_header = restored + .user_defined + .get(s3s::header::X_AMZ_RESTORE.as_str()) + .expect("successful multipart restore must persist restore status"); + let restore_status = parse_restore_obj_status(restore_header).expect("successful restore status must be valid"); + assert!(!restore_status.on_going(), "successful multipart restore must not remain in progress"); + assert!( + restore_status.expiry().is_some(), + "successful multipart restore must retain its expiry date" + ); + } + #[tokio::test] #[serial_test::serial] async fn local_commit_failure_returns_error_and_preserves_remote_candidate() { @@ -5730,11 +6182,13 @@ mod transition_upload_integrity_tests { fn drop(&mut self) { let previous = self.previous.clone(); let handle = tokio::runtime::Handle::current(); - tokio::task::block_in_place(|| { - handle.block_on(async move { + std::thread::spawn(move || { + handle.block_on(async { runtime_sources::set_setup_type(previous).await; }); - }); + }) + .join() + .expect("setup type restore thread should not panic"); } } @@ -6139,8 +6593,7 @@ mod transition_upload_integrity_tests { let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await; let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; - let previous_setup_type = runtime_sources::current_setup_type().await; - runtime_sources::set_setup_type(SetupType::DistErasure).await; + let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; let mut opts = transition_options(&original, tier_name); opts.no_lock = false; let barrier = TransitionCommitBarrier::install_before_lock_lost_check(bucket, object); @@ -6164,7 +6617,6 @@ mod transition_upload_integrity_tests { matches!(error, StorageError::NamespaceLockQuorumUnavailable { .. }), "unexpected transition lock-lost error: {error:?}" ); - runtime_sources::set_setup_type(previous_setup_type).await; assert_eq!(backend.put_count().await, 1); assert_eq!( backend.remove_count().await, @@ -6369,20 +6821,21 @@ mod transition_upload_integrity_tests { #[tokio::test] #[serial_test::serial] - async fn opaque_remote_version_is_persisted_exactly() { + async fn unversioned_remote_version_is_persisted_without_version_id() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; - let bucket = "transition-unknown-version-bucket"; + let bucket = "transition-unversioned-tier-bucket"; let object = "object.bin"; - let payload = b"unknown remote version must retain local data".repeat(1024); + let payload = b"unversioned remote tier must commit without a version id".repeat(1024); let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await; let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; - backend.set_put_remote_version(Some("opaque-version-token".to_string())).await; + backend.set_put_remote_version(Some(String::new())).await; + let save_probe = TransitionUploadedSaveProbe::install(bucket, object); set_disks .transition_object(bucket, object, &transition_options(&original, tier_name)) .await - .expect("an accepted opaque remote version must commit"); + .expect("an unversioned remote version must commit"); let (fi, _, _) = set_disks .get_object_fileinfo( bucket, @@ -6396,13 +6849,70 @@ mod transition_upload_integrity_tests { false, ) .await - .expect("committed opaque transition metadata should be readable"); + .expect("committed unversioned transition metadata should be readable"); assert_eq!(fi.transition_version_id, None); - assert_eq!(fi.transition_version.as_deref(), Some("opaque-version-token")); + assert_eq!(fi.transition_version, None); + assert_eq!(fi.transition_version_state, rustfs_filemeta::TransitionVersionState::KnownDisabled); + assert_eq!(save_probe.attempts(), 1); assert_eq!(backend.remove_count().await, 0); assert_eq!(backend.object_count().await, 1); } + #[tokio::test] + #[serial_test::serial] + async fn opaque_remote_version_is_cleaned_before_parse_failure() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transition-unknown-version-bucket"; + let object = "object.bin"; + let payload = b"unknown remote version must retain local data".repeat(1024); + let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await; + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + backend.set_put_remote_version(Some("opaque-version-token".to_string())).await; + + set_disks + .transition_object(bucket, object, &transition_options(&original, tier_name)) + .await + .expect_err("an opaque remote version must fail closed until the capability gate is active"); + let removed_versions = backend.remove_versions().await; + assert_eq!(removed_versions.len(), 1); + assert_eq!(removed_versions[0].1, "opaque-version-token"); + assert_eq!(backend.object_count().await, 0); + assert_local_source_intact(&set_disks, bucket, object, &payload).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn nil_remote_version_is_cleaned_exactly_before_transaction_persistence() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transition-nil-version-bucket"; + let object = "object.bin"; + let payload = b"nil remote version must retain local data".repeat(1024); + let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await; + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let remote_version = Uuid::nil().to_string(); + let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + backend.set_put_remote_version(Some(remote_version.clone())).await; + let save_probe = TransitionUploadedSaveProbe::install(bucket, object); + + set_disks + .transition_object(bucket, object, &transition_options(&original, tier_name)) + .await + .expect_err("a nil remote version must fail closed before transaction persistence"); + let put_versions = backend.put_versions().await; + let removed_versions = backend.remove_versions().await; + assert_eq!(removed_versions, put_versions); + assert_eq!(removed_versions.len(), 1); + assert_eq!( + removed_versions.first().map(|(_, version)| version.as_str()), + Some(remote_version.as_str()) + ); + assert_eq!(save_probe.attempts(), 0, "nil remote version must be rejected before saving Uploaded"); + assert_eq!(backend.exact_remove_count(), 1); + assert_eq!(backend.object_count().await, 0); + assert_local_source_intact(&set_disks, bucket, object, &payload).await; + } + #[tokio::test] #[serial_test::serial] async fn authoritative_read_failure_after_upload_cleans_exact_candidate_and_preserves_source() { @@ -6510,6 +7020,63 @@ mod transition_upload_integrity_tests { assert_local_source_intact(&set_disks, bucket, object, &payload).await; } + #[tokio::test(flavor = "current_thread", start_paused = true)] + #[serial_test::serial] + async fn tagging_lock_lost_before_metadata_write_fails_closed() { + let refresh_calls = Arc::new(AtomicUsize::new(0)); + let lockers: Vec> = (0..4) + .map(|_| Arc::new(LockLostRefreshClient::new(Arc::clone(&refresh_calls))) as Arc) + .collect(); + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await; + let bucket = "tagging-lock-lost-bucket"; + let object = "object.bin"; + write_source(&set_disks, &disk_stores, bucket, object, b"tagging source").await; + + let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; + let barrier = ObjectTaggingCommitBarrier::install(bucket, object); + let tagging_set = Arc::clone(&set_disks); + let tagging = tokio::spawn(async move { + tagging_set + .put_object_tags(bucket, object, "must=not-commit", &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + assert!( + refresh_calls.load(Ordering::SeqCst) > 0, + "test must drive the real distributed-lock heartbeat before the tagging commit fence" + ); + barrier.release(); + + let error = tagging + .await + .expect("tagging task should not panic") + .expect_err("tagging must fail after its namespace lock loses refresh quorum"); + assert!( + matches!(error, StorageError::NamespaceLockQuorumUnavailable { .. }), + "unexpected tagging lock-lost error: {error:?}" + ); + let (fi, _, _) = set_disks + .get_object_fileinfo( + bucket, + object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + false, + false, + ) + .await + .expect("source metadata should remain readable"); + assert!( + !fi.metadata.contains_key(AMZ_OBJECT_TAGGING), + "a stale tagging writer must not write metadata after refresh-quorum loss" + ); + } + #[tokio::test] #[serial_test::serial] async fn remote_cleanup_failure_after_version_rejection_preserves_source_and_candidate() { @@ -6995,6 +7562,217 @@ mod put_object_tags_early_stop_regression_tests { } } +#[cfg(test)] +mod object_tagging_namespace_lock_tests { + use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; + use super::*; + use crate::disk::{DiskAPI as _, ReadOptions}; + use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use tokio::io::AsyncReadExt as _; + + #[derive(Clone, Copy, Debug)] + enum CompetingMutation { + Put, + Delete, + } + + async fn read_body(set_disks: &SetDisks, bucket: &str, object: &str, opts: &ObjectOptions) -> Result> { + let mut body = Vec::new(); + set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), opts) + .await? + .stream + .read_to_end(&mut body) + .await?; + Ok(body) + } + + #[tokio::test] + async fn tagging_honors_an_inherited_namespace_write_lock() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "tag-lock-inherited"; + let object = "object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut reader = PutObjReader::from_vec(b"body".to_vec()); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("object should be written"); + + let outer_lock = set_disks + .new_ns_lock(bucket, object) + .await + .expect("outer namespace lock should be created") + .get_write_lock(Duration::from_secs(1)) + .await + .expect("outer namespace write lock should be acquired"); + set_disks + .put_object_tags( + bucket, + object, + "lock=inherited", + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("tagging should not reacquire an inherited namespace lock"); + drop(outer_lock); + + assert_eq!( + set_disks + .get_object_tags(bucket, object, &ObjectOptions::default()) + .await + .expect("persisted tags should remain readable"), + "lock=inherited" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn tagging_serializes_with_put_and_delete_for_versioned_and_unversioned_objects() { + for versioned in [false, true] { + for mutation in [CompetingMutation::Put, CompetingMutation::Delete] { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = format!("tag-lock-{}-{mutation:?}", if versioned { "versioned" } else { "plain" }).to_lowercase(); + let object = "object"; + for disk in &disk_stores { + disk.make_volume(&bucket).await.expect("bucket volume should be created"); + } + + let object_opts = ObjectOptions { + versioned, + ..Default::default() + }; + let original_body = b"original body".to_vec(); + let mut original_reader = PutObjReader::from_vec(original_body.clone()); + let original = set_disks + .put_object(&bucket, object, &mut original_reader, &object_opts) + .await + .expect("original object should be written"); + let original_version = original.version_id.map(|version| version.to_string()); + set_disks + .put_object_tags(&bucket, object, "stage=initial", &object_opts) + .await + .expect("initial tags should be written"); + + let barrier = ObjectTaggingCommitBarrier::install(&bucket, object); + let tagging_set = Arc::clone(&set_disks); + let tagging_bucket = bucket.clone(); + let tagging_opts = object_opts.clone(); + let tagging = tokio::spawn(async move { + match mutation { + CompetingMutation::Put => { + tagging_set + .put_object_tags(&tagging_bucket, object, "stage=before-mutation", &tagging_opts) + .await + } + CompetingMutation::Delete => tagging_set.delete_object_tags(&tagging_bucket, object, &tagging_opts).await, + } + }); + barrier.wait_until_paused().await; + + let mutation_set = Arc::clone(&set_disks); + let mutation_bucket = bucket.clone(); + let mutation_opts = object_opts.clone(); + let (mutation_started_tx, mutation_started_rx) = tokio::sync::oneshot::channel(); + let competing = tokio::spawn(async move { + mutation_started_tx + .send(()) + .expect("tagging test should wait for the competing mutation"); + match mutation { + CompetingMutation::Put => { + let mut replacement = PutObjReader::from_vec(b"replacement body".to_vec()); + mutation_set + .put_object(&mutation_bucket, object, &mut replacement, &mutation_opts) + .await + .map(Some) + } + CompetingMutation::Delete => mutation_set + .delete_object(&mutation_bucket, object, mutation_opts) + .await + .map(|_| None), + } + }); + mutation_started_rx + .await + .expect("competing mutation should reach the namespace operation while tagging is paused"); + + barrier.release(); + tagging + .await + .expect("tagging task should not panic") + .expect("tagging should commit before the queued mutation"); + let competing_result = competing + .await + .expect("competing mutation task should not panic") + .expect("competing mutation should commit after tagging releases the lock"); + + match mutation { + CompetingMutation::Put => { + let replacement = competing_result.expect("put mutation should return the replacement object"); + let current = set_disks + .get_object_info(&bucket, object, &object_opts) + .await + .expect("replacement metadata should remain readable"); + assert_eq!( + read_body(&set_disks, &bucket, object, &object_opts) + .await + .expect("replacement version should remain readable"), + b"replacement body" + ); + assert_eq!(current.etag, replacement.etag, "tagging must not restore the previous version's ETag"); + assert!( + current.user_tags.is_empty(), + "a tag update ordered before the replacement must not leak onto the replacement version" + ); + } + CompetingMutation::Delete if versioned => { + let old_version_opts = ObjectOptions { + versioned: true, + version_id: original_version, + ..Default::default() + }; + assert_eq!( + read_body(&set_disks, &bucket, object, &old_version_opts) + .await + .expect("the version hidden by the delete marker should remain readable"), + original_body + ); + let old_version = set_disks + .get_object_info(&bucket, object, &old_version_opts) + .await + .expect("the historical version metadata should remain readable"); + assert!( + old_version.user_tags.is_empty(), + "delete-tagging ordered before the delete marker must persist on the historical version" + ); + for disk in &disk_stores { + let current = disk + .read_version("", &bucket, object, "", &ReadOptions::default()) + .await + .expect("the current delete marker should remain visible on every disk"); + assert!(current.deleted); + } + } + CompetingMutation::Delete => { + let error = read_body(&set_disks, &bucket, object, &object_opts) + .await + .expect_err("unversioned deletion must not be undone by a stale tagging write"); + assert!( + is_err_object_not_found(&error) || is_err_version_not_found(&error), + "unexpected error after unversioned delete: {error:?}" + ); + } + } + } + } + } +} + #[cfg(test)] mod delete_objects_lock_gating_tests { //! Regression coverage for backlog#929 (HP-8): the batch-delete per-object diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 1a33716f4..64369b5d5 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -207,11 +207,9 @@ impl SetDisks { version_id: &str, opts: &ReadOptions, ) -> Result> { - // Use existing disk selection logic - let disks = self.disks.read().await; - let required_reads = self.format.erasure.sets.len(); + let disks = self.disks.read().await.clone(); + let required_reads = self.default_read_quorum(); - // Clone parameters outside the closure to avoid lifetime issues let bucket = bucket.to_string(); let object = object.to_string(); let version_id = version_id.to_string(); @@ -220,7 +218,6 @@ impl SetDisks { let processor = runtime_sources::batch_processors().read_processor(); let tasks: Vec<_> = disks .iter() - .take(required_reads + 2) // Read a few extra for reliability .filter_map(|disk| { disk.as_ref().map(|d| { let disk = d.clone(); @@ -234,10 +231,10 @@ impl SetDisks { }) .collect(); - match processor.execute_batch_with_quorum(tasks, required_reads).await { - Ok(results) => Ok(results), - Err(_) => Err(DiskError::FileNotFound.into()), // Use existing error type - } + processor + .execute_batch_with_quorum(tasks, required_reads) + .await + .map_err(Into::into) } #[tracing::instrument(level = "debug", skip(self))] @@ -1960,6 +1957,42 @@ mod metadata_cache_tests { (dir, disk) } + async fn read_version_quorum_test_set( + bucket: &str, + object: &str, + pool_set_count: usize, + readable_disks: usize, + ) -> (Vec, Arc) { + let mut dirs = Vec::new(); + let mut disks = Vec::new(); + for disk_index in 0..4 { + let (dir, disk) = new_read_version_test_disk(bucket).await; + if disk_index < readable_disks { + let mut fi = valid_test_fileinfo(object); + fi.mod_time = Some(OffsetDateTime::now_utc()); + fi.erasure.index = fi.erasure.distribution[disk_index]; + disk.write_metadata(bucket, bucket, object, fi) + .await + .expect("metadata should be written before quorum read"); + } + dirs.push(dir); + disks.push(Some(disk)); + } + let set = SetDisks::new( + "read-version-quorum-test".to_string(), + Arc::new(RwLock::new(disks)), + 4, + 2, + 0, + 0, + Vec::new(), + FormatV3::new(pool_set_count, 4), + Vec::new(), + ) + .await; + (dirs, set) + } + #[test] #[serial] fn get_object_metadata_cache_capacity_uses_default_and_env_override() { @@ -1986,6 +2019,33 @@ mod metadata_cache_tests { fi } + #[tokio::test] + async fn get_object_with_fileinfo_rejects_positive_size_without_parts() { + let mut output = Vec::new(); + let err = SetDisks::get_object_with_fileinfo( + "bucket", + "object", + 0, + 1, + &mut output, + valid_test_fileinfo("object"), + Vec::new(), + &[], + 0, + 0, + false, + false, + GET_OBJECT_PATH_SET_DISK, + "plain", + "small", + ) + .await + .expect_err("positive-size metadata without parts must fail without panicking"); + + assert_eq!(err, Error::FileCorrupt); + assert!(output.is_empty()); + } + #[tokio::test] async fn get_object_with_fileinfo_rejects_invalid_ranges_before_reader_setup() { let bucket = "bucket"; @@ -2059,6 +2119,12 @@ mod metadata_cache_tests { let mut invalid_erasure = valid_test_fileinfo(object); invalid_erasure.erasure.block_size = 0; + invalid_erasure.parts.push(ObjectPartInfo { + number: 1, + size: 1, + actual_size: 1, + ..Default::default() + }); let err = SetDisks::get_object_with_fileinfo( bucket, object, @@ -2087,6 +2153,37 @@ mod metadata_cache_tests { assert!(output.is_empty()); } + #[tokio::test] + async fn get_object_with_fileinfo_accepts_zero_size_without_parts() { + let bucket = "bucket"; + let object = "empty"; + let mut fi = valid_test_fileinfo(object); + fi.size = 0; + + let mut output = Vec::new(); + SetDisks::get_object_with_fileinfo( + bucket, + object, + 0, + 0, + &mut output, + fi, + Vec::new(), + &[], + 0, + 0, + false, + false, + GET_OBJECT_PATH_SET_DISK, + "plain", + "empty", + ) + .await + .expect("zero-byte object without parts must remain readable"); + + assert!(output.is_empty()); + } + #[tokio::test] async fn get_object_with_fileinfo_fails_closed_without_read_quorum() { let bucket = "bucket"; @@ -2133,6 +2230,7 @@ mod metadata_cache_tests { let object = "object"; let (_dir, disk) = new_read_version_test_disk(bucket).await; let mut fi = valid_test_fileinfo(object); + fi.size = 0; fi.mod_time = Some(OffsetDateTime::now_utc()); disk.write_metadata(bucket, bucket, object, fi.clone()) .await @@ -2165,11 +2263,70 @@ mod metadata_cache_tests { .await .expect_err("empty disk set must fail closed"); assert!( - missing_quorum.to_string().to_ascii_lowercase().contains("file"), - "optimized read failure should map to file-not-found style error: {missing_quorum}" + missing_quorum.to_string().contains("Insufficient successful results"), + "optimized read failure should preserve the batch quorum diagnostic: {missing_quorum}" ); } + #[tokio::test] + async fn read_version_optimized_uses_current_set_drive_quorum_not_pool_set_count() { + let bucket = "read-version-layout-quorum-bucket"; + let object = "object"; + + let (_single_set_dirs, single_set) = read_version_quorum_test_set(bucket, object, 1, 1).await; + single_set + .read_version_optimized(bucket, object, "", &ReadOptions::default()) + .await + .expect_err("one metadata copy must not satisfy a four-drive 2+2 set"); + + let (_multi_set_dirs, multi_set) = read_version_quorum_test_set(bucket, object, 3, 2).await; + let versions = multi_set + .read_version_optimized(bucket, object, "", &ReadOptions::default()) + .await + .expect("two metadata copies should satisfy the current set's read quorum"); + assert_eq!(versions.len(), 2); + assert!(versions.iter().all(|version| version.name == object)); + } + + #[tokio::test] + async fn read_version_optimized_counts_only_valid_metadata_toward_quorum() { + let bucket = "read-version-corrupt-quorum-bucket"; + let object = "object"; + + let (quorum_minus_one_dirs, quorum_minus_one) = read_version_quorum_test_set(bucket, object, 1, 2).await; + tokio::fs::write( + quorum_minus_one_dirs[1] + .path() + .join(bucket) + .join(object) + .join(crate::disk::STORAGE_FORMAT_FILE), + b"corrupt metadata", + ) + .await + .expect("metadata should be corrupted for the test"); + quorum_minus_one + .read_version_optimized(bucket, object, "", &ReadOptions::default()) + .await + .expect_err("one valid and one corrupt metadata copy must not satisfy read quorum"); + + let (quorum_dirs, quorum) = read_version_quorum_test_set(bucket, object, 1, 3).await; + tokio::fs::write( + quorum_dirs[2] + .path() + .join(bucket) + .join(object) + .join(crate::disk::STORAGE_FORMAT_FILE), + b"corrupt metadata", + ) + .await + .expect("metadata should be corrupted for the test"); + let versions = quorum + .read_version_optimized(bucket, object, "", &ReadOptions::default()) + .await + .expect("two valid metadata copies must satisfy read quorum despite one corrupt copy"); + assert_eq!(versions.len(), 2); + } + #[tokio::test] async fn get_object_info_and_quorum_maps_delete_marker_and_purge_states() { let bucket = "get-object-info-marker-bucket"; diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 801ccd411..1ce3bbd7f 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -2722,10 +2722,12 @@ mod tests { #[serial_test::serial(storage_class_env)] async fn transition_transaction_recovery_deletes_provider_recovered_unknown_upload() { let versioned_remote = uuid::Uuid::new_v4().to_string(); + let nil_remote = uuid::Uuid::nil().to_string(); for (case, tier_name, remote_version) in [ ("missing", "TXPROBEMISSING", None), ("unversioned", "TXPROBEUNVERSIONED", Some(String::new())), ("versioned", "TXPROBEVERSIONED", Some(versioned_remote)), + ("nil-version", "TXPROBENILVERSION", Some(nil_remote)), ] { let temp_dir = tempfile::tempdir().expect("create temp store dir"); let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store( diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 758b6279c..bdfab4fe8 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -6696,7 +6696,7 @@ mod test { use crate::object_api::ObjectInfo; use rustfs_filemeta::{ FileInfo, FileMeta, FileMetaVersion, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, MetaDeleteMarker, - VersionType, + ObjectPartInfo, VersionType, }; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; @@ -6899,6 +6899,12 @@ mod test { fi.volume = "bucket".to_owned(); fi.name = "object".to_owned(); fi.size = 1; + fi.parts = vec![ObjectPartInfo { + number: 1, + size: 1, + actual_size: 1, + ..Default::default() + }]; fi.fresh = true; fi.erasure.index = 1; fi.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")); @@ -6954,6 +6960,12 @@ mod test { fi.version_id = Some(Uuid::from_u128(version_idx)); fi.versioned = true; fi.size = 1; + fi.parts = vec![ObjectPartInfo { + number: 1, + size: 1, + actual_size: 1, + ..Default::default() + }]; fi.mod_time = Some(*mod_time); fi.metadata = metadata; diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index bf2841e3a..e1f1f2e1b 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -429,6 +429,10 @@ impl FileInfo { } fn validate_collection_contents(&self, layout: Option<&ValidatedErasureLayout>) -> Result<()> { + if layout.is_some() && self.size > 0 && self.parts.is_empty() { + return Err(Error::FileCorrupt); + } + let mut part_numbers = [0u64; FILEINFO_PART_BITMAP_WORDS]; for part in &self.parts { if let Some(layout) = layout { @@ -730,6 +734,10 @@ impl FileInfo { // to_part_offset gets the part index where offset is located, returns part index and offset pub fn to_part_offset(&self, offset: usize) -> Result<(usize, usize)> { + if self.size > 0 && self.parts.is_empty() { + return Err(Error::FileCorrupt); + } + if offset == 0 { return Ok((0, 0)); } @@ -1225,6 +1233,21 @@ mod tests { assert_eq!(layout.expect("strict layout must be present").data_blocks, 16); } + #[test] + fn validate_require_erasure_rejects_positive_size_without_parts() { + let mut fi = FileInfo::new("bucket/object", 4, 2); + fi.erasure.index = 1; + fi.size = 1; + + assert_eq!(fi.validate_for_metadata_read(), Err(Error::FileCorrupt)); + assert_eq!(fi.to_part_offset(0), Err(Error::FileCorrupt)); + + fi.size = 0; + fi.validate_for_metadata_read() + .expect("zero-byte object metadata may omit parts"); + assert_eq!(fi.to_part_offset(0), Ok((0, 0))); + } + #[test] fn validate_for_erasure_write_only_relaxes_pending_shard_index() { let mut fi = validation_test_fileinfo(); diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index f6eb519ca..41b799c28 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -182,14 +182,9 @@ impl FileMeta { // TODO: use old buf let meta_buf = ver.marshal_msg()?; - let pre_mod_time = self.versions[idx].header.mod_time; - self.versions[idx].header = ver.header(); self.versions[idx].meta = meta_buf; - - if pre_mod_time != self.versions[idx].header.mod_time { - self.sort_by_mod_time(); - } + self.sort_by_mod_time(); Ok(()) } @@ -356,15 +351,10 @@ impl FileMeta { return self.set_idx(fidx, version); } - let mod_time = version.get_mod_time(); let new_shallow = FileMetaShallowVersion::try_from(version)?; - let insert_pos = match mod_time { - Some(nm) => self.versions.partition_point(|exist| match exist.header.mod_time { - Some(em) => em > nm, - None => false, - }), - None => self.versions.partition_point(|exist| exist.header.mod_time.is_some()), - }; + let insert_pos = self + .versions + .partition_point(|existing| existing.header.sorts_before(&new_shallow.header)); self.versions.insert(insert_pos, new_shallow); Ok(()) @@ -1112,6 +1102,119 @@ mod test { assert!(fm.is_sorted_by_mod_time()); } + fn version_for_ordering( + version_type: VersionType, + version_id: Uuid, + mod_time: OffsetDateTime, + marker: u8, + ) -> FileMetaVersion { + match version_type { + VersionType::Object => FileMetaVersion { + version_type, + object: Some(MetaObject { + version_id: Some(version_id), + erasure_algorithm: ErasureAlgo::ReedSolomon, + erasure_m: 2, + erasure_n: 2, + erasure_block_size: 1 << 20, + bitrot_checksum_algo: ChecksumAlgo::HighwayHash, + mod_time: Some(mod_time), + meta_sys: HashMap::from([("ordering-marker".to_string(), vec![marker])]), + ..Default::default() + }), + ..Default::default() + }, + VersionType::Delete => FileMetaVersion { + version_type, + delete_marker: Some(MetaDeleteMarker { + version_id: Some(version_id), + mod_time: Some(mod_time), + meta_sys: HashMap::from([("ordering-marker".to_string(), vec![marker])]), + }), + ..Default::default() + }, + _ => unreachable!("ordering regression only constructs object and delete versions"), + } + } + + #[test] + fn add_version_filemata_uses_canonical_equal_time_order() { + let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp"); + let object = version_for_ordering(VersionType::Object, Uuid::from_u128(1), mod_time, 1); + let delete_marker = version_for_ordering(VersionType::Delete, Uuid::from_u128(2), mod_time, 2); + + for versions in [[object.clone(), delete_marker.clone()], [delete_marker, object]] { + let mut fm = FileMeta::new(); + for version in versions { + fm.add_version_filemata(version).expect("add equal-time version"); + } + + assert_eq!(fm.versions[0].header.version_type, VersionType::Object); + assert_eq!(fm.versions[1].header.version_type, VersionType::Delete); + assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header)); + } + } + + #[test] + fn add_version_filemata_preserves_tie_breaks_after_reload() { + let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp"); + let versions = [ + version_for_ordering(VersionType::Object, Uuid::from_u128(10), mod_time, 10), + version_for_ordering(VersionType::Object, Uuid::from_u128(20), mod_time, 20), + version_for_ordering(VersionType::Delete, Uuid::from_u128(30), mod_time, 30), + ]; + let mut expected = versions.iter().map(FileMetaVersion::header).collect::>(); + expected.sort_by(|a, b| { + if a.sorts_before(b) { + Ordering::Less + } else if b.sorts_before(a) { + Ordering::Greater + } else { + Ordering::Equal + } + }); + + let mut fm = FileMeta::new(); + for version in versions.into_iter().rev() { + fm.add_version_filemata(version).expect("add equal-time version"); + } + let loaded = FileMeta::load(&fm.marshal_msg().expect("serialize file metadata")).expect("reload file metadata"); + let actual = loaded + .versions + .iter() + .map(|version| version.header.clone()) + .collect::>(); + + assert_ne!(expected[0].signature, expected[1].signature, "fixture must exercise signature ordering"); + assert_eq!(actual, expected); + assert!(actual.windows(2).all(|pair| pair[0].sorts_before(&pair[1]))); + } + + #[test] + fn add_version_filemata_reorders_equal_time_replacement() { + let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp"); + let first = version_for_ordering(VersionType::Object, Uuid::from_u128(40), mod_time, 40); + let second = version_for_ordering(VersionType::Object, Uuid::from_u128(50), mod_time, 50); + let (target, peer) = if first.header().sorts_before(&second.header()) { + (first, second) + } else { + (second, first) + }; + let target_id = target.header().version_id.expect("test version id"); + + let mut fm = FileMeta::new(); + fm.add_version_filemata(peer).expect("add peer object"); + fm.add_version_filemata(target).expect("add target object"); + assert_eq!(fm.versions[0].header.version_id, Some(target_id)); + + fm.add_version_filemata(version_for_ordering(VersionType::Delete, target_id, mod_time, 60)) + .expect("replace target with equal-time delete marker"); + + assert_eq!(fm.versions[0].header.version_type, VersionType::Object); + assert_eq!(fm.versions[1].header.version_type, VersionType::Delete); + assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header)); + } + /// Regression for backlog#799 B16: replication reset state must be /// persisted under both internal prefixes, never as a bare ARN. A bare-ARN /// key (produced by `ObjectInfo::replication_state`) has no internal prefix, diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index 3ecc4dfcc..091bfb527 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -2509,6 +2509,8 @@ impl MetaObject { ); if let Some(transition_version) = transitioned_version_bytes(fi) { insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version); + } else { + remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID); } set_transition_version_state(&mut self.meta_sys, fi.transition_version_state); insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec()); @@ -4248,6 +4250,37 @@ mod tests { assert_eq!(decoded.transition_version.as_deref(), Some(expected_version.as_str())); } + #[test] + fn set_transition_known_disabled_removes_stale_version_dual_keys() { + let mut meta_sys = HashMap::new(); + insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, b"stale-legacy-version".to_vec()); + let mut object = make_meta_object_with_sys(meta_sys); + object.set_transition(&FileInfo { + transition_status: TRANSITION_COMPLETE.to_string(), + transitioned_objname: "remote/object".to_string(), + transition_version_state: TransitionVersionState::KnownDisabled, + transition_tier: "WARM".to_string(), + ..Default::default() + }); + + assert_eq!(get_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID), None); + assert!( + !object + .meta_sys + .contains_key(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}")) + ); + assert!( + !object + .meta_sys + .contains_key(&format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX)) + ); + let decoded = object + .into_fileinfo("b", "k", false) + .expect("known-disabled transition must remain readable after replacing stale metadata"); + assert_eq!(decoded.transition_version, None); + assert_eq!(decoded.transition_version_state, TransitionVersionState::KnownDisabled); + } + #[test] fn meta_object_transition_version_state_conflict_fails_closed() { let mut sys = HashMap::new(); diff --git a/crates/heal/src/error.rs b/crates/heal/src/error.rs index 7888e02c3..8ab4ff9e5 100644 --- a/crates/heal/src/error.rs +++ b/crates/heal/src/error.rs @@ -144,6 +144,7 @@ fn is_recoverable_heal_error_message(error: &str) -> bool { "connection refused", "operation canceled", "quorum not reached", + "heal rename incomplete", ] .iter() .any(|pattern| error.contains(pattern)) @@ -154,3 +155,22 @@ impl From for std::io::Error { std::io::Error::other(err) } } + +#[cfg(test)] +mod tests { + use super::Error; + use crate::heal::EcstoreError; + + #[test] + fn incomplete_target_rename_is_recoverable() { + let task_error = Error::TaskExecutionFailed { + message: "heal rename incomplete: 1 of 2 targets committed".to_string(), + }; + let storage_error = Error::Storage(EcstoreError::Io(std::io::Error::other( + "heal rename incomplete: 1 of 2 targets committed", + ))); + + assert!(task_error.is_recoverable_heal()); + assert!(storage_error.is_recoverable_heal()); + } +} diff --git a/crates/kms/src/api_types.rs b/crates/kms/src/api_types.rs index dba7e5de7..3af643c48 100644 --- a/crates/kms/src/api_types.rs +++ b/crates/kms/src/api_types.rs @@ -161,6 +161,14 @@ pub struct ConfigureStaticKmsRequest { pub allow_insecure_dev_defaults: Option, } +impl Drop for ConfigureStaticKmsRequest { + fn drop(&mut self) { + use zeroize::Zeroize; + + self.secret_key.zeroize(); + } +} + impl fmt::Debug for ConfigureStaticKmsRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ConfigureStaticKmsRequest") diff --git a/crates/kms/src/backends/static_kms.rs b/crates/kms/src/backends/static_kms.rs index 0dc36f456..8372429a9 100644 --- a/crates/kms/src/backends/static_kms.rs +++ b/crates/kms/src/backends/static_kms.rs @@ -23,16 +23,17 @@ use crate::backends::{BackendInfo, KmsBackend, KmsClient}; use crate::config::{BackendConfig, KmsConfig}; +use crate::encryption::DataKeyEnvelope; use crate::error::{KmsError, Result}; use crate::types::*; use aes_gcm::{ Aes256Gcm, Key, Nonce, - aead::{Aead, KeyInit}, + aead::{Aead, KeyInit, Payload}, }; use async_trait::async_trait; use jiff::Zoned; use rand::RngExt; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use tracing::debug; use zeroize::Zeroizing; @@ -41,6 +42,11 @@ const NONCE_SIZE: usize = 12; /// AES-256 key size in bytes. const KEY_SIZE: usize = 32; +fn context_aad(context: &HashMap) -> Result> { + let canonical: BTreeMap<&str, &str> = context.iter().map(|(key, value)| (key.as_str(), value.as_str())).collect(); + serde_json::to_vec(&canonical).map_err(Into::into) +} + /// Static single-key KMS backend. /// /// Uses a pre-configured AES-256 key to derive data encryption keys. This is a @@ -111,21 +117,35 @@ impl KmsClient for StaticKmsBackend { let key = Key::::from(*self.key); let cipher = Aes256Gcm::new(&key); let nonce = Nonce::from(nonce_bytes); + let aad = context_aad(&request.encryption_context)?; let encrypted = cipher - .encrypt(&nonce, plaintext.as_ref()) + .encrypt( + &nonce, + Payload { + msg: plaintext.as_ref(), + aad: &aad, + }, + ) .map_err(|e| KmsError::cryptographic_error("AES-256-GCM encrypt", e.to_string()))?; - // Ciphertext format: encrypted_dek || nonce - let mut ciphertext = encrypted; - ciphertext.extend_from_slice(&nonce_bytes); + let envelope = DataKeyEnvelope { + key_id: uuid::Uuid::new_v4().to_string(), + master_key_id: request.master_key_id.clone(), + key_spec: request.key_spec.clone(), + encrypted_key: encrypted, + nonce: nonce_bytes.to_vec(), + encryption_context: request.encryption_context.clone(), + created_at: Zoned::now(), + }; + let ciphertext = serde_json::to_vec(&envelope)?; Ok(DataKeyInfo::new( self.key_id.clone(), - 0, // version is always 0 for static KMS + 0, Some(plaintext.to_vec()), ciphertext, - "AES_256".to_string(), + request.key_spec.clone(), )) } @@ -141,14 +161,28 @@ impl KmsClient for StaticKmsBackend { let key = Key::::from(*self.key); let cipher = Aes256Gcm::new(&key); let nonce = Nonce::from(nonce_bytes); + let aad = context_aad(&request.encryption_context)?; let encrypted = cipher - .encrypt(&nonce, request.plaintext.as_ref()) + .encrypt( + &nonce, + Payload { + msg: request.plaintext.as_ref(), + aad: &aad, + }, + ) .map_err(|e| KmsError::cryptographic_error("AES-256-GCM encrypt", e.to_string()))?; - // Ciphertext format: encrypted_data || nonce - let mut ciphertext = encrypted; - ciphertext.extend_from_slice(&nonce_bytes); + let envelope = DataKeyEnvelope { + key_id: uuid::Uuid::new_v4().to_string(), + master_key_id: request.key_id.clone(), + key_spec: "AES_256".to_string(), + encrypted_key: encrypted, + nonce: nonce_bytes.to_vec(), + encryption_context: request.encryption_context.clone(), + created_at: Zoned::now(), + }; + let ciphertext = serde_json::to_vec(&envelope)?; Ok(EncryptResponse { ciphertext, @@ -159,21 +193,39 @@ impl KmsClient for StaticKmsBackend { } async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result> { - if request.ciphertext.len() < NONCE_SIZE + 1 { - return Err(KmsError::cryptographic_error("decrypt", "Ciphertext too short for static KMS format")); + let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext) + .map_err(|error| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {error}")))?; + if envelope.master_key_id != self.key_id { + return Err(KmsError::key_not_found(&envelope.master_key_id)); } - // Split ciphertext: encrypted_data || nonce(12) - let split_at = request.ciphertext.len() - NONCE_SIZE; - let encrypted = &request.ciphertext[..split_at]; - let nonce_slice = &request.ciphertext[split_at..]; + for (key, expected_value) in &envelope.encryption_context { + match request.encryption_context.get(key) { + Some(actual_value) if actual_value == expected_value => {} + Some(actual_value) => { + return Err(KmsError::context_mismatch(format!( + "Context mismatch for key '{key}': expected '{expected_value}', got '{actual_value}'" + ))); + } + None if request.encryption_context.is_empty() => {} + None => return Err(KmsError::context_mismatch(format!("Missing context key '{key}'"))), + } + } let key = Key::::from(*self.key); let cipher = Aes256Gcm::new(&key); - let nonce = Nonce::try_from(nonce_slice).map_err(|_| KmsError::cryptographic_error("nonce", "invalid nonce length"))?; + let nonce = Nonce::try_from(envelope.nonce.as_slice()) + .map_err(|_| KmsError::cryptographic_error("nonce", "invalid nonce length"))?; + let aad = context_aad(&envelope.encryption_context)?; let plaintext = cipher - .decrypt(&nonce, encrypted) + .decrypt( + &nonce, + Payload { + msg: envelope.encrypted_key.as_ref(), + aad: &aad, + }, + ) .map_err(|e| KmsError::cryptographic_error("AES-256-GCM decrypt", e.to_string()))?; Ok(plaintext) @@ -317,7 +369,13 @@ impl KmsBackend for StaticKmsBackend { } async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result { - let gen_req = GenerateKeyRequest::new(request.key_id.clone(), request.key_spec.as_str().to_string()); + let gen_req = GenerateKeyRequest { + master_key_id: request.key_id.clone(), + key_spec: request.key_spec.as_str().to_string(), + key_length: None, + encryption_context: request.encryption_context, + grant_tokens: Vec::new(), + }; let data_key = ::generate_data_key(self, &gen_req, None).await?; let plaintext_key = data_key @@ -378,8 +436,9 @@ impl KmsBackend for StaticKmsBackend { #[cfg(test)] mod tests { use super::*; - use crate::backends::KmsClient; + use crate::backends::{KmsBackend as KmsBackendTrait, KmsClient}; use crate::config::{BackendConfig, KmsBackend, StaticConfig}; + use crate::encryption::is_data_key_envelope; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; @@ -430,8 +489,12 @@ mod tests { assert_eq!(data_key.version, 0); assert!(data_key.plaintext.is_some()); assert_eq!(data_key.plaintext.as_ref().expect("plaintext should be set").len(), 32); - // Ciphertext should be: encrypted(32) + tag(16) + nonce(12) - assert_eq!(data_key.ciphertext.len(), 32 + 16 + NONCE_SIZE); + let envelope: DataKeyEnvelope = + serde_json::from_slice(&data_key.ciphertext).expect("static data key should use a KMS envelope"); + assert_eq!(envelope.master_key_id, key_id); + assert_eq!(envelope.encrypted_key.len(), 32 + 16); + assert_eq!(envelope.nonce.len(), NONCE_SIZE); + assert_eq!(envelope.encryption_context.get("bucket").map(String::as_str), Some("test-bucket")); // Decrypt the data key let decrypt_request = @@ -443,6 +506,75 @@ mod tests { assert_eq!(decrypted.as_slice(), data_key.plaintext.as_deref().expect("plaintext should exist")); } + #[tokio::test] + async fn generated_data_key_uses_kms_envelope_for_sse_read_routing() { + let (backend, key_id, _key) = create_test_backend().await; + let request = GenerateKeyRequest::new(key_id, "AES_256".to_string()) + .with_context("bucket".to_string(), "source-bucket".to_string()) + .with_context("object".to_string(), "source-object".to_string()); + + let data_key = KmsClient::generate_data_key(&backend, &request, None) + .await + .expect("generate static KMS data key"); + + assert!( + is_data_key_envelope(&data_key.ciphertext), + "static KMS ciphertext must use the KMS envelope recognized by the SSE read path" + ); + } + + #[tokio::test] + async fn generated_data_key_rejects_a_different_encryption_context() { + let (backend, key_id, _key) = create_test_backend().await; + let generate_request = GenerateDataKeyRequest { + key_id, + key_spec: KeySpec::Aes256, + encryption_context: HashMap::from([ + ("bucket".to_string(), "source-bucket".to_string()), + ("object".to_string(), "source-object".to_string()), + ]), + }; + let generated = KmsBackendTrait::generate_data_key(&backend, generate_request) + .await + .expect("generate context-bound static KMS data key"); + let decrypt_request = DecryptRequest { + ciphertext: generated.ciphertext_blob, + encryption_context: HashMap::from([ + ("bucket".to_string(), "different-bucket".to_string()), + ("object".to_string(), "different-object".to_string()), + ]), + grant_tokens: Vec::new(), + }; + + let error = KmsBackendTrait::decrypt(&backend, decrypt_request) + .await + .expect_err("a static KMS data key must not decrypt under a different object context"); + + assert!(matches!(error, KmsError::ContextMismatch { .. })); + } + + #[tokio::test] + async fn generated_data_key_rejects_tampered_envelope_context() { + let (backend, key_id, _key) = create_test_backend().await; + let request = GenerateKeyRequest::new(key_id, "AES_256".to_string()) + .with_context("bucket".to_string(), "source-bucket".to_string()); + let generated = KmsClient::generate_data_key(&backend, &request, None) + .await + .expect("generate context-bound data key"); + let mut envelope: DataKeyEnvelope = serde_json::from_slice(&generated.ciphertext).expect("parse static KMS envelope"); + envelope + .encryption_context + .insert("bucket".to_string(), "different-bucket".to_string()); + let decrypt_request = DecryptRequest::new(serde_json::to_vec(&envelope).expect("serialize tampered envelope")) + .with_context("bucket".to_string(), "different-bucket".to_string()); + + let error = KmsClient::decrypt(&backend, &decrypt_request, None) + .await + .expect_err("tampering with authenticated envelope context must fail"); + + assert!(matches!(error, KmsError::CryptographicError { .. })); + } + #[tokio::test] async fn test_generate_data_key_wrong_key_id() { let (backend, _key_id, _key) = create_test_backend().await; diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index cf6cc2177..3d2a469a5 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -214,9 +214,18 @@ pub struct StaticConfig { /// Key identifier (name) for the single configured key pub key_id: String, /// Base64-encoded 32-byte AES-256 key material (zeroed on drop) + #[serde(skip_serializing, default)] pub secret_key: String, } +impl Drop for StaticConfig { + fn drop(&mut self) { + use zeroize::Zeroize; + + self.secret_key.zeroize(); + } +} + impl fmt::Debug for StaticConfig { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("StaticConfig") @@ -1054,6 +1063,21 @@ mod tests { assert!(serialized.contains("persisted-token-secret")); } + #[test] + fn static_kms_config_serialization_does_not_expose_key_material() { + use base64::Engine as _; + + let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]); + let config = KmsConfig::static_kms("static-key".to_string(), encoded_key.clone()); + + let serialized = serde_json::to_string(&config).expect("static KMS config should serialize"); + + assert!( + !serialized.contains(&encoded_key), + "persisted static KMS configuration must not contain plaintext key material" + ); + } + #[test] fn test_config_validation() { let mut config = KmsConfig { diff --git a/crates/kms/src/manager.rs b/crates/kms/src/manager.rs index 1fb489e1a..3cfc8612c 100644 --- a/crates/kms/src/manager.rs +++ b/crates/kms/src/manager.rs @@ -31,19 +31,25 @@ use tokio::sync::RwLock; pub struct KmsManager { backend: Arc, cache: Arc>, - config: KmsConfig, + default_key_id: Option, + enable_cache: bool, } impl KmsManager { /// Create a new KMS manager with the given backend and config pub fn new(backend: Arc, config: KmsConfig) -> Self { let cache = Arc::new(RwLock::new(KmsCache::new(config.cache_config.max_keys as u64))); - Self { backend, cache, config } + Self { + backend, + cache, + default_key_id: config.default_key_id, + enable_cache: config.enable_cache, + } } /// Get the default key ID if configured pub fn get_default_key_id(&self) -> Option<&String> { - self.config.default_key_id.as_ref() + self.default_key_id.as_ref() } /// Create a new master key @@ -51,7 +57,7 @@ impl KmsManager { let response = self.backend.create_key(request).await?; // Cache the key metadata if enabled - if self.config.enable_cache { + if self.enable_cache { let mut cache = self.cache.write().await; cache.put_key_metadata(&response.key_id, &response.key_metadata).await; } @@ -77,7 +83,7 @@ impl KmsManager { /// Describe a key pub async fn describe_key(&self, request: DescribeKeyRequest) -> Result { // Check cache first if enabled - if self.config.enable_cache { + if self.enable_cache { let cache = self.cache.read().await; if let Some(cached_metadata) = cache.get_key_metadata(&request.key_id).await { return Ok(DescribeKeyResponse { @@ -89,7 +95,7 @@ impl KmsManager { // Get from backend and cache let response = self.backend.describe_key(request).await?; - if self.config.enable_cache { + if self.enable_cache { let mut cache = self.cache.write().await; cache .put_key_metadata(&response.key_metadata.key_id, &response.key_metadata) @@ -106,7 +112,7 @@ impl KmsManager { /// Get cache statistics pub async fn cache_stats(&self) -> Option<(u64, u64)> { - if self.config.enable_cache { + if self.enable_cache { let cache = self.cache.read().await; Some(cache.stats()) } else { @@ -116,7 +122,7 @@ impl KmsManager { /// Clear the cache pub async fn clear_cache(&self) -> Result<()> { - if self.config.enable_cache { + if self.enable_cache { let mut cache = self.cache.write().await; cache.clear().await; } @@ -128,7 +134,7 @@ impl KmsManager { let response = self.backend.delete_key(request).await?; // Remove from cache if enabled and key is being deleted - if self.config.enable_cache { + if self.enable_cache { let mut cache = self.cache.write().await; cache.remove_key_metadata(&response.key_id).await; } @@ -141,7 +147,7 @@ impl KmsManager { let response = self.backend.cancel_key_deletion(request).await?; // Update cache if enabled - if self.config.enable_cache { + if self.enable_cache { let mut cache = self.cache.write().await; cache.put_key_metadata(&response.key_id, &response.key_metadata).await; } diff --git a/crates/kms/src/service.rs b/crates/kms/src/service.rs index a7f91c645..2cac74953 100644 --- a/crates/kms/src/service.rs +++ b/crates/kms/src/service.rs @@ -350,11 +350,7 @@ impl ObjectEncryptionService { encryption_context: context.clone(), }; - let data_key = self - .kms_manager - .generate_data_key(request) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to generate data key: {e}")))?; + let data_key = self.kms_manager.generate_data_key(request).await?; let plaintext_key = data_key.plaintext_key; @@ -431,11 +427,7 @@ impl ObjectEncryptionService { grant_tokens: Vec::new(), }; - let decrypt_response = self - .kms_manager - .decrypt(decrypt_request) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to decrypt data key: {e}")))?; + let decrypt_response = self.kms_manager.decrypt(decrypt_request).await?; // Create cipher let cipher = create_cipher(&algorithm, &decrypt_response.plaintext)?; diff --git a/crates/kms/src/service_manager.rs b/crates/kms/src/service_manager.rs index f81d799fb..b3a37ca22 100644 --- a/crates/kms/src/service_manager.rs +++ b/crates/kms/src/service_manager.rs @@ -94,6 +94,16 @@ impl KmsServiceManager { self.config.read().await.clone() } + /// Get configuration for status and management responses without static key material. + pub async fn get_redacted_config(&self) -> Option { + let mut config = self.config.read().await.clone()?; + if let BackendConfig::Static(static_config) = &mut config.backend_config { + use zeroize::Zeroize; + static_config.secret_key.zeroize(); + } + Some(config) + } + /// Configure KMS with new configuration pub async fn configure(&self, new_config: KmsConfig) -> Result<()> { new_config.validate()?; @@ -449,4 +459,22 @@ mod tests { assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured); assert!(manager.get_config().await.is_none()); } + + #[tokio::test] + async fn redacted_config_omits_static_key_material() { + use base64::Engine as _; + + let manager = KmsServiceManager::new(); + let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]); + manager + .configure(KmsConfig::static_kms("static-key".to_string(), encoded_key)) + .await + .expect("configure static KMS"); + + let config = manager.get_redacted_config().await.expect("redacted config"); + let BackendConfig::Static(static_config) = config.backend_config else { + panic!("expected static config"); + }; + assert!(static_config.secret_key.is_empty()); + } } diff --git a/crates/lifecycle/Cargo.toml b/crates/lifecycle/Cargo.toml index 6e0e007e6..43dfac6aa 100644 --- a/crates/lifecycle/Cargo.toml +++ b/crates/lifecycle/Cargo.toml @@ -27,6 +27,7 @@ documentation = "https://docs.rs/rustfs-lifecycle/latest/rustfs_lifecycle/" [dependencies] async-trait.workspace = true +metrics.workspace = true rustfs-common.workspace = true rustfs-config = { workspace = true, features = ["constants"] } rustfs-replication.workspace = true @@ -38,6 +39,7 @@ url.workspace = true uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } [dev-dependencies] +metrics-util = { version = "0.20", features = ["debugging"] } proptest = "1" serial_test.workspace = true temp-env.workspace = true diff --git a/crates/lifecycle/src/core.rs b/crates/lifecycle/src/core.rs index bfed5b270..9de9bdfd6 100644 --- a/crates/lifecycle/src/core.rs +++ b/crates/lifecycle/src/core.rs @@ -34,6 +34,7 @@ const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; const EVENT_LIFECYCLE_EXPIRY_COMPUTED: &str = "lifecycle_expiry_computed"; const EVENT_LIFECYCLE_DEBUG_DAY_SECS: &str = "lifecycle_debug_day_secs"; const EVENT_LIFECYCLE_NONCURRENT_EXPIRY_SKIPPED: &str = "lifecycle_noncurrent_expiry_skipped"; +const METRIC_LIFECYCLE_INVALID_MOD_TIME_TOTAL: &str = "rustfs_lifecycle_invalid_mod_time_total"; const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule"; const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule"; const _ERR_XML_NOT_WELL_FORMED: &str = @@ -204,6 +205,21 @@ fn lifecycle_rule_prefix(rule: &LifecycleRule) -> Option<&str> { filter.and.as_ref().and_then(|and| and.prefix.as_deref()) } +fn lifecycle_mod_time(obj: &ObjectOpts) -> Option { + let reason = match obj.mod_time { + None => "missing", + Some(mod_time) if mod_time < OffsetDateTime::UNIX_EPOCH => "before_unix_epoch", + Some(mod_time) if mod_time == OffsetDateTime::UNIX_EPOCH => "unix_epoch", + Some(mod_time) => return Some(mod_time), + }; + metrics::counter!( + METRIC_LIFECYCLE_INVALID_MOD_TIME_TOTAL, + "reason" => reason + ) + .increment(1); + None +} + #[async_trait::async_trait] pub trait Lifecycle { async fn has_transition(&self) -> bool; @@ -434,14 +450,8 @@ impl Lifecycle for BucketLifecycleConfiguration { } async fn predict_expiration(&self, obj: &ObjectOpts) -> Event { - let mod_time = match obj.mod_time { - Some(time) => { - if time.unix_timestamp() == 0 { - return Event::default(); - } - time - } - None => return Event::default(), + let Some(mod_time) = lifecycle_mod_time(obj) else { + return Event::default(); }; if obj.delete_marker || !(obj.is_latest || obj.version_id.is_none_or(|v| v.is_nil())) { @@ -495,19 +505,9 @@ impl Lifecycle for BucketLifecycleConfiguration { obj.name, obj.mod_time, obj.successor_mod_time, now, obj.is_latest, obj.delete_marker ); - // Gracefully handle missing mod_time instead of panicking - let mod_time = match obj.mod_time { - Some(t) => t, - None => { - debug!("eval_inner: mod_time is None for object={}, returning default event", obj.name); - return Event::default(); - } - }; - - if mod_time.unix_timestamp() == 0 { - debug!("eval_inner: mod_time is 0, returning default event"); + let Some(mod_time) = lifecycle_mod_time(obj) else { return Event::default(); - } + }; if let Some(restore_expires) = obj.restore_expires && restore_expires.unix_timestamp() != 0 @@ -1082,6 +1082,8 @@ impl Default for TransitionOptions { #[cfg(test)] mod tests { use super::*; + use metrics_util::MetricKind; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; use s3s::dto::{LifecycleRuleFilter, TransitionStorageClass}; use serial_test::serial; use std::sync::Arc; @@ -1097,6 +1099,99 @@ mod tests { }); } + #[test] + fn eval_inner_reports_invalid_mod_time_without_expiring_object() { + let lifecycle = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(1), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("expire-after-one-day".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .expect("current-thread runtime should build"); + + let actions = metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + let now = OffsetDateTime::from_unix_timestamp(10 * 86_400).expect("fixed timestamp should be valid"); + let mut opts = ObjectOpts { + name: "legacy-object".to_string(), + is_latest: true, + ..Default::default() + }; + let mut actions = Vec::new(); + for mod_time in [ + None, + Some(OffsetDateTime::UNIX_EPOCH), + Some(OffsetDateTime::UNIX_EPOCH - Duration::nanoseconds(1)), + Some(OffsetDateTime::UNIX_EPOCH + Duration::nanoseconds(1)), + ] { + opts.mod_time = mod_time; + actions.push(( + lifecycle.eval_inner(&opts, now, 0).await.action, + lifecycle.predict_expiration(&opts).await.action, + )); + } + actions + }) + }); + + assert_eq!( + actions, + [ + (IlmAction::NoneAction, IlmAction::NoneAction), + (IlmAction::NoneAction, IlmAction::NoneAction), + (IlmAction::NoneAction, IlmAction::NoneAction), + (IlmAction::DeleteAction, IlmAction::DeleteAction) + ] + ); + + let invalid_mod_time_metrics = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter_map(|(composite, _unit, _description, value)| { + (composite.kind() == MetricKind::Counter && composite.key().name() == METRIC_LIFECYCLE_INVALID_MOD_TIME_TOTAL) + .then(|| { + let reason = composite + .key() + .labels() + .find(|label| label.key() == "reason") + .map(|label| label.value().to_string()) + .expect("invalid mod_time metric should have a reason"); + (reason, value) + }) + }) + .collect::>(); + assert_eq!(invalid_mod_time_metrics.len(), 3); + assert!( + invalid_mod_time_metrics + .iter() + .all(|(_, value)| matches!(value, DebugValue::Counter(2))) + ); + assert!(invalid_mod_time_metrics.iter().any(|(reason, _)| reason == "missing")); + assert!(invalid_mod_time_metrics.iter().any(|(reason, _)| reason == "unix_epoch")); + assert!( + invalid_mod_time_metrics + .iter() + .any(|(reason, _)| reason == "before_unix_epoch") + ); + } + #[tokio::test] #[serial] async fn validate_rejects_zero_expiration_days() { diff --git a/crates/log-analyzer/src/rules/seed/heal.rs b/crates/log-analyzer/src/rules/seed/heal.rs index 2c2f8256f..3ddc7f034 100644 --- a/crates/log-analyzer/src/rules/seed/heal.rs +++ b/crates/log-analyzer/src/rules/seed/heal.rs @@ -45,15 +45,15 @@ pub(super) fn rules() -> Vec { ) }, Rule { - anchors: strings(["all healed data rename attempts failed"]), + anchors: strings(["heal rename incomplete"]), ..base( "heal-rename-failed", P1Unavailable, "heal", "heal 数据落位失败", - contains("all healed data rename attempts failed"), - "heal 数据落位(rename)全部失败。", - "检查盘写权限与空间。", + contains("heal rename incomplete"), + "heal 数据落位(rename)未在全部目标盘完成。", + "检查失败目标盘的写权限与空间,修复后重跑 heal。", ) }, Rule { diff --git a/crates/log-analyzer/src/rules/seed/tests.rs b/crates/log-analyzer/src/rules/seed/tests.rs index 8152257cc..3e8c596ee 100644 --- a/crates/log-analyzer/src/rules/seed/tests.rs +++ b/crates/log-analyzer/src/rules/seed/tests.rs @@ -177,7 +177,7 @@ fn every_rule_has_a_positive_sample() { msg("heal: latest metadata for b/o has no data_dir, cannot heal object data"), ), ("heal-all-writes-failed", msg("all drives had write errors, unable to heal b/o")), - ("heal-rename-failed", msg("all healed data rename attempts failed for b/o")), + ("heal-rename-failed", msg("heal rename incomplete: 1 of 2 targets committed for b/o")), ( "heal-xlmeta-regen-failed", msg("heal_object: failed to regenerate recoverable xl.meta on disk"), diff --git a/crates/protocols/Cargo.toml b/crates/protocols/Cargo.toml index 2323a1d0e..d9ea48b41 100644 --- a/crates/protocols/Cargo.toml +++ b/crates/protocols/Cargo.toml @@ -49,6 +49,8 @@ swift = [ "dep:hmac", "dep:sha1", "dep:hex", + "dep:ipnetwork", + "dep:rustfs-trusted-proxies", "dep:astral-tokio-tar", "dep:base64", "dep:async-compression", @@ -111,6 +113,8 @@ quick-xml = { workspace = true, optional = true, features = ["serialize"] } hmac = { workspace = true, optional = true } sha1 = { workspace = true, optional = true } hex = { workspace = true, optional = true } +ipnetwork = { workspace = true, optional = true } +rustfs-trusted-proxies = { workspace = true, optional = true } astral-tokio-tar = { workspace = true, optional = true } base64 = { workspace = true, optional = true } async-compression = { workspace = true, optional = true, features = ["tokio", "gzip", "bzip2"] } diff --git a/crates/protocols/src/swift/container.rs b/crates/protocols/src/swift/container.rs index 8c9cf0e7c..ffbc24498 100644 --- a/crates/protocols/src/swift/container.rs +++ b/crates/protocols/src/swift/container.rs @@ -22,12 +22,19 @@ use super::storage_api::container::{ }; use super::types::Container; use super::{SwiftError, SwiftResult}; -use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata}; +use super::{get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, set_swift_bucket_metadata}; use rustfs_credentials::Credentials; use s3s::dto::{Tag, Tagging}; use sha2::{Digest, Sha256}; use tracing::{debug, error}; +fn required_bucket_usage(usage: &std::collections::HashMap, bucket: &str) -> SwiftResult<(u64, u64)> { + usage + .get(bucket) + .copied() + .ok_or_else(|| SwiftError::ServiceUnavailable("Container usage is unavailable".to_string())) +} + const LOG_COMPONENT_PROTOCOLS: &str = "protocols"; const LOG_SUBSYSTEM_SWIFT_CONTAINER: &str = "swift_container"; const EVENT_SWIFT_CONTAINER_STORAGE_STATE: &str = "swift_container_storage_state"; @@ -252,13 +259,24 @@ pub async fn list_containers(account: &str, credentials: &Credentials) -> SwiftR .list_bucket(&BucketOptions::default()) .await .map_err(|e| sanitize_storage_error("Container listing", e))?; + let usage = get_swift_bucket_usage() + .await + .map_err(|e| sanitize_storage_error("Container usage retrieval", e))? + .ok_or_else(|| SwiftError::ServiceUnavailable("Container usage is unavailable".to_string()))?; // Filter and convert buckets to containers let containers: Vec = bucket_infos .iter() .filter(|info| mapper.bucket_belongs_to_project(&info.name, &project_id)) - .filter_map(|info| bucket_info_to_container(info, &mapper, &project_id)) - .collect(); + .filter_map(|info| { + bucket_info_to_container(info, &mapper, &project_id).map(|mut container| { + let (count, bytes) = required_bucket_usage(&usage, &info.name)?; + container.count = count; + container.bytes = bytes; + Ok(container) + }) + }) + .collect::>>()?; debug!( event = EVENT_SWIFT_CONTAINER_STORAGE_STATE, @@ -368,6 +386,44 @@ pub struct ContainerMetadata { pub custom_metadata: std::collections::HashMap, } +async fn get_container_metadata_base( + account: &str, + container: &str, + credentials: &Credentials, +) -> SwiftResult<(String, BucketInfo, std::collections::HashMap)> { + let project_id = validate_account_access(account, credentials)?; + validate_container_name(container)?; + let bucket_name = ContainerMapper::default().swift_to_s3_bucket(container, &project_id); + let Some(store) = resolve_swift_object_store_handle() else { + return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string())); + }; + let bucket_info = store + .get_bucket_info(&bucket_name, &BucketOptions::default()) + .await + .map_err(|e| { + if e.to_string().contains("not found") || e.to_string().contains("NoSuchBucket") { + SwiftError::NotFound(format!("Container '{}' not found", container)) + } else { + sanitize_storage_error("Container metadata retrieval", e) + } + })?; + let custom_metadata = get_swift_bucket_metadata(&bucket_name) + .await + .ok() + .and_then(|bucket_meta| bucket_meta.tagging_config.as_ref().map(s3_tags_to_swift_metadata)) + .unwrap_or_default(); + Ok((bucket_name, bucket_info, custom_metadata)) +} + +pub(crate) async fn get_container_custom_metadata( + account: &str, + container: &str, + credentials: &Credentials, +) -> SwiftResult> { + let (_, _, custom_metadata) = get_container_metadata_base(account, container, credentials).await?; + Ok(custom_metadata) +} + /// Get container metadata (for HEAD operation) /// /// This function: @@ -382,60 +438,22 @@ pub struct ContainerMetadata { /// - Returns 404 Not Found if container doesn't exist #[allow(dead_code)] // Used by handler pub async fn get_container_metadata(account: &str, container: &str, credentials: &Credentials) -> SwiftResult { - // Validate account access and extract project_id - let project_id = validate_account_access(account, credentials)?; + let (bucket_name, bucket_info, custom_metadata) = get_container_metadata_base(account, container, credentials).await?; - // Validate container name - validate_container_name(container)?; - - // Create mapper with default config (tenant prefixing enabled) - let mapper = ContainerMapper::default(); - - // Convert Swift container name to S3 bucket name - let bucket_name = mapper.swift_to_s3_bucket(container, &project_id); - - // Get storage layer - let Some(store) = resolve_swift_object_store_handle() else { - return Err(SwiftError::InternalServerError("Storage layer not initialized".to_string())); - }; - - // Get bucket info - let bucket_info = store - .get_bucket_info(&bucket_name, &BucketOptions::default()) + // Ecstore serves the persisted scanner snapshot through a bounded in-process cache. + // Single-node deployments overlay successful in-process mutations; distributed + // deployments use only the cluster-wide persisted snapshot. + let usage = get_swift_bucket_usage() .await - .map_err(|e| { - // Check if bucket not found - if e.to_string().contains("not found") || e.to_string().contains("NoSuchBucket") { - SwiftError::NotFound(format!("Container '{}' not found", container)) - } else { - sanitize_storage_error("Container metadata retrieval", e) - } - })?; + .map_err(|e| sanitize_storage_error("Container usage retrieval", e))? + .ok_or_else(|| SwiftError::ServiceUnavailable("Container usage is unavailable".to_string()))?; + let (object_count, bytes_used) = required_bucket_usage(&usage, &bucket_name)?; - // Load bucket metadata to get custom metadata from tags - let custom_metadata = match get_swift_bucket_metadata(&bucket_name).await { - Ok(bucket_meta) => { - if let Some(tagging) = &bucket_meta.tagging_config { - s3_tags_to_swift_metadata(tagging) - } else { - std::collections::HashMap::new() - } - } - Err(_) => { - // If metadata not available, return empty (container may be newly created) - std::collections::HashMap::new() - } - }; - - // Currently returns basic metadata with limitations: - // 1. Object count requires iterating all objects (expensive) - // 2. Bytes used requires summing all object sizes (expensive) - // 3. Custom metadata is now loaded from bucket tags ✅ Ok(ContainerMetadata { - object_count: 0, // TODO: implement object counting in backend - bytes_used: 0, // TODO: implement size aggregation in backend + object_count, + bytes_used, created: bucket_info.created, - custom_metadata, // ✅ Now populated from bucket tags! + custom_metadata, }) } @@ -1843,4 +1861,24 @@ mod tests { assert_eq!(tagging.tag_set.len(), 1); assert_eq!(tagging.tag_set[0].value.as_deref(), Some("new-archive")); } + + #[test] + fn nonempty_container_usage_preserves_authoritative_totals() { + let usage = std::collections::HashMap::from([("tenant-container".to_string(), (7, 4097))]); + + assert_eq!( + required_bucket_usage(&usage, "tenant-container").expect("authoritative bucket usage"), + (7, 4097) + ); + } + + #[test] + fn missing_container_usage_fails_closed() { + let usage = std::collections::HashMap::new(); + + assert!(matches!( + required_bucket_usage(&usage, "tenant-container"), + Err(SwiftError::ServiceUnavailable(_)) + )); + } } diff --git a/crates/protocols/src/swift/errors.rs b/crates/protocols/src/swift/errors.rs index 4526243b1..25275dc41 100644 --- a/crates/protocols/src/swift/errors.rs +++ b/crates/protocols/src/swift/errors.rs @@ -32,6 +32,8 @@ pub enum SwiftError { NotFound(String), /// 409 Conflict Conflict(String), + /// 411 Length Required + LengthRequired(String), /// 413 Request Entity Too Large (Payload Too Large) RequestEntityTooLarge(String), /// 422 Unprocessable Entity @@ -54,6 +56,7 @@ impl fmt::Display for SwiftError { SwiftError::Forbidden(msg) => write!(f, "Forbidden: {}", msg), SwiftError::NotFound(msg) => write!(f, "Not Found: {}", msg), SwiftError::Conflict(msg) => write!(f, "Conflict: {}", msg), + SwiftError::LengthRequired(msg) => write!(f, "Length Required: {}", msg), SwiftError::RequestEntityTooLarge(msg) => write!(f, "Request Entity Too Large: {}", msg), SwiftError::UnprocessableEntity(msg) => write!(f, "Unprocessable Entity: {}", msg), SwiftError::TooManyRequests { retry_after, .. } => { @@ -76,6 +79,7 @@ impl SwiftError { SwiftError::Forbidden(_) => StatusCode::FORBIDDEN, SwiftError::NotFound(_) => StatusCode::NOT_FOUND, SwiftError::Conflict(_) => StatusCode::CONFLICT, + SwiftError::LengthRequired(_) => StatusCode::LENGTH_REQUIRED, SwiftError::RequestEntityTooLarge(_) => StatusCode::PAYLOAD_TOO_LARGE, SwiftError::UnprocessableEntity(_) => StatusCode::UNPROCESSABLE_ENTITY, SwiftError::TooManyRequests { .. } => StatusCode::TOO_MANY_REQUESTS, diff --git a/crates/protocols/src/swift/handler.rs b/crates/protocols/src/swift/handler.rs index 4ad9294de..9961cddf2 100644 --- a/crates/protocols/src/swift/handler.rs +++ b/crates/protocols/src/swift/handler.rs @@ -28,7 +28,9 @@ use axum::http::{Method, Request, Response, StatusCode}; use futures::Future; use rustfs_credentials::Credentials; use rustfs_keystone::KEYSTONE_CREDENTIALS; +use rustfs_trusted_proxies::ClientInfo; use s3s::Body; +use std::net::{IpAddr, SocketAddr}; use std::pin::Pin; use std::task::{Context, Poll}; use tokio_util::io::StreamReader; @@ -40,6 +42,14 @@ const LOG_SUBSYSTEM_SWIFT_HANDLER: &str = "swift_handler"; const EVENT_SWIFT_ROUTE_STATE: &str = "swift_route_state"; const EVENT_SWIFT_TEMPURL_STATE: &str = "swift_tempurl_state"; +fn trusted_client_ip(req: &Request) -> Option { + req.extensions() + .get::() + .map(|info| info.real_ip) + .or_else(|| req.extensions().get::().map(SocketAddr::ip)) + .filter(|ip| !ip.is_unspecified()) +} + /// Swift-aware service that routes to Swift handlers or S3 service #[derive(Clone)] pub struct SwiftService { @@ -128,6 +138,7 @@ async fn handle_swift_request( route: SwiftRoute, credentials: Option, ) -> Result, SwiftError> { + let client_ip = trusted_client_ip(&req); // Extract parts let (parts, body) = req.into_parts(); let method = parts.method.clone(); @@ -164,7 +175,7 @@ async fn handle_swift_request( let tempurl = tempurl::TempURL::new(key); let path = uri.path(); - tempurl.validate_request(method.as_str(), path, &tempurl_params)?; + tempurl.validate_request(method.as_str(), path, &tempurl_params, client_ip)?; // TempURL is valid - proceed with request (no credentials needed) debug!( @@ -641,14 +652,11 @@ async fn handle_authenticated_request( .await; } - // Check quota before upload (if Content-Length provided) - if let Some(content_length) = headers.get("content-length") - && let Ok(size_str) = content_length.to_str() - && let Ok(object_size) = size_str.parse::() - { - // Check if upload would exceed quota - super::quota::check_upload_quota(&account, &container, object_size, &credentials).await?; - } + let object_size = headers + .get("content-length") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + super::quota::check_upload_quota(&account, &container, object_size, &credentials).await?; // Check if versioning is enabled for this container if let Some(archive_container) = container::get_versions_location(&account, &container, &credentials).await? { @@ -1475,6 +1483,7 @@ fn swift_error_to_response(error: SwiftError) -> Response { SwiftError::Forbidden(msg) => (StatusCode::FORBIDDEN, msg.as_str()), SwiftError::NotFound(msg) => (StatusCode::NOT_FOUND, msg.as_str()), SwiftError::Conflict(msg) => (StatusCode::CONFLICT, msg.as_str()), + SwiftError::LengthRequired(msg) => (StatusCode::LENGTH_REQUIRED, msg.as_str()), SwiftError::RequestEntityTooLarge(msg) => (StatusCode::PAYLOAD_TOO_LARGE, msg.as_str()), SwiftError::UnprocessableEntity(msg) => (StatusCode::UNPROCESSABLE_ENTITY, msg.as_str()), SwiftError::TooManyRequests { .. } => (StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded"), @@ -1497,7 +1506,10 @@ fn swift_error_to_response(error: SwiftError) -> Response { #[cfg(test)] mod tests { - use super::parse_range_header; + use super::{parse_range_header, trusted_client_ip}; + use axum::http::Request; + use rustfs_trusted_proxies::ClientInfo; + use std::net::SocketAddr; #[test] fn test_parse_range_header_start_end() { // bytes=100-199 @@ -1588,4 +1600,41 @@ mod tests { let result = parse_range_header("bytes=0-999", 1000); assert_eq!(result, Some((0, 999))); } + + #[test] + fn trusted_client_ip_ignores_untrusted_forwarded_headers() { + let mut request = Request::builder() + .header("x-forwarded-for", "198.51.100.9") + .header("x-real-ip", "198.51.100.10") + .body(()) + .expect("request"); + request + .extensions_mut() + .insert("192.0.2.7:9000".parse::().expect("socket address")); + + assert_eq!(trusted_client_ip(&request), Some("192.0.2.7".parse().expect("peer IP address"))); + } + + #[test] + fn trusted_client_ip_prefers_validated_proxy_identity() { + let mut request = Request::new(()); + request + .extensions_mut() + .insert("192.0.2.7:9000".parse::().expect("socket address")); + request.extensions_mut().insert(ClientInfo::direct( + "203.0.113.8:443".parse::().expect("validated client address"), + )); + + assert_eq!(trusted_client_ip(&request), Some("203.0.113.8".parse().expect("validated client IP"))); + } + + #[test] + fn trusted_client_ip_rejects_unspecified_fallback() { + let mut request = Request::new(()); + request + .extensions_mut() + .insert(ClientInfo::direct("0.0.0.0:0".parse::().expect("unspecified fallback"))); + + assert_eq!(trusted_client_ip(&request), None); + } } diff --git a/crates/protocols/src/swift/mod.rs b/crates/protocols/src/swift/mod.rs index 528cce52b..2dba11b8d 100644 --- a/crates/protocols/src/swift/mod.rs +++ b/crates/protocols/src/swift/mod.rs @@ -61,7 +61,7 @@ pub use router::{SwiftRoute, SwiftRouter}; // Note: Container, Object, and SwiftMetadata types used by Swift implementation pub use storage_api::public_api::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader}; pub(crate) use storage_api::public_api::{ - get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata, + get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, set_swift_bucket_metadata, }; #[allow(unused_imports)] pub use types::{Container, Object, SwiftMetadata}; diff --git a/crates/protocols/src/swift/quota.rs b/crates/protocols/src/swift/quota.rs index 9c79c3a9d..308161592 100644 --- a/crates/protocols/src/swift/quota.rs +++ b/crates/protocols/src/swift/quota.rs @@ -87,18 +87,17 @@ pub struct QuotaConfig { impl QuotaConfig { /// Load quota configuration from container metadata pub async fn load(account: &str, container_name: &str, credentials: &Credentials) -> SwiftResult { - // Get container metadata - let container_info = container::get_container_metadata(account, container_name, credentials).await?; + let custom_metadata = container::get_container_custom_metadata(account, container_name, credentials).await?; let mut config = QuotaConfig::default(); // Parse Quota-Bytes - if let Some(quota_bytes_str) = container_info.custom_metadata.get("x-container-meta-quota-bytes") { + if let Some(quota_bytes_str) = custom_metadata.get("x-container-meta-quota-bytes") { config.quota_bytes = quota_bytes_str.parse().ok(); } // Parse Quota-Count - if let Some(quota_count_str) = container_info.custom_metadata.get("x-container-meta-quota-count") { + if let Some(quota_count_str) = custom_metadata.get("x-container-meta-quota-count") { config.quota_count = quota_count_str.parse().ok(); } @@ -113,9 +112,12 @@ impl QuotaConfig { /// Check if adding an object would exceed quotas /// /// Returns Ok(()) if within quota, Err with 413 if exceeded - pub fn check_quota(&self, current_bytes: u64, current_count: u64, additional_bytes: u64) -> SwiftResult<()> { + pub fn check_quota(&self, current_bytes: u64, current_count: u64, additional_bytes: Option) -> SwiftResult<()> { // Check byte quota if let Some(max_bytes) = self.quota_bytes { + let additional_bytes = additional_bytes.ok_or_else(|| { + SwiftError::LengthRequired("Content-Length is required when a byte quota is configured".to_string()) + })?; let new_bytes = current_bytes.saturating_add(additional_bytes); if new_bytes > max_bytes { return Err(SwiftError::RequestEntityTooLarge(format!( @@ -147,7 +149,7 @@ impl QuotaConfig { pub async fn check_upload_quota( account: &str, container_name: &str, - object_size: u64, + object_size: Option, credentials: &Credentials, ) -> SwiftResult<()> { // Load quota config @@ -157,7 +159,6 @@ pub async fn check_upload_quota( if !quota.is_enabled() { return Ok(()); } - // Get current container usage let metadata = container::get_container_metadata(account, container_name, credentials).await?; @@ -173,7 +174,7 @@ pub async fn check_upload_quota( quota_bytes = ?quota.quota_bytes, current_count = metadata.object_count, quota_count = ?quota.quota_count, - object_size, + object_size = ?object_size, "swift quota state changed" ); @@ -235,7 +236,7 @@ mod tests { }; // Current: 500 bytes, adding 400 bytes = 900 total (within 1000 limit) - let result = config.check_quota(500, 0, 400); + let result = config.check_quota(500, 0, Some(400)); assert!(result.is_ok()); } @@ -247,7 +248,7 @@ mod tests { }; // Current: 500 bytes, adding 600 bytes = 1100 total (exceeds 1000 limit) - let result = config.check_quota(500, 0, 600); + let result = config.check_quota(500, 0, Some(600)); assert!(result.is_err()); match result { Err(SwiftError::RequestEntityTooLarge(msg)) => { @@ -265,7 +266,7 @@ mod tests { }; // Current: 500 bytes, adding 500 bytes = 1000 total (exactly at limit) - let result = config.check_quota(500, 0, 500); + let result = config.check_quota(500, 0, Some(500)); assert!(result.is_ok()); } @@ -277,7 +278,7 @@ mod tests { }; // Current: 5 objects, adding 1 = 6 total (within 10 limit) - let result = config.check_quota(0, 5, 100); + let result = config.check_quota(0, 5, Some(100)); assert!(result.is_ok()); } @@ -289,7 +290,7 @@ mod tests { }; // Current: 10 objects, adding 1 = 11 total (exceeds 10 limit) - let result = config.check_quota(0, 10, 100); + let result = config.check_quota(0, 10, Some(100)); assert!(result.is_err()); match result { Err(SwiftError::RequestEntityTooLarge(msg)) => { @@ -307,7 +308,7 @@ mod tests { }; // Current: 9 objects, adding 1 = 10 total (exactly at limit) - let result = config.check_quota(0, 9, 100); + let result = config.check_quota(0, 9, Some(100)); assert!(result.is_ok()); } @@ -319,7 +320,7 @@ mod tests { }; // Both within limits - let result = config.check_quota(500, 5, 400); + let result = config.check_quota(500, 5, Some(400)); assert!(result.is_ok()); } @@ -331,7 +332,7 @@ mod tests { }; // Bytes exceeded, count within - let result = config.check_quota(500, 5, 600); + let result = config.check_quota(500, 5, Some(600)); assert!(result.is_err()); } @@ -343,7 +344,7 @@ mod tests { }; // Count exceeded, bytes within - let result = config.check_quota(500, 10, 100); + let result = config.check_quota(500, 10, Some(100)); assert!(result.is_err()); } @@ -355,7 +356,7 @@ mod tests { }; // No limits, should always pass - let result = config.check_quota(999999, 999999, 999999); + let result = config.check_quota(999999, 999999, Some(999999)); assert!(result.is_ok()); } @@ -367,7 +368,7 @@ mod tests { }; // Zero limit means no uploads allowed - let result = config.check_quota(0, 0, 1); + let result = config.check_quota(0, 0, Some(1)); assert!(result.is_err()); } @@ -379,7 +380,7 @@ mod tests { }; // Zero limit means no objects allowed - let result = config.check_quota(0, 0, 100); + let result = config.check_quota(0, 0, Some(100)); assert!(result.is_err()); } @@ -391,8 +392,28 @@ mod tests { }; // Test saturating_add protection - let result = config.check_quota(u64::MAX - 100, 0, 200); + let result = config.check_quota(u64::MAX - 100, 0, Some(200)); // Should saturate to u64::MAX and compare against quota assert!(result.is_ok()); } + + #[test] + fn byte_quota_rejects_upload_without_content_length() { + let config = QuotaConfig { + quota_bytes: Some(1000), + quota_count: None, + }; + + assert!(matches!(config.check_quota(500, 0, None), Err(SwiftError::LengthRequired(_)))); + } + + #[test] + fn count_only_quota_checks_upload_without_content_length() { + let config = QuotaConfig { + quota_bytes: None, + quota_count: Some(10), + }; + + assert!(matches!(config.check_quota(0, 10, None), Err(SwiftError::RequestEntityTooLarge(_)))); + } } diff --git a/crates/protocols/src/swift/storage_api.rs b/crates/protocols/src/swift/storage_api.rs index 8dc6d3430..b65c711e5 100644 --- a/crates/protocols/src/swift/storage_api.rs +++ b/crates/protocols/src/swift/storage_api.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::sync::Arc; pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata as SwiftBucketMetadata; @@ -43,7 +44,9 @@ pub(crate) mod object { pub(crate) mod public_api { pub use super::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader}; - pub(crate) use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata}; + pub(crate) use super::{ + get_swift_bucket_metadata, get_swift_bucket_usage, resolve_swift_object_store_handle, set_swift_bucket_metadata, + }; } pub(crate) mod versioning { @@ -62,3 +65,18 @@ pub(crate) async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResul pub(crate) async fn set_swift_bucket_metadata(bucket: String, metadata: SwiftBucketMetadata) -> SwiftStorageResult<()> { set_swift_bucket_metadata_in_backend(bucket, metadata).await } + +pub(crate) async fn get_swift_bucket_usage() -> SwiftStorageResult>> { + let Some(store) = resolve_swift_object_store_handle() else { + return Ok(None); + }; + let mut data_usage = rustfs_ecstore::api::data_usage::load_data_usage_from_backend_cached(store).await?; + rustfs_ecstore::api::data_usage::apply_bucket_usage_memory_overlay(&mut data_usage).await; + Ok(Some( + data_usage + .buckets_usage + .into_iter() + .map(|(bucket, usage)| (bucket, (usage.objects_count, usage.size))) + .collect(), + )) +} diff --git a/crates/protocols/src/swift/tempurl.rs b/crates/protocols/src/swift/tempurl.rs index 8c96b17c2..2c0e7fbad 100644 --- a/crates/protocols/src/swift/tempurl.rs +++ b/crates/protocols/src/swift/tempurl.rs @@ -21,7 +21,11 @@ use crate::swift::errors::SwiftError; use hmac::{Hmac, KeyInit, Mac}; +use ipnetwork::IpNetwork; +use percent_encoding::percent_decode_str; use sha1::Sha1; +use std::net::IpAddr; +use std::str::FromStr; use std::time::{SystemTime, UNIX_EPOCH}; type HmacSha1 = Hmac; @@ -50,12 +54,12 @@ impl TempURLParams { let mut ip_range = None; for param in query.split('&') { - let parts: Vec<&str> = param.split('=').collect(); - if parts.len() == 2 { - match parts[0] { - "temp_url_sig" => sig = Some(parts[1].to_string()), - "temp_url_expires" => expires = parts[1].parse().ok(), - "temp_url_ip_range" => ip_range = Some(parts[1].to_string()), + if let Some((name, value)) = param.split_once('=') { + let value = percent_decode_str(value).decode_utf8().ok()?; + match name { + "temp_url_sig" => sig = Some(value.into_owned()), + "temp_url_expires" => expires = value.parse().ok(), + "temp_url_ip_range" => ip_range = Some(value.into_owned()), _ => {} } } @@ -97,9 +101,22 @@ impl TempURL { /// # Returns /// Hex-encoded HMAC-SHA1 signature pub fn generate_signature(&self, method: &str, expires: u64, path: &str) -> Result { + self.generate_signature_with_ip_range(method, expires, path, None) + } + + /// Generate a TempURL signature, optionally binding it to an IP range. + pub fn generate_signature_with_ip_range( + &self, + method: &str, + expires: u64, + path: &str, + ip_range: Option<&str>, + ) -> Result { // Construct message for HMAC - // Format: "{METHOD}\n{expires}\n{path}" - let message = format!("{}\n{}\n{}", method.to_uppercase(), expires, path); + let message = match ip_range { + Some(ip_range) => format!("ip={}\n{}\n{}\n{}", ip_range, method.to_uppercase(), expires, path), + None => format!("{}\n{}\n{}", method.to_uppercase(), expires, path), + }; // Calculate HMAC-SHA1 let mut mac = HmacSha1::new_from_slice(self.key.as_bytes()) @@ -127,7 +144,13 @@ impl TempURL { /// # Returns /// - `Ok(())` if signature is valid and not expired /// - `Err(SwiftError::Unauthorized)` if invalid or expired - pub fn validate_request(&self, method: &str, path: &str, params: &TempURLParams) -> Result<(), SwiftError> { + pub fn validate_request( + &self, + method: &str, + path: &str, + params: &TempURLParams, + client_ip: Option, + ) -> Result<(), SwiftError> { // 1. Check expiration first (fast path for expired URLs) let now = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -139,17 +162,25 @@ impl TempURL { } // 2. Generate expected signature - let expected_sig = self.generate_signature(method, params.temp_url_expires, path)?; + let expected_sig = + self.generate_signature_with_ip_range(method, params.temp_url_expires, path, params.temp_url_ip_range.as_deref())?; // 3. Constant-time comparison to prevent timing attacks if !constant_time_compare(params.temp_url_sig.as_bytes(), expected_sig.as_bytes()) { return Err(SwiftError::Unauthorized("Invalid TempURL signature".to_string())); } - // 4. TODO: Validate IP range if specified (future enhancement) - // if let Some(ip_range) = ¶ms.temp_url_ip_range { - // validate_ip_range(client_ip, ip_range)?; - // } + if let Some(ip_range) = ¶ms.temp_url_ip_range { + let client_ip = + client_ip.ok_or_else(|| SwiftError::Unauthorized("Trusted client address unavailable".to_string()))?; + let allowed = IpAddr::from_str(ip_range) + .map(|ip| ip == client_ip) + .or_else(|_| IpNetwork::from_str(ip_range).map(|network| network.contains(client_ip))) + .map_err(|_| SwiftError::Unauthorized("Invalid TempURL IP range".to_string()))?; + if !allowed { + return Err(SwiftError::Unauthorized("Client address is outside the TempURL IP range".to_string())); + } + } Ok(()) } @@ -308,7 +339,7 @@ mod tests { // Should validate successfully assert!( tempurl - .validate_request("GET", "/v1/AUTH_test/container/object", ¶ms) + .validate_request("GET", "/v1/AUTH_test/container/object", ¶ms, None) .is_ok() ); } @@ -331,7 +362,7 @@ mod tests { }; // Should reject expired URL - let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms); + let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms, None); assert!(result.is_err()); assert!(matches!(result.unwrap_err(), SwiftError::Unauthorized(_))); } @@ -349,7 +380,7 @@ mod tests { }; // Should reject invalid signature - let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms); + let result = tempurl.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms, None); assert!(result.is_err()); assert!(matches!(result.unwrap_err(), SwiftError::Unauthorized(_))); } @@ -372,7 +403,7 @@ mod tests { }; // Try to validate with PUT method - let result = tempurl.validate_request("PUT", "/v1/AUTH_test/container/object", ¶ms); + let result = tempurl.validate_request("PUT", "/v1/AUTH_test/container/object", ¶ms, None); assert!(result.is_err()); assert!(matches!(result.unwrap_err(), SwiftError::Unauthorized(_))); } @@ -423,6 +454,14 @@ mod tests { assert_eq!(params.temp_url_ip_range.as_deref(), Some("192.168.1.0/24")); } + #[test] + fn test_parse_percent_encoded_ipv6_range() { + let query = "temp_url_sig=abc123&temp_url_expires=1609459200&temp_url_ip_range=2001%3Adb8%3A%3A%2F32"; + let params = TempURLParams::from_query(query).expect("encoded IPv6 TempURL parameters"); + + assert_eq!(params.temp_url_ip_range.as_deref(), Some("2001:db8::/32")); + } + #[test] fn test_parse_tempurl_params_missing_sig() { let query = "temp_url_expires=1609459200"; @@ -475,4 +514,103 @@ mod tests { .unwrap(); assert_eq!(sig, sig2); } + + #[test] + fn ip_bound_signature_matches_openstack_message_format() { + let tempurl = TempURL::new("mykey".to_string()); + let signature = tempurl + .generate_signature_with_ip_range("GET", 1648082711, "/v1/AUTH_account/container/object", Some("1.2.3.0/24")) + .expect("IP-bound signature generation"); + + assert_eq!(signature, "8698990d811e64cff1c8a2151340874fd5bda0c5"); + } + + fn ip_bound_params(tempurl: &TempURL, ip_range: &str) -> TempURLParams { + let expires = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock after Unix epoch") + .as_secs() + + 3600; + let signature = tempurl + .generate_signature_with_ip_range("GET", expires, "/v1/AUTH_test/container/object", Some(ip_range)) + .expect("IP-bound signature generation"); + TempURLParams { + temp_url_sig: signature, + temp_url_expires: expires, + temp_url_ip_range: Some(ip_range.to_string()), + } + } + + #[test] + fn ip_bound_tempurl_accepts_ipv4_inside_range_and_rejects_outside() { + let tempurl = TempURL::new("mykey".to_string()); + let params = ip_bound_params(&tempurl, "192.0.2.0/24"); + + assert!( + tempurl + .validate_request( + "GET", + "/v1/AUTH_test/container/object", + ¶ms, + Some("192.0.2.42".parse().expect("IPv4 address")), + ) + .is_ok() + ); + assert!(matches!( + tempurl.validate_request( + "GET", + "/v1/AUTH_test/container/object", + ¶ms, + Some("198.51.100.42".parse().expect("IPv4 address")), + ), + Err(SwiftError::Unauthorized(_)) + )); + } + + #[test] + fn ip_bound_tempurl_accepts_ipv6_inside_range_and_rejects_outside() { + let tempurl = TempURL::new("mykey".to_string()); + let params = ip_bound_params(&tempurl, "2001:db8::/32"); + + assert!( + tempurl + .validate_request( + "GET", + "/v1/AUTH_test/container/object", + ¶ms, + Some("2001:db8::42".parse().expect("IPv6 address")), + ) + .is_ok() + ); + assert!(matches!( + tempurl.validate_request( + "GET", + "/v1/AUTH_test/container/object", + ¶ms, + Some("2001:db9::42".parse().expect("IPv6 address")), + ), + Err(SwiftError::Unauthorized(_)) + )); + } + + #[test] + fn ip_bound_tempurl_rejects_missing_client_ip_and_invalid_range() { + let tempurl = TempURL::new("mykey".to_string()); + let params = ip_bound_params(&tempurl, "192.0.2.0/24"); + assert!(matches!( + tempurl.validate_request("GET", "/v1/AUTH_test/container/object", ¶ms, None), + Err(SwiftError::Unauthorized(_)) + )); + + let invalid_params = ip_bound_params(&tempurl, "not-a-network"); + assert!(matches!( + tempurl.validate_request( + "GET", + "/v1/AUTH_test/container/object", + &invalid_params, + Some("192.0.2.42".parse().expect("IPv4 address")), + ), + Err(SwiftError::Unauthorized(_)) + )); + } } diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 6c7fc7fd5..8bc8a386f 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -233,6 +233,46 @@ pub struct RenamePartResponse { pub error: ::core::option::Option, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PreparePartTransactionRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub src_volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub src_path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub dst_volume: ::prost::alloc::string::String, + #[prost(string, tag = "5")] + pub dst_path: ::prost::alloc::string::String, + #[prost(bytes = "bytes", tag = "6")] + pub meta: ::prost::bytes::Bytes, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PreparePartTransactionResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(message, optional, tag = "2")] + pub error: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SettlePartTransactionRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(bool, tag = "4")] + pub rollback: bool, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SettlePartTransactionResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(message, optional, tag = "2")] + pub error: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RenameFileRequest { #[prost(string, tag = "1")] pub disk: ::prost::alloc::string::String, @@ -1600,6 +1640,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "CheckParts")); self.inner.unary(req, path, codec).await } + pub async fn prepare_part_transaction( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/PreparePartTransaction"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "PreparePartTransaction")); + self.inner.unary(req, path, codec).await + } pub async fn rename_part( &mut self, request: impl tonic::IntoRequest, @@ -1615,6 +1670,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "RenamePart")); self.inner.unary(req, path, codec).await } + pub async fn settle_part_transaction( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/SettlePartTransaction"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "SettlePartTransaction")); + self.inner.unary(req, path, codec).await + } pub async fn rename_file( &mut self, request: impl tonic::IntoRequest, @@ -2726,10 +2796,18 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn prepare_part_transaction( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn rename_part( &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn settle_part_transaction( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn rename_file( &self, request: tonic::Request, @@ -3432,6 +3510,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/PreparePartTransaction" => { + #[allow(non_camel_case_types)] + struct PreparePartTransactionSvc(pub Arc); + impl tonic::server::UnaryService for PreparePartTransactionSvc { + type Response = super::PreparePartTransactionResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::prepare_part_transaction(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = PreparePartTransactionSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/RenamePart" => { #[allow(non_camel_case_types)] struct RenamePartSvc(pub Arc); @@ -3460,6 +3566,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/SettlePartTransaction" => { + #[allow(non_camel_case_types)] + struct SettlePartTransactionSvc(pub Arc); + impl tonic::server::UnaryService for SettlePartTransactionSvc { + type Response = super::SettlePartTransactionResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::settle_part_transaction(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = SettlePartTransactionSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/RenameFile" => { #[allow(non_camel_case_types)] struct RenameFileSvc(pub Arc); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 6a1c96d69..165ac3c52 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -600,6 +600,30 @@ pub fn canonical_rename_part_request_body( Ok(body.finish()) } +pub fn canonical_prepare_part_transaction_request_body( + request: &proto_gen::node_service::PreparePartTransactionRequest, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-prepare-part-transaction-request-v1\0"); + body.push_str(&request.disk)?; + body.push_str(&request.src_volume)?; + body.push_str(&request.src_path)?; + body.push_str(&request.dst_volume)?; + body.push_str(&request.dst_path)?; + body.push_bytes(&request.meta)?; + Ok(body.finish()) +} + +pub fn canonical_settle_part_transaction_request_body( + request: &proto_gen::node_service::SettlePartTransactionRequest, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-settle-part-transaction-request-v1\0"); + body.push_str(&request.disk)?; + body.push_str(&request.volume)?; + body.push_str(&request.path)?; + body.push_bool(request.rollback); + Ok(body.finish()) +} + pub fn canonical_delete_volume_request_body( request: &proto_gen::node_service::DeleteVolumeRequest, ) -> Result, std::num::TryFromIntError> { @@ -637,8 +661,8 @@ pub fn canonical_make_volumes_request_body( mod disk_mutation_canonical_tests { use super::proto_gen::node_service::{ DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, MakeVolumeRequest, - MakeVolumesRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, UpdateMetadataRequest, WriteAllRequest, - WriteMetadataRequest, + MakeVolumesRequest, PreparePartTransactionRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, + SettlePartTransactionRequest, UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest, }; use super::*; @@ -886,6 +910,48 @@ mod disk_mutation_canonical_tests { } assert_all_distinct(&bodies); + let prepare_part = PreparePartTransactionRequest { + disk: "d".into(), + src_volume: "sv".into(), + src_path: "sp".into(), + dst_volume: "dv".into(), + dst_path: "dp".into(), + meta: vec![0x01].into(), + }; + let mut bodies = vec![canonical_prepare_part_transaction_request_body(&prepare_part).unwrap()]; + for mutate in [ + |r: &mut PreparePartTransactionRequest| r.disk = "d2".into(), + |r: &mut PreparePartTransactionRequest| r.src_volume = "sv2".into(), + |r: &mut PreparePartTransactionRequest| r.src_path = "sp2".into(), + |r: &mut PreparePartTransactionRequest| r.dst_volume = "dv2".into(), + |r: &mut PreparePartTransactionRequest| r.dst_path = "dp2".into(), + |r: &mut PreparePartTransactionRequest| r.meta = vec![0x02].into(), + ] { + let mut request = prepare_part.clone(); + mutate(&mut request); + bodies.push(canonical_prepare_part_transaction_request_body(&request).unwrap()); + } + assert_all_distinct(&bodies); + + let settle_part = SettlePartTransactionRequest { + disk: "d".into(), + volume: "v".into(), + path: "p".into(), + rollback: false, + }; + let mut bodies = vec![canonical_settle_part_transaction_request_body(&settle_part).unwrap()]; + for mutate in [ + |r: &mut SettlePartTransactionRequest| r.disk = "d2".into(), + |r: &mut SettlePartTransactionRequest| r.volume = "v2".into(), + |r: &mut SettlePartTransactionRequest| r.path = "p2".into(), + |r: &mut SettlePartTransactionRequest| r.rollback = true, + ] { + let mut request = settle_part.clone(); + mutate(&mut request); + bodies.push(canonical_settle_part_transaction_request_body(&request).unwrap()); + } + assert_all_distinct(&bodies); + let rename_part = RenamePartRequest { disk: "d".into(), src_volume: "sv".into(), diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index 3a0fc66ed..36df6b5cb 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -170,6 +170,32 @@ message RenamePartResponse { optional Error error = 2; } +message PreparePartTransactionRequest { + string disk = 1; + string src_volume = 2; + string src_path = 3; + string dst_volume = 4; + string dst_path = 5; + bytes meta = 6; +} + +message PreparePartTransactionResponse { + bool success = 1; + optional Error error = 2; +} + +message SettlePartTransactionRequest { + string disk = 1; + string volume = 2; + string path = 3; + bool rollback = 4; +} + +message SettlePartTransactionResponse { + bool success = 1; + optional Error error = 2; +} + message RenameFileRequest { string disk = 1; string src_volume = 2; @@ -943,7 +969,9 @@ service NodeService { rpc VerifyFile(VerifyFileRequest) returns (VerifyFileResponse) {}; rpc ReadParts(ReadPartsRequest) returns (ReadPartsResponse) {}; rpc CheckParts(CheckPartsRequest) returns (CheckPartsResponse) {}; + rpc PreparePartTransaction(PreparePartTransactionRequest) returns (PreparePartTransactionResponse) {}; rpc RenamePart(RenamePartRequest) returns (RenamePartResponse) {}; + rpc SettlePartTransaction(SettlePartTransactionRequest) returns (SettlePartTransactionResponse) {}; rpc RenameFile(RenameFileRequest) returns (RenameFileResponse) {}; rpc Write(WriteRequest) returns (WriteResponse) {}; rpc WriteStream(stream WriteRequest) returns (stream WriteResponse) {}; diff --git a/crates/s3-types/src/event_name.rs b/crates/s3-types/src/event_name.rs index 427cc9d20..fe0ed12f4 100644 --- a/crates/s3-types/src/event_name.rs +++ b/crates/s3-types/src/event_name.rs @@ -271,7 +271,11 @@ impl EventName { EventName::ObjectCreatedPut, ], EventName::ObjectTaggingAll => vec![EventName::ObjectTaggingPut, EventName::ObjectTaggingDelete], - EventName::ObjectRemovedAll => vec![EventName::ObjectRemovedDelete, EventName::ObjectRemovedDeleteMarkerCreated], + EventName::ObjectRemovedAll => vec![ + EventName::ObjectRemovedDelete, + EventName::ObjectRemovedDeleteMarkerCreated, + EventName::ObjectRemovedNoOP, + ], EventName::ObjectReplicationAll => vec![ EventName::ObjectReplicationFailed, EventName::ObjectReplicationComplete, @@ -639,6 +643,15 @@ mod tests { assert_eq!(EventName::parse("s3:Scanner:*").unwrap(), EventName::ObjectScannerAll); } + #[test] + fn test_object_removed_all_includes_noop_extension() { + let expanded = EventName::ObjectRemovedAll.expand(); + + assert!(expanded.contains(&EventName::ObjectRemovedDelete)); + assert!(expanded.contains(&EventName::ObjectRemovedDeleteMarkerCreated)); + assert!(expanded.contains(&EventName::ObjectRemovedNoOP)); + } + /// `is_removed` must be true for every `ObjectRemoved*` variant and false /// for everything else. #[test] diff --git a/deny.toml b/deny.toml index b676b20a9..0929d8659 100644 --- a/deny.toml +++ b/deny.toml @@ -40,6 +40,9 @@ allow-git = [ # SigV4 payload-checksum fix until it is available in a crates.io release. # owner: marshawcoco review: 2026-10 "https://github.com/s3s-project/s3s.git", + # Presigned URL expiry validation pending upstream submission. + # owner: cxymds review: 2026-10 + "https://github.com/cxymds/s3s.git", "https://github.com/apache/datafusion.git", ] diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index e89e4b8cd..cdd62a4b1 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -20,6 +20,7 @@ for later deletion. - `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus. - `backlog-1316` legacy encrypted multipart range seek: the feature remains opt-in until every server that can initiate, write, or complete multipart uploads supports the candidate-to-final marker protocol and uploadId commit lock, and pre-upgrade multipart uploads have drained. Remove the RUSTFS_ENCRYPTED_RANGE_SEEK switch after the minimum supported release does so; keep the quorum marker and malformed-layout full-read guards permanently. - `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection. +- `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object. ## Review Checklist diff --git a/docs/operations/rolling-restart.md b/docs/operations/rolling-restart.md index 076a0bc9f..83a52bfef 100644 --- a/docs/operations/rolling-restart.md +++ b/docs/operations/rolling-restart.md @@ -9,6 +9,16 @@ failure pattern reported in rustfs/rustfs#4304. > format. Replacing the executable and restarting is safe; no migration step > runs on startup. +> [!WARNING] +> The release that switches local SSE wrapped DEKs from the legacy +> `base64(nonce):base64(ciphertext)` representation to the versioned JSON +> envelope is a deliberate exception. Do not run that release together with +> an older RustFS version: older nodes cannot read objects written with the +> JSON envelope. Freeze every source of object mutation, including client +> writes and background lifecycle or replication work, upgrade every node, +> and then resume traffic. Downgrading or rolling back after new encrypted +> objects are written is not supported. + ## TL;DR - **Rolling restart (no downtime):** restart **one node at a time**, and wait diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index edcff88b1..e4753542d 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -36,6 +36,8 @@ use tracing::{error, info, instrument, warn}; /// Path to store KMS configuration in the cluster metadata const KMS_CONFIG_PATH: &str = "config/kms_config.json"; +const STATIC_KMS_LOCAL_CONFIG_REQUIRED: &str = + "Static KMS must be configured through RUSTFS_KMS_STATIC_SECRET_KEY or RUSTFS_KMS_STATIC_SECRET_KEY_FILE"; const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_KMS: &str = "kms"; const EVENT_ADMIN_KMS_DYNAMIC_STATE: &str = "admin_kms_dynamic_state"; @@ -106,9 +108,25 @@ fn normalize_configure_request_auth( Ok(()) } +fn ensure_kms_config_persistable(config: &KmsConfig) -> Result<(), String> { + if matches!(&config.backend_config, rustfs_kms::BackendConfig::Static(_)) { + return Err(STATIC_KMS_LOCAL_CONFIG_REQUIRED.to_string()); + } + Ok(()) +} + +fn ensure_kms_request_persistable(request: &ConfigureKmsRequest) -> Result<(), String> { + if matches!(request, ConfigureKmsRequest::Static(_)) { + return Err(STATIC_KMS_LOCAL_CONFIG_REQUIRED.to_string()); + } + Ok(()) +} + /// Save KMS configuration to cluster storage #[instrument(skip(config))] async fn save_kms_config(config: &KmsConfig) -> Result<(), String> { + ensure_kms_config_persistable(config)?; + let context = current_app_context(); let Some(store) = current_object_store_handle_for_context(context.as_deref()) else { return Err("Storage layer not initialized".to_string()); @@ -332,12 +350,16 @@ impl Operation for ConfigureKmsHandler { ); let service_manager = kms_service_manager_from_context(); - let existing_config = service_manager.get_config().await; + let existing_config = service_manager.get_redacted_config().await; if let Err(e) = normalize_configure_request_auth(&mut configure_request, existing_config.as_ref()) { return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e)))); } + if let Err(e) = ensure_kms_request_persistable(&configure_request) { + return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e)))); + } + // Convert request to KmsConfig let kms_config = configure_request.to_kms_config(); @@ -753,7 +775,7 @@ impl Operation for GetKmsStatusHandler { let service_manager = kms_service_manager_from_context(); let status = service_manager.get_status().await; - let config = service_manager.get_config().await; + let config = service_manager.get_redacted_config().await; // Get backend type and health status let backend_type = config.as_ref().map(|c| c.backend.clone()); @@ -873,12 +895,16 @@ impl Operation for ReconfigureKmsHandler { ); let service_manager = kms_service_manager_from_context(); - let existing_config = service_manager.get_config().await; + let existing_config = service_manager.get_redacted_config().await; if let Err(e) = normalize_configure_request_auth(&mut configure_request, existing_config.as_ref()) { return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e)))); } + if let Err(e) = ensure_kms_request_persistable(&configure_request) { + return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e)))); + } + // Convert request to KmsConfig let kms_config = configure_request.to_kms_config(); @@ -960,7 +986,7 @@ impl Operation for ReconfigureKmsHandler { #[cfg(test)] mod tests { - use super::{decode_persisted_kms_config, kms_configure_actions, kms_service_control_actions}; + use super::{decode_persisted_kms_config, ensure_kms_config_persistable, kms_configure_actions, kms_service_control_actions}; use rustfs_policy::policy::action::{Action, AdminAction, KmsAction}; use tempfile::TempDir; @@ -1046,4 +1072,16 @@ mod tests { assert!(!config.allow_insecure_dev_defaults); assert!(config.validate().is_ok()); } + + #[test] + fn static_kms_config_is_not_persisted_with_cluster_configuration() { + use base64::Engine as _; + + let config = rustfs_kms::KmsConfig::static_kms( + "static-key".to_string(), + base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]), + ); + + assert!(ensure_kms_config_persistable(&config).is_err()); + } } diff --git a/rustfs/src/admin/handlers/kms_management.rs b/rustfs/src/admin/handlers/kms_management.rs index b958654ab..bbeca1d90 100644 --- a/rustfs/src/admin/handlers/kms_management.rs +++ b/rustfs/src/admin/handlers/kms_management.rs @@ -193,7 +193,7 @@ impl Operation for KmsStatusHandler { hit_count: hits, miss_count: misses, }); - let config = kms_service_manager_from_context().get_config().await; + let config = kms_service_manager_from_context().get_redacted_config().await; let response = KmsStatusResponse { backend_type: config @@ -243,7 +243,7 @@ impl Operation for KmsConfigHandler { }; let config = kms_service_manager_from_context() - .get_config() + .get_redacted_config() .await .ok_or_else(|| s3_error!(InternalError, "KMS config not available"))?; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index dd94e0a15..0fe1474a1 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -1810,7 +1810,7 @@ impl DefaultBucketUsecase { client_info, ); - let read_allowed = PolicySys::is_allowed(&BucketPolicyArgs { + let read_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs { bucket: &bucket, action: Action::S3Action(S3Action::ListBucketAction), is_owner: false, @@ -1819,9 +1819,10 @@ impl DefaultBucketUsecase { conditions: &conditions, object: "", }) - .await; + .await + .map_err(ApiError::from)?; - let write_allowed = PolicySys::is_allowed(&BucketPolicyArgs { + let write_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs { bucket: &bucket, action: Action::S3Action(S3Action::PutObjectAction), is_owner: false, @@ -1830,7 +1831,8 @@ impl DefaultBucketUsecase { conditions: &conditions, object: "", }) - .await; + .await + .map_err(ApiError::from)?; let mut is_public = read_allowed || write_allowed; let ignore_public_acls = match metadata_sys::get_public_access_block_config(&bucket).await { diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 5d4e208d1..a926be8f3 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -440,6 +440,12 @@ impl DefaultMultipartUsecase { } let Some(multipart_upload) = multipart_upload else { return Err(s3_error!(InvalidPart)) }; + let Some(parts) = multipart_upload.parts.filter(|parts| !parts.is_empty()) else { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "You must specify at least one part".to_string(), + )); + }; let mut opts = get_complete_multipart_upload_opts(&req.headers).map_err(ApiError::from)?; let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; @@ -449,12 +455,7 @@ impl DefaultMultipartUsecase { let capacity_scope_token = Uuid::new_v4(); opts.capacity_scope_token = Some(capacity_scope_token); - let uploaded_parts_vec = multipart_upload - .parts - .unwrap_or_default() - .into_iter() - .map(complete_part_from_s3) - .collect::>(); + let uploaded_parts_vec = parts.into_iter().map(complete_part_from_s3).collect::>(); let uploaded_parts = normalize_complete_multipart_parts(uploaded_parts_vec)?; @@ -1793,6 +1794,127 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InvalidPart); } + #[tokio::test] + async fn execute_complete_multipart_upload_rejects_missing_parts_list() { + use s3s::xml::Deserialize as _; + + let mut deserializer = s3s::xml::Deserializer::new(b""); + let multipart_upload = + CompletedMultipartUpload::deserialize(&mut deserializer).expect("empty multipart XML should decode"); + deserializer + .expect_eof() + .expect("empty multipart XML should be fully consumed"); + assert!(multipart_upload.parts.is_none()); + + let input = CompleteMultipartUploadInput::builder() + .bucket("bucket".to_string()) + .key("object".to_string()) + .upload_id("upload-id".to_string()) + .multipart_upload(Some(multipart_upload)) + .build() + .expect("complete multipart input should build"); + let req = build_request(input, Method::POST); + + let err = Box::pin(make_usecase().execute_complete_multipart_upload(req)) + .await + .expect_err("missing parts list must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[tokio::test] + #[serial_test::serial] + async fn rejected_empty_parts_preserve_existing_object_and_staging() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + + let bucket = format!("empty-complete-{}", Uuid::new_v4()); + let object = "existing-object"; + let existing_payload = b"existing object must survive"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create multipart regression bucket"); + + let mut existing_reader = PutObjReader::from_vec(existing_payload.to_vec()); + let existing_info = store + .put_object(&bucket, object, &mut existing_reader, &ObjectOptions::default()) + .await + .expect("write existing object"); + + let upload = store + .new_multipart_upload(&bucket, object, &ObjectOptions::default()) + .await + .expect("create multipart staging upload"); + let mut part_reader = PutObjReader::from_vec(b"staged part".to_vec()); + let staged_part = store + .put_object_part(&bucket, object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default()) + .await + .expect("write staged multipart part"); + + let input = CompleteMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .upload_id(upload.upload_id.clone()) + .multipart_upload(Some(CompletedMultipartUpload { parts: Some(Vec::new()) })) + .build() + .expect("complete multipart input should build"); + let err = DefaultMultipartUsecase::from_global() + .execute_complete_multipart_upload(build_request(input, Method::POST)) + .await + .expect_err("empty parts list must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + + let current_info = store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("existing object should remain readable"); + assert_eq!(current_info.size, existing_info.size); + assert_eq!(current_info.etag, existing_info.etag); + + let staged = store + .list_object_parts(&bucket, object, &upload.upload_id, None, 1000, &ObjectOptions::default()) + .await + .expect("multipart staging should remain available"); + assert_eq!(staged.parts.len(), 1); + assert_eq!(staged.parts[0].part_num, 1); + assert_eq!(staged.parts[0].etag, staged_part.etag); + + let staged_etag = staged_part.etag.expect("staged part should have an ETag"); + let input = CompleteMultipartUploadInput::builder() + .bucket(bucket) + .key(object.to_string()) + .upload_id(upload.upload_id) + .multipart_upload(Some(CompletedMultipartUpload { + parts: Some(vec![CompletedPart { + part_number: Some(1), + e_tag: Some(to_s3s_etag(&staged_etag)), + ..Default::default() + }]), + })) + .build() + .expect("complete multipart input should build"); + let completed = DefaultMultipartUsecase::from_global() + .execute_complete_multipart_upload(build_request(input, Method::POST)) + .await + .expect("staged part should remain completable after empty request rejection"); + assert!(completed.output.e_tag.is_some()); + let completed_info = store + .get_object_info( + completed + .output + .bucket + .as_deref() + .expect("complete response should include bucket"), + object, + &ObjectOptions::default(), + ) + .await + .expect("completed object should be readable"); + assert_eq!(completed_info.size, b"staged part".len() as i64); + } + #[tokio::test] async fn execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence() { let multipart_upload = CompletedMultipartUpload { diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 14f6e4796..a9828919e 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -763,7 +763,7 @@ async fn enqueue_transitioned_delete_cleanup( let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup); let je = if opts.delete_prefix { - tier_sweeper::transitioned_force_delete_journal_entry(&existing.transitioned_object) + tier_sweeper::transitioned_force_delete_journal_entry(&existing.transitioned_object, existing.transition_version_state) } else { let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok()); tier_sweeper::transitioned_delete_journal_entry( @@ -771,6 +771,7 @@ async fn enqueue_transitioned_delete_cleanup( opts.versioned, opts.version_suspended, &existing.transitioned_object, + existing.transition_version_state, ) }; let Some(mut je) = je else { @@ -3051,6 +3052,25 @@ fn can_skip_delete_objects_pre_stat( !bucket_lock_enabled && !replicate_deletes && delete_creates_delete_marker(opts) && accounting_creates_delete_marker } +fn complete_delete_noop( + helper: OperationHelper, + bucket: String, + key: String, + version_id: Option, +) -> (S3Result>, OperationHelper) { + let helper = helper + .event_name(EventName::ObjectRemovedNoOP) + .object(ObjectInfo { + name: key, + bucket, + ..Default::default() + }) + .version_id(version_id.unwrap_or_default()); + let result = Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT)); + let helper = helper.complete(&result); + (result, helper) +} + fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result { let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER) .map(|value| normalize_snowball_prefix(&value)) @@ -7057,7 +7077,7 @@ impl DefaultObjectUsecase { // TODO: Future optimization (separate PR) - If performance becomes critical under high delete load: // 1. Add a lightweight get_object_lock_info() that only fetches retention metadata // 2. Or use combined get-and-delete in storage layer with retention check callback - let get_opts: ObjectOptions = get_opts(&bucket, &key, version_id_clone, None, &req.headers) + let get_opts: ObjectOptions = get_opts(&bucket, &key, version_id_clone.clone(), None, &req.headers) .await .map_err(ApiError::from)?; let existing_object_info = match store.get_object_info(&bucket, &key, &get_opts).await { @@ -7100,9 +7120,8 @@ impl DefaultObjectUsecase { } if is_err_object_not_found(&err) || is_err_version_not_found(&err) { - // TODO: send event - - return Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT)); + let (result, _helper) = complete_delete_noop(helper, bucket, key, version_id_clone); + return result; } return Err(ApiError::from(err).into()); @@ -9665,7 +9684,7 @@ mod tests { #[tokio::test] #[serial_test::serial] - async fn transitioned_delete_cleanup_persists_identity_bound_and_legacy_journals() { + async fn transitioned_delete_cleanup_persists_known_state_and_rejects_unknown_state() { let store = crate::app::gating_test_env::shared_gating_ecstore().await; if current_app_context().is_none() { crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; @@ -9685,8 +9704,9 @@ mod tests { current.transitioned_object.tier = "WARM".to_string(); current.transitioned_object.name = "remote/identity-bound".to_string(); current.transitioned_object.version_id = "remote-version".to_string(); + current.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact; - let journal_name = |remote_object: &str, backend_identity: Option<[u8; 32]>| { + let journal_name = |remote_object: &str, backend_identity: Option<[u8; 32]>, version_id_exact: bool| { use sha2::{Digest, Sha256}; let mut hasher = Sha256::new(); @@ -9699,6 +9719,10 @@ mod tests { hasher.update([0]); hasher.update(backend_identity); } + if version_id_exact { + hasher.update([0]); + hasher.update(b"exact-version-id"); + } format!("ilm/tier-delete-journal/{}.json", rustfs_utils::crypto::hex(hasher.finalize().as_slice())) }; @@ -9708,7 +9732,7 @@ mod tests { let mut identity_bound = store .get_object_reader( ".rustfs.sys", - &journal_name("remote/identity-bound", Some(identity)), + &journal_name("remote/identity-bound", Some(identity), true), None, http::HeaderMap::new(), &ObjectOptions::default(), @@ -9721,11 +9745,14 @@ mod tests { .expect("identity-bound journal body should be readable"); let identity_bound: serde_json::Value = serde_json::from_slice(&identity_bound_data).expect("identity-bound journal should decode as JSON"); - assert_eq!(identity_bound["version"], serde_json::json!(2)); + assert_eq!(identity_bound["version"], serde_json::json!(4)); assert_eq!(identity_bound["backend_identity"], serde_json::json!(identity)); + assert_eq!(identity_bound["version_id_exact"], serde_json::json!(true)); + assert_eq!(identity_bound["version_state"], serde_json::json!("exact")); current.user_defined = Arc::new(HashMap::new()); current.transitioned_object.name = "remote/legacy".to_string(); + current.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown; enqueue_transitioned_delete_cleanup( store.clone(), "bucket", @@ -9737,24 +9764,21 @@ mod tests { Some(¤t), ) .await - .expect("legacy force-delete cleanup should persist a fail-closed v1 journal"); - let mut legacy = store + .expect("unknown force-delete cleanup should fail closed without a journal"); + let legacy_err = match store .get_object_reader( ".rustfs.sys", - &journal_name("remote/legacy", None), + &journal_name("remote/legacy", None, false), None, http::HeaderMap::new(), &ObjectOptions::default(), ) .await - .expect("legacy journal should be readable"); - let mut legacy_data = Vec::new(); - tokio::io::AsyncReadExt::read_to_end(&mut legacy.stream, &mut legacy_data) - .await - .expect("legacy journal body should be readable"); - let legacy: serde_json::Value = serde_json::from_slice(&legacy_data).expect("legacy journal should decode as JSON"); - assert_eq!(legacy["version"], serde_json::json!(1)); - assert_eq!(legacy["backend_identity"], serde_json::Value::Null); + { + Ok(_) => panic!("unknown remote version state must not persist a delete journal"), + Err(err) => err, + }; + assert!(matches!(legacy_err, StorageError::FileNotFound)); } async fn put_real_cold_fill_object(store: &Arc, bucket: &str, object: &str, body: &[u8]) -> ObjectInfo { @@ -13087,6 +13111,40 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); } + #[test] + fn delete_not_found_completes_noop_event_with_version_context() { + temp_env::with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"), || { + crate::server::refresh_notify_module_enabled(); + for (version_id, expected_version) in [(None, ""), (Some("requested-version".to_string()), "requested-version")] { + let input = DeleteObjectInput::builder() + .bucket("test-bucket".to_string()) + .key("missing-key".to_string()) + .version_id(version_id.clone()) + .build() + .expect("delete input should build"); + let mut req = build_request(input, Method::DELETE); + req.extensions.insert(crate::storage::access::ReqInfo { + bucket: Some("test-bucket".to_string()), + object: Some("missing-key".to_string()), + version_id: version_id.clone(), + ..Default::default() + }); + let helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObject); + + let (result, helper) = + complete_delete_noop(helper, "test-bucket".to_string(), "missing-key".to_string(), version_id); + let event = helper.event_args().expect("successful no-op delete should retain an event"); + + assert_eq!(result.expect("no-op delete should succeed").status, Some(StatusCode::NO_CONTENT)); + assert_eq!(event.event_name, EventName::ObjectRemovedNoOP); + assert_eq!(event.bucket_name, "test-bucket"); + assert_eq!(event.object.name, "missing-key"); + assert_eq!(event.version_id, expected_version); + } + }); + crate::server::refresh_notify_module_enabled(); + } + #[test] fn expected_current_version_header_normalizes_uuid_and_null() { let version = Uuid::new_v4(); diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 06765ac96..b97477b9a 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -409,20 +409,24 @@ pub(crate) mod bucket { versioned: bool, suspended: bool, transitioned: &super::super::super::storage_contracts::TransitionedObject, + transition_version_state: rustfs_filemeta::TransitionVersionState, ) -> Option { crate::storage::storage_api::ecstore_bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry( version_id, versioned, suspended, transitioned, + transition_version_state, ) } pub(crate) fn transitioned_force_delete_journal_entry( transitioned: &super::super::super::storage_contracts::TransitionedObject, + transition_version_state: rustfs_filemeta::TransitionVersionState, ) -> Option { crate::storage::storage_api::ecstore_bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry( transitioned, + transition_version_state, ) } } diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 831f7314a..63e6f3bf8 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -245,6 +245,21 @@ impl From for ApiError { }; } + if let StorageError::Io(ref io_err) = err + && matches!( + io_err + .get_ref() + .and_then(|inner| inner.downcast_ref::()), + Some(rustfs_kms::KmsError::BackendError { .. }) + ) + { + return ApiError { + code: S3ErrorCode::ServiceUnavailable, + message: ApiError::error_code_to_message(&S3ErrorCode::ServiceUnavailable), + source: Some(Box::new(err)), + }; + } + let code = match &err { StorageError::NotImplemented => S3ErrorCode::NotImplemented, StorageError::InvalidArgument(_, _, _) => S3ErrorCode::InvalidArgument, @@ -479,6 +494,14 @@ mod tests { assert!(downcast_storage_error.is_some()); } + #[test] + fn policy_metadata_read_failure_maps_to_internal_error() { + let api_error = ApiError::from(StorageError::Io(IoError::other("policy read failed"))); + + assert_eq!(api_error.code, S3ErrorCode::InternalError); + assert!(api_error.source.is_some()); + } + #[test] fn test_kms_service_unavailable_maps_to_retryable_error() { let api_error = ApiError::from(StorageError::other(KmsUnavailableError)); @@ -487,6 +510,14 @@ mod tests { assert_eq!(api_error.message, "The service is unavailable. Please retry."); } + #[test] + fn test_kms_backend_unavailable_maps_to_retryable_error() { + let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::backend_error("Vault connection refused"))); + + assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable); + assert_eq!(api_error.message, "The service is unavailable. Please retry."); + } + #[test] fn test_unknown_authoritative_quota_usage_maps_to_retryable_error() { let api_error = ApiError::from(QuotaError::UsageUnavailable { @@ -497,6 +528,16 @@ mod tests { assert_eq!(api_error.message, "The service is unavailable. Please retry."); } + #[test] + fn test_kms_cryptographic_error_is_not_retryable() { + let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::cryptographic_error( + "decrypt", + "authentication failed", + ))); + + assert_eq!(api_error.code, S3ErrorCode::InternalError); + } + #[test] fn test_api_error_from_storage_error_mappings() { let test_cases = vec![ diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 35360763f..a5df755ee 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -505,7 +505,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R let owner_can_bypass_deny = owner_can_bypass_policy_deny(is_owner, &action); if !bucket_name.is_empty() && !owner_can_bypass_deny - && !PolicySys::is_allowed(&BucketPolicyArgs { + && !PolicySys::try_is_allowed(&BucketPolicyArgs { bucket: bucket_name, action, // Early explicit-deny gate for bucket policy: use owner short-circuit path so @@ -517,6 +517,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R object: object.as_str(), }) .await + .map_err(ApiError::from)? { return Err(s3_error!(AccessDenied, "Access Denied")); } @@ -535,7 +536,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R }; let delete_version_allowed = iam_store.eval_prepared(&prepared, &delete_version_args).await; if !delete_version_allowed - && !PolicySys::is_allowed(&BucketPolicyArgs { + && !PolicySys::try_is_allowed(&BucketPolicyArgs { bucket: bucket.as_str(), action: Action::S3Action(S3Action::DeleteObjectVersionAction), is_owner, @@ -545,6 +546,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R object: object.as_str(), }) .await + .map_err(ApiError::from)? { return Err(s3_error!(AccessDenied, "Access Denied")); } @@ -571,7 +573,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R return Ok(()); } - let policy_allowed_fallback = PolicySys::is_allowed(&BucketPolicyArgs { + let policy_allowed_fallback = PolicySys::try_is_allowed(&BucketPolicyArgs { bucket: bucket.as_str(), action, is_owner, @@ -580,7 +582,8 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R conditions: &conditions, object: object.as_str(), }) - .await; + .await + .map_err(ApiError::from)?; if policy_allowed_fallback { authorize_table_data_plane_if_needed(action, bucket.as_str(), object.as_str(), cred, is_owner, &conditions, claims) @@ -605,7 +608,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R return Ok(()); } - if PolicySys::is_allowed(&BucketPolicyArgs { + if PolicySys::try_is_allowed(&BucketPolicyArgs { bucket: bucket.as_str(), action: Action::S3Action(S3Action::ListBucketAction), is_owner, @@ -615,6 +618,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R object: object.as_str(), }) .await + .map_err(ApiError::from)? { return Ok(()); } @@ -688,7 +692,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R let bucket_name = bucket.as_str(); if !bucket_name.is_empty() - && !PolicySys::is_allowed(&BucketPolicyArgs { + && !PolicySys::try_is_allowed(&BucketPolicyArgs { bucket: bucket_name, action, // Early explicit-deny gate for bucket policy in anonymous path. @@ -699,13 +703,14 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R object: object.as_str(), }) .await + .map_err(ApiError::from)? { return Err(s3_error!(AccessDenied, "Access Denied")); } if action != Action::S3Action(S3Action::ListAllMyBucketsAction) { if action == Action::S3Action(S3Action::DeleteObjectAction) && version_id.is_some() { - let delete_version_allowed = PolicySys::is_allowed(&BucketPolicyArgs { + let delete_version_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs { bucket: bucket.as_str(), action: Action::S3Action(S3Action::DeleteObjectVersionAction), is_owner: false, @@ -714,13 +719,14 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R conditions: &conditions, object: object.as_str(), }) - .await; + .await + .map_err(ApiError::from)?; if !delete_version_allowed { return Err(s3_error!(AccessDenied, "Access Denied")); } } - let policy_allowed = PolicySys::is_allowed(&BucketPolicyArgs { + let policy_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs { bucket: bucket.as_str(), action, is_owner: false, @@ -729,7 +735,8 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R conditions: &conditions, object: object.as_str(), }) - .await; + .await + .map_err(ApiError::from)?; // A bucket policy granting s3:ListBucket also covers listing versions. This // fallback has to feed the same post-authorization gates as the direct grant @@ -737,7 +744,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R // ListObjectVersions after RestrictPublicBuckets is turned on. let policy_allowed = policy_allowed || (action == Action::S3Action(S3Action::ListBucketVersionsAction) - && PolicySys::is_allowed(&BucketPolicyArgs { + && PolicySys::try_is_allowed(&BucketPolicyArgs { bucket: bucket.as_str(), action: Action::S3Action(S3Action::ListBucketAction), is_owner: false, @@ -746,7 +753,8 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R conditions: &conditions, object: "", }) - .await); + .await + .map_err(ApiError::from)?); if policy_allowed { deny_anonymous_table_data_plane_if_needed(action, bucket.as_str(), object.as_str()).await?; @@ -2763,7 +2771,7 @@ mod tests { } #[tokio::test] - async fn abort_multipart_upload_rejects_unauthorized_request() { + async fn abort_multipart_upload_fails_closed_when_policy_metadata_is_unavailable() { let fs = FS::new(); let mut req = build_request( AbortMultipartUploadInput::builder() @@ -2779,12 +2787,12 @@ mod tests { let err = fs .abort_multipart_upload(&mut req) .await - .expect_err("missing credentials should reject access"); - assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + .expect_err("unavailable policy metadata must fail closed"); + assert_eq!(err.code(), &S3ErrorCode::InternalError); } #[tokio::test] - async fn complete_multipart_upload_rejects_unauthorized_request() { + async fn complete_multipart_upload_fails_closed_when_policy_metadata_is_unavailable() { let fs = FS::new(); let mut req = build_request( CompleteMultipartUploadInput::builder() @@ -2801,12 +2809,12 @@ mod tests { let err = fs .complete_multipart_upload(&mut req) .await - .expect_err("missing credentials should reject access"); - assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + .expect_err("unavailable policy metadata must fail closed"); + assert_eq!(err.code(), &S3ErrorCode::InternalError); } #[tokio::test] - async fn upload_part_copy_rejects_unauthorized_request() { + async fn upload_part_copy_fails_closed_when_policy_metadata_is_unavailable() { let fs = FS::new(); let mut req = build_request( UploadPartCopyInput::builder() @@ -2828,7 +2836,7 @@ mod tests { let err = fs .upload_part_copy(&mut req) .await - .expect_err("missing credentials should reject access"); - assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + .expect_err("unavailable policy metadata must fail closed"); + assert_eq!(err.code(), &S3ErrorCode::InternalError); } } diff --git a/rustfs/src/storage/helper.rs b/rustfs/src/storage/helper.rs index ffe3ea700..768471e21 100644 --- a/rustfs/src/storage/helper.rs +++ b/rustfs/src/storage/helper.rs @@ -243,6 +243,14 @@ impl OperationHelper { matches!(self, Self::Enabled(state) if state.event_builder.is_some()) } + #[cfg(test)] + pub(crate) fn event_args(&self) -> Option { + match self { + Self::Enabled(state) => state.event_builder.clone().map(|builder| builder.build()), + Self::Disabled => None, + } + } + /// Sets the ObjectInfo for event notification. pub fn object(mut self, object_info: ObjectInfo) -> Self { if let Self::Enabled(state) = &mut self diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index fe50ccbde..a7d2b272c 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -926,10 +926,24 @@ impl Node for NodeService { self.handle_check_parts(request).await } + async fn prepare_part_transaction( + &self, + request: Request, + ) -> Result, Status> { + self.handle_prepare_part_transaction(request).await + } + async fn rename_part(&self, request: Request) -> Result, Status> { self.handle_rename_part(request).await } + async fn settle_part_transaction( + &self, + request: Request, + ) -> Result, Status> { + self.handle_settle_part_transaction(request).await + } + async fn rename_file(&self, request: Request) -> Result, Status> { self.handle_rename_file(request).await } @@ -2046,12 +2060,12 @@ mod tests { HealBucketRequest, HealControlRequest, ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest, - MakeVolumesRequest, Mss, PingRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, - ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, - RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, - StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState, TierMutationPrepareRequest, - UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, - WriteRequest, + MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, + ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, + RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ServerInfoRequest, SettlePartTransactionRequest, + SignalServiceRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationPeerState, + TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, WriteRequest, heal_control_service_client::HealControlServiceClient, heal_control_service_server::{HealControlService as _, HealControlServiceServer}, node_service_client::NodeServiceClient, @@ -2598,6 +2612,28 @@ mod tests { }, rustfs_protos::canonical_rename_part_request_body ); + assert_gated!( + prepare_part_transaction, + PreparePartTransactionRequest { + disk: disk.clone(), + src_volume: "src".into(), + src_path: "sp".into(), + dst_volume: "dst".into(), + dst_path: "dp".into(), + meta: vec![0x04].into(), + }, + rustfs_protos::canonical_prepare_part_transaction_request_body + ); + assert_gated!( + settle_part_transaction, + SettlePartTransactionRequest { + disk: disk.clone(), + volume: "dst".into(), + path: "dp".into(), + rollback: true, + }, + rustfs_protos::canonical_settle_part_transaction_request_body + ); assert_gated!( delete_volume, DeleteVolumeRequest { @@ -3316,6 +3352,39 @@ mod tests { assert!(rename_response.error.is_some()); } + #[tokio::test] + async fn test_part_transaction_invalid_disk() { + let service = create_test_node_service(); + + let prepare = service + .prepare_part_transaction(Request::new(PreparePartTransactionRequest { + disk: "invalid-disk-path".to_string(), + src_volume: "src-volume".to_string(), + src_path: "src-path".to_string(), + dst_volume: "dst-volume".to_string(), + dst_path: "dst-path".to_string(), + meta: Bytes::new(), + })) + .await + .expect("prepare RPC should return a structured disk error") + .into_inner(); + assert!(!prepare.success); + assert!(prepare.error.is_some()); + + let settle = service + .settle_part_transaction(Request::new(SettlePartTransactionRequest { + disk: "invalid-disk-path".to_string(), + volume: "dst-volume".to_string(), + path: "dst-path".to_string(), + rollback: true, + })) + .await + .expect("settle RPC should return a structured disk error") + .into_inner(); + assert!(!settle.success); + assert!(settle.error.is_some()); + } + #[tokio::test] async fn test_rename_file_invalid_disk() { let service = create_test_node_service(); diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index fef403ed0..e6c3fb60d 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -18,7 +18,7 @@ use crate::storage::storage_api::rpc_consumer::node_service::{ ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count, }; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; -use crate::storage::storage_api::verify_tonic_mutation_body_digest; +use crate::storage::storage_api::{PartTransactionAction, verify_tonic_mutation_body_digest}; use bytes::Bytes; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::{ @@ -1038,6 +1038,78 @@ impl NodeService { } } + pub(super) async fn handle_prepare_part_transaction( + &self, + request: Request, + ) -> Result, Status> { + verify_disk_mutation_digest( + &request, + rustfs_protos::canonical_prepare_part_transaction_request_body(request.get_ref()), + "prepare_part_transaction", + )?; + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk + .prepare_part_transaction( + &request.src_volume, + &request.src_path, + &request.dst_volume, + &request.dst_path, + request.meta, + ) + .await + { + Ok(()) => Ok(Response::new(PreparePartTransactionResponse { + success: true, + error: None, + })), + Err(err) => Ok(Response::new(PreparePartTransactionResponse { + success: false, + error: Some(err.into()), + })), + } + } else { + Ok(Response::new(PreparePartTransactionResponse { + success: false, + error: Some(DiskError::other("cannot find disk".to_string()).into()), + })) + } + } + + pub(super) async fn handle_settle_part_transaction( + &self, + request: Request, + ) -> Result, Status> { + verify_disk_mutation_digest( + &request, + rustfs_protos::canonical_settle_part_transaction_request_body(request.get_ref()), + "settle_part_transaction", + )?; + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let action = if request.rollback { + PartTransactionAction::Rollback + } else { + PartTransactionAction::Commit + }; + match disk.settle_part_transaction(&request.volume, &request.path, action).await { + Ok(()) => Ok(Response::new(SettlePartTransactionResponse { + success: true, + error: None, + })), + Err(err) => Ok(Response::new(SettlePartTransactionResponse { + success: false, + error: Some(err.into()), + })), + } + } else { + Ok(Response::new(SettlePartTransactionResponse { + success: false, + error: Some(DiskError::other("cannot find disk".to_string()).into()), + })) + } + } + pub(super) async fn handle_check_parts( &self, request: Request, diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index e24ee5c90..36aa2116b 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -91,6 +91,7 @@ use rustfs_kms::{DataKey, KmsUnavailableError, is_data_key_envelope, types::Obje use rustfs_utils::get_env_opt_str; use s3s::S3ErrorCode; use s3s::dto::ServerSideEncryption; +use serde::{Deserialize, Serialize}; #[cfg(feature = "rio-v2")] use sha2::Sha256; use std::collections::HashMap; @@ -1885,6 +1886,10 @@ struct KmsSseDekProvider { service_manager: Option>, } +fn kms_operation_error(error: rustfs_kms::KmsError) -> ApiError { + ApiError::from(StorageError::other(error)) +} + impl KmsSseDekProvider { /// Create a new KMS-backed provider pub async fn new() -> Result { @@ -1936,7 +1941,7 @@ impl SseDekProvider for KmsSseDekProvider { let (data_key, encrypted_data_key) = service .create_data_key(&kms_key_option, context) .await - .map_err(|e| ApiError::from(StorageError::other(format!("Failed to create data key: {}", e))))?; + .map_err(kms_operation_error)?; Ok((data_key, encrypted_data_key)) } @@ -1954,7 +1959,7 @@ impl SseDekProvider for KmsSseDekProvider { let data_key = service .decrypt_data_key(encrypted_dek, context) .await - .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt data key: {}", e))))?; + .map_err(kms_operation_error)?; Ok(data_key.plaintext_key) } @@ -1973,7 +1978,7 @@ impl SseDekProvider for KmsSseDekProvider { let data_key = service .decrypt_legacy_data_key(encrypted_dek) .await - .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt legacy data key: {e}"))))?; + .map_err(kms_operation_error)?; Ok(data_key.plaintext_key) } @@ -1998,6 +2003,16 @@ pub(crate) struct LocalSseDekProvider { master_key: [u8; 32], } +const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1; + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct LocalSseDekEnvelope<'a> { + version: u8, + nonce: &'a str, + ciphertext: &'a str, +} + /// Test-only alias so existing test code that references `TestSseDekProvider` /// continues to compile without changes. #[cfg(test)] @@ -2099,22 +2114,51 @@ impl LocalSseDekProvider { .encrypt(&nonce, dek.as_slice()) .map_err(|_| ApiError::from(StorageError::other("Failed to encrypt DEK")))?; - // nonce:ciphertext - Ok(format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext))) + let nonce = BASE64_STANDARD.encode(nonce); + let ciphertext = BASE64_STANDARD.encode(ciphertext); + serde_json::to_string(&LocalSseDekEnvelope { + version: LOCAL_SSE_DEK_FORMAT_VERSION, + nonce: &nonce, + ciphertext: &ciphertext, + }) + .map_err(|e| ApiError::from(StorageError::other(format!("Failed to serialize encrypted DEK: {e}")))) } // Simple decryption of DEK pub(crate) fn decrypt_dek(encrypted_dek: &str, cmk_value: [u8; 32]) -> Result<[u8; 32], ApiError> { - let parts: Vec<&str> = encrypted_dek.split(':').collect(); - if parts.len() != 2 { - return Err(ApiError::from(StorageError::other("Invalid encrypted DEK format"))); - } - + let envelope = serde_json::from_str::>(encrypted_dek); + let (nonce, ciphertext) = match envelope { + Ok(envelope) => { + if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION { + return Err(ApiError::from(StorageError::other(format!( + "Unsupported encrypted DEK format version: {}", + envelope.version + )))); + } + (envelope.nonce, envelope.ciphertext) + } + Err(json_error) if encrypted_dek.trim_start().starts_with('{') => { + return Err(ApiError::from(StorageError::other(format!( + "Invalid encrypted DEK JSON format: {json_error}" + )))); + } + Err(_) => { + // DEPRECATED: read-only compatibility for persisted colon-delimited DEKs. + // RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs. + let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else { + return Err(ApiError::from(StorageError::other("Invalid encrypted DEK format"))); + }; + if ciphertext.contains(':') { + return Err(ApiError::from(StorageError::other("Invalid encrypted DEK format"))); + } + (nonce, ciphertext) + } + }; let nonce_vec = BASE64_STANDARD - .decode(parts[0]) + .decode(nonce) .map_err(|_| ApiError::from(StorageError::other("Invalid nonce format")))?; let ciphertext = BASE64_STANDARD - .decode(parts[1]) + .decode(ciphertext) .map_err(|_| ApiError::from(StorageError::other("Invalid ciphertext format")))?; let key = Key::::from(cmk_value); @@ -2596,10 +2640,10 @@ mod tests { SseDekProvider, SsecParams, StorageError, TestSseDekProvider, apply_managed_decryption_material, apply_managed_encryption_material, encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse, - map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata, reset_sse_dek_provider, - resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, - validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params, - verify_ssec_key_match, + kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata, + reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption, + strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, + validate_ssec_params, verify_ssec_key_match, }; #[cfg(feature = "rio-v2")] use super::{ @@ -2622,6 +2666,15 @@ mod tests { assert!(super::parse_simple_sse_cmk(&zero).is_err()); } + #[test] + fn kms_operation_errors_preserve_retryability_classification() { + let unavailable = kms_operation_error(rustfs_kms::KmsError::backend_error("connection refused")); + let corrupt = kms_operation_error(rustfs_kms::KmsError::cryptographic_error("decrypt", "authentication failed")); + + assert_eq!(unavailable.code, S3ErrorCode::ServiceUnavailable); + assert_eq!(corrupt.code, S3ErrorCode::InternalError); + } + #[test] fn parse_simple_sse_cmk_accepts_valid_32_byte_key() { let mut key = [0u8; 32]; @@ -4023,17 +4076,25 @@ mod tests { } #[test] - fn test_encrypt_dek_uses_random_nonce_prefixes() { + fn test_encrypt_dek_writes_json_with_random_nonces() { let dek = [0x11u8; 32]; let cmk = [0x22u8; 32]; let encrypted_a = TestSseDekProvider::encrypt_dek(dek, cmk).expect("first DEK wrap should succeed"); let encrypted_b = TestSseDekProvider::encrypt_dek(dek, cmk).expect("second DEK wrap should succeed"); - let nonce_a = encrypted_a.split(':').next().expect("wrapped DEK should contain nonce"); - let nonce_b = encrypted_b.split(':').next().expect("wrapped DEK should contain nonce"); + let envelope_a: super::LocalSseDekEnvelope = + serde_json::from_str(&encrypted_a).expect("first wrapped DEK should be a JSON envelope"); + let envelope_b: super::LocalSseDekEnvelope = + serde_json::from_str(&encrypted_b).expect("second wrapped DEK should be a JSON envelope"); - assert_ne!(nonce_a, nonce_b, "each DEK wrap should use a distinct nonce prefix"); + assert_eq!(envelope_a.version, super::LOCAL_SSE_DEK_FORMAT_VERSION); + assert_eq!(envelope_b.version, super::LOCAL_SSE_DEK_FORMAT_VERSION); + assert_ne!(envelope_a.nonce, envelope_b.nonce, "each DEK wrap should use a distinct nonce"); + assert_eq!( + TestSseDekProvider::decrypt_dek(&encrypted_a, cmk).expect("first JSON envelope should decrypt"), + dek + ); } #[test] @@ -4051,6 +4112,20 @@ mod tests { assert_eq!(decrypted, dek); } + #[test] + fn test_decrypt_dek_rejects_unknown_json_version() { + let envelope = serde_json::json!({ + "version": super::LOCAL_SSE_DEK_FORMAT_VERSION + 1, + "nonce": BASE64_STANDARD.encode([0u8; 12]), + "ciphertext": BASE64_STANDARD.encode([0u8; 48]), + }) + .to_string(); + + let error = TestSseDekProvider::decrypt_dek(&envelope, [0x55u8; 32]) + .expect_err("unknown JSON envelope versions must fail closed"); + assert!(error.message.contains("Unsupported encrypted DEK format version")); + } + #[tokio::test] async fn test_sse_encryption_fails_closed_without_local_sse_master_key() { let _guard = lock_sse_test_state().await; diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 1892d29bf..1735ce2b4 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -430,9 +430,9 @@ pub(crate) mod ecstore_data_usage { pub(crate) mod ecstore_disk { pub(crate) use rustfs_ecstore::api::disk::{ BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore, - FileInfoVersions, FileReader, FileWriter, OldCurrentSize, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, - ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, - validate_batch_read_version_item_count, + FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, + ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, + get_object_disk_read_timeout, validate_batch_read_version_item_count, }; pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce}; } @@ -629,6 +629,7 @@ pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq; pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp; pub(crate) type ReadOptions = ecstore_disk::ReadOptions; pub(crate) type OldCurrentSize = ecstore_disk::OldCurrentSize; +pub(crate) type PartTransactionAction = ecstore_disk::PartTransactionAction; pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp; pub(crate) type ReplicationStatusType = ecstore_bucket::replication::ReplicationStatusType; pub(crate) type ReplicationStats = StorageReplicationStatsHandle; @@ -1097,6 +1098,15 @@ pub(crate) trait StorageDiskRpcExt { dst_path: &str, meta: bytes::Bytes, ) -> DiskResult<()>; + async fn prepare_part_transaction( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + meta: bytes::Bytes, + ) -> DiskResult<()>; + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()>; async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()>; async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; @@ -1241,6 +1251,21 @@ where ecstore_disk::DiskAPI::rename_part(self, src_volume, src_path, dst_volume, dst_path, meta).await } + async fn prepare_part_transaction( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + meta: bytes::Bytes, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::prepare_part_transaction(self, src_volume, src_path, dst_volume, dst_path, meta).await + } + + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()> { + ecstore_disk::DiskAPI::settle_part_transaction(self, volume, path, action).await + } + async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()> { ecstore_disk::DiskAPI::delete(self, volume, path, options).await }