From de6ef096356082099537ce15d2e8bd9d4e8da2bf Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 09:24:29 +0800 Subject: [PATCH 01/12] fix(ecstore): drain durable control-plane write tails --- .../bucket/lifecycle/manual_transition_job.rs | 5 + .../bucket/lifecycle/tier_delete_journal.rs | 6 + .../lifecycle/transition_transaction.rs | 2 + crates/ecstore/src/core/pools.rs | 4 + crates/ecstore/src/object_api/types.rs | 16 + .../src/services/tier/tier_mutation_intent.rs | 2 + .../src/services/tier/tier_probe_intent.rs | 2 + crates/ecstore/src/set_disk/ops/object.rs | 283 +++++++++++++++++- .../src/set_disk/transition_matrix_tests.rs | 5 +- crates/ecstore/src/store/init.rs | 9 +- .../ecstore-validation-suite-design.md | 16 + 11 files changed, 337 insertions(+), 13 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs index b480ba468..050228b25 100644 --- a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs +++ b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs @@ -1170,6 +1170,7 @@ pub async fn save_manual_transition_job_record_if_current( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current_etag.to_string()), ..Default::default() @@ -1242,6 +1243,7 @@ pub(crate) async fn save_manual_transition_worker_result_if_absent( data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1270,6 +1272,7 @@ pub(crate) async fn save_manual_transition_task_if_absent( data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1621,6 +1624,7 @@ pub async fn save_manual_transition_scope_admission_if_absent( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1672,6 +1676,7 @@ pub async fn save_manual_transition_scope_admission_if_current( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current_etag.to_string()), ..Default::default() diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 3a6a7e451..2270bce3b 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -1733,6 +1733,7 @@ async fn save_config_if_none_fenced( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -1832,6 +1833,7 @@ async fn save_decommission_manifest_checkpoint_if_match( let mut opts = ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, no_lock: true, http_preconditions: Some(HTTPPreconditions { if_match: Some(observed_etag), @@ -1960,6 +1962,7 @@ async fn save_config_if_match_fenced( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag.to_string()), ..Default::default() @@ -3780,6 +3783,7 @@ where data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -3869,6 +3873,7 @@ where data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag), ..Default::default() @@ -3893,6 +3898,7 @@ where data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index 87df9fe0d..82e32f598 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -612,6 +612,7 @@ pub(crate) async fn save_transition_transaction_record( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -658,6 +659,7 @@ pub(crate) async fn save_transition_transaction_record_if_current( data.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag), ..Default::default() diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 4973f6831..94f3a6344 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -5493,6 +5493,7 @@ where fence.ensure_held()?; let mut opts = ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, no_lock: true, http_preconditions: Some(pool_meta_cas_preconditions(token, object)?), ..Default::default() @@ -14412,6 +14413,7 @@ impl ECStore { encoded.clone(), &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -14566,6 +14568,7 @@ impl ECStore { encoded, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(http_preconditions), ..Default::default() }, @@ -14957,6 +14960,7 @@ impl ECStore { encoded, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(etag), ..Default::default() diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 85e270679..701596cbf 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -870,6 +870,18 @@ impl TierFreeVersionReceiptSink { } } +/// Internal PUT completion boundary; this does not change fsync or write quorum. +#[doc(hidden)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum WriteCompletion { + /// Return at write quorum when the commit owner can retain its guards. + #[default] + Quorum, + /// Drain the rename fan-out before returning. Minority failures still heal + /// after a successful quorum commit; this does not require every disk to succeed. + TailDrained, +} + #[derive(Default, Clone)] pub struct ObjectOptions { // Use the maximum parity (N/2), used when saving server configuration files @@ -896,6 +908,10 @@ pub struct ObjectOptions { /// Persisted bucket incarnation observed before authorization. pub expected_bucket_incarnation_id: Option, pub no_lock: bool, + /// Control-plane writers that immediately read or CAS the same namespace + /// key use TailDrained without changing namespace lock ownership. + #[doc(hidden)] + pub write_completion: WriteCompletion, /// True when an upper layer already holds the object read lock before /// forwarding a no_lock read to the set layer. pub metadata_cache_safe: bool, diff --git a/crates/ecstore/src/services/tier/tier_mutation_intent.rs b/crates/ecstore/src/services/tier/tier_mutation_intent.rs index 288b02969..eef2dcf7a 100644 --- a/crates/ecstore/src/services/tier/tier_mutation_intent.rs +++ b/crates/ecstore/src/services/tier/tier_mutation_intent.rs @@ -460,6 +460,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -556,6 +557,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current_etag.to_string()), ..Default::default() diff --git a/crates/ecstore/src/services/tier/tier_probe_intent.rs b/crates/ecstore/src/services/tier/tier_probe_intent.rs index 3d11402a5..b3d96dc05 100644 --- a/crates/ecstore/src/services/tier/tier_probe_intent.rs +++ b/crates/ecstore/src/services/tier/tier_probe_intent.rs @@ -494,6 +494,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), ..Default::default() @@ -549,6 +550,7 @@ where data, &ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(HTTPPreconditions { if_match: Some(current.record_etag.clone()), ..Default::default() diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 5a220e432..8db359d3f 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -299,7 +299,7 @@ use crate::error::is_err_invalid_upload_id; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; use crate::object_api::{ NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, - SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, + SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion, }; use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; @@ -4266,13 +4266,17 @@ impl SetDisks { // complete rename fan-out drains. Keep this path synchronous so // its terminal state is known before the coordinator releases // remote leases. - let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation()) - && (commit_object_lock_guard.is_some() - || commit_decommission_object_lock_guard.is_some() - || commit_publication_guard.is_some()) + let commit_owns_namespace_guard = commit_object_lock_guard.is_some() + || commit_decommission_object_lock_guard.is_some() + || commit_publication_guard.is_some(); + let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum + && !(opts.data_movement && opts.has_decommission_capacity_reservation()) + && commit_owns_namespace_guard && commit_scanner_publication_scope.is_none(); + // Full-tail callers also transfer owned guards to the coordinator: + // cancelling their ACK waiter must not cancel an in-flight rename. let detach_commit_owner = commit_scanner_publication_scope.is_some() - || commit_allows_early_ack + || commit_owns_namespace_guard || commit_bucket_lifecycle_guard.is_some() || quota_mutation_fence; let commit_write_path_label = write_path.metric_label(); @@ -4617,9 +4621,8 @@ impl SetDisks { request.object_version_id = committed_version_id .or_else(|| commit_version_suspended.then(Uuid::nil)) .map(|version_id| version_id.to_string()); - tokio::spawn(async move { - let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await; - }); + let heal_set = commit_set.clone(); + tokio::spawn(async move { heal_set.submit_rename_tail_heal(request).await }); } let rename_stage_elapsed = rename_stage_start.elapsed(); @@ -18157,6 +18160,256 @@ mod put_object_tmp_cleanup_tests { .await; } + async fn make_completion_test_bucket(disks: &[DiskStore], bucket: &str) { + for disk in disks { + disk.make_volume(bucket) + .await + .expect("completion test bucket should be created"); + } + } + + /// Observe the actual metadata quorum while the remaining rename is parked. + /// A completed task count alone can race tasks that have not started yet. + async fn wait_for_paused_tail_metadata_quorum(disks: &[DiskStore], bucket: &str, object: &str) { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let mut committed = 0; + for disk in disks { + match disk.read_version("", bucket, object, "", &ReadOptions::default()).await { + Ok(_) => committed += 1, + Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {} + Err(err) => panic!("unexpected metadata error while observing {bucket}/{object}: {err}"), + } + } + if committed == 3 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("three disks must publish metadata while the fourth rename remains paused"); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for size in [4096, 1024 * 1024] { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-full-tail-cas"; + let object = "full-tail-cas-object"; + make_completion_test_bucket(&disks, bucket).await; + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; size]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("full-tail PUT must reach the rename barrier"); + wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await; + assert!(!put.is_finished(), "full-tail PUT must remain pending after metadata quorum"); + let mut lock_probe = Box::pin(set.acquire_write_lock_diag("full_tail_probe", bucket, object)); + assert!( + futures::poll!(lock_probe.as_mut()).is_pending(), + "the owned namespace guard must remain held" + ); + barrier.release(); + let written = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("full-tail PUT should finish after release") + .expect("full-tail PUT task should join") + .expect("full-tail PUT must commit"); + assert_eq!(tasks.running(), 0, "full-tail response must follow every rename task"); + drop( + tokio::time::timeout(Duration::from_secs(5), lock_probe) + .await + .expect("same-key lock should be available on return") + .expect("same-key lock probe should succeed"), + ); + for disk in &disks { + disk.read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("successful full-tail PUT must publish on every healthy disk"); + } + drop(barrier); + let mut replacement = PutObjReader::from_vec(b"cas successor".to_vec()); + set.put_object( + bucket, + object, + &mut replacement, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_match: written.etag, + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .expect("immediate same-key CAS must acquire the namespace guard"); + let mut read = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("CAS successor must be immediately readable"); + let mut body = Vec::new(); + read.stream.read_to_end(&mut body).await.expect("successor body must drain"); + assert_eq!(body, b"cas successor"); + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_preserves_quorum_success_and_heals_failed_tail() { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-full-tail-heal"; + let object = "full-tail-heal-object"; + make_completion_test_bucket(&disks, bucket).await; + let mut heals = set.capture_test_rename_tail_heals(); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let _fault = rename_fault_injection::fail_rename_on(object, &[0]); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("failed tail must first reach the rename barrier"); + wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await; + assert!(!put.is_finished(), "committed quorum must still wait for the failing tail"); + barrier.release(); + tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("failed tail should drain") + .expect("PUT task should join") + .expect("a minority tail error must not negate committed quorum"); + assert_eq!(tasks.running(), 0); + let heal = tokio::time::timeout(Duration::from_secs(30), heals.recv()) + .await + .expect("failed tail must schedule heal") + .expect("heal capture must remain connected"); + assert_eq!(heal.bucket, bucket); + assert_eq!(heal.object_prefix.as_deref(), Some(object)); + let info = set + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("committed object must remain readable despite the failed tail"); + assert_eq!(info.size, TEST_OBJECT_SIZE as i64); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_rejects_quorum_minus_one() { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-full-tail-no-quorum"; + let object = "full-tail-no-quorum-object"; + make_completion_test_bucket(&disks, bucket).await; + let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]); + let tasks = rename_fanout_barrier::observe_tasks(object); + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + let err = set + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect_err("draining two successful disks cannot satisfy write quorum three"); + assert!( + matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)), + "original quorum error expected: {err}" + ); + assert_eq!(tasks.running(), 0, "failed fan-out and rollback must complete before return"); + assert!( + set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(), + "failed fresh write must not become visible" + ); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn tail_drained_put_owned_commit_survives_waiter_cancellation() { + let (dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = RUSTFS_META_BUCKET; + let object = "full-tail-cancelled-receipt"; + // Internal config writes do not own a bucket lifecycle guard. The object + // guard alone must keep the full-tail coordinator alive after cancellation. + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("cancelled receipt must first reach the rename barrier"); + wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await; + put.abort(); + assert!(put.await.expect_err("ACK waiter should cancel").is_cancelled()); + let mut lock_probe = Box::pin(set.acquire_write_lock_diag("cancelled_full_tail_probe", bucket, object)); + assert!( + futures::poll!(lock_probe.as_mut()).is_pending(), + "owned coordinator must retain the namespace guard after waiter cancellation" + ); + barrier.release(); + drop( + tokio::time::timeout(Duration::from_secs(30), lock_probe) + .await + .expect("cancelled coordinator must eventually release its guard") + .expect("post-commit lock probe should succeed"), + ); + assert_eq!(tasks.running(), 0, "cancelled coordinator must reap every rename task"); + for disk in &disks { + disk.read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("caller cancellation must not interrupt committed receipt materialization"); + } + wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await; + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn no_lock_put_waits_for_rename_tail_under_outer_guard() { @@ -18184,6 +18437,7 @@ mod put_object_tmp_cleanup_tests { &mut reader, &ObjectOptions { no_lock: true, + write_completion: WriteCompletion::TailDrained, ..Default::default() }, ) @@ -18209,7 +18463,18 @@ mod put_object_tmp_cleanup_tests { put.await .expect("no-lock PUT task should join") .expect("no-lock PUT should commit after the rename tail releases"); + let mut lock_probe = Box::pin(set_disks.acquire_write_lock_diag("borrowed_full_tail_probe", bucket, object)); + assert!( + futures::poll!(lock_probe.as_mut()).is_pending(), + "full-tail PUT must not release the caller's outer guard" + ); drop(outer_guard); + drop( + tokio::time::timeout(Duration::from_secs(5), lock_probe) + .await + .expect("outer owner releasing its guard should unblock the probe") + .expect("post-outer-guard probe should succeed"), + ); }) .await; } diff --git a/crates/ecstore/src/set_disk/transition_matrix_tests.rs b/crates/ecstore/src/set_disk/transition_matrix_tests.rs index 8924b2a4a..9b5912ce7 100644 --- a/crates/ecstore/src/set_disk/transition_matrix_tests.rs +++ b/crates/ecstore/src/set_disk/transition_matrix_tests.rs @@ -18,6 +18,7 @@ use super::{ }; use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time}; use crate::ecstore_validation_blackbox::make_local_set_disks; +use crate::object_api::WriteCompletion; use crate::services::tier::test_util::register_mock_tier; use crate::storage_api_contracts::bucket::BucketOperations; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; @@ -72,7 +73,7 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() { object, &mut reader, &ObjectOptions { - no_lock: true, + write_completion: WriteCompletion::TailDrained, ..Default::default() }, ) @@ -185,7 +186,7 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot object, &mut reader, &ObjectOptions { - no_lock: true, + write_completion: WriteCompletion::TailDrained, ..Default::default() }, ) diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 22d7e37c3..73905389f 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -8045,10 +8045,15 @@ mod tests { ); assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok()); - com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone()) + let full_tail = ObjectOptions { + max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }; + com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail) .await .expect("second page receipt should restore"); - com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec()) + com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail) .await .expect("second page receipt should corrupt deterministically"); let corrupt = store diff --git a/docs/testing/ecstore-validation-suite-design.md b/docs/testing/ecstore-validation-suite-design.md index e9bec1fe4..de717d1b9 100644 --- a/docs/testing/ecstore-validation-suite-design.md +++ b/docs/testing/ecstore-validation-suite-design.md @@ -54,6 +54,22 @@ Fail-closed invariants every row enforces: Fault injection is explicit and deterministic: local disk mocks for unit tests, process-level disk manipulation (`crates/e2e_test/src/chaos.rs`) for e2e tests. Property tests replay a fixed seed for payload, range, and missing-shard selection. +### PUT completion fixtures + +`ObjectOptions::default()` uses `WriteCompletion::Quorum`: a namespace-lock-owning PUT may acknowledge write quorum while its rename tail retains the lock. A fixture that immediately inspects every disk or primes a metadata generation must set `write_completion: WriteCompletion::TailDrained` and keep normal locking. TailDrained waits for the existing rename fan-out; it does not require every disk to succeed or change fsync policy. Codec-only `no_lock` fixtures do not cover namespace locking. + +The object tests reuse `rename_fanout_barrier::arm(object, disk_slot, phase)` and `observe_tasks(object)`. Wait for the barrier with a deadline, observe actual metadata quorum with `wait_for_paused_tail_metadata_quorum`, then release or cancel. The metadata check distinguishes a real quorum from disk tasks that have not started. Assert zero remaining rename tasks after the owned coordinator releases its lock; cancellation tests also wait for staging cleanup. + +| Fixture | Completion boundary | +|---|---| +| `early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes` | Default PUT returns before the parked tail; a second writer remains blocked. | +| `tail_drained_put_*` | Explicit full-tail PUT retains its guard, preserves quorum success with a failed minority, rejects quorum-minus-one, and survives ACK waiter cancellation. | +| `transition_and_restore_reclaim_prior_metadata_generations` | Both source fixtures use TailDrained before cache priming, with normal namespace locks. | +| `object_transaction_fencing_persists_epoch_on_multipart_commit` | Multipart completion already always drains rename before inspecting all per-disk transaction UUIDs. | +| `decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page`, `dispatch_completion_cas_is_bounded_and_reaches_the_tail` | Durable receipt, journal, and manifest writers choose TailDrained; the pagination fixture also drains deliberate receipt replacement writes. | + +Select these checks with `cargo nextest list -p rustfs-ecstore --features test-util -E 'test(tail_drained_put) | test(early_ack_tail_drain) | test(no_lock_put_waits_for_rename_tail) | test(object_transaction_fencing_persists_epoch_on_multipart_commit) | test(transition_and_restore_reclaim) | test(decommission_durable_ilm_receipt_pagination) | test(dispatch_completion_cas)'`, then run the same expression under the default and CI profiles without retries. Remaining crash, reopen, rollback, and lock-loss schedules use the existing domain tests; this completion fixture is not a replacement for those checks. + ### Coverage gate `full` and `destructive` run `cargo llvm-cov -p rustfs-ecstore --lib` and fail when line coverage of the gate scope is below `--unit-coverage-min`. The default minimum and the 100% target for EC read, write, decode, heal, metadata-quorum, and rollback paths are the `UNIT_COVERAGE_*` constants at the top of the runner. `cargo-llvm-cov` must be installed unless `--skip-coverage` is passed explicitly. The default scope `ec-critical` is: From 70be76c54e6438a945c6d20c7e2b52aa2796029c Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 09:26:12 +0800 Subject: [PATCH 02/12] fix(ecstore): retain PUT staging after incomplete rollback --- crates/ecstore/src/set_disk/ops/object.rs | 107 +++++++++++++++++++++- 1 file changed, 104 insertions(+), 3 deletions(-) diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 8db359d3f..ce9f95d46 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -303,7 +303,7 @@ use crate::object_api::{ }; use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; -use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal}; +use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal}; #[cfg(test)] use crate::storage_api_contracts::namespace::NamespaceLocking; #[cfg(test)] @@ -3548,6 +3548,7 @@ impl SetDisks { (None, None, None) }; let mut tmp_cleanup_owned = false; + let rollback_receipt = RenameRollbackReceipt::default(); let operation = async { let erasure = Arc::new(erasure_from_file_info(&fi, false)?); @@ -4256,6 +4257,7 @@ impl SetDisks { let commit_bucket = bucket.to_owned(); let commit_object = object.to_owned(); let commit_tmp_dir = tmp_dir.clone(); + let commit_rollback_receipt = rollback_receipt.clone(); let commit_object_lock_guard = object_lock_guard.take(); let commit_decommission_object_lock_guard = decommission_object_lock_guard.take(); let commit_publication_guard = publication_commit_guard.take(); @@ -4456,7 +4458,8 @@ impl SetDisks { write_quorum, commit_scanner_publication_lease_tokens.as_ref(), ) - .with_publication_scope(commit_scanner_publication_scope.clone()), + .with_publication_scope(commit_scanner_publication_scope.clone()) + .with_rollback_receipt(commit_rollback_receipt.clone()), ) .await; if let Some(scope) = commit_scanner_publication_scope.as_ref() { @@ -4589,6 +4592,11 @@ impl SetDisks { let rename_commit = match rename_result { Ok(commit) => commit, Err(err) => { + if commit_rollback_receipt.is_incomplete() { + // Incomplete undo retains the staging source and + // rollback backup for recovery; cleanup is unsafe. + return Err(err.into()); + } if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { @@ -4888,7 +4896,7 @@ impl SetDisks { ); } }); - } else { + } else if !rollback_receipt.is_incomplete() { // Failure path (quorum loss / rollback): keep the cleanup inline so // a failed PUT never returns while its tmp shards are still on disk // (state-residue hardening tracked by backlog#864 / backlog#898). @@ -18358,6 +18366,99 @@ mod put_object_tmp_cleanup_tests { ); } + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() { + use crate::set_disk::core::io_primitives::rollback_fault_injection; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] { + let (dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-incomplete-undo"; + let object = "incomplete-undo-object"; + make_completion_test_bucket(&disks, bucket).await; + let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]); + set.put_object( + bucket, + object, + &mut old_reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("old generation should be completely committed"); + wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await; + let old = disks[0] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("old metadata must be readable"); + let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory"); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("overwrite must enter the actual rename fan-out before failure injection"); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("incomplete undo must return without hanging") + .expect("PUT task should join") + .expect_err("two renamed disks cannot satisfy write quorum three"); + assert!( + matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)), + "original quorum error expected: {err}" + ); + assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return"); + let leftovers = non_trash_tmp_entries(&dirs).await; + assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery"); + let backups = dirs + .iter() + .filter(|dir| { + dir.path() + .join(bucket) + .join(object) + .join(old_data_dir.to_string()) + .join(crate::disk::STORAGE_FORMAT_FILE_BACKUP) + .exists() + }) + .count(); + assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup"); + // The remaining three disks still serve the old generation; + // the failed minority must never become an acknowledged write. + let mut read = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("old generation must remain readable after incomplete rollback"); + let mut body = Vec::new(); + read.stream + .read_to_end(&mut body) + .await + .expect("old generation should stream"); + assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]); + } + }) + .await; + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn tail_drained_put_owned_commit_survives_waiter_cancellation() { From e83ace9533e7288f6010891c7e463c8790c982c5 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 09:29:01 +0800 Subject: [PATCH 03/12] fix(ecstore): drain backfill checkpoint before confirmation --- crates/ecstore/src/bucket/on_demand_migration/backfill.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs b/crates/ecstore/src/bucket/on_demand_migration/backfill.rs index 6768c9717..ddcbce8da 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/backfill.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/backfill.rs @@ -684,6 +684,7 @@ async fn write_checkpoint( }; let opts = ObjectOptions { max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, http_preconditions: Some(preconditions), ..Default::default() }; From dacb617ff1beee78521e6cbba2d97a5dc332f382 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 09:37:50 +0800 Subject: [PATCH 04/12] refactor(ecstore): isolate local object rename commit --- crates/ecstore/src/disk/local.rs | 1145 +-------------------- crates/ecstore/src/disk/local/commit.rs | 1205 +++++++++++++++++++++++ 2 files changed, 1212 insertions(+), 1138 deletions(-) create mode 100644 crates/ecstore/src/disk/local/commit.rs diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 6baa92a3e..adef17e85 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(test)] +use self::commit::lock_rename_commit_directories; + +mod commit; + use crate::crash_inject::{self, CrashPoint}; use crate::data_usage::local_snapshot::ensure_data_usage_layout; use crate::diagnostics::get::{ @@ -184,65 +189,6 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r } } -fn rollback_committed_rename_std( - dst_file_path: &Path, - new_data_path: Option<&Path>, - rollback_data_dir: Option, -) -> std::io::Result<()> { - if let Some(old_data_dir) = rollback_data_dir { - let Some(dst_parent) = dst_file_path.parent() else { - return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent")); - }; - let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); - std::fs::rename(backup_path, dst_file_path)?; - } else { - remove_file_if_exists(dst_file_path)?; - } - - if let Some(new_data_path) = new_data_path { - remove_dir_all_if_exists(new_data_path)?; - } - - Ok(()) -} - -fn rollback_inline_metadata_commit_std( - dst_file_path: &Path, - rollback_data_dir: Option, - local_rollback_path: Option<&Path>, -) -> std::io::Result<()> { - if let Some(backup_path) = local_rollback_path { - // The commit immediately before this rollback renamed the staged - // xl.meta from the same directory as `backup_path` onto - // `dst_file_path`, proving both paths are on the same filesystem. - // Unix rename atomically replaces the committed destination; never - // unlink it first or an interrupted rollback could lose xl.meta. - std::fs::rename(backup_path, dst_file_path)?; - } else { - rollback_committed_rename_std(dst_file_path, None, rollback_data_dir)?; - } - Ok(()) -} - -fn create_local_inline_rollback_backup( - dst_file_path: &Path, - staging_file_path: &Path, - old_metadata: &[u8], -) -> std::io::Result { - let Some(staging_parent) = staging_file_path.parent() else { - return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent")); - }; - let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP); - remove_file_if_exists(&backup_path)?; - if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err()) - && let Err(err) = std::fs::write(&backup_path, old_metadata) - { - let _ = remove_file_if_exists(&backup_path); - return Err(err); - } - Ok(backup_path) -} - async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> { let backup_dir = object_dir.join(rollback_dir.to_string()); fs::create_dir_all(&backup_dir).await.map_err(to_file_error)?; @@ -269,126 +215,6 @@ async fn restore_metadata_backup( Ok(()) } -async fn lock_rename_commit_directories( - source_parent: &Path, - destination_parent: &Path, - base_dir: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result { - #[cfg(windows)] - let result = { - let source_parent = source_parent.to_path_buf(); - let destination_parent = destination_parent.to_path_buf(); - let base_dir = base_dir.to_path_buf(); - let publication_root = publication_root.clone(); - os::run_blocking_namespace_operation(mutation_lease, move || { - let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root); - #[cfg(test)] - if result.is_ok() { - run_destination_commit_directory_preparation(&destination_parent); - } - result - }) - .await - }; - #[cfg(not(windows))] - let result = { - let _ = mutation_lease; - os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) - }; - - let result = result.map_err(|err| match std::fs::symlink_metadata(base_dir) { - Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err, - _ => err, - }); - - result.map_err(to_file_error).map_err(DiskError::from) -} - -async fn read_rename_destination_metadata( - file_path: &Path, - rename_commit_guard: &os::RenameCommitGuard, - mutation_lease: Arc, -) -> Result> { - #[cfg(windows)] - let result = { - let file_path = file_path.to_path_buf(); - let rename_commit_guard = rename_commit_guard.clone(); - os::run_blocking_namespace_operation(mutation_lease, move || { - os::read_destination_file_with_commit_guard(&file_path, &rename_commit_guard) - }) - .await - }; - #[cfg(not(windows))] - let _ = (rename_commit_guard, mutation_lease); - #[cfg(not(windows))] - let result = match super::fs::read_file(file_path).await { - Ok(data) => Ok(Some(data)), - Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), - Err(err) => Err(err), - }; - - result - .map(|data| data.map(Bytes::from)) - .map_err(to_file_error) - .map_err(DiskError::from) -} - -async fn restore_renamed_data_source( - src_volume_dir: &Path, - src_data_path: &Path, - dst_data_path: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result<()> { - if fs::symlink_metadata(src_data_path).await.is_ok() { - return Ok(()); - } - let result = - match os::rename_all_with_lease(dst_data_path, src_data_path, src_volume_dir, publication_root, mutation_lease).await { - Ok(()) => Ok(()), - Err(DiskError::FileNotFound) => { - let source_exists = fs::symlink_metadata(src_data_path).await.is_ok(); - let destination_missing = matches!( - fs::symlink_metadata(dst_data_path).await, - Err(err) if err.kind() == ErrorKind::NotFound - ); - if source_exists && destination_missing { - Ok(()) - } else { - Err(DiskError::FileNotFound) - } - } - Err(err) => Err(err), - }; - if let Err(err) = &result { - warn!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "restore_staged_data_source_failed", - src_path = ?src_data_path, - dst_path = ?dst_data_path, - error = ?err, - "Failed to restore staged data after a metadata commit was rejected" - ); - } - result -} - -async fn restore_published_data_source( - data_paths: Option<&(PathBuf, PathBuf)>, - src_volume_dir: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result<()> { - let Some((src_data_path, dst_data_path)) = data_paths else { - return Ok(()); - }; - restore_renamed_data_source(src_volume_dir, src_data_path, dst_data_path, publication_root, mutation_lease).await -} - async fn restore_delete_rollback( object_dir: &Path, xl_path: &Path, @@ -9050,965 +8876,7 @@ impl DiskAPI for LocalDisk { dst_path: &str, ) -> Result { crate::hp_guard!("LocalDisk::rename_data"); - let mut fi = fi; - // A non-force DeleteBucket must not remove a directory while a local - // object commit is publishing into it. The peer's empty scan remains - // optimistic; this lease establishes the local commit/delete order and - // remains owned by any blocking syscall that outlives async cancellation. - let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; - let quota_fence_token = - match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { - Some(value) => { - let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; - Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) - } - None if rustfs_utils::http::metadata_compat::contains_key_str( - &fi.metadata, - QUOTA_MUTATION_FENCE_METADATA_SUFFIX, - ) => - { - return Err(DiskError::FileCorrupt); - } - None => None, - }; - rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); - let quota_fence_claim = match quota_fence_token { - Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), - None => None, - }; - let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; - if let Some(claim) = quota_fence_claim { - mutation_lease.attach_external_guard(claim); - } - if fi.is_legacy_indexed_delete_marker() { - fi.erasure.index = 0; - } - fi.validate_for_metadata_read()?; - // Snapshot the destination part paths before `fi` is consumed below. These - // are the descriptors a reader may hold for the version this call is about - // to replace (backlog#1145); readers build the identical string in - // `io_primitives`. An inline-data version has no parts and yields none. - let invalidate_part_paths: Vec = { - let data_dir = fi.data_dir.unwrap_or_default(); - fi.parts - .iter() - .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) - .collect() - }; - let src_volume_dir = self.io_get_bucket_path(src_volume)?; - if !skip_access_checks(src_volume) - && let Err(e) = super::fs::access_std(&src_volume_dir) - { - info!( - event = EVENT_DISK_LOCAL_ACCESS_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - path = ?src_volume_dir, - operation = "rename_data_src_access", - error = %e, - "Disk local access check failed" - ); - return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); - } - - let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; - if !skip_access_checks(dst_volume) - && let Err(e) = super::fs::access_std(&dst_volume_dir) - { - info!( - event = EVENT_DISK_LOCAL_ACCESS_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - path = ?dst_volume_dir, - operation = "rename_data_dst_access", - error = %e, - "Disk local access check failed" - ); - return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); - } - - // xl.meta path - let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; - let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; - - // data_dir path - let has_data_dir_path = { - let has_data_dir = { - if !fi.is_remote() { - fi.data_dir - .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) - } else { - None - } - }; - - if let Some(data_dir) = has_data_dir { - let src_data_path = self.io_get_object_path( - src_volume, - rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), - )?; - let dst_data_path = self.io_get_object_path( - dst_volume, - rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), - )?; - - Some((src_data_path, dst_data_path)) - } else { - None - } - }; - - check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; - check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; - - let no_inline = fi.data.is_none() && fi.size > 0; - // Captured before `fi` is consumed by add_version; gates the stale - // destination purge below. - let fi_healing = fi.is_healing(); - - // Resolved once for the whole commit so a concurrent configuration - // change can never leave a single rename_data half-synced. The tier is - // keyed on the destination volume: user data staged in scratch - // namespaces follows the configured tier, while commits into - // system-critical namespaces (IAM, config, bucket metadata) stay - // pinned to strict. - let durability = effective_durability(dst_volume); - - let src_file_parent = src_file_path - .parent() - .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; - let dst_file_parent = dst_file_path - .parent() - .ok_or_else(|| DiskError::other("missing object metadata parent"))?; - if !no_inline { - fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; - } - // Acquire the common trees before reading destination metadata. On - // Windows this pins the object directory identity across metadata - // preparation, data publication, rollback backup, and final commit. - let rename_commit_guard = lock_rename_commit_directories( - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; - - if no_inline { - // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta - let mut xlmeta = FileMeta::new(); - // An existing dst xl.meta that fails to parse leaves `xlmeta` empty - // and gets overwritten by the commit below (pre-existing behavior); - // track that so the old-size observation reports unknown instead of - // a false `Absent` (rustfs/backlog#1009). - let mut dst_meta_unparsable = false; - if let Some(dst_buf) = has_dst_buf.as_ref() { - if FileMeta::is_xl2_v1_format(dst_buf) - && let Ok(nmeta) = FileMeta::load(dst_buf) - { - xlmeta = nmeta - } else { - dst_meta_unparsable = true; - } - } - - let old_current_size = if dst_meta_unparsable { - None - } else { - observe_old_current_size(has_dst_buf.is_some(), &xlmeta) - }; - - let mut skip_parent = dst_volume_dir.clone(); - if has_dst_buf.as_ref().is_some() - && let Some(parent) = dst_file_path.parent() - { - skip_parent = parent.to_path_buf(); - } - - let version_id = fi.version_id.unwrap_or_default(); - let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); - let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); - let rollback_data_dir = has_old_data_dir.or_else(|| { - if old_version_exists && has_dst_buf.is_some() { - Some(inline_metadata_rollback_dir(version_id, &xlmeta)) - } else { - None - } - }); - if let Some(old_data_dir) = has_old_data_dir.as_ref() { - let _ = xlmeta.data.remove_two(version_id, *old_data_dir); - } - xlmeta.add_version(fi)?; - let version_signature = rename_data_versions_signature(&xlmeta); - let new_dst_buf = xlmeta.marshal_msg()?; - - // This tmp xl.meta is renamed onto dst_file_path at the commit - // point below, so only its contents must be durable before the - // rename (SyncMode::FileOnly); the dst parent directory is fsynced - // after the commit rename, and a crash before the rename means the - // PUT was never acknowledged. A metadata commit: relaxed tiers - // leave it to the page cache. - let tmp_meta_sync = if durability.syncs_commit_metadata() { - SyncMode::FileOnly - } else { - SyncMode::None - }; - // The tmp xl.meta write and the shard-file fdatasync are independent - // (disjoint paths) and both only need to be durable before the commit - // renames below, so run them concurrently to drop a blocking - // round-trip from the PUT commit critical path (rustfs/backlog#922 - // step 2). The "contents durable -> rename -> dst dir fsync" ordering - // is unchanged — both futures complete before any rename — which the - // rename_data crash-consistency harness (backlog#935) exercises. - // - // Shard durability: once rename_data succeeds the write is - // acknowledged, so data must not live only in the page cache. - // Multipart parts were already synced during rename_part, so their - // fdatasync here is a cheap no-op. A missing source dir is left for the - // rename below to report through the existing rollback path. Payload - // durability is kept by both strict and relaxed. - let tmp_meta_write = { - let src_file_path = src_file_path.clone(); - let dst_file_path = dst_file_path.clone(); - let rename_commit_guard = rename_commit_guard.clone(); - let mutation_lease = mutation_lease.clone(); - async move { - os::run_blocking_namespace_operation(mutation_lease, move || { - #[cfg(test)] - run_owned_file_write_before_open(&src_file_path); - let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( - &src_file_path, - &dst_file_path, - &rename_commit_guard, - )?; - prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; - Ok(prepared_metadata_source) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from) - } - }; - let shard_sync = async { - if durability.syncs_data_shards() - && let Some((src_data_path, _)) = has_data_dir_path.as_ref() - && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await - && err.kind() != ErrorKind::NotFound - { - return Err::<(), DiskError>(to_file_error(err).into()); - } - Ok(()) - }; - let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); - // Surface a tmp-meta failure first (its prior serial position), then a - // shard-sync failure; either aborts before any rename, exactly as the - // sequential version did. - let prepared_metadata_source = tmp_meta_res?; - shard_sync_res?; - let rename_commit_guard = remove_dst_base_before_commit( - dst_path, - rename_commit_guard, - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - if should_remove_staged_meta_before_commit(dst_path) { - drop(prepared_metadata_source); - std::fs::remove_file(&src_file_path).map_err(to_file_error)?; - return Err(DiskError::FileNotFound); - } - - // Heal reuses the version's data_dir, so for in-place corruption - // the destination dir still exists — and rename(2) cannot replace - // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge - // it first, healing commits only; fresh PUTs mint a new data_dir - // and never collide. Best effort: a real failure surfaces in the - // rename below. - if fi_healing - && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = self.move_to_trash(dst_data_path, true, false).await - { - warn!( - event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - dst_path = ?dst_data_path, - error = ?err, - "Healing commit could not purge the stale destination data dir" - ); - } - if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = os::rename_all_with_commit_guard( - src_data_path, - dst_data_path, - &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "rename_all_data_path_failed", - src_path = ?src_data_path, - dst_path = ?dst_data_path, - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - #[cfg(test)] - if has_data_dir_path.is_some() { - run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); - } - - // Crash-consistency injection: hard power loss after the data dir - // is in place but before xl.meta commits. No cleanup — the harness - // reopens the disk and asserts the object still reads as the old - // version (the staged data dir is a harmless orphan for GC). - if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { - return Err(DiskError::Unexpected); - } - - if should_fail_before_old_metadata_backup(dst_path) { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "test_fail_before_old_metadata_backup", - "Disk local rename flow failed before metadata commit" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(DiskError::Unexpected); - } - - // The rollback backup stays where it is written (no rename) and is - // the sole restore source for a later undo_write, so under strict - // it keeps SyncMode::FileAndDir: contents and directory entry both - // durable. It is part of the metadata commit machinery, so relaxed - // tiers leave it to the page cache like the xl.meta it mirrors. - let backup_sync = if durability.syncs_commit_metadata() { - SyncMode::FileAndDir - } else { - SyncMode::None - }; - if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { - let backup_parent = dst_file_parent.join(old_data_dir.to_string()); - #[cfg(not(windows))] - if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { - Ok(guard) => guard, - Err(err) => { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(DiskError::from(to_file_error(err))); - } - }; - let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); - if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { - #[cfg(windows)] - drop(backup_path_guard); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - let backup_bytes = dst_buf.clone(); - // Keep the volume, commit-tree, and exact destination-path - // guards in this task until the backup write and durability - // sync finish. A detached spawn_blocking writer could survive - // cancellation and later truncate a newer transaction's - // deterministic rollback backup. - let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { - #[cfg(test)] - run_owned_file_write_before_open(&backup_path); - backup_path_guard.write_file_for_path_access( - &backup_path, - backup_bytes.as_ref(), - backup_sync != SyncMode::None, - backup_sync == SyncMode::FileAndDir, - ) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from); - if let Err(err) = write_result { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "write_old_metadata_backup_failed", - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - } - - // Crash-consistency injection: hard power loss after the rollback - // backup is durable but before the xl.meta commit rename. No - // cleanup — the harness asserts the object still reads as the old - // version, since the destination xl.meta is untouched here. - if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { - return Err(DiskError::Unexpected); - } - - if let Err(err) = os::rename_all_with_prepared_source( - prepared_metadata_source, - &src_file_path, - &dst_file_path, - &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "rename_all_metadata_failed", - src_path = ?src_file_path, - dst_path = ?dst_file_path, - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - - let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); - if should_fail_after_metadata_commit(dst_path) { - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - return Err(DiskError::Unexpected); - } - - // Crash-consistency injection: hard power loss immediately after the - // xl.meta commit rename but before the durability fsync. Unlike the - // graceful failpoint above, no rollback runs — the commit rename is - // already on disk, so the harness asserts the object reads back as - // the new version. - if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { - return Err(DiskError::Unexpected); - } - - // Persist the directory entries for both the data dir and xl.meta renames; - // without this the commit itself can vanish on power loss. Relaxed tiers - // accept that window (documented in docs/operations/durability-modes.md). - if durability.syncs_commit_metadata() - && let Some(parent) = dst_file_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // The commit rename changed the dst part inodes before this fsync - // failed and rolled them back; drop any fd cached during that - // window so readers re-open the restored inode (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(to_file_error(err).into()); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - } - - // First PUT of an object creates its directory (and any missing prefix - // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The - // commit fsync above persists the object dir's *contents*, not its own - // entry in the bucket/prefix dir, so on power loss after ack the whole - // object dir could vanish (rustfs/backlog#922 step 4). For a new object - // (no prior xl.meta) fsync the ancestor chain from the object dir's - // parent up to and including the bucket so those new directory entries - // are durable. Overwrites already have a durable object dir. The - // starts_with guard bounds the walk to the bucket subtree. Relaxed/none - // accept the wider window, like the commit fsync above. - if has_dst_buf.is_none() && durability.syncs_commit_metadata() { - let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); - while let Some(dir) = ancestor { - if !dir.starts_with(&dst_volume_dir) { - break; - } - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dir(dir).await { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // Same post-commit rollback window as above — drop cached - // dst part fds so readers re-open the restored inode - // (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(to_file_error(err).into()); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - if dir == dst_volume_dir.as_path() { - break; - } - ancestor = dir.parent(); - } - } - - // Publication and every rollback-capable durability step are now - // complete. Do not retain the Windows object identity guard while - // cleaning staging paths or invalidating cached descriptors. - #[cfg(windows)] - drop(rename_commit_guard); - - if let Some(src_file_path_parent) = src_file_path.parent() { - if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { - let _ = std::fs::remove_dir(src_file_path_parent); - } else { - let _ = self - .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) - .await; - } - } - - // Heal reuses a version's `data_dir` and lands the rebuilt shard on - // the SAME `//part.N` path. Without this, a cached - // descriptor would keep serving the pre-heal inode, defeating the heal - // and eroding read quorum (backlog#1145). - // - // The exact keys are derivable here, and this runs on every write, so - // use them rather than registering a predicate the read path would then - // have to evaluate. Readers build the same string - // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every - // part of the version now at `dst_path` — any part path absent from it - // no longer exists for readers to ask for. - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - - Ok(RenameDataResp { - old_data_dir: has_old_data_dir, - rollback_data_dir, - cleanup_data_dir: has_old_data_dir, - sign: version_signature, - old_current_size, - }) - } else { - // Inline metadata preparation is blocking. The transaction lease is - // moved into that work so a timeout can release the async waiter without - // allowing a retry to reuse the deterministic staging path too early. - let src = src_file_path.clone(); - let dst = dst_file_path.clone(); - let cleanup_path = if src_volume == super::RUSTFS_META_MULTIPART_BUCKET { - src_file_path.parent().map(|p| p.to_path_buf()) - } else { - None - }; - let dst_path_for_failpoint = dst_path.to_string(); - #[cfg(windows)] - let source_parent = src_file_parent.to_path_buf(); - let rename_commit_guard_for_preparation = rename_commit_guard.clone(); - let sync = durability.syncs_commit_metadata(); - #[cfg(test)] - run_inline_before_file_sync_admission(dst_path); - let mut file_sync_admission = if sync { - Some( - os::acquire_file_sync_admission(self.file_sync_permits.clone()) - .await - .map_err(to_file_error) - .map_err(DiskError::from)?, - ) - } else { - None - }; - let prepare_inline_metadata = move || { - let mut prepared_metadata_source = - os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; - #[cfg(windows)] - let source_metadata_guard = - rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; - let mut xlmeta = FileMeta::new(); - // Same as the non-inline branch: an unparsable existing dst - // xl.meta must surface as unknown, not `Absent` - // (rustfs/backlog#1009). - let mut dst_meta_unparsable = false; - if let Some(ref buf) = has_dst_buf { - if FileMeta::is_xl2_v1_format(buf) - && let Ok(nmeta) = FileMeta::load(buf) - { - xlmeta = nmeta - } else { - dst_meta_unparsable = true; - } - } - - let old_current_size = if dst_meta_unparsable { - None - } else { - observe_old_current_size(has_dst_buf.is_some(), &xlmeta) - }; - - let version_id = fi.version_id.unwrap_or_default(); - let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); - let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); - let rollback_data_dir = old_data_dir.or_else(|| { - if old_version_exists && has_dst_buf.is_some() { - Some(inline_metadata_rollback_dir(version_id, &xlmeta)) - } else { - None - } - }); - let mut staged_rollback_path = None; - if let Some(d) = old_data_dir.as_ref() { - let _ = xlmeta.data.remove_two(version_id, *d); - } - xlmeta.add_version(fi)?; - let version_signature = rename_data_versions_signature(&xlmeta); - let new_buf = xlmeta.marshal_msg()?; - // Write the staged xl.meta. Inline objects carry their data inside - // xl.meta, so this is the durable preparation for the metadata commit: - // relaxed tiers do no per-object fsync here at all (aligned - // with MinIO's default), trading a documented power-loss - // window for latency. - prepared_metadata_source.write_all(&new_buf, sync)?; - run_inline_preparation_before_backup(&dst_path_for_failpoint); - if let Some(ref old_metadata) = has_dst_buf - && (rollback_data_dir.is_some() || sync || cfg!(test)) - { - #[cfg(windows)] - let backup_path = { - let backup_path = src - .parent() - .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? - .join(STORAGE_FORMAT_FILE_BACKUP); - source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; - backup_path - }; - #[cfg(not(windows))] - let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; - #[cfg(not(windows))] - if sync { - std::fs::File::open(&backup_path)?.sync_data()?; - } - staged_rollback_path = Some(backup_path); - } - - Ok::<_, std::io::Error>(( - rollback_data_dir, - old_data_dir, - version_signature, - old_current_size, - staged_rollback_path, - has_dst_buf.is_none(), - prepared_metadata_source, - )) - }; - let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { - os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await - } else { - os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await - } - .map_err(to_file_error) - .map_err(DiskError::from); - - let ( - rollback_data_dir, - cleanup_data_dir, - version_signature, - old_current_size, - mut local_rollback_path, - destination_was_absent, - prepared_metadata_source, - ) = match inline_preparation { - Ok(prepared) => prepared, - Err(err) => { - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(err); - } - }; - - let rename_commit_guard = remove_dst_base_before_commit( - dst_path, - rename_commit_guard, - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - - if should_remove_staged_meta_before_commit(dst_path) { - drop(prepared_metadata_source); - let remove_result = std::fs::remove_file(&src_file_path); - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - remove_result.map_err(to_file_error)?; - return Err(DiskError::FileNotFound); - } - - if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { - let Some(dst_parent) = dst_file_path.parent() else { - return Err(DiskError::other("missing object metadata parent")); - }; - let backup_path = dst_parent - .join(rollback_data_dir.to_string()) - .join(STORAGE_FORMAT_FILE_BACKUP); - // rename_all acquires the backup path's namespace lease. Do not - // hold a disk admission while acquiring another namespace lock. - drop(file_sync_admission.take()); - if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { - let _ = remove_file_if_exists(staged_backup); - return Err(err); - } - #[cfg(test)] - run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); - if sync { - file_sync_admission = Some( - os::acquire_file_sync_admission(self.file_sync_permits.clone()) - .await - .map_err(to_file_error) - .map_err(DiskError::from)?, - ); - } - if let Some(admission) = file_sync_admission.as_ref() - && let Some(backup_parent) = backup_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, - fsync_started, - ); - return Err(DiskError::from(to_file_error(err))); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, - fsync_started, - ); - } - local_rollback_path = None; - } - - let commit_result = if should_fail_commit_rename(dst_path) { - Err(DiskError::other("test fail during metadata commit rename")) - } else { - os::rename_all_with_prepared_source( - prepared_metadata_source, - &src_file_path, - &dst_file_path, - &dst_volume_dir, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - }; - if let Err(err) = commit_result { - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(err); - } - - let post_commit = async { - if should_fail_after_metadata_commit(dst_path) { - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(std::io::Error::other("test fail after metadata commit")); - } - - // Persist the commit rename's directory entry across power loss. - if let Some(admission) = file_sync_admission.as_ref() - && let Some(dst_parent) = dst_file_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) - .await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(err); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - } - - // Same power-loss gap as the non-inline path (rustfs/backlog#922 - // step 4): a first PUT creates the object dir (and any missing - // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all - // never fsynced. The fsync above persists the object dir's contents, - // not its own entry, so for a new inline object fsync the ancestor - // chain up to and including the bucket. Overwrites already have a - // durable object dir; the starts_with guard bounds the walk. - if let Some(admission) = file_sync_admission.as_ref() - && destination_was_absent - { - let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); - while let Some(ancestor_dir) = ancestor { - if !ancestor_dir.starts_with(&dst_volume_dir) { - break; - } - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - rollback_inline_metadata_commit_std( - &dst_file_path, - rollback_data_dir, - local_rollback_path.as_deref(), - )?; - return Err(err); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - if ancestor_dir == dst_volume_dir.as_path() { - break; - } - ancestor = ancestor_dir.parent(); - } - } - - Ok::<(), std::io::Error>(()) - } - .await; - - // The disk admission protects the durability chain, not staging - // cleanup or cache invalidation after that chain has completed. - drop(file_sync_admission.take()); - - // A post-commit rollback (for example, a commit-metadata fsync - // failure under strict durability) restores the old metadata; drop any - // descriptors cached during the committed window before propagating the - // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so - // this is mostly defensive and keeps both commit branches consistent. - if let Err(err) = post_commit { - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(DiskError::from(err)); - } - - // The commit no longer has a rollback path. Release the Windows - // object identity guard before best-effort staging cleanup. - #[cfg(windows)] - drop(rename_commit_guard); - - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - - // Cleanup - if let Some(ref cleanup) = cleanup_path { - let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; - } else if let Some(parent) = src_file_path.parent() { - let _ = std::fs::remove_dir(parent); - } - - // Heal reuses a version's `data_dir` and lands the rebuilt shard on - // the SAME `//part.N` path. Without this, a cached - // descriptor would keep serving the pre-heal inode, defeating the heal - // and eroding read quorum (backlog#1145). - // - // The exact keys are derivable here, and this runs on every write, so - // use them rather than registering a predicate the read path would then - // have to evaluate. Readers build the same string - // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every - // part of the version now at `dst_path` — any part path absent from it - // no longer exists for readers to ask for. - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - - Ok(RenameDataResp { - old_data_dir: cleanup_data_dir, - rollback_data_dir, - cleanup_data_dir, - sign: version_signature, - old_current_size, - }) - } + self.rename_data_commit(src_volume, src_path, fi, dst_volume, dst_path).await } #[tracing::instrument(level = "trace", skip_all)] @@ -11055,6 +9923,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf #[cfg(test)] mod test { + use super::commit::create_local_inline_rollback_backup; use super::*; use rustfs_filemeta::ErasureInfo; use std::io::{self, Write}; diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs new file mode 100644 index 000000000..12663bc5f --- /dev/null +++ b/crates/ecstore/src/disk/local/commit.rs @@ -0,0 +1,1205 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Single-disk object rename publication and rollback. The caller retains the +//! DiskAPI instrumentation; mutation leases and commit guards follow the syscall. + +#[cfg(all(test, windows))] +use super::run_destination_commit_directory_preparation; +use super::{ + EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE, + LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size, + remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature, + run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup, + should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit, + skip_access_checks, +}; +#[cfg(test)] +use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication}; +use crate::crash_inject::{self, CrashPoint}; +use crate::disk::{ + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, + error::{DiskError, Result}, + error_conv::{to_access_error, to_file_error}, + os, + os::{check_path_length, rename_all}, +}; +use bytes::Bytes; +use rustfs_filemeta::{FileInfo, FileMeta}; +use std::{ + io::ErrorKind, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::fs; +use tracing::{info, warn}; +use uuid::Uuid; + +fn rollback_committed_rename_std( + dst_file_path: &Path, + new_data_path: Option<&Path>, + rollback_data_dir: Option, +) -> std::io::Result<()> { + if let Some(old_data_dir) = rollback_data_dir { + let Some(dst_parent) = dst_file_path.parent() else { + return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent")); + }; + let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); + std::fs::rename(backup_path, dst_file_path)?; + } else { + remove_file_if_exists(dst_file_path)?; + } + + if let Some(new_data_path) = new_data_path { + remove_dir_all_if_exists(new_data_path)?; + } + + Ok(()) +} + +fn rollback_inline_metadata_commit_std( + dst_file_path: &Path, + rollback_data_dir: Option, + local_rollback_path: Option<&Path>, +) -> std::io::Result<()> { + if let Some(backup_path) = local_rollback_path { + // The commit immediately before this rollback renamed the staged + // xl.meta from the same directory as `backup_path` onto + // `dst_file_path`, proving both paths are on the same filesystem. + // Unix rename atomically replaces the committed destination; never + // unlink it first or an interrupted rollback could lose xl.meta. + std::fs::rename(backup_path, dst_file_path)?; + } else { + rollback_committed_rename_std(dst_file_path, None, rollback_data_dir)?; + } + Ok(()) +} + +pub(super) fn create_local_inline_rollback_backup( + dst_file_path: &Path, + staging_file_path: &Path, + old_metadata: &[u8], +) -> std::io::Result { + let Some(staging_parent) = staging_file_path.parent() else { + return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent")); + }; + let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP); + remove_file_if_exists(&backup_path)?; + if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err()) + && let Err(err) = std::fs::write(&backup_path, old_metadata) + { + let _ = remove_file_if_exists(&backup_path); + return Err(err); + } + Ok(backup_path) +} + +pub(super) async fn lock_rename_commit_directories( + source_parent: &Path, + destination_parent: &Path, + base_dir: &Path, + publication_root: &os::PublicationRoot, + mutation_lease: Arc, +) -> Result { + #[cfg(windows)] + let result = { + let source_parent = source_parent.to_path_buf(); + let destination_parent = destination_parent.to_path_buf(); + let base_dir = base_dir.to_path_buf(); + let publication_root = publication_root.clone(); + os::run_blocking_namespace_operation(mutation_lease, move || { + let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root); + #[cfg(test)] + if result.is_ok() { + run_destination_commit_directory_preparation(&destination_parent); + } + result + }) + .await + }; + #[cfg(not(windows))] + let result = { + let _ = mutation_lease; + os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) + }; + + let result = result.map_err(|err| match std::fs::symlink_metadata(base_dir) { + Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err, + _ => err, + }); + + result.map_err(to_file_error).map_err(DiskError::from) +} + +async fn read_rename_destination_metadata( + file_path: &Path, + rename_commit_guard: &os::RenameCommitGuard, + mutation_lease: Arc, +) -> Result> { + #[cfg(windows)] + let result = { + let file_path = file_path.to_path_buf(); + let rename_commit_guard = rename_commit_guard.clone(); + os::run_blocking_namespace_operation(mutation_lease, move || { + os::read_destination_file_with_commit_guard(&file_path, &rename_commit_guard) + }) + .await + }; + #[cfg(not(windows))] + let _ = (rename_commit_guard, mutation_lease); + #[cfg(not(windows))] + let result = match super::super::fs::read_file(file_path).await { + Ok(data) => Ok(Some(data)), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + }; + + result + .map(|data| data.map(Bytes::from)) + .map_err(to_file_error) + .map_err(DiskError::from) +} + +async fn restore_renamed_data_source( + src_volume_dir: &Path, + src_data_path: &Path, + dst_data_path: &Path, + publication_root: &os::PublicationRoot, + mutation_lease: Arc, +) -> Result<()> { + if fs::symlink_metadata(src_data_path).await.is_ok() { + return Ok(()); + } + let result = + match os::rename_all_with_lease(dst_data_path, src_data_path, src_volume_dir, publication_root, mutation_lease).await { + Ok(()) => Ok(()), + Err(DiskError::FileNotFound) => { + let source_exists = fs::symlink_metadata(src_data_path).await.is_ok(); + let destination_missing = matches!( + fs::symlink_metadata(dst_data_path).await, + Err(err) if err.kind() == ErrorKind::NotFound + ); + if source_exists && destination_missing { + Ok(()) + } else { + Err(DiskError::FileNotFound) + } + } + Err(err) => Err(err), + }; + if let Err(err) = &result { + warn!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "restore_staged_data_source_failed", + src_path = ?src_data_path, + dst_path = ?dst_data_path, + error = ?err, + "Failed to restore staged data after a metadata commit was rejected" + ); + } + result +} + +async fn restore_published_data_source( + data_paths: Option<&(PathBuf, PathBuf)>, + src_volume_dir: &Path, + publication_root: &os::PublicationRoot, + mutation_lease: Arc, +) -> Result<()> { + let Some((src_data_path, dst_data_path)) = data_paths else { + return Ok(()); + }; + restore_renamed_data_source(src_volume_dir, src_data_path, dst_data_path, publication_root, mutation_lease).await +} + +impl LocalDisk { + pub(super) async fn rename_data_commit( + &self, + src_volume: &str, + src_path: &str, + fi: FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result { + let mut fi = fi; + // A non-force DeleteBucket must not remove a directory while a local + // object commit is publishing into it. The peer's empty scan remains + // optimistic; this lease establishes the local commit/delete order and + // remains owned by any blocking syscall that outlives async cancellation. + let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; + let quota_fence_token = + match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { + Some(value) => { + let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; + Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) + } + None if rustfs_utils::http::metadata_compat::contains_key_str( + &fi.metadata, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + ) => + { + return Err(DiskError::FileCorrupt); + } + None => None, + }; + rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); + let quota_fence_claim = match quota_fence_token { + Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), + None => None, + }; + let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; + if let Some(claim) = quota_fence_claim { + mutation_lease.attach_external_guard(claim); + } + if fi.is_legacy_indexed_delete_marker() { + fi.erasure.index = 0; + } + fi.validate_for_metadata_read()?; + // Snapshot the destination part paths before `fi` is consumed below. These + // are the descriptors a reader may hold for the version this call is about + // to replace (backlog#1145); readers build the identical string in + // `io_primitives`. An inline-data version has no parts and yields none. + let invalidate_part_paths: Vec = { + let data_dir = fi.data_dir.unwrap_or_default(); + fi.parts + .iter() + .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) + .collect() + }; + let src_volume_dir = self.io_get_bucket_path(src_volume)?; + if !skip_access_checks(src_volume) + && let Err(e) = super::super::fs::access_std(&src_volume_dir) + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?src_volume_dir, + operation = "rename_data_src_access", + error = %e, + "Disk local access check failed" + ); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; + if !skip_access_checks(dst_volume) + && let Err(e) = super::super::fs::access_std(&dst_volume_dir) + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?dst_volume_dir, + operation = "rename_data_dst_access", + error = %e, + "Disk local access check failed" + ); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + // xl.meta path + let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; + let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; + + // data_dir path + let has_data_dir_path = { + let has_data_dir = { + if !fi.is_remote() { + fi.data_dir + .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) + } else { + None + } + }; + + if let Some(data_dir) = has_data_dir { + let src_data_path = self.io_get_object_path( + src_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), + )?; + let dst_data_path = self.io_get_object_path( + dst_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), + )?; + + Some((src_data_path, dst_data_path)) + } else { + None + } + }; + + check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; + check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; + + let no_inline = fi.data.is_none() && fi.size > 0; + // Captured before `fi` is consumed by add_version; gates the stale + // destination purge below. + let fi_healing = fi.is_healing(); + + // Resolved once for the whole commit so a concurrent configuration + // change can never leave a single rename_data half-synced. The tier is + // keyed on the destination volume: user data staged in scratch + // namespaces follows the configured tier, while commits into + // system-critical namespaces (IAM, config, bucket metadata) stay + // pinned to strict. + let durability = effective_durability(dst_volume); + + let src_file_parent = src_file_path + .parent() + .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; + let dst_file_parent = dst_file_path + .parent() + .ok_or_else(|| DiskError::other("missing object metadata parent"))?; + if !no_inline { + fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; + } + // Acquire the common trees before reading destination metadata. On + // Windows this pins the object directory identity across metadata + // preparation, data publication, rollback backup, and final commit. + let rename_commit_guard = lock_rename_commit_directories( + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; + + if no_inline { + // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta + let mut xlmeta = FileMeta::new(); + // An existing dst xl.meta that fails to parse leaves `xlmeta` empty + // and gets overwritten by the commit below (pre-existing behavior); + // track that so the old-size observation reports unknown instead of + // a false `Absent` (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(dst_buf) = has_dst_buf.as_ref() { + if FileMeta::is_xl2_v1_format(dst_buf) + && let Ok(nmeta) = FileMeta::load(dst_buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let mut skip_parent = dst_volume_dir.clone(); + if has_dst_buf.as_ref().is_some() + && let Some(parent) = dst_file_path.parent() + { + skip_parent = parent.to_path_buf(); + } + + let version_id = fi.version_id.unwrap_or_default(); + let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = has_old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + if let Some(old_data_dir) = has_old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *old_data_dir); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_dst_buf = xlmeta.marshal_msg()?; + + // This tmp xl.meta is renamed onto dst_file_path at the commit + // point below, so only its contents must be durable before the + // rename (SyncMode::FileOnly); the dst parent directory is fsynced + // after the commit rename, and a crash before the rename means the + // PUT was never acknowledged. A metadata commit: relaxed tiers + // leave it to the page cache. + let tmp_meta_sync = if durability.syncs_commit_metadata() { + SyncMode::FileOnly + } else { + SyncMode::None + }; + // The tmp xl.meta write and the shard-file fdatasync are independent + // (disjoint paths) and both only need to be durable before the commit + // renames below, so run them concurrently to drop a blocking + // round-trip from the PUT commit critical path (rustfs/backlog#922 + // step 2). The "contents durable -> rename -> dst dir fsync" ordering + // is unchanged — both futures complete before any rename — which the + // rename_data crash-consistency harness (backlog#935) exercises. + // + // Shard durability: once rename_data succeeds the write is + // acknowledged, so data must not live only in the page cache. + // Multipart parts were already synced during rename_part, so their + // fdatasync here is a cheap no-op. A missing source dir is left for the + // rename below to report through the existing rollback path. Payload + // durability is kept by both strict and relaxed. + let tmp_meta_write = { + let src_file_path = src_file_path.clone(); + let dst_file_path = dst_file_path.clone(); + let rename_commit_guard = rename_commit_guard.clone(); + let mutation_lease = mutation_lease.clone(); + async move { + os::run_blocking_namespace_operation(mutation_lease, move || { + #[cfg(test)] + run_owned_file_write_before_open(&src_file_path); + let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( + &src_file_path, + &dst_file_path, + &rename_commit_guard, + )?; + prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; + Ok(prepared_metadata_source) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from) + } + }; + let shard_sync = async { + if durability.syncs_data_shards() + && let Some((src_data_path, _)) = has_data_dir_path.as_ref() + && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await + && err.kind() != ErrorKind::NotFound + { + return Err::<(), DiskError>(to_file_error(err).into()); + } + Ok(()) + }; + let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); + // Surface a tmp-meta failure first (its prior serial position), then a + // shard-sync failure; either aborts before any rename, exactly as the + // sequential version did. + let prepared_metadata_source = tmp_meta_res?; + shard_sync_res?; + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + std::fs::remove_file(&src_file_path).map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + // Heal reuses the version's data_dir, so for in-place corruption + // the destination dir still exists — and rename(2) cannot replace + // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge + // it first, healing commits only; fresh PUTs mint a new data_dir + // and never collide. Best effort: a real failure surfaces in the + // rename below. + if fi_healing + && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = self.move_to_trash(dst_data_path, true, false).await + { + warn!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + dst_path = ?dst_data_path, + error = ?err, + "Healing commit could not purge the stale destination data dir" + ); + } + if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = os::rename_all_with_commit_guard( + src_data_path, + dst_data_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_data_path_failed", + src_path = ?src_data_path, + dst_path = ?dst_data_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + #[cfg(test)] + if has_data_dir_path.is_some() { + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + } + + // Crash-consistency injection: hard power loss after the data dir + // is in place but before xl.meta commits. No cleanup — the harness + // reopens the disk and asserts the object still reads as the old + // version (the staged data dir is a harmless orphan for GC). + if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { + return Err(DiskError::Unexpected); + } + + if should_fail_before_old_metadata_backup(dst_path) { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "test_fail_before_old_metadata_backup", + "Disk local rename flow failed before metadata commit" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::Unexpected); + } + + // The rollback backup stays where it is written (no rename) and is + // the sole restore source for a later undo_write, so under strict + // it keeps SyncMode::FileAndDir: contents and directory entry both + // durable. It is part of the metadata commit machinery, so relaxed + // tiers leave it to the page cache like the xl.meta it mirrors. + let backup_sync = if durability.syncs_commit_metadata() { + SyncMode::FileAndDir + } else { + SyncMode::None + }; + if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { + let backup_parent = dst_file_parent.join(old_data_dir.to_string()); + #[cfg(not(windows))] + if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { + Ok(guard) => guard, + Err(err) => { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::from(to_file_error(err))); + } + }; + let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); + if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { + #[cfg(windows)] + drop(backup_path_guard); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_bytes = dst_buf.clone(); + // Keep the volume, commit-tree, and exact destination-path + // guards in this task until the backup write and durability + // sync finish. A detached spawn_blocking writer could survive + // cancellation and later truncate a newer transaction's + // deterministic rollback backup. + let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { + #[cfg(test)] + run_owned_file_write_before_open(&backup_path); + backup_path_guard.write_file_for_path_access( + &backup_path, + backup_bytes.as_ref(), + backup_sync != SyncMode::None, + backup_sync == SyncMode::FileAndDir, + ) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from); + if let Err(err) = write_result { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "write_old_metadata_backup_failed", + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + } + + // Crash-consistency injection: hard power loss after the rollback + // backup is durable but before the xl.meta commit rename. No + // cleanup — the harness asserts the object still reads as the old + // version, since the destination xl.meta is untouched here. + if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + if let Err(err) = os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_metadata_failed", + src_path = ?src_file_path, + dst_path = ?dst_file_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + + let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); + if should_fail_after_metadata_commit(dst_path) { + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + return Err(DiskError::Unexpected); + } + + // Crash-consistency injection: hard power loss immediately after the + // xl.meta commit rename but before the durability fsync. Unlike the + // graceful failpoint above, no rollback runs — the commit rename is + // already on disk, so the harness asserts the object reads back as + // the new version. + if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + // Persist the directory entries for both the data dir and xl.meta renames; + // without this the commit itself can vanish on power loss. Relaxed tiers + // accept that window (documented in docs/operations/durability-modes.md). + if durability.syncs_commit_metadata() + && let Some(parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // The commit rename changed the dst part inodes before this fsync + // failed and rolled them back; drop any fd cached during that + // window so readers re-open the restored inode (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // First PUT of an object creates its directory (and any missing prefix + // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The + // commit fsync above persists the object dir's *contents*, not its own + // entry in the bucket/prefix dir, so on power loss after ack the whole + // object dir could vanish (rustfs/backlog#922 step 4). For a new object + // (no prior xl.meta) fsync the ancestor chain from the object dir's + // parent up to and including the bucket so those new directory entries + // are durable. Overwrites already have a durable object dir. The + // starts_with guard bounds the walk to the bucket subtree. Relaxed/none + // accept the wider window, like the commit fsync above. + if has_dst_buf.is_none() && durability.syncs_commit_metadata() { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(dir) = ancestor { + if !dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dir(dir).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // Same post-commit rollback window as above — drop cached + // dst part fds so readers re-open the restored inode + // (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if dir == dst_volume_dir.as_path() { + break; + } + ancestor = dir.parent(); + } + } + + // Publication and every rollback-capable durability step are now + // complete. Do not retain the Windows object identity guard while + // cleaning staging paths or invalidating cached descriptors. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(src_file_path_parent) = src_file_path.parent() { + if src_volume != super::super::RUSTFS_META_MULTIPART_BUCKET { + let _ = std::fs::remove_dir(src_file_path_parent); + } else { + let _ = self + .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) + .await; + } + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: has_old_data_dir, + rollback_data_dir, + cleanup_data_dir: has_old_data_dir, + sign: version_signature, + old_current_size, + }) + } else { + // Inline metadata preparation is blocking. The transaction lease is + // moved into that work so a timeout can release the async waiter without + // allowing a retry to reuse the deterministic staging path too early. + let src = src_file_path.clone(); + let dst = dst_file_path.clone(); + let cleanup_path = if src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET { + src_file_path.parent().map(|p| p.to_path_buf()) + } else { + None + }; + let dst_path_for_failpoint = dst_path.to_string(); + #[cfg(windows)] + let source_parent = src_file_parent.to_path_buf(); + let rename_commit_guard_for_preparation = rename_commit_guard.clone(); + let sync = durability.syncs_commit_metadata(); + #[cfg(test)] + run_inline_before_file_sync_admission(dst_path); + let mut file_sync_admission = if sync { + Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ) + } else { + None + }; + let prepare_inline_metadata = move || { + let mut prepared_metadata_source = + os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; + #[cfg(windows)] + let source_metadata_guard = + rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; + let mut xlmeta = FileMeta::new(); + // Same as the non-inline branch: an unparsable existing dst + // xl.meta must surface as unknown, not `Absent` + // (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(ref buf) = has_dst_buf { + if FileMeta::is_xl2_v1_format(buf) + && let Ok(nmeta) = FileMeta::load(buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let version_id = fi.version_id.unwrap_or_default(); + let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + let mut staged_rollback_path = None; + if let Some(d) = old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *d); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_buf = xlmeta.marshal_msg()?; + // Write the staged xl.meta. Inline objects carry their data inside + // xl.meta, so this is the durable preparation for the metadata commit: + // relaxed tiers do no per-object fsync here at all (aligned + // with MinIO's default), trading a documented power-loss + // window for latency. + prepared_metadata_source.write_all(&new_buf, sync)?; + run_inline_preparation_before_backup(&dst_path_for_failpoint); + if let Some(ref old_metadata) = has_dst_buf + && (rollback_data_dir.is_some() || sync || cfg!(test)) + { + #[cfg(windows)] + let backup_path = { + let backup_path = src + .parent() + .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? + .join(STORAGE_FORMAT_FILE_BACKUP); + source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; + backup_path + }; + #[cfg(not(windows))] + let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; + #[cfg(not(windows))] + if sync { + std::fs::File::open(&backup_path)?.sync_data()?; + } + staged_rollback_path = Some(backup_path); + } + + Ok::<_, std::io::Error>(( + rollback_data_dir, + old_data_dir, + version_signature, + old_current_size, + staged_rollback_path, + has_dst_buf.is_none(), + prepared_metadata_source, + )) + }; + let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { + os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await + } else { + os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await + } + .map_err(to_file_error) + .map_err(DiskError::from); + + let ( + rollback_data_dir, + cleanup_data_dir, + version_signature, + old_current_size, + mut local_rollback_path, + destination_was_absent, + prepared_metadata_source, + ) = match inline_preparation { + Ok(prepared) => prepared, + Err(err) => { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + }; + + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + let remove_result = std::fs::remove_file(&src_file_path); + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + remove_result.map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { + let Some(dst_parent) = dst_file_path.parent() else { + return Err(DiskError::other("missing object metadata parent")); + }; + let backup_path = dst_parent + .join(rollback_data_dir.to_string()) + .join(STORAGE_FORMAT_FILE_BACKUP); + // rename_all acquires the backup path's namespace lease. Do not + // hold a disk admission while acquiring another namespace lock. + drop(file_sync_admission.take()); + if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { + let _ = remove_file_if_exists(staged_backup); + return Err(err); + } + #[cfg(test)] + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + if sync { + file_sync_admission = Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ); + } + if let Some(admission) = file_sync_admission.as_ref() + && let Some(backup_parent) = backup_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + return Err(DiskError::from(to_file_error(err))); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + } + local_rollback_path = None; + } + + let commit_result = if should_fail_commit_rename(dst_path) { + Err(DiskError::other("test fail during metadata commit rename")) + } else { + os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &dst_volume_dir, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + }; + if let Err(err) = commit_result { + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + + let post_commit = async { + if should_fail_after_metadata_commit(dst_path) { + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(std::io::Error::other("test fail after metadata commit")); + } + + // Persist the commit rename's directory entry across power loss. + if let Some(admission) = file_sync_admission.as_ref() + && let Some(dst_parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) + .await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // Same power-loss gap as the non-inline path (rustfs/backlog#922 + // step 4): a first PUT creates the object dir (and any missing + // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all + // never fsynced. The fsync above persists the object dir's contents, + // not its own entry, so for a new inline object fsync the ancestor + // chain up to and including the bucket. Overwrites already have a + // durable object dir; the starts_with guard bounds the walk. + if let Some(admission) = file_sync_admission.as_ref() + && destination_was_absent + { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(ancestor_dir) = ancestor { + if !ancestor_dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std( + &dst_file_path, + rollback_data_dir, + local_rollback_path.as_deref(), + )?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if ancestor_dir == dst_volume_dir.as_path() { + break; + } + ancestor = ancestor_dir.parent(); + } + } + + Ok::<(), std::io::Error>(()) + } + .await; + + // The disk admission protects the durability chain, not staging + // cleanup or cache invalidation after that chain has completed. + drop(file_sync_admission.take()); + + // A post-commit rollback (for example, a commit-metadata fsync + // failure under strict durability) restores the old metadata; drop any + // descriptors cached during the committed window before propagating the + // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so + // this is mostly defensive and keeps both commit branches consistent. + if let Err(err) = post_commit { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(DiskError::from(err)); + } + + // The commit no longer has a rollback path. Release the Windows + // object identity guard before best-effort staging cleanup. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + + // Cleanup + if let Some(ref cleanup) = cleanup_path { + let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; + } else if let Some(parent) = src_file_path.parent() { + let _ = std::fs::remove_dir(parent); + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: cleanup_data_dir, + rollback_data_dir, + cleanup_data_dir, + sign: version_signature, + old_current_size, + }) + } + } +} From 2a6b2f31c7393b4fe97ee11909ab9641e123d1ce Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 09:56:41 +0800 Subject: [PATCH 05/12] refactor(ecstore): remove moved quota fence import --- crates/ecstore/src/disk/local.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index adef17e85..774450201 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -31,10 +31,10 @@ use crate::disk::{ BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, - PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, - QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, - ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, - SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int, + PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, + RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, + STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, + conv_part_err_to_int, endpoint::Endpoint, error::{DiskError, Error, FileAccessDeniedWithContext, Result}, error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error}, From dbe007336264581ca3221a7e02e48c52dc713c94 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 09:25:07 +0800 Subject: [PATCH 06/12] fix(ecstore): retain per-disk rename rollback outcomes --- .../src/set_disk/core/io_primitives.rs | 656 ++++++++++++++++-- 1 file changed, 580 insertions(+), 76 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index c07a013de..6c653daa8 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -1492,6 +1492,7 @@ pub(in crate::set_disk) fn record_read_repair_dedup(reason: &'static str) { counter!("rustfs_heal_read_repair_dedup_total", "reason" => reason).increment(1); } +#[derive(Debug)] pub(in crate::set_disk) enum ReadRepairAdmissionOutcome { Response(HealAdmissionResult), Failed(String), @@ -3838,6 +3839,211 @@ pub(in crate::set_disk) struct RenameTailOutcome { pub(in crate::set_disk) cleanup: Vec, } +const EVENT_SET_DISK_RENAME_ROLLBACK: &str = "set_disk_rename_rollback"; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum RenameRollbackOutcome { + NotAttempted(DiskError), + Succeeded, + Failed(DiskError), + Panicked, + Cancelled, +} + +impl RenameRollbackOutcome { + fn stage(&self) -> &'static str { + match self { + Self::NotAttempted(_) => "rename_failed", + Self::Succeeded => "undo_succeeded", + Self::Failed(_) => "undo_failed", + Self::Panicked => "undo_panicked", + Self::Cancelled => "undo_cancelled", + } + } + + fn failed(&self) -> bool { + matches!(self, Self::Failed(_) | Self::Panicked | Self::Cancelled) + } +} + +#[derive(Debug, Clone)] +struct RenameRollbackDiskOutcome { + disk_index: usize, + rollback_dir: Option, + outcome: RenameRollbackOutcome, +} + +#[derive(Debug)] +struct RenameRollbackReport { + disks: Vec, +} + +/// Shares rollback completion with the staging owner without replacing the +/// original disk/quorum error returned by the rename operation. +#[derive(Clone, Default)] +pub(in crate::set_disk) struct RenameRollbackReceipt(Arc>); + +impl RenameRollbackReceipt { + pub(in crate::set_disk) fn is_incomplete(&self) -> bool { + self.0 + .get() + .is_some_and(|report| report.disks.iter().any(|disk| disk.outcome.failed())) + } +} + +async fn inspect_incomplete_rename_rollback( + disks: &[Option], + bucket: &str, + object: &str, + submitter: ReadRepairAdmissionSubmitter, +) -> ReadRepairAdmissionOutcome { + let location = disks.iter().flatten().next().map(|disk| disk.get_disk_location()); + let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options( + bucket.to_string(), + Some(object.to_string()), + false, + Some(HealChannelPriority::High), + location.as_ref().and_then(|location| location.pool_idx), + location.as_ref().and_then(|location| location.set_idx), + ); + // A failed write's surviving minority is not an authoritative heal source. + // Request inspection only: MRF PartialWrite would schedule mutating repair. + request.dry_run = Some(true); + request.remove_corrupted = Some(false); + request.recreate_missing = Some(false); + request.update_parity = Some(false); + request.recursive = Some(false); + match tokio::time::timeout(Duration::from_secs(1), submitter(request)).await { + Ok(result) => result, + Err(_) => ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string()), + } +} + +fn rename_rollback_task_outcome( + result: std::result::Result, tokio::task::JoinError>, +) -> RenameRollbackOutcome { + match result { + Ok(Ok(())) => RenameRollbackOutcome::Succeeded, + Ok(Err(err)) => RenameRollbackOutcome::Failed(err), + Err(err) if err.is_panic() => RenameRollbackOutcome::Panicked, + Err(_) => RenameRollbackOutcome::Cancelled, + } +} + +async fn rollback_failed_rename( + disks: &[Option], + mut file_infos: Vec, + errs: &[Option], + rollback_dirs: &[Option], + dst: (&str, &str), + receipt: Option, +) { + let (bucket, object) = dst; + let mut outcomes = Vec::with_capacity(disks.len()); + let mut tasks = Vec::with_capacity(disks.len()); + for (disk_index, disk) in disks.iter().enumerate() { + let rollback_dir = rollback_dirs[disk_index]; + let outcome = match &errs[disk_index] { + Some(err) => RenameRollbackOutcome::NotAttempted(err.clone()), + None => RenameRollbackOutcome::Failed(DiskError::DiskNotFound), + }; + outcomes.push(RenameRollbackDiskOutcome { + disk_index, + rollback_dir, + outcome, + }); + if errs[disk_index].is_some() { + continue; + } + let Some(disk) = disk.clone() else { + continue; + }; + let fi = std::mem::take(&mut file_infos[disk_index]); + let bucket = bucket.to_string(); + let object = object.to_string(); + let task = tokio::spawn(async move { + #[allow(clippy::let_unit_value)] + let _task_guard = SetDisks::rename_fanout_task_guard(&object); + SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await; + #[cfg(test)] + rollback_fault_injection::before_undo(&object, disk_index)?; + disk.delete_version( + &bucket, + &object, + fi, + false, + DeleteOptions { + undo_write: true, + old_data_dir: rollback_dir, + ..Default::default() + }, + ) + .await + }); + tasks.push(async move { (disk_index, task.await) }); + } + for (disk_index, result) in join_all(tasks).await { + outcomes[disk_index].outcome = rename_rollback_task_outcome(result); + } + + let attempted = outcomes + .iter() + .filter(|disk| !matches!(disk.outcome, RenameRollbackOutcome::NotAttempted(_))) + .count(); + let failed = outcomes.iter().filter(|disk| disk.outcome.failed()).count(); + let succeeded = attempted - failed; + for disk in &outcomes { + counter!("rustfs_rename_rollback_disks_total", "stage" => disk.outcome.stage()).increment(1); + if disk.outcome.failed() { + let location = disks[disk.disk_index].as_ref().map(|disk| disk.get_disk_location()); + warn!( + event = EVENT_SET_DISK_RENAME_ROLLBACK, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + state = "recovery_required", + stage = disk.outcome.stage(), + pool_index = ?location.as_ref().and_then(|location| location.pool_idx), + set_index = ?location.as_ref().and_then(|location| location.set_idx), + disk_index = disk.disk_index, + bucket, + object, + rollback_dir = ?disk.rollback_dir, + outcome = ?disk.outcome, + attempted, + succeeded, + failed, + "rename rollback incomplete; preserve recovery material" + ); + } + } + if let Some(receipt) = receipt { + let _ = receipt.0.set(RenameRollbackReport { disks: outcomes }); + } + if failed > 0 { + let result = inspect_incomplete_rename_rollback(disks, bucket, object, send_read_repair_heal_request).await; + let admission = match &result { + ReadRepairAdmissionOutcome::Response(response) => response.result_label(), + ReadRepairAdmissionOutcome::Failed(_) => "failed", + }; + counter!("rustfs_rename_rollback_inspection_total", "admission" => admission).increment(1); + warn!( + event = EVENT_SET_DISK_RENAME_ROLLBACK, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + state = "recovery_required", + stage = "inspection_admission", + bucket, + object, + attempted, + succeeded, + failed, + admission, + outcome = ?result, + "rename rollback inspection requested; recovery remains incomplete" + ); + } +} + /// Options shared by the normal and early-ack rename fanouts. Keeping the /// quorum and optional scanner lease map together avoids widening either /// fanout helper's argument list while preserving the fence semantics. @@ -3845,6 +4051,7 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> { write_quorum: usize, scanner_publication_lease_tokens: Option<&'a HashMap>, scanner_publication_commit_scope: Option, + rollback_receipt: Option, } impl<'a> RenameDataFenceOptions<'a> { @@ -3856,9 +4063,15 @@ impl<'a> RenameDataFenceOptions<'a> { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope: None, + rollback_receipt: None, } } + pub(in crate::set_disk) fn with_rollback_receipt(mut self, receipt: RenameRollbackReceipt) -> Self { + self.rollback_receipt = Some(receipt); + self + } + pub(in crate::set_disk) fn with_publication_scope( mut self, scanner_publication_commit_scope: Option, @@ -4224,6 +4437,7 @@ impl SetDisks { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope: _scanner_publication_commit_scope, + rollback_receipt, } = fence_options; if let Some(file_info) = disks .iter() @@ -4405,36 +4619,15 @@ impl SetDisks { if !sent_commit { let ret_err = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum).unwrap_or(DiskError::Unexpected); - let mut rollbacks = Vec::new(); - let mut rollback_file_infos = file_infos; - for (i, err) in errs.iter().enumerate() { - if err.is_some() { - continue; - } - - if let Some(disk) = coordinator_disks[i].as_ref() { - let fi = std::mem::take(&mut rollback_file_infos[i]); - let old_data_dir = data_dirs[i]; - let disk = disk.clone(); - let dst_bucket = fanout_dst_bucket.clone(); - let dst_object = fanout_dst_object.clone(); - rollbacks.push(tokio::spawn(async move { - disk.delete_version( - &dst_bucket, - &dst_object, - fi, - false, - DeleteOptions { - undo_write: true, - old_data_dir, - ..Default::default() - }, - ) - .await - })); - } - } - let _ = join_all(rollbacks).await; + rollback_failed_rename( + &coordinator_disks, + file_infos, + &errs, + &data_dirs, + (&fanout_dst_bucket, &fanout_dst_object), + rollback_receipt, + ) + .await; if let Some(commit_tx) = commit_tx.take() { let _ = commit_tx.send(Err(ret_err)); } @@ -4582,6 +4775,7 @@ impl SetDisks { write_quorum, scanner_publication_lease_tokens, scanner_publication_commit_scope, + rollback_receipt, } = fence_options; if let Some(file_info) = disks .iter() @@ -4793,36 +4987,7 @@ impl SetDisks { ); } - let mut futures = Vec::with_capacity(disks.len()); if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - for (i, err) in errs.iter().enumerate() { - if err.is_some() { - continue; - } - - if let Some(disk) = disks[i].as_ref() { - let fi = std::mem::take(&mut file_infos[i]); - let old_data_dir = data_dirs[i]; - let disk = disk.clone(); - let dst_bucket = dst_bucket.clone(); - let dst_object = dst_object.clone(); - futures.push(tokio::spawn(async move { - disk.delete_version( - &dst_bucket, - &dst_object, - fi, - false, - DeleteOptions { - undo_write: true, - old_data_dir, - ..Default::default() - }, - ) - .await - })); - } - } - if issue3031_diag_enabled() { warn!( target: "rustfs_ecstore::set_disk", @@ -4838,23 +5003,7 @@ impl SetDisks { ); } - let undo_results = join_all(futures).await; - let undo_error_count = undo_results - .iter() - .filter(|result| match result { - Err(_) | Ok(Err(_)) => true, - Ok(Ok(_)) => false, - }) - .count(); - if undo_error_count > 0 { - warn!( - target: "rustfs_ecstore::set_disk", - dst_bucket = %dst_bucket, - dst_object = %dst_object, - undo_error_count, - "rename_data quorum rollback reported errors" - ); - } + rollback_failed_rename(disks, file_infos, &errs, &data_dirs, (&dst_bucket, &dst_object), rollback_receipt).await; return Err(ret_err); } @@ -6800,6 +6949,57 @@ pub(in crate::set_disk) mod rename_fault_injection { } } +#[cfg(test)] +pub(in crate::set_disk) mod rollback_fault_injection { + use super::DiskError; + use std::{ + collections::HashMap, + sync::{Mutex, OnceLock}, + }; + + #[derive(Clone, Copy, Debug)] + pub(in crate::set_disk) enum Fault { + Io, + Panic, + } + + fn registry() -> &'static Mutex> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(Mutex::default) + } + + pub(in crate::set_disk) struct Guard(String); + + impl Drop for Guard { + fn drop(&mut self) { + if let Ok(mut registry) = registry().lock() { + registry.remove(&self.0); + } + } + } + + pub(in crate::set_disk) fn arm(object: &str, disk_index: usize, fault: Fault) -> Guard { + registry() + .lock() + .expect("rollback registry should not poison") + .insert(object.to_string(), (disk_index, fault)); + Guard(object.to_string()) + } + + pub(super) fn before_undo(object: &str, disk_index: usize) -> Result<(), DiskError> { + let fault = registry() + .lock() + .expect("rollback registry should not poison") + .get(object) + .copied(); + match fault { + Some((target, Fault::Io)) if target == disk_index => Err(DiskError::FaultyDisk), + Some((target, Fault::Panic)) if target == disk_index => panic!("injected rollback panic"), + _ => Ok(()), + } + } +} + /// Test-only per-disk call counters for the metadata fan-out (backlog#1325, /// serving the RPC-count assertions of #1309 / #1314 / #1315). /// @@ -6911,6 +7111,7 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase { pub const RENAME: &str = "rename"; /// The per-disk old-data-dir cleanup phase of the commit fan-out. pub const CLEANUP: &str = "cleanup"; + pub const ROLLBACK: &str = "rollback"; /// The per-disk `read_version` phase of metadata read fan-out. #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub const READ_VERSION: &str = "read_version"; @@ -10422,6 +10623,309 @@ mod tests { .await; } + #[tokio::test] + async fn rename_rollback_incomplete_inspection_rejection_is_not_recovery() { + fn reject_inspection(request: rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture { + assert_eq!(request.bucket, "rollback-inspection"); + assert_eq!(request.object_prefix.as_deref(), Some("object")); + assert_eq!(request.dry_run, Some(true), "failed minority must never become a mutating heal source"); + assert_eq!(request.remove_corrupted, Some(false)); + assert_eq!(request.recreate_missing, Some(false)); + assert_eq!(request.recursive, Some(false)); + Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Full) }) + } + let result = inspect_incomplete_rename_rollback(&[], "rollback-inspection", "object", reject_inspection).await; + assert!(matches!(result, ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Full))); + } + + #[tokio::test] + async fn rename_rollback_incomplete_cancelled_task_is_not_success() { + let task = tokio::spawn(std::future::pending::>()); + task.abort(); + assert_eq!(rename_rollback_task_outcome(task.await), RenameRollbackOutcome::Cancelled); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_incomplete_matches_early_ack_and_full_wait_after_reopen() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + const DISKS: usize = 4; + const WRITE_QUORUM: usize = 3; + for overwrite in [false, true] { + for success_count in [0, WRITE_QUORUM - 1, WRITE_QUORUM] { + for fault in [ + None, + Some(rollback_fault_injection::Fault::Io), + Some(rollback_fault_injection::Fault::Panic), + ] { + let mut previous = None; + for early_ack in [false, true] { + let bucket = "rename-rollback-matrix"; + let object = format!("object-{overwrite}-{success_count}-{fault:?}-{early_ack}"); + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + if overwrite { + let mut old = metadata_test_fileinfo(&object); + old.mod_time = Some(OffsetDateTime::now_utc()); + old.data = Some(Bytes::from_static(b"old-inline-body")); + old.metadata.insert("etag".to_string(), "old-etag".to_string()); + for disk in disks.iter().flatten() { + disk.write_metadata(bucket, bucket, &object, old.clone()) + .await + .expect("old version must be staged"); + } + } + let _rename_fault = + rename_fault_injection::fail_rename_on(&object, &(success_count..DISKS).collect::>()); + let _undo_fault = fault.map(|fault| rollback_fault_injection::arm(&object, 0, fault)); + let receipt = RenameRollbackReceipt::default(); + let result = SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(&object, DISKS, "new-etag"), + (bucket, &object), + early_ack, + RenameDataFenceOptions::new(WRITE_QUORUM, None).with_rollback_receipt(receipt.clone()), + ) + .await; + let actual_error = match result { + Ok(commit) => { + assert_eq!(success_count, WRITE_QUORUM); + if let Some(tail) = commit.tail_drain { + tail.await + .expect("committed tail should join") + .expect("committed tail should converge"); + } + assert!( + receipt.0.get().is_none(), + "a quorum commit must not enter rollback even when undo faults are armed" + ); + None + } + Err(err) => { + assert!(success_count < WRITE_QUORUM); + let report = receipt + .0 + .get() + .expect("both failure paths must publish per-disk rollback evidence"); + assert_eq!(report.disks.len(), DISKS); + for (idx, outcome) in report.disks.iter().enumerate() { + assert_eq!(outcome.disk_index, idx); + let expected = if idx >= success_count { + RenameRollbackOutcome::NotAttempted(DiskError::other( + "injected rename failure (test-only)", + )) + } else if idx == 0 { + match fault { + Some(rollback_fault_injection::Fault::Io) => { + RenameRollbackOutcome::Failed(DiskError::FaultyDisk) + } + Some(rollback_fault_injection::Fault::Panic) => RenameRollbackOutcome::Panicked, + None => RenameRollbackOutcome::Succeeded, + } + } else { + RenameRollbackOutcome::Succeeded + }; + assert_eq!(outcome.outcome, expected); + if overwrite && idx < success_count { + let backup = dirs[idx] + .path() + .join(bucket) + .join(&object) + .join(outcome.rollback_dir.expect("overwrite needs rollback dir").to_string()) + .join(STORAGE_FORMAT_FILE_BACKUP); + assert_eq!( + backup.exists(), + outcome.outcome.failed(), + "failed undo must retain its only old-version backup" + ); + } + } + assert_eq!(receipt.is_incomplete(), success_count > 0 && fault.is_some()); + Some(err) + } + }; + if let Some(expected_error) = previous.as_ref() { + assert_eq!( + &actual_error, expected_error, + "early ACK must preserve the original full-wait quorum error" + ); + } + previous = Some(actual_error); + for (idx, dir) in dirs.iter().enumerate() { + let reopened = reopen_local_disk(dir).await; + let read = reopened + .read_version( + "", + bucket, + &object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await; + let keeps_new = + idx < success_count && (success_count == WRITE_QUORUM || (idx == 0 && fault.is_some())); + if keeps_new || overwrite { + let stored = read.expect("old or committed version must survive reopen"); + assert_eq!( + stored.metadata.get("etag").map(String::as_str), + Some(if keeps_new { "new-etag" } else { "old-etag" }) + ); + assert_eq!( + stored.data.as_deref(), + Some(if keeps_new { + b"inline-body".as_slice() + } else { + b"old-inline-body".as_slice() + }) + ); + } else { + assert!( + matches!(read, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)), + "fresh rollback must not expose data: {read:?}" + ); + } + } + } + } + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for early_ack in [false, true] { + let bucket = "rollback-data-dirs"; + let object = format!("object-{early_ack}"); + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let old_data_dir = Uuid::new_v4(); + let new_data_dir = Uuid::new_v4(); + let mut old = metadata_test_fileinfo(&object); + old.data_dir = Some(old_data_dir); + old.mod_time = Some(OffsetDateTime::now_utc()); + let mut infos = Vec::new(); + for (idx, disk) in disks.iter().enumerate() { + let disk = disk.as_ref().expect("fixture disk should be present"); + disk.write_metadata(bucket, bucket, &object, old.clone()) + .await + .expect("old metadata should be staged"); + let old_dir = dirs[idx].path().join(bucket).join(&object).join(old_data_dir.to_string()); + tokio::fs::create_dir_all(&old_dir) + .await + .expect("old data directory should exist"); + tokio::fs::write(old_dir.join("part.1"), b"old-data") + .await + .expect("old shard should exist"); + let source = dirs[idx] + .path() + .join(RUSTFS_META_TMP_BUCKET) + .join("source") + .join(new_data_dir.to_string()); + tokio::fs::create_dir_all(&source) + .await + .expect("new data directory should be staged"); + tokio::fs::write(source.join("part.1"), b"new-data") + .await + .expect("new shard should be staged"); + let mut fi = metadata_test_fileinfo(&object); + fi.data_dir = Some(new_data_dir); + fi.erasure.index = idx + 1; + fi.mod_time = Some(OffsetDateTime::now_utc()); + infos.push(fi); + } + let _rename_fault = rename_fault_injection::fail_rename_on(&object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(&object, 0, rollback_fault_injection::Fault::Io); + let receipt = RenameRollbackReceipt::default(); + assert!( + SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + infos, + (bucket, &object), + early_ack, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + ) + .await + .is_err() + ); + assert!(receipt.is_incomplete()); + for (idx, dir) in dirs.iter().enumerate() { + let root = dir.path().join(bucket).join(&object); + assert_eq!( + tokio::fs::read(root.join(old_data_dir.to_string()).join("part.1")) + .await + .expect("old data must survive failed overwrite"), + b"old-data" + ); + let backup = root.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); + assert_eq!(backup.exists(), idx == 0, "only the failed undo retains its backup"); + if idx >= 2 { + let staged = dir + .path() + .join(RUSTFS_META_TMP_BUCKET) + .join("source") + .join(new_data_dir.to_string()) + .join("part.1"); + assert_eq!( + tokio::fs::read(staged) + .await + .expect("failed-write staging must remain available"), + b"new-data" + ); + } + let reopened = reopen_local_disk(dir).await; + let stored = reopened + .read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("metadata should survive reopen"); + assert_eq!(stored.data_dir, Some(if idx == 0 { new_data_dir } else { old_data_dir })); + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() { + let bucket = "rename-rollback-barrier"; + let object = "rollback-barrier-object"; + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK); + let receipt = RenameRollbackReceipt::default(); + let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(object, 4, "new-etag"), + (bucket, object), + false, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + )); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + tokio::select! { + () = barrier.wait_until_paused() => {} + _ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"), + } + }) + .await + .expect("undo must reach its disk barrier"); + assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success"); + barrier.release(); + assert!(rename.await.is_err()); + assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt"); + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn rename_data_early_ack_strict_quorum_failure_rolls_back_fresh_after_reopen() { From 19690eba7b7790e62184ad8e2610c87b0c9bed24 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 09:31:56 +0800 Subject: [PATCH 07/12] fix(ecstore): retain indeterminate rename recovery evidence --- .../src/set_disk/core/io_primitives.rs | 765 ++++++++++++------ 1 file changed, 522 insertions(+), 243 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 6c653daa8..d07651ba8 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3844,6 +3844,7 @@ const EVENT_SET_DISK_RENAME_ROLLBACK: &str = "set_disk_rename_rollback"; #[derive(Debug, Clone, PartialEq, Eq)] enum RenameRollbackOutcome { NotAttempted(DiskError), + Indeterminate(DiskError), Succeeded, Failed(DiskError), Panicked, @@ -3853,7 +3854,8 @@ enum RenameRollbackOutcome { impl RenameRollbackOutcome { fn stage(&self) -> &'static str { match self { - Self::NotAttempted(_) => "rename_failed", + Self::NotAttempted(_) => "rename_not_dispatched", + Self::Indeterminate(_) => "rename_indeterminate", Self::Succeeded => "undo_succeeded", Self::Failed(_) => "undo_failed", Self::Panicked => "undo_panicked", @@ -3861,8 +3863,12 @@ impl RenameRollbackOutcome { } } - fn failed(&self) -> bool { - matches!(self, Self::Failed(_) | Self::Panicked | Self::Cancelled) + fn undo_attempted(&self) -> bool { + matches!(self, Self::Succeeded | Self::Failed(_) | Self::Panicked | Self::Cancelled) + } + + fn needs_recovery(&self) -> bool { + matches!(self, Self::Indeterminate(_) | Self::Failed(_) | Self::Panicked | Self::Cancelled) } } @@ -3887,7 +3893,7 @@ impl RenameRollbackReceipt { pub(in crate::set_disk) fn is_incomplete(&self) -> bool { self.0 .get() - .is_some_and(|report| report.disks.iter().any(|disk| disk.outcome.failed())) + .is_some_and(|report| report.disks.iter().any(|disk| disk.outcome.needs_recovery())) } } @@ -3932,69 +3938,122 @@ fn rename_rollback_task_outcome( async fn rollback_failed_rename( disks: &[Option], - mut file_infos: Vec, + file_infos: Vec, errs: &[Option], + dispatched: &[bool], rollback_dirs: &[Option], dst: (&str, &str), receipt: Option, ) { - let (bucket, object) = dst; - let mut outcomes = Vec::with_capacity(disks.len()); - let mut tasks = Vec::with_capacity(disks.len()); - for (disk_index, disk) in disks.iter().enumerate() { - let rollback_dir = rollback_dirs[disk_index]; - let outcome = match &errs[disk_index] { - Some(err) => RenameRollbackOutcome::NotAttempted(err.clone()), - None => RenameRollbackOutcome::Failed(DiskError::DiskNotFound), - }; - outcomes.push(RenameRollbackDiskOutcome { - disk_index, - rollback_dir, - outcome, - }); - if errs[disk_index].is_some() { - continue; - } - let Some(disk) = disk.clone() else { - continue; - }; - let fi = std::mem::take(&mut file_infos[disk_index]); - let bucket = bucket.to_string(); - let object = object.to_string(); - let task = tokio::spawn(async move { - #[allow(clippy::let_unit_value)] - let _task_guard = SetDisks::rename_fanout_task_guard(&object); - SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await; - #[cfg(test)] - rollback_fault_injection::before_undo(&object, disk_index)?; - disk.delete_version( - &bucket, - &object, - fi, - false, - DeleteOptions { - undo_write: true, - old_data_dir: rollback_dir, - ..Default::default() - }, - ) - .await - }); - tasks.push(async move { (disk_index, task.await) }); - } - for (disk_index, result) in join_all(tasks).await { - outcomes[disk_index].outcome = rename_rollback_task_outcome(result); - } + let owned_disks = disks.to_vec(); + let owned_errs = errs.to_vec(); + let owned_dispatched = dispatched.to_vec(); + let owned_dirs = rollback_dirs.to_vec(); + let owned_dst = (dst.0.to_string(), dst.1.to_string()); + let coordinator_failure_receipt = receipt.clone(); + // Own both undo mutations and their accounting: a cancelled requester must + // not leave detached disk tasks without the recovery evidence they produce. + let rollback = tokio::spawn(async move { + let disks = owned_disks.as_slice(); + let errs = owned_errs.as_slice(); + let dispatched = owned_dispatched.as_slice(); + let rollback_dirs = owned_dirs.as_slice(); + let dst = (owned_dst.0.as_str(), owned_dst.1.as_str()); + let mut file_infos = file_infos; - let attempted = outcomes + let (bucket, object) = dst; + let mut outcomes = Vec::with_capacity(disks.len()); + let mut tasks = Vec::with_capacity(disks.len()); + for (disk_index, disk) in disks.iter().enumerate() { + let rollback_dir = rollback_dirs[disk_index]; + let outcome = match &errs[disk_index] { + Some(err) if dispatched[disk_index] => RenameRollbackOutcome::Indeterminate(err.clone()), + Some(err) => RenameRollbackOutcome::NotAttempted(err.clone()), + None => RenameRollbackOutcome::Failed(DiskError::DiskNotFound), + }; + outcomes.push(RenameRollbackDiskOutcome { + disk_index, + rollback_dir, + outcome, + }); + if errs[disk_index].is_some() { + continue; + } + let Some(disk) = disk.clone() else { + continue; + }; + let fi = std::mem::take(&mut file_infos[disk_index]); + let bucket = bucket.to_string(); + let object = object.to_string(); + let task = tokio::spawn(async move { + #[allow(clippy::let_unit_value)] + let _task_guard = SetDisks::rename_fanout_task_guard(&object); + SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await; + #[cfg(test)] + rollback_fault_injection::before_undo(&object, disk_index)?; + disk.delete_version( + &bucket, + &object, + fi, + false, + DeleteOptions { + undo_write: true, + old_data_dir: rollback_dir, + ..Default::default() + }, + ) + .await + }); + tasks.push(async move { (disk_index, task.await) }); + } + for (disk_index, result) in join_all(tasks).await { + outcomes[disk_index].outcome = rename_rollback_task_outcome(result); + } + + record_rename_rollback_outcomes(disks, outcomes, dst, receipt).await; + }); + if rollback.await.is_err() { + record_indeterminate_rename(disks, dst, coordinator_failure_receipt).await; + } +} + +async fn record_indeterminate_rename(disks: &[Option], dst: (&str, &str), receipt: Option) { + let outcomes = disks .iter() - .filter(|disk| !matches!(disk.outcome, RenameRollbackOutcome::NotAttempted(_))) + .enumerate() + .map(|(disk_index, disk)| RenameRollbackDiskOutcome { + disk_index, + rollback_dir: None, + outcome: if disk.is_some() { + RenameRollbackOutcome::Indeterminate(DiskError::Unexpected) + } else { + RenameRollbackOutcome::NotAttempted(DiskError::DiskNotFound) + }, + }) + .collect(); + record_rename_rollback_outcomes(disks, outcomes, dst, receipt).await; +} + +async fn record_rename_rollback_outcomes( + disks: &[Option], + outcomes: Vec, + dst: (&str, &str), + receipt: Option, +) { + let (bucket, object) = dst; + let attempted = outcomes.iter().filter(|disk| disk.outcome.undo_attempted()).count(); + let failed = outcomes + .iter() + .filter(|disk| disk.outcome.undo_attempted() && disk.outcome.needs_recovery()) + .count(); + let indeterminate = outcomes + .iter() + .filter(|disk| matches!(disk.outcome, RenameRollbackOutcome::Indeterminate(_))) .count(); - let failed = outcomes.iter().filter(|disk| disk.outcome.failed()).count(); let succeeded = attempted - failed; for disk in &outcomes { counter!("rustfs_rename_rollback_disks_total", "stage" => disk.outcome.stage()).increment(1); - if disk.outcome.failed() { + if disk.outcome.needs_recovery() { let location = disks[disk.disk_index].as_ref().map(|disk| disk.get_disk_location()); warn!( event = EVENT_SET_DISK_RENAME_ROLLBACK, @@ -4012,6 +4071,7 @@ async fn rollback_failed_rename( attempted, succeeded, failed, + indeterminate, "rename rollback incomplete; preserve recovery material" ); } @@ -4019,7 +4079,7 @@ async fn rollback_failed_rename( if let Some(receipt) = receipt { let _ = receipt.0.set(RenameRollbackReport { disks: outcomes }); } - if failed > 0 { + if failed > 0 || indeterminate > 0 { let result = inspect_incomplete_rename_rollback(disks, bucket, object, send_read_repair_heal_request).await; let admission = match &result { ReadRepairAdmissionOutcome::Response(response) => response.result_label(), @@ -4037,6 +4097,7 @@ async fn rollback_failed_rename( attempted, succeeded, failed, + indeterminate, admission, outcome = ?result, "rename rollback inspection requested; recovery remains incomplete" @@ -4461,6 +4522,7 @@ impl SetDisks { let dst_object = Arc::new(dst_object.to_string()); let (commit_tx, commit_rx) = tokio::sync::oneshot::channel(); + let coordinator_failure_receipt = rollback_receipt.clone(); let tail_drain = tokio::spawn({ let fanout_src_bucket = src_bucket.clone(); let fanout_src_object = src_object.clone(); @@ -4483,7 +4545,8 @@ impl SetDisks { let file_info = file_info.clone(); let successful_rename_completion_rank = successful_rename_completion_rank.clone(); tasks.spawn(async move { - let result = std::panic::AssertUnwindSafe(async move { + let mut dispatched = false; + let result = std::panic::AssertUnwindSafe(async { #[allow(clippy::let_unit_value)] let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object); @@ -4508,6 +4571,7 @@ impl SetDisks { } let disk_wait_started = rustfs_io_metrics::put_stage_timer(); + dispatched = true; let result = disk .rename_data_borrowed_with_fence( &src_bucket, @@ -4518,6 +4582,10 @@ impl SetDisks { scanner_publication_lease_token, ) .await; + #[cfg(test)] + if result.is_ok() { + rollback_fault_injection::after_rename(&dst_object, i)?; + } if let Some(disk_wait_started) = disk_wait_started { let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0; rustfs_io_metrics::record_put_object_stage_duration( @@ -4543,7 +4611,7 @@ impl SetDisks { }) .catch_unwind() .await; - (i, result) + (i, dispatched, result) }); } @@ -4553,6 +4621,8 @@ impl SetDisks { let mut fanout_panic = 0usize; let mut results_seen = 0usize; let mut errs = vec![Some(DiskError::DiskNotFound); disk_count]; + // Missing task results cannot prove that a disk mutation never ran. + let mut dispatched = vec![true; disk_count]; let mut disk_versions = vec![None; disk_count]; let mut data_dirs = vec![None; disk_count]; let mut cleanup_data_dirs = vec![None; disk_count]; @@ -4563,7 +4633,8 @@ impl SetDisks { while let Some(joined) = tasks.join_next().await { results_seen += 1; match joined { - Ok((idx, Ok(Ok(res)))) => { + Ok((idx, was_dispatched, Ok(Ok(res)))) => { + dispatched[idx] = was_dispatched; data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir); cleanup_data_dirs[idx] = res.cleanup_data_dir; disk_versions[idx] = res.sign; @@ -4571,10 +4642,12 @@ impl SetDisks { errs[idx] = None; success_count += 1; } - Ok((idx, Ok(Err(err)))) => { + Ok((idx, was_dispatched, Ok(Err(err)))) => { + dispatched[idx] = was_dispatched; errs[idx] = Some(err); } - Ok((idx, Err(_))) => { + Ok((idx, was_dispatched, Err(_))) => { + dispatched[idx] = was_dispatched; errs[idx] = Some(DiskError::Unexpected); fanout_panic += 1; } @@ -4604,6 +4677,8 @@ impl SetDisks { } } + #[cfg(test)] + rollback_fault_injection::after_fanout(&fanout_dst_object); if rustfs_io_metrics::put_stage_metrics_enabled() { let fanout_success = errs.iter().filter(|err| err.is_none()).count(); let fanout_error = errs.len().saturating_sub(fanout_success + fanout_panic); @@ -4623,6 +4698,7 @@ impl SetDisks { &coordinator_disks, file_infos, &errs, + &dispatched, &data_dirs, (&fanout_dst_bucket, &fanout_dst_object), rollback_receipt, @@ -4717,7 +4793,13 @@ impl SetDisks { }); let quorum_wait_started = rustfs_io_metrics::put_stage_timer(); - let commit = commit_rx.await.map_err(|_| DiskError::Unexpected)?; + let commit = match commit_rx.await { + Ok(commit) => commit, + Err(_) => { + record_indeterminate_rename(disks, (&dst_bucket, &dst_object), coordinator_failure_receipt).await; + return Err(DiskError::Unexpected); + } + }; rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, quorum_wait_started, @@ -4831,81 +4913,95 @@ impl SetDisks { let successful_rename_completion_rank = successful_rename_completion_rank.clone(); let publication_scope = scanner_publication_commit_scope.clone(); - std::panic::AssertUnwindSafe(async move { - // Test-only introspection guard: counts this operation as - // in-flight for the whole body. Compiles to `()` in production. - #[allow(clippy::let_unit_value)] - let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object); + async move { + let mut dispatched = false; + let result = std::panic::AssertUnwindSafe(async { + // Test-only introspection guard: counts this operation as + // in-flight for the whole body. Compiles to `()` in production. + #[allow(clippy::let_unit_value)] + let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object); - let Some(disk) = disk else { - return Err(DiskError::DiskNotFound); - }; - - let is_delete_marker = file_info.is_canonical_delete_marker(); - let mut local_file_info; - let file_info = if file_info.erasure.index == 0 { - local_file_info = file_info.clone(); - local_file_info.erasure.index = i + 1; - &local_file_info - } else { - file_info - }; - if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) { - return Err(DiskError::FileCorrupt); - } - - // Test-only awaitable pause point right before the disk rename. - // A no-op immediately-ready future in production. - Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await; - - if let Some(err) = Self::rename_injected_error(&dst_object, i) { - return Err(err); - } - - if let Some(scope) = publication_scope.as_ref() - && !scope.can_commit() - { - let _ = scope.mark_indeterminate(); - return Err(DiskError::other("scanner publication commit scope deadline or cancellation reached")); - } - - let disk_wait_started = rustfs_io_metrics::put_stage_timer(); - let result = disk - .rename_data_borrowed_with_fence( - &src_bucket, - &src_object, - file_info, - &dst_bucket, - &dst_object, - scanner_publication_lease_token, - ) - .await; - if let Some(disk_wait_started) = disk_wait_started { - let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0; - rustfs_io_metrics::record_put_object_stage_duration( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, - duration_ms, - ); - let position = if result.is_ok() { - let rank = successful_rename_completion_rank - .as_ref() - .map(|rank| rank.fetch_add(1, Ordering::Relaxed) + 1) - .unwrap_or(1); - if rank <= write_quorum { - rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST - } else { - rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL - } - } else { - rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR + let Some(disk) = disk else { + return Err(DiskError::DiskNotFound); }; - rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms); - } - result - }) - .catch_unwind() + + let is_delete_marker = file_info.is_canonical_delete_marker(); + let mut local_file_info; + let file_info = if file_info.erasure.index == 0 { + local_file_info = file_info.clone(); + local_file_info.erasure.index = i + 1; + &local_file_info + } else { + file_info + }; + if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) { + return Err(DiskError::FileCorrupt); + } + + // Test-only awaitable pause point right before the disk rename. + // A no-op immediately-ready future in production. + Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await; + + if let Some(err) = Self::rename_injected_error(&dst_object, i) { + return Err(err); + } + + if let Some(scope) = publication_scope.as_ref() + && !scope.can_commit() + { + let _ = scope.mark_indeterminate(); + return Err(DiskError::other( + "scanner publication commit scope deadline or cancellation reached", + )); + } + + let disk_wait_started = rustfs_io_metrics::put_stage_timer(); + dispatched = true; + let result = disk + .rename_data_borrowed_with_fence( + &src_bucket, + &src_object, + file_info, + &dst_bucket, + &dst_object, + scanner_publication_lease_token, + ) + .await; + #[cfg(test)] + if result.is_ok() { + rollback_fault_injection::after_rename(&dst_object, i)?; + } + if let Some(disk_wait_started) = disk_wait_started { + let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, + duration_ms, + ); + let position = if result.is_ok() { + let rank = successful_rename_completion_rank + .as_ref() + .map(|rank| rank.fetch_add(1, Ordering::Relaxed) + 1) + .unwrap_or(1); + if rank <= write_quorum { + rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST + } else { + rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL + } + } else { + rustfs_io_metrics::PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR + }; + rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms); + } + result + }) + .catch_unwind() + .await; + (dispatched, result) + } }); let results = join_all(futures).await; + #[cfg(test)] + rollback_fault_injection::after_fanout(&fanout_dst_object); (results, fanout_file_infos) }); @@ -4920,12 +5016,18 @@ impl SetDisks { rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, quorum_wait_started, ); - let (results, mut file_infos) = fanout_result.map_err(|_| DiskError::Unexpected)?; + let (results, mut file_infos) = match fanout_result { + Ok(result) => result, + Err(_) => { + record_indeterminate_rename(disks, (&dst_bucket, &dst_object), rollback_receipt).await; + return Err(DiskError::Unexpected); + } + }; if rustfs_io_metrics::put_stage_metrics_enabled() { let mut fanout_success = 0; let mut fanout_error = 0; let mut fanout_panic = 0; - for result in &results { + for (_, result) in &results { match result { Ok(Ok(_)) => fanout_success += 1, Ok(Err(_)) => fanout_error += 1, @@ -4941,7 +5043,9 @@ impl SetDisks { ); } - for (idx, result) in results.iter().enumerate() { + let mut dispatched = Vec::with_capacity(results.len()); + for (idx, (was_dispatched, result)) in results.iter().enumerate() { + dispatched.push(*was_dispatched); match result { Ok(Ok(res)) => { data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir); @@ -5003,7 +5107,16 @@ impl SetDisks { ); } - rollback_failed_rename(disks, file_infos, &errs, &data_dirs, (&dst_bucket, &dst_object), rollback_receipt).await; + rollback_failed_rename( + disks, + file_infos, + &errs, + &dispatched, + &data_dirs, + (&dst_bucket, &dst_object), + rollback_receipt, + ) + .await; return Err(ret_err); } @@ -6961,6 +7074,9 @@ pub(in crate::set_disk) mod rollback_fault_injection { pub(in crate::set_disk) enum Fault { Io, Panic, + IoAfterRename, + PanicAfterRename, + CoordinatorPanic, } fn registry() -> &'static Mutex> { @@ -6998,6 +7114,30 @@ pub(in crate::set_disk) mod rollback_fault_injection { _ => Ok(()), } } + + pub(super) fn after_rename(object: &str, disk_index: usize) -> Result<(), DiskError> { + let fault = registry() + .lock() + .expect("rollback registry should not poison") + .get(object) + .copied(); + match fault { + Some((target, Fault::IoAfterRename)) if target == disk_index => Err(DiskError::FaultyDisk), + Some((target, Fault::PanicAfterRename)) if target == disk_index => panic!("injected panic after rename mutation"), + _ => Ok(()), + } + } + + pub(super) fn after_fanout(object: &str) { + let fault = registry() + .lock() + .expect("rollback registry should not poison") + .get(object) + .copied(); + if matches!(fault, Some((_, Fault::CoordinatorPanic))) { + panic!("injected rename coordinator panic"); + } + } } /// Test-only per-disk call counters for the metadata fan-out (backlog#1325, @@ -10722,6 +10862,7 @@ mod tests { } Some(rollback_fault_injection::Fault::Panic) => RenameRollbackOutcome::Panicked, None => RenameRollbackOutcome::Succeeded, + Some(_) => unreachable!("matrix only injects undo faults"), } } else { RenameRollbackOutcome::Succeeded @@ -10736,7 +10877,7 @@ mod tests { .join(STORAGE_FORMAT_FILE_BACKUP); assert_eq!( backup.exists(), - outcome.outcome.failed(), + outcome.outcome.needs_recovery(), "failed undo must retain its only old-version backup" ); } @@ -10799,94 +10940,188 @@ mod tests { #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] - async fn rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging() { + async fn rename_data_early_ack_post_mutation_tail_error_never_rolls_back_commit() { temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { - for early_ack in [false, true] { - let bucket = "rollback-data-dirs"; - let object = format!("object-{early_ack}"); + for fault in [ + rollback_fault_injection::Fault::IoAfterRename, + rollback_fault_injection::Fault::PanicAfterRename, + ] { + let bucket = "rename-tail-unknown"; + let object = format!("tail-{fault:?}"); let (dirs, disks) = call_counter_local_disks(bucket, 4).await; prepare_rename_source_dirs(&dirs, &disks, "source").await; - let old_data_dir = Uuid::new_v4(); - let new_data_dir = Uuid::new_v4(); - let mut old = metadata_test_fileinfo(&object); - old.data_dir = Some(old_data_dir); - old.mod_time = Some(OffsetDateTime::now_utc()); - let mut infos = Vec::new(); - for (idx, disk) in disks.iter().enumerate() { - let disk = disk.as_ref().expect("fixture disk should be present"); - disk.write_metadata(bucket, bucket, &object, old.clone()) - .await - .expect("old metadata should be staged"); - let old_dir = dirs[idx].path().join(bucket).join(&object).join(old_data_dir.to_string()); - tokio::fs::create_dir_all(&old_dir) - .await - .expect("old data directory should exist"); - tokio::fs::write(old_dir.join("part.1"), b"old-data") - .await - .expect("old shard should exist"); - let source = dirs[idx] - .path() - .join(RUSTFS_META_TMP_BUCKET) - .join("source") - .join(new_data_dir.to_string()); - tokio::fs::create_dir_all(&source) - .await - .expect("new data directory should be staged"); - tokio::fs::write(source.join("part.1"), b"new-data") - .await - .expect("new shard should be staged"); - let mut fi = metadata_test_fileinfo(&object); - fi.data_dir = Some(new_data_dir); - fi.erasure.index = idx + 1; - fi.mod_time = Some(OffsetDateTime::now_utc()); - infos.push(fi); - } - let _rename_fault = rename_fault_injection::fail_rename_on(&object, &[2, 3]); - let _undo_fault = rollback_fault_injection::arm(&object, 0, rollback_fault_injection::Fault::Io); let receipt = RenameRollbackReceipt::default(); - assert!( - SetDisks::rename_data_owned_with_fence( - &disks, - (RUSTFS_META_TMP_BUCKET, "source"), - infos, - (bucket, &object), - early_ack, - RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), - ) + let _fault = rollback_fault_injection::arm(&object, 0, fault); + let barrier = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier_phase::RENAME); + let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(&object, 4, "new-etag"), + (bucket, &object), + true, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + )); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + tokio::select! { + () = barrier.wait_until_paused() => {} + _ = rename.as_mut() => panic!("tail barrier must precede quorum ACK"), + } + }) + .await + .expect("tail reaches the barrier"); + let commit = tokio::time::timeout(BARRIER_PAUSE_GUARD, rename) .await - .is_err() - ); - assert!(receipt.is_incomplete()); - for (idx, dir) in dirs.iter().enumerate() { - let root = dir.path().join(bucket).join(&object); - assert_eq!( - tokio::fs::read(root.join(old_data_dir.to_string()).join("part.1")) + .expect("quorum must ACK before tail release") + .expect("three disks commit"); + barrier.release(); + let tail = commit + .tail_drain + .expect("early ACK owns a tail") + .await + .expect("tail coordinator joins") + .expect("committed tail reports convergence"); + assert_eq!(tail.convergence, RenameConvergence::PartialCommit); + assert!(receipt.0.get().is_none(), "post-ACK errors must never start rollback"); + for dir in &dirs { + let reopened = reopen_local_disk(dir).await; + let stored = reopened + .read_version( + "", + bucket, + &object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("all actual writes survive despite a lost tail acknowledgement"); + assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice())); + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging() { + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for fault in [ + rollback_fault_injection::Fault::Io, + rollback_fault_injection::Fault::IoAfterRename, + rollback_fault_injection::Fault::PanicAfterRename, + rollback_fault_injection::Fault::CoordinatorPanic, + ] { + for early_ack in [false, true] { + let bucket = "rollback-data-dirs"; + let object = format!("object-{early_ack}-{fault:?}"); + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let old_data_dir = Uuid::new_v4(); + let new_data_dir = Uuid::new_v4(); + let mut old = metadata_test_fileinfo(&object); + old.data_dir = Some(old_data_dir); + old.mod_time = Some(OffsetDateTime::now_utc()); + let mut infos = Vec::new(); + for (idx, disk) in disks.iter().enumerate() { + let disk = disk.as_ref().expect("fixture disk should be present"); + disk.write_metadata(bucket, bucket, &object, old.clone()) .await - .expect("old data must survive failed overwrite"), - b"old-data" - ); - let backup = root.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); - assert_eq!(backup.exists(), idx == 0, "only the failed undo retains its backup"); - if idx >= 2 { - let staged = dir + .expect("old metadata should be staged"); + let old_dir = dirs[idx].path().join(bucket).join(&object).join(old_data_dir.to_string()); + tokio::fs::create_dir_all(&old_dir) + .await + .expect("old data directory should exist"); + tokio::fs::write(old_dir.join("part.1"), b"old-data") + .await + .expect("old shard should exist"); + let source = dirs[idx] .path() .join(RUSTFS_META_TMP_BUCKET) .join("source") - .join(new_data_dir.to_string()) - .join("part.1"); - assert_eq!( - tokio::fs::read(staged) - .await - .expect("failed-write staging must remain available"), - b"new-data" + .join(new_data_dir.to_string()); + tokio::fs::create_dir_all(&source) + .await + .expect("new data directory should be staged"); + tokio::fs::write(source.join("part.1"), b"new-data") + .await + .expect("new shard should be staged"); + let mut fi = metadata_test_fileinfo(&object); + fi.data_dir = Some(new_data_dir); + fi.erasure.index = idx + 1; + fi.mod_time = Some(OffsetDateTime::now_utc()); + infos.push(fi); + } + let _rename_fault = rename_fault_injection::fail_rename_on(&object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(&object, 0, fault); + let receipt = RenameRollbackReceipt::default(); + assert!( + SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + infos, + (bucket, &object), + early_ack, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + ) + .await + .is_err() + ); + assert!(receipt.is_incomplete()); + if !matches!(fault, rollback_fault_injection::Fault::Io) { + assert!( + matches!( + receipt.0.get().expect("indeterminate report").disks[0].outcome, + RenameRollbackOutcome::Indeterminate(_) + ), + "post-mutation failure must not be classified as unattempted" + ); + } + let coordinator_failed = matches!(fault, rollback_fault_injection::Fault::CoordinatorPanic); + for (idx, dir) in dirs.iter().enumerate() { + let root = dir.path().join(bucket).join(&object); + assert_eq!( + tokio::fs::read(root.join(old_data_dir.to_string()).join("part.1")) + .await + .expect("old data must survive failed overwrite"), + b"old-data" + ); + let backup = root.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); + assert_eq!( + backup.exists(), + idx == 0 || (coordinator_failed && idx == 1), + "unknown mutations must retain the old-version backup" + ); + if idx >= 2 { + let staged = dir + .path() + .join(RUSTFS_META_TMP_BUCKET) + .join("source") + .join(new_data_dir.to_string()) + .join("part.1"); + assert_eq!( + tokio::fs::read(staged) + .await + .expect("failed-write staging must remain available"), + b"new-data" + ); + } + let reopened = reopen_local_disk(dir).await; + let stored = reopened + .read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("metadata should survive reopen"); + assert_eq!( + stored.data_dir, + Some(if idx == 0 || (coordinator_failed && idx == 1) { + new_data_dir + } else { + old_data_dir + }) ); } - let reopened = reopen_local_disk(dir).await; - let stored = reopened - .read_version("", bucket, &object, "", &ReadOptions::default()) - .await - .expect("metadata should survive reopen"); - assert_eq!(stored.data_dir, Some(if idx == 0 { new_data_dir } else { old_data_dir })); } } }) @@ -10896,34 +11131,78 @@ mod tests { #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() { - let bucket = "rename-rollback-barrier"; - let object = "rollback-barrier-object"; - let (dirs, disks) = call_counter_local_disks(bucket, 4).await; - prepare_rename_source_dirs(&dirs, &disks, "source").await; - let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); - let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io); - let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK); - let receipt = RenameRollbackReceipt::default(); - let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( - &disks, - (RUSTFS_META_TMP_BUCKET, "source"), - rename_commit_fileinfos(object, 4, "new-etag"), - (bucket, object), - false, - RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), - )); - tokio::time::timeout(BARRIER_PAUSE_GUARD, async { - tokio::select! { - () = barrier.wait_until_paused() => {} - _ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"), + for cancel_caller in [false, true] { + let bucket = "rename-rollback-barrier"; + let object = if cancel_caller { + "rollback-barrier-cancelled" + } else { + "rollback-barrier-object" + }; + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let mut old = metadata_test_fileinfo(object); + old.mod_time = Some(OffsetDateTime::now_utc()); + old.data = Some(Bytes::from_static(b"old-inline-body")); + old.metadata.insert("etag".to_string(), "old-etag".to_string()); + for disk in disks.iter().flatten() { + disk.write_metadata(bucket, bucket, object, old.clone()) + .await + .expect("old metadata should be staged"); } - }) - .await - .expect("undo must reach its disk barrier"); - assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success"); - barrier.release(); - assert!(rename.await.is_err()); - assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt"); + let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK); + let receipt = RenameRollbackReceipt::default(); + let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + rename_commit_fileinfos(object, 4, "new-etag"), + (bucket, object), + false, + RenameDataFenceOptions::new(3, None).with_rollback_receipt(receipt.clone()), + )); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + tokio::select! { + () = barrier.wait_until_paused() => {} + _ = rename.as_mut() => panic!("rename returned before the armed rollback barrier"), + } + }) + .await + .expect("undo must reach its disk barrier"); + assert!(receipt.0.get().is_none(), "pending undo must not be recorded as success"); + if cancel_caller { + drop(rename); + barrier.release(); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + while receipt.0.get().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled caller must not cancel rollback accounting"); + } else { + barrier.release(); + assert!(rename.await.is_err()); + } + assert!(receipt.is_incomplete(), "drained undo failure must survive in the receipt"); + for dir in dirs.iter().skip(1) { + let reopened = reopen_local_disk(dir).await; + let restored = reopened + .read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("old version must remain readable after caller cancellation"); + assert_eq!(restored.data.as_deref(), Some(b"old-inline-body".as_slice())); + } + } } #[tokio::test] From 5513dadb78da2936be16598009c0b9144ab3c74b Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 09:48:45 +0800 Subject: [PATCH 08/12] test(ecstore): mark rollback fixtures as inline data --- crates/ecstore/src/set_disk/core/io_primitives.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index d07651ba8..4d8315302 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -10009,6 +10009,7 @@ mod tests { file_info.mod_time = Some(OffsetDateTime::now_utc()); file_info.erasure.index = idx + 1; file_info.data = Some(Bytes::from_static(b"inline-body")); + file_info.set_inline_data(); file_info.metadata.insert("etag".to_string(), etag.to_string()); file_info }) @@ -10808,6 +10809,7 @@ mod tests { let mut old = metadata_test_fileinfo(&object); old.mod_time = Some(OffsetDateTime::now_utc()); old.data = Some(Bytes::from_static(b"old-inline-body")); + old.set_inline_data(); old.metadata.insert("etag".to_string(), "old-etag".to_string()); for disk in disks.iter().flatten() { disk.write_metadata(bucket, bucket, &object, old.clone()) @@ -10921,7 +10923,8 @@ mod tests { b"inline-body".as_slice() } else { b"old-inline-body".as_slice() - }) + }), + "object={object}, disk={idx}, keeps_new={keeps_new}" ); } else { assert!( @@ -11143,6 +11146,7 @@ mod tests { let mut old = metadata_test_fileinfo(object); old.mod_time = Some(OffsetDateTime::now_utc()); old.data = Some(Bytes::from_static(b"old-inline-body")); + old.set_inline_data(); old.metadata.insert("etag".to_string(), "old-etag".to_string()); for disk in disks.iter().flatten() { disk.write_metadata(bucket, bucket, object, old.clone()) From daba4f7b328fc7323af61843bbe83acee1086ed2 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 09:28:46 +0800 Subject: [PATCH 09/12] test(ecstore): match sealed context fixture map type --- crates/ecstore/src/bucket/sealed_credentials.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ecstore/src/bucket/sealed_credentials.rs b/crates/ecstore/src/bucket/sealed_credentials.rs index 46a0f79fb..a5b264fc0 100644 --- a/crates/ecstore/src/bucket/sealed_credentials.rs +++ b/crates/ecstore/src/bucket/sealed_credentials.rs @@ -203,7 +203,7 @@ mod tests { use parking_lot::Mutex; use std::collections::BTreeMap; - fn encode_context(context: &HashMap) -> String { + fn encode_context(context: &BTreeMap) -> String { let ordered = context.iter().collect::>(); serde_json::to_string(&ordered).expect("context serializes") } From ccf8e2362c819cd78085ef05fef07017c300349b Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 09:28:46 +0800 Subject: [PATCH 10/12] test(ecstore): match sealed context fixture map type --- crates/ecstore/src/bucket/sealed_credentials.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ecstore/src/bucket/sealed_credentials.rs b/crates/ecstore/src/bucket/sealed_credentials.rs index 46a0f79fb..a5b264fc0 100644 --- a/crates/ecstore/src/bucket/sealed_credentials.rs +++ b/crates/ecstore/src/bucket/sealed_credentials.rs @@ -203,7 +203,7 @@ mod tests { use parking_lot::Mutex; use std::collections::BTreeMap; - fn encode_context(context: &HashMap) -> String { + fn encode_context(context: &BTreeMap) -> String { let ordered = context.iter().collect::>(); serde_json::to_string(&ordered).expect("context serializes") } From 0658b228e1751d8072ae78b5900b5ce736f85050 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 12:08:28 +0800 Subject: [PATCH 11/12] fix(ecstore): preserve known preflight rename rejections --- crates/ecstore/src/disk/disk_store.rs | 85 +- crates/ecstore/src/disk/local.rs | 1958 +++++++++-------- crates/ecstore/src/disk/mod.rs | 49 + .../src/set_disk/core/io_primitives.rs | 82 +- crates/ecstore/src/set_disk/ops/object.rs | 243 +- 5 files changed, 1324 insertions(+), 1093 deletions(-) diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index e5eccba32..b98b294ca 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -317,6 +317,22 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper { dst_path: &str, external_guard: Option>, ) -> Result { + self.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, external_guard) + .await + .result + } +} + +impl LocalDiskWrapper { + pub(in crate::disk) async fn rename_data_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + external_guard: Option>, + ) -> super::RenameDataObservation { let operation = self.clone(); let src_volume = src_volume.to_owned(); let src_path = src_path.to_owned(); @@ -333,22 +349,35 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper { } else { get_max_timeout_duration() }; - run_owned_mutation(external_guard, move || async move { - operation + let observed = run_owned_mutation(external_guard, move || async move { + let mut preflight_rejection = None; + let result = operation .track_disk_health_mutation( "rename_data", DiskMetricMutation::Write, || async { - operation - .disk - .rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path) - .await + // Preserve the former DiskAPI future's single boxing boundary. + let observed = + Box::pin( + operation + .disk + .rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path), + ) + .await; + preflight_rejection = observed.preflight_rejection; + observed.result }, timeout_duration, ) - .await + .await; + // Health tracking must observe the real disk error, not an Ok tuple. + Ok(super::RenameDataObservation { + result, + preflight_rejection, + }) }) - .await + .await; + observed.unwrap_or_else(|error| super::RenameDataObservation::unknown(Err(error))) } } @@ -2588,6 +2617,46 @@ mod tests { assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1)); } + #[tokio::test] + async fn rename_preflight_evidence_preserves_health_errors_and_owned_reply() { + for source_exists in [false, true] { + for guarded in [false, true] { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")) + .expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + if source_exists { + disk.make_volume("source").await.expect("source volume should exist"); + } + let wrapper = LocalDiskWrapper::new(disk, false); + let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let external_guard = guarded.then(|| Arc::new(DropProbe(Arc::clone(&drops))) as Arc); + let mut file_info = FileInfo::new("object", 1, 0); + file_info.mod_time = Some(::time::OffsetDateTime::now_utc()); + file_info.erasure.index = 1; + let observed = wrapper + .rename_data_observed("source", "object", &file_info, "missing-destination", "object", external_guard) + .await; + assert!(observed.rejected_before_publication(), "normal access rejection must carry proof"); + assert!(matches!(observed.result, Err(DiskError::VolumeNotFound))); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1)); + assert_eq!(snapshot.total_writes, 0, "health tracking must not observe the rejection as Ok"); + assert_eq!(drops.load(Ordering::SeqCst), usize::from(guarded)); + + wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline); + let observed = wrapper + .rename_data_observed("source", "object", &file_info, "missing-destination", "object", None) + .await; + assert!(!observed.rejected_before_publication(), "wrapper errors carry no local preflight proof"); + assert!(matches!(observed.result, Err(DiskError::FaultyDisk))); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.total_errors_availability, 1); + assert_eq!(snapshot.total_writes, 0); + } + } + } + #[tokio::test] async fn local_disk_health_wrapper_counts_returned_availability_errors() { let dir = tempfile::tempdir().expect("temp dir should be created"); diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 6baa92a3e..d9f5956b3 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -9040,7 +9040,6 @@ impl DiskAPI for LocalDisk { Ok(()) } - #[tracing::instrument(level = "trace", skip_all)] async fn rename_data( &self, src_volume: &str, @@ -9049,966 +9048,8 @@ impl DiskAPI for LocalDisk { dst_volume: &str, dst_path: &str, ) -> Result { - crate::hp_guard!("LocalDisk::rename_data"); - let mut fi = fi; - // A non-force DeleteBucket must not remove a directory while a local - // object commit is publishing into it. The peer's empty scan remains - // optimistic; this lease establishes the local commit/delete order and - // remains owned by any blocking syscall that outlives async cancellation. - let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; - let quota_fence_token = - match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { - Some(value) => { - let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; - Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) - } - None if rustfs_utils::http::metadata_compat::contains_key_str( - &fi.metadata, - QUOTA_MUTATION_FENCE_METADATA_SUFFIX, - ) => - { - return Err(DiskError::FileCorrupt); - } - None => None, - }; - rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); - let quota_fence_claim = match quota_fence_token { - Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), - None => None, - }; - let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; - if let Some(claim) = quota_fence_claim { - mutation_lease.attach_external_guard(claim); - } - if fi.is_legacy_indexed_delete_marker() { - fi.erasure.index = 0; - } - fi.validate_for_metadata_read()?; - // Snapshot the destination part paths before `fi` is consumed below. These - // are the descriptors a reader may hold for the version this call is about - // to replace (backlog#1145); readers build the identical string in - // `io_primitives`. An inline-data version has no parts and yields none. - let invalidate_part_paths: Vec = { - let data_dir = fi.data_dir.unwrap_or_default(); - fi.parts - .iter() - .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) - .collect() - }; - let src_volume_dir = self.io_get_bucket_path(src_volume)?; - if !skip_access_checks(src_volume) - && let Err(e) = super::fs::access_std(&src_volume_dir) - { - info!( - event = EVENT_DISK_LOCAL_ACCESS_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - path = ?src_volume_dir, - operation = "rename_data_src_access", - error = %e, - "Disk local access check failed" - ); - return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); - } - - let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; - if !skip_access_checks(dst_volume) - && let Err(e) = super::fs::access_std(&dst_volume_dir) - { - info!( - event = EVENT_DISK_LOCAL_ACCESS_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - path = ?dst_volume_dir, - operation = "rename_data_dst_access", - error = %e, - "Disk local access check failed" - ); - return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); - } - - // xl.meta path - let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; - let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; - - // data_dir path - let has_data_dir_path = { - let has_data_dir = { - if !fi.is_remote() { - fi.data_dir - .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) - } else { - None - } - }; - - if let Some(data_dir) = has_data_dir { - let src_data_path = self.io_get_object_path( - src_volume, - rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), - )?; - let dst_data_path = self.io_get_object_path( - dst_volume, - rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), - )?; - - Some((src_data_path, dst_data_path)) - } else { - None - } - }; - - check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; - check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; - - let no_inline = fi.data.is_none() && fi.size > 0; - // Captured before `fi` is consumed by add_version; gates the stale - // destination purge below. - let fi_healing = fi.is_healing(); - - // Resolved once for the whole commit so a concurrent configuration - // change can never leave a single rename_data half-synced. The tier is - // keyed on the destination volume: user data staged in scratch - // namespaces follows the configured tier, while commits into - // system-critical namespaces (IAM, config, bucket metadata) stay - // pinned to strict. - let durability = effective_durability(dst_volume); - - let src_file_parent = src_file_path - .parent() - .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; - let dst_file_parent = dst_file_path - .parent() - .ok_or_else(|| DiskError::other("missing object metadata parent"))?; - if !no_inline { - fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; - } - // Acquire the common trees before reading destination metadata. On - // Windows this pins the object directory identity across metadata - // preparation, data publication, rollback backup, and final commit. - let rename_commit_guard = lock_rename_commit_directories( - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; - - if no_inline { - // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta - let mut xlmeta = FileMeta::new(); - // An existing dst xl.meta that fails to parse leaves `xlmeta` empty - // and gets overwritten by the commit below (pre-existing behavior); - // track that so the old-size observation reports unknown instead of - // a false `Absent` (rustfs/backlog#1009). - let mut dst_meta_unparsable = false; - if let Some(dst_buf) = has_dst_buf.as_ref() { - if FileMeta::is_xl2_v1_format(dst_buf) - && let Ok(nmeta) = FileMeta::load(dst_buf) - { - xlmeta = nmeta - } else { - dst_meta_unparsable = true; - } - } - - let old_current_size = if dst_meta_unparsable { - None - } else { - observe_old_current_size(has_dst_buf.is_some(), &xlmeta) - }; - - let mut skip_parent = dst_volume_dir.clone(); - if has_dst_buf.as_ref().is_some() - && let Some(parent) = dst_file_path.parent() - { - skip_parent = parent.to_path_buf(); - } - - let version_id = fi.version_id.unwrap_or_default(); - let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); - let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); - let rollback_data_dir = has_old_data_dir.or_else(|| { - if old_version_exists && has_dst_buf.is_some() { - Some(inline_metadata_rollback_dir(version_id, &xlmeta)) - } else { - None - } - }); - if let Some(old_data_dir) = has_old_data_dir.as_ref() { - let _ = xlmeta.data.remove_two(version_id, *old_data_dir); - } - xlmeta.add_version(fi)?; - let version_signature = rename_data_versions_signature(&xlmeta); - let new_dst_buf = xlmeta.marshal_msg()?; - - // This tmp xl.meta is renamed onto dst_file_path at the commit - // point below, so only its contents must be durable before the - // rename (SyncMode::FileOnly); the dst parent directory is fsynced - // after the commit rename, and a crash before the rename means the - // PUT was never acknowledged. A metadata commit: relaxed tiers - // leave it to the page cache. - let tmp_meta_sync = if durability.syncs_commit_metadata() { - SyncMode::FileOnly - } else { - SyncMode::None - }; - // The tmp xl.meta write and the shard-file fdatasync are independent - // (disjoint paths) and both only need to be durable before the commit - // renames below, so run them concurrently to drop a blocking - // round-trip from the PUT commit critical path (rustfs/backlog#922 - // step 2). The "contents durable -> rename -> dst dir fsync" ordering - // is unchanged — both futures complete before any rename — which the - // rename_data crash-consistency harness (backlog#935) exercises. - // - // Shard durability: once rename_data succeeds the write is - // acknowledged, so data must not live only in the page cache. - // Multipart parts were already synced during rename_part, so their - // fdatasync here is a cheap no-op. A missing source dir is left for the - // rename below to report through the existing rollback path. Payload - // durability is kept by both strict and relaxed. - let tmp_meta_write = { - let src_file_path = src_file_path.clone(); - let dst_file_path = dst_file_path.clone(); - let rename_commit_guard = rename_commit_guard.clone(); - let mutation_lease = mutation_lease.clone(); - async move { - os::run_blocking_namespace_operation(mutation_lease, move || { - #[cfg(test)] - run_owned_file_write_before_open(&src_file_path); - let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( - &src_file_path, - &dst_file_path, - &rename_commit_guard, - )?; - prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; - Ok(prepared_metadata_source) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from) - } - }; - let shard_sync = async { - if durability.syncs_data_shards() - && let Some((src_data_path, _)) = has_data_dir_path.as_ref() - && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await - && err.kind() != ErrorKind::NotFound - { - return Err::<(), DiskError>(to_file_error(err).into()); - } - Ok(()) - }; - let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); - // Surface a tmp-meta failure first (its prior serial position), then a - // shard-sync failure; either aborts before any rename, exactly as the - // sequential version did. - let prepared_metadata_source = tmp_meta_res?; - shard_sync_res?; - let rename_commit_guard = remove_dst_base_before_commit( - dst_path, - rename_commit_guard, - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - if should_remove_staged_meta_before_commit(dst_path) { - drop(prepared_metadata_source); - std::fs::remove_file(&src_file_path).map_err(to_file_error)?; - return Err(DiskError::FileNotFound); - } - - // Heal reuses the version's data_dir, so for in-place corruption - // the destination dir still exists — and rename(2) cannot replace - // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge - // it first, healing commits only; fresh PUTs mint a new data_dir - // and never collide. Best effort: a real failure surfaces in the - // rename below. - if fi_healing - && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = self.move_to_trash(dst_data_path, true, false).await - { - warn!( - event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - dst_path = ?dst_data_path, - error = ?err, - "Healing commit could not purge the stale destination data dir" - ); - } - if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = os::rename_all_with_commit_guard( - src_data_path, - dst_data_path, - &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "rename_all_data_path_failed", - src_path = ?src_data_path, - dst_path = ?dst_data_path, - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - #[cfg(test)] - if has_data_dir_path.is_some() { - run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); - } - - // Crash-consistency injection: hard power loss after the data dir - // is in place but before xl.meta commits. No cleanup — the harness - // reopens the disk and asserts the object still reads as the old - // version (the staged data dir is a harmless orphan for GC). - if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { - return Err(DiskError::Unexpected); - } - - if should_fail_before_old_metadata_backup(dst_path) { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "test_fail_before_old_metadata_backup", - "Disk local rename flow failed before metadata commit" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(DiskError::Unexpected); - } - - // The rollback backup stays where it is written (no rename) and is - // the sole restore source for a later undo_write, so under strict - // it keeps SyncMode::FileAndDir: contents and directory entry both - // durable. It is part of the metadata commit machinery, so relaxed - // tiers leave it to the page cache like the xl.meta it mirrors. - let backup_sync = if durability.syncs_commit_metadata() { - SyncMode::FileAndDir - } else { - SyncMode::None - }; - if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { - let backup_parent = dst_file_parent.join(old_data_dir.to_string()); - #[cfg(not(windows))] - if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { - Ok(guard) => guard, - Err(err) => { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(DiskError::from(to_file_error(err))); - } - }; - let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); - if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { - #[cfg(windows)] - drop(backup_path_guard); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - let backup_bytes = dst_buf.clone(); - // Keep the volume, commit-tree, and exact destination-path - // guards in this task until the backup write and durability - // sync finish. A detached spawn_blocking writer could survive - // cancellation and later truncate a newer transaction's - // deterministic rollback backup. - let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { - #[cfg(test)] - run_owned_file_write_before_open(&backup_path); - backup_path_guard.write_file_for_path_access( - &backup_path, - backup_bytes.as_ref(), - backup_sync != SyncMode::None, - backup_sync == SyncMode::FileAndDir, - ) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from); - if let Err(err) = write_result { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "write_old_metadata_backup_failed", - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - } - - // Crash-consistency injection: hard power loss after the rollback - // backup is durable but before the xl.meta commit rename. No - // cleanup — the harness asserts the object still reads as the old - // version, since the destination xl.meta is untouched here. - if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { - return Err(DiskError::Unexpected); - } - - if let Err(err) = os::rename_all_with_prepared_source( - prepared_metadata_source, - &src_file_path, - &dst_file_path, - &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) + self.rename_data_inner(src_volume, src_path, fi, dst_volume, dst_path, &mut None) .await - { - info!( - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "rename_all_metadata_failed", - src_path = ?src_file_path, - dst_path = ?dst_file_path, - error = ?err, - "Disk local rename flow failed" - ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - return Err(err); - } - - let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); - if should_fail_after_metadata_commit(dst_path) { - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - return Err(DiskError::Unexpected); - } - - // Crash-consistency injection: hard power loss immediately after the - // xl.meta commit rename but before the durability fsync. Unlike the - // graceful failpoint above, no rollback runs — the commit rename is - // already on disk, so the harness asserts the object reads back as - // the new version. - if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { - return Err(DiskError::Unexpected); - } - - // Persist the directory entries for both the data dir and xl.meta renames; - // without this the commit itself can vanish on power loss. Relaxed tiers - // accept that window (documented in docs/operations/durability-modes.md). - if durability.syncs_commit_metadata() - && let Some(parent) = dst_file_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // The commit rename changed the dst part inodes before this fsync - // failed and rolled them back; drop any fd cached during that - // window so readers re-open the restored inode (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(to_file_error(err).into()); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - } - - // First PUT of an object creates its directory (and any missing prefix - // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The - // commit fsync above persists the object dir's *contents*, not its own - // entry in the bucket/prefix dir, so on power loss after ack the whole - // object dir could vanish (rustfs/backlog#922 step 4). For a new object - // (no prior xl.meta) fsync the ancestor chain from the object dir's - // parent up to and including the bucket so those new directory entries - // are durable. Overwrites already have a durable object dir. The - // starts_with guard bounds the walk to the bucket subtree. Relaxed/none - // accept the wider window, like the commit fsync above. - if has_dst_buf.is_none() && durability.syncs_commit_metadata() { - let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); - while let Some(dir) = ancestor { - if !dir.starts_with(&dst_volume_dir) { - break; - } - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dir(dir).await { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // Same post-commit rollback window as above — drop cached - // dst part fds so readers re-open the restored inode - // (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(to_file_error(err).into()); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - if dir == dst_volume_dir.as_path() { - break; - } - ancestor = dir.parent(); - } - } - - // Publication and every rollback-capable durability step are now - // complete. Do not retain the Windows object identity guard while - // cleaning staging paths or invalidating cached descriptors. - #[cfg(windows)] - drop(rename_commit_guard); - - if let Some(src_file_path_parent) = src_file_path.parent() { - if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { - let _ = std::fs::remove_dir(src_file_path_parent); - } else { - let _ = self - .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) - .await; - } - } - - // Heal reuses a version's `data_dir` and lands the rebuilt shard on - // the SAME `//part.N` path. Without this, a cached - // descriptor would keep serving the pre-heal inode, defeating the heal - // and eroding read quorum (backlog#1145). - // - // The exact keys are derivable here, and this runs on every write, so - // use them rather than registering a predicate the read path would then - // have to evaluate. Readers build the same string - // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every - // part of the version now at `dst_path` — any part path absent from it - // no longer exists for readers to ask for. - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - - Ok(RenameDataResp { - old_data_dir: has_old_data_dir, - rollback_data_dir, - cleanup_data_dir: has_old_data_dir, - sign: version_signature, - old_current_size, - }) - } else { - // Inline metadata preparation is blocking. The transaction lease is - // moved into that work so a timeout can release the async waiter without - // allowing a retry to reuse the deterministic staging path too early. - let src = src_file_path.clone(); - let dst = dst_file_path.clone(); - let cleanup_path = if src_volume == super::RUSTFS_META_MULTIPART_BUCKET { - src_file_path.parent().map(|p| p.to_path_buf()) - } else { - None - }; - let dst_path_for_failpoint = dst_path.to_string(); - #[cfg(windows)] - let source_parent = src_file_parent.to_path_buf(); - let rename_commit_guard_for_preparation = rename_commit_guard.clone(); - let sync = durability.syncs_commit_metadata(); - #[cfg(test)] - run_inline_before_file_sync_admission(dst_path); - let mut file_sync_admission = if sync { - Some( - os::acquire_file_sync_admission(self.file_sync_permits.clone()) - .await - .map_err(to_file_error) - .map_err(DiskError::from)?, - ) - } else { - None - }; - let prepare_inline_metadata = move || { - let mut prepared_metadata_source = - os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; - #[cfg(windows)] - let source_metadata_guard = - rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; - let mut xlmeta = FileMeta::new(); - // Same as the non-inline branch: an unparsable existing dst - // xl.meta must surface as unknown, not `Absent` - // (rustfs/backlog#1009). - let mut dst_meta_unparsable = false; - if let Some(ref buf) = has_dst_buf { - if FileMeta::is_xl2_v1_format(buf) - && let Ok(nmeta) = FileMeta::load(buf) - { - xlmeta = nmeta - } else { - dst_meta_unparsable = true; - } - } - - let old_current_size = if dst_meta_unparsable { - None - } else { - observe_old_current_size(has_dst_buf.is_some(), &xlmeta) - }; - - let version_id = fi.version_id.unwrap_or_default(); - let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); - let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); - let rollback_data_dir = old_data_dir.or_else(|| { - if old_version_exists && has_dst_buf.is_some() { - Some(inline_metadata_rollback_dir(version_id, &xlmeta)) - } else { - None - } - }); - let mut staged_rollback_path = None; - if let Some(d) = old_data_dir.as_ref() { - let _ = xlmeta.data.remove_two(version_id, *d); - } - xlmeta.add_version(fi)?; - let version_signature = rename_data_versions_signature(&xlmeta); - let new_buf = xlmeta.marshal_msg()?; - // Write the staged xl.meta. Inline objects carry their data inside - // xl.meta, so this is the durable preparation for the metadata commit: - // relaxed tiers do no per-object fsync here at all (aligned - // with MinIO's default), trading a documented power-loss - // window for latency. - prepared_metadata_source.write_all(&new_buf, sync)?; - run_inline_preparation_before_backup(&dst_path_for_failpoint); - if let Some(ref old_metadata) = has_dst_buf - && (rollback_data_dir.is_some() || sync || cfg!(test)) - { - #[cfg(windows)] - let backup_path = { - let backup_path = src - .parent() - .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? - .join(STORAGE_FORMAT_FILE_BACKUP); - source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; - backup_path - }; - #[cfg(not(windows))] - let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; - #[cfg(not(windows))] - if sync { - std::fs::File::open(&backup_path)?.sync_data()?; - } - staged_rollback_path = Some(backup_path); - } - - Ok::<_, std::io::Error>(( - rollback_data_dir, - old_data_dir, - version_signature, - old_current_size, - staged_rollback_path, - has_dst_buf.is_none(), - prepared_metadata_source, - )) - }; - let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { - os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await - } else { - os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await - } - .map_err(to_file_error) - .map_err(DiskError::from); - - let ( - rollback_data_dir, - cleanup_data_dir, - version_signature, - old_current_size, - mut local_rollback_path, - destination_was_absent, - prepared_metadata_source, - ) = match inline_preparation { - Ok(prepared) => prepared, - Err(err) => { - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(err); - } - }; - - let rename_commit_guard = remove_dst_base_before_commit( - dst_path, - rename_commit_guard, - src_file_parent, - dst_file_parent, - &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; - - if should_remove_staged_meta_before_commit(dst_path) { - drop(prepared_metadata_source); - let remove_result = std::fs::remove_file(&src_file_path); - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - remove_result.map_err(to_file_error)?; - return Err(DiskError::FileNotFound); - } - - if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { - let Some(dst_parent) = dst_file_path.parent() else { - return Err(DiskError::other("missing object metadata parent")); - }; - let backup_path = dst_parent - .join(rollback_data_dir.to_string()) - .join(STORAGE_FORMAT_FILE_BACKUP); - // rename_all acquires the backup path's namespace lease. Do not - // hold a disk admission while acquiring another namespace lock. - drop(file_sync_admission.take()); - if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { - let _ = remove_file_if_exists(staged_backup); - return Err(err); - } - #[cfg(test)] - run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); - if sync { - file_sync_admission = Some( - os::acquire_file_sync_admission(self.file_sync_permits.clone()) - .await - .map_err(to_file_error) - .map_err(DiskError::from)?, - ); - } - if let Some(admission) = file_sync_admission.as_ref() - && let Some(backup_parent) = backup_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, - fsync_started, - ); - return Err(DiskError::from(to_file_error(err))); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, - fsync_started, - ); - } - local_rollback_path = None; - } - - let commit_result = if should_fail_commit_rename(dst_path) { - Err(DiskError::other("test fail during metadata commit rename")) - } else { - os::rename_all_with_prepared_source( - prepared_metadata_source, - &src_file_path, - &dst_file_path, - &dst_volume_dir, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), - ) - .await - }; - if let Err(err) = commit_result { - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(err); - } - - let post_commit = async { - if should_fail_after_metadata_commit(dst_path) { - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(std::io::Error::other("test fail after metadata commit")); - } - - // Persist the commit rename's directory entry across power loss. - if let Some(admission) = file_sync_admission.as_ref() - && let Some(dst_parent) = dst_file_path.parent() - { - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) - .await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(err); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, - fsync_started, - ); - } - - // Same power-loss gap as the non-inline path (rustfs/backlog#922 - // step 4): a first PUT creates the object dir (and any missing - // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all - // never fsynced. The fsync above persists the object dir's contents, - // not its own entry, so for a new inline object fsync the ancestor - // chain up to and including the bucket. Overwrites already have a - // durable object dir; the starts_with guard bounds the walk. - if let Some(admission) = file_sync_admission.as_ref() - && destination_was_absent - { - let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); - while let Some(ancestor_dir) = ancestor { - if !ancestor_dir.starts_with(&dst_volume_dir) { - break; - } - let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await - { - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - rollback_inline_metadata_commit_std( - &dst_file_path, - rollback_data_dir, - local_rollback_path.as_deref(), - )?; - return Err(err); - } - rustfs_io_metrics::record_put_object_stage_duration_from( - rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, - fsync_started, - ); - if ancestor_dir == dst_volume_dir.as_path() { - break; - } - ancestor = ancestor_dir.parent(); - } - } - - Ok::<(), std::io::Error>(()) - } - .await; - - // The disk admission protects the durability chain, not staging - // cleanup or cache invalidation after that chain has completed. - drop(file_sync_admission.take()); - - // A post-commit rollback (for example, a commit-metadata fsync - // failure under strict durability) restores the old metadata; drop any - // descriptors cached during the committed window before propagating the - // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so - // this is mostly defensive and keeps both commit branches consistent. - if let Err(err) = post_commit { - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - return Err(DiskError::from(err)); - } - - // The commit no longer has a rollback path. Release the Windows - // object identity guard before best-effort staging cleanup. - #[cfg(windows)] - drop(rename_commit_guard); - - if let Some(backup_path) = local_rollback_path.as_deref() { - let _ = remove_file_if_exists(backup_path); - } - - // Cleanup - if let Some(ref cleanup) = cleanup_path { - let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; - } else if let Some(parent) = src_file_path.parent() { - let _ = std::fs::remove_dir(parent); - } - - // Heal reuses a version's `data_dir` and lands the rebuilt shard on - // the SAME `//part.N` path. Without this, a cached - // descriptor would keep serving the pre-heal inode, defeating the heal - // and eroding read quorum (backlog#1145). - // - // The exact keys are derivable here, and this runs on every write, so - // use them rather than registering a predicate the read path would then - // have to evaluate. Readers build the same string - // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every - // part of the version now at `dst_path` — any part path absent from it - // no longer exists for readers to ask for. - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; - } - - Ok(RenameDataResp { - old_data_dir: cleanup_data_dir, - rollback_data_dir, - cleanup_data_dir, - sign: version_signature, - old_current_size, - }) - } } #[tracing::instrument(level = "trace", skip_all)] @@ -10994,7 +10035,1004 @@ fn should_read_legacy_inline_part(fi: &FileInfo, storage_class_config: &crate::c storage_class_config.should_inline(shard_size, fi.erasure.data_blocks, versioned) } +/// Proof produced only when the local rename returns at an existing access +/// preflight, before metadata, backups, or object data can be published. +#[derive(Debug)] +pub(in crate::disk) struct LocalRenamePreflightRejection(()); + impl LocalDisk { + #[tracing::instrument(name = "rename_data", level = "trace", skip_all)] + async fn rename_data_inner( + &self, + src_volume: &str, + src_path: &str, + fi: FileInfo, + dst_volume: &str, + dst_path: &str, + preflight_rejection: &mut Option, + ) -> Result { + crate::hp_guard!("LocalDisk::rename_data"); + let mut fi = fi; + // A non-force DeleteBucket must not remove a directory while a local + // object commit is publishing into it. The peer's empty scan remains + // optimistic; this lease establishes the local commit/delete order and + // remains owned by any blocking syscall that outlives async cancellation. + let destination_object_path = self.io_get_object_path(dst_volume, dst_path)?; + let quota_fence_token = + match rustfs_utils::http::metadata_compat::get_consistent_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX) { + Some(value) => { + let token = Uuid::parse_str(value).map_err(|_| DiskError::FileCorrupt)?; + Some(SnapshotLeaseToken::from_slice(token.as_bytes())?) + } + None if rustfs_utils::http::metadata_compat::contains_key_str( + &fi.metadata, + QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + ) => + { + return Err(DiskError::FileCorrupt); + } + None => None, + }; + rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX); + let quota_fence_claim = match quota_fence_token { + Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), + None => None, + }; + let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; + if let Some(claim) = quota_fence_claim { + mutation_lease.attach_external_guard(claim); + } + if fi.is_legacy_indexed_delete_marker() { + fi.erasure.index = 0; + } + fi.validate_for_metadata_read()?; + // Snapshot the destination part paths before `fi` is consumed below. These + // are the descriptors a reader may hold for the version this call is about + // to replace (backlog#1145); readers build the identical string in + // `io_primitives`. An inline-data version has no parts and yields none. + let invalidate_part_paths: Vec = { + let data_dir = fi.data_dir.unwrap_or_default(); + fi.parts + .iter() + .map(|part| format!("{dst_path}/{data_dir}/part.{}", part.number)) + .collect() + }; + let src_volume_dir = self.io_get_bucket_path(src_volume)?; + if !skip_access_checks(src_volume) + && let Err(e) = super::fs::access_std(&src_volume_dir) + { + info!( + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?src_volume_dir, + operation = "rename_data_src_access", + error = %e, + "Disk local access check failed" + ); + *preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + let dst_volume_dir = self.io_get_bucket_path(dst_volume)?; + if !skip_access_checks(dst_volume) + && let Err(e) = super::fs::access_std(&dst_volume_dir) + { + info!( + event = EVENT_DISK_LOCAL_ACCESS_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + path = ?dst_volume_dir, + operation = "rename_data_dst_access", + error = %e, + "Disk local access check failed" + ); + *preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); + } + + // xl.meta path + let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; + let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; + + // data_dir path + let has_data_dir_path = { + let has_data_dir = { + if !fi.is_remote() { + fi.data_dir + .map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str())) + } else { + None + } + }; + + if let Some(data_dir) = has_data_dir { + let src_data_path = self.io_get_object_path( + src_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(), + )?; + let dst_data_path = self.io_get_object_path( + dst_volume, + rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(), + )?; + + Some((src_data_path, dst_data_path)) + } else { + None + } + }; + + check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; + check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; + + let no_inline = fi.data.is_none() && fi.size > 0; + // Captured before `fi` is consumed by add_version; gates the stale + // destination purge below. + let fi_healing = fi.is_healing(); + + // Resolved once for the whole commit so a concurrent configuration + // change can never leave a single rename_data half-synced. The tier is + // keyed on the destination volume: user data staged in scratch + // namespaces follows the configured tier, while commits into + // system-critical namespaces (IAM, config, bucket metadata) stay + // pinned to strict. + let durability = effective_durability(dst_volume); + + let src_file_parent = src_file_path + .parent() + .ok_or_else(|| DiskError::other("missing staged metadata parent"))?; + let dst_file_parent = dst_file_path + .parent() + .ok_or_else(|| DiskError::other("missing object metadata parent"))?; + if !no_inline { + fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; + } + // Acquire the common trees before reading destination metadata. On + // Windows this pins the object directory identity across metadata + // preparation, data publication, rollback backup, and final commit. + let rename_commit_guard = lock_rename_commit_directories( + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + let has_dst_buf = read_rename_destination_metadata(&dst_file_path, &rename_commit_guard, mutation_lease.clone()).await?; + + if no_inline { + // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta + let mut xlmeta = FileMeta::new(); + // An existing dst xl.meta that fails to parse leaves `xlmeta` empty + // and gets overwritten by the commit below (pre-existing behavior); + // track that so the old-size observation reports unknown instead of + // a false `Absent` (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(dst_buf) = has_dst_buf.as_ref() { + if FileMeta::is_xl2_v1_format(dst_buf) + && let Ok(nmeta) = FileMeta::load(dst_buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let mut skip_parent = dst_volume_dir.clone(); + if has_dst_buf.as_ref().is_some() + && let Some(parent) = dst_file_path.parent() + { + skip_parent = parent.to_path_buf(); + } + + let version_id = fi.version_id.unwrap_or_default(); + let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = has_old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + if let Some(old_data_dir) = has_old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *old_data_dir); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_dst_buf = xlmeta.marshal_msg()?; + + // This tmp xl.meta is renamed onto dst_file_path at the commit + // point below, so only its contents must be durable before the + // rename (SyncMode::FileOnly); the dst parent directory is fsynced + // after the commit rename, and a crash before the rename means the + // PUT was never acknowledged. A metadata commit: relaxed tiers + // leave it to the page cache. + let tmp_meta_sync = if durability.syncs_commit_metadata() { + SyncMode::FileOnly + } else { + SyncMode::None + }; + // The tmp xl.meta write and the shard-file fdatasync are independent + // (disjoint paths) and both only need to be durable before the commit + // renames below, so run them concurrently to drop a blocking + // round-trip from the PUT commit critical path (rustfs/backlog#922 + // step 2). The "contents durable -> rename -> dst dir fsync" ordering + // is unchanged — both futures complete before any rename — which the + // rename_data crash-consistency harness (backlog#935) exercises. + // + // Shard durability: once rename_data succeeds the write is + // acknowledged, so data must not live only in the page cache. + // Multipart parts were already synced during rename_part, so their + // fdatasync here is a cheap no-op. A missing source dir is left for the + // rename below to report through the existing rollback path. Payload + // durability is kept by both strict and relaxed. + let tmp_meta_write = { + let src_file_path = src_file_path.clone(); + let dst_file_path = dst_file_path.clone(); + let rename_commit_guard = rename_commit_guard.clone(); + let mutation_lease = mutation_lease.clone(); + async move { + os::run_blocking_namespace_operation(mutation_lease, move || { + #[cfg(test)] + run_owned_file_write_before_open(&src_file_path); + let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( + &src_file_path, + &dst_file_path, + &rename_commit_guard, + )?; + prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; + Ok(prepared_metadata_source) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from) + } + }; + let shard_sync = async { + if durability.syncs_data_shards() + && let Some((src_data_path, _)) = has_data_dir_path.as_ref() + && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await + && err.kind() != ErrorKind::NotFound + { + return Err::<(), DiskError>(to_file_error(err).into()); + } + Ok(()) + }; + let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync); + // Surface a tmp-meta failure first (its prior serial position), then a + // shard-sync failure; either aborts before any rename, exactly as the + // sequential version did. + let prepared_metadata_source = tmp_meta_res?; + shard_sync_res?; + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + std::fs::remove_file(&src_file_path).map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + // Heal reuses the version's data_dir, so for in-place corruption + // the destination dir still exists — and rename(2) cannot replace + // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge + // it first, healing commits only; fresh PUTs mint a new data_dir + // and never collide. Best effort: a real failure surfaces in the + // rename below. + if fi_healing + && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = self.move_to_trash(dst_data_path, true, false).await + { + warn!( + event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + dst_path = ?dst_data_path, + error = ?err, + "Healing commit could not purge the stale destination data dir" + ); + } + if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() + && let Err(err) = os::rename_all_with_commit_guard( + src_data_path, + dst_data_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_data_path_failed", + src_path = ?src_data_path, + dst_path = ?dst_data_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + #[cfg(test)] + if has_data_dir_path.is_some() { + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + } + + // Crash-consistency injection: hard power loss after the data dir + // is in place but before xl.meta commits. No cleanup — the harness + // reopens the disk and asserts the object still reads as the old + // version (the staged data dir is a harmless orphan for GC). + if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, dst_path) { + return Err(DiskError::Unexpected); + } + + if should_fail_before_old_metadata_backup(dst_path) { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "test_fail_before_old_metadata_backup", + "Disk local rename flow failed before metadata commit" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::Unexpected); + } + + // The rollback backup stays where it is written (no rename) and is + // the sole restore source for a later undo_write, so under strict + // it keeps SyncMode::FileAndDir: contents and directory entry both + // durable. It is part of the metadata commit machinery, so relaxed + // tiers leave it to the page cache like the xl.meta it mirrors. + let backup_sync = if durability.syncs_commit_metadata() { + SyncMode::FileAndDir + } else { + SyncMode::None + }; + if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) { + let backup_parent = dst_file_parent.join(old_data_dir.to_string()); + #[cfg(not(windows))] + if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { + Ok(guard) => guard, + Err(err) => { + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(DiskError::from(to_file_error(err))); + } + }; + let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP); + if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { + #[cfg(windows)] + drop(backup_path_guard); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + let backup_bytes = dst_buf.clone(); + // Keep the volume, commit-tree, and exact destination-path + // guards in this task until the backup write and durability + // sync finish. A detached spawn_blocking writer could survive + // cancellation and later truncate a newer transaction's + // deterministic rollback backup. + let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { + #[cfg(test)] + run_owned_file_write_before_open(&backup_path); + backup_path_guard.write_file_for_path_access( + &backup_path, + backup_bytes.as_ref(), + backup_sync != SyncMode::None, + backup_sync == SyncMode::FileAndDir, + ) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from); + if let Err(err) = write_result { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "write_old_metadata_backup_failed", + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + } + + // Crash-consistency injection: hard power loss after the rollback + // backup is durable but before the xl.meta commit rename. No + // cleanup — the harness asserts the object still reads as the old + // version, since the destination xl.meta is untouched here. + if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + if let Err(err) = os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &skip_parent, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + { + info!( + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "rename_all_metadata_failed", + src_path = ?src_file_path, + dst_path = ?dst_file_path, + error = ?err, + "Disk local rename flow failed" + ); + restore_published_data_source( + has_data_dir_path.as_ref(), + &src_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + return Err(err); + } + + let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); + if should_fail_after_metadata_commit(dst_path) { + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + return Err(DiskError::Unexpected); + } + + // Crash-consistency injection: hard power loss immediately after the + // xl.meta commit rename but before the durability fsync. Unlike the + // graceful failpoint above, no rollback runs — the commit rename is + // already on disk, so the harness asserts the object reads back as + // the new version. + if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, dst_path) { + return Err(DiskError::Unexpected); + } + + // Persist the directory entries for both the data dir and xl.meta renames; + // without this the commit itself can vanish on power loss. Relaxed tiers + // accept that window (documented in docs/operations/durability-modes.md). + if durability.syncs_commit_metadata() + && let Some(parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // The commit rename changed the dst part inodes before this fsync + // failed and rolled them back; drop any fd cached during that + // window so readers re-open the restored inode (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // First PUT of an object creates its directory (and any missing prefix + // dirs) via reliable_mkdir_all, which never fsyncs the parent chain. The + // commit fsync above persists the object dir's *contents*, not its own + // entry in the bucket/prefix dir, so on power loss after ack the whole + // object dir could vanish (rustfs/backlog#922 step 4). For a new object + // (no prior xl.meta) fsync the ancestor chain from the object dir's + // parent up to and including the bucket so those new directory entries + // are durable. Overwrites already have a durable object dir. The + // starts_with guard bounds the walk to the bucket subtree. Relaxed/none + // accept the wider window, like the commit fsync above. + if has_dst_buf.is_none() && durability.syncs_commit_metadata() { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(dir) = ancestor { + if !dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dir(dir).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // Same post-commit rollback window as above — drop cached + // dst part fds so readers re-open the restored inode + // (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if dir == dst_volume_dir.as_path() { + break; + } + ancestor = dir.parent(); + } + } + + // Publication and every rollback-capable durability step are now + // complete. Do not retain the Windows object identity guard while + // cleaning staging paths or invalidating cached descriptors. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(src_file_path_parent) = src_file_path.parent() { + if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { + let _ = std::fs::remove_dir(src_file_path_parent); + } else { + let _ = self + .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) + .await; + } + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: has_old_data_dir, + rollback_data_dir, + cleanup_data_dir: has_old_data_dir, + sign: version_signature, + old_current_size, + }) + } else { + // Inline metadata preparation is blocking. The transaction lease is + // moved into that work so a timeout can release the async waiter without + // allowing a retry to reuse the deterministic staging path too early. + let src = src_file_path.clone(); + let dst = dst_file_path.clone(); + let cleanup_path = if src_volume == super::RUSTFS_META_MULTIPART_BUCKET { + src_file_path.parent().map(|p| p.to_path_buf()) + } else { + None + }; + let dst_path_for_failpoint = dst_path.to_string(); + #[cfg(windows)] + let source_parent = src_file_parent.to_path_buf(); + let rename_commit_guard_for_preparation = rename_commit_guard.clone(); + let sync = durability.syncs_commit_metadata(); + #[cfg(test)] + run_inline_before_file_sync_admission(dst_path); + let mut file_sync_admission = if sync { + Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ) + } else { + None + }; + let prepare_inline_metadata = move || { + let mut prepared_metadata_source = + os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?; + #[cfg(windows)] + let source_metadata_guard = + rename_commit_guard_for_preparation.lock_source_directory_for_path_access(&source_parent)?; + let mut xlmeta = FileMeta::new(); + // Same as the non-inline branch: an unparsable existing dst + // xl.meta must surface as unknown, not `Absent` + // (rustfs/backlog#1009). + let mut dst_meta_unparsable = false; + if let Some(ref buf) = has_dst_buf { + if FileMeta::is_xl2_v1_format(buf) + && let Ok(nmeta) = FileMeta::load(buf) + { + xlmeta = nmeta + } else { + dst_meta_unparsable = true; + } + } + + let old_current_size = if dst_meta_unparsable { + None + } else { + observe_old_current_size(has_dst_buf.is_some(), &xlmeta) + }; + + let version_id = fi.version_id.unwrap_or_default(); + let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); + let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); + let rollback_data_dir = old_data_dir.or_else(|| { + if old_version_exists && has_dst_buf.is_some() { + Some(inline_metadata_rollback_dir(version_id, &xlmeta)) + } else { + None + } + }); + let mut staged_rollback_path = None; + if let Some(d) = old_data_dir.as_ref() { + let _ = xlmeta.data.remove_two(version_id, *d); + } + xlmeta.add_version(fi)?; + let version_signature = rename_data_versions_signature(&xlmeta); + let new_buf = xlmeta.marshal_msg()?; + // Write the staged xl.meta. Inline objects carry their data inside + // xl.meta, so this is the durable preparation for the metadata commit: + // relaxed tiers do no per-object fsync here at all (aligned + // with MinIO's default), trading a documented power-loss + // window for latency. + prepared_metadata_source.write_all(&new_buf, sync)?; + run_inline_preparation_before_backup(&dst_path_for_failpoint); + if let Some(ref old_metadata) = has_dst_buf + && (rollback_data_dir.is_some() || sync || cfg!(test)) + { + #[cfg(windows)] + let backup_path = { + let backup_path = src + .parent() + .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"))? + .join(STORAGE_FORMAT_FILE_BACKUP); + source_metadata_guard.write_file_for_path_access(&backup_path, old_metadata, sync, false)?; + backup_path + }; + #[cfg(not(windows))] + let backup_path = create_local_inline_rollback_backup(&dst, &src, old_metadata)?; + #[cfg(not(windows))] + if sync { + std::fs::File::open(&backup_path)?.sync_data()?; + } + staged_rollback_path = Some(backup_path); + } + + Ok::<_, std::io::Error>(( + rollback_data_dir, + old_data_dir, + version_signature, + old_current_size, + staged_rollback_path, + has_dst_buf.is_none(), + prepared_metadata_source, + )) + }; + let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { + os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await + } else { + os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await + } + .map_err(to_file_error) + .map_err(DiskError::from); + + let ( + rollback_data_dir, + cleanup_data_dir, + version_signature, + old_current_size, + mut local_rollback_path, + destination_was_absent, + prepared_metadata_source, + ) = match inline_preparation { + Ok(prepared) => prepared, + Err(err) => { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + }; + + let rename_commit_guard = remove_dst_base_before_commit( + dst_path, + rename_commit_guard, + src_file_parent, + dst_file_parent, + &dst_volume_dir, + &self.publication_root, + mutation_lease.clone(), + ) + .await?; + + if should_remove_staged_meta_before_commit(dst_path) { + drop(prepared_metadata_source); + let remove_result = std::fs::remove_file(&src_file_path); + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + remove_result.map_err(to_file_error)?; + return Err(DiskError::FileNotFound); + } + + if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { + let Some(dst_parent) = dst_file_path.parent() else { + return Err(DiskError::other("missing object metadata parent")); + }; + let backup_path = dst_parent + .join(rollback_data_dir.to_string()) + .join(STORAGE_FORMAT_FILE_BACKUP); + // rename_all acquires the backup path's namespace lease. Do not + // hold a disk admission while acquiring another namespace lock. + drop(file_sync_admission.take()); + if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { + let _ = remove_file_if_exists(staged_backup); + return Err(err); + } + #[cfg(test)] + run_rename_data_after_first_publication(&self.root, dst_volume, dst_path); + if sync { + file_sync_admission = Some( + os::acquire_file_sync_admission(self.file_sync_permits.clone()) + .await + .map_err(to_file_error) + .map_err(DiskError::from)?, + ); + } + if let Some(admission) = file_sync_admission.as_ref() + && let Some(backup_parent) = backup_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + return Err(DiskError::from(to_file_error(err))); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + } + local_rollback_path = None; + } + + let commit_result = if should_fail_commit_rename(dst_path) { + Err(DiskError::other("test fail during metadata commit rename")) + } else { + os::rename_all_with_prepared_source( + prepared_metadata_source, + &src_file_path, + &dst_file_path, + &dst_volume_dir, + &self.publication_root, + &rename_commit_guard, + mutation_lease.clone(), + ) + .await + }; + if let Err(err) = commit_result { + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(err); + } + + let post_commit = async { + if should_fail_after_metadata_commit(dst_path) { + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(std::io::Error::other("test fail after metadata commit")); + } + + // Persist the commit rename's directory entry across power loss. + if let Some(admission) = file_sync_admission.as_ref() + && let Some(dst_parent) = dst_file_path.parent() + { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission) + .await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + } + + // Same power-loss gap as the non-inline path (rustfs/backlog#922 + // step 4): a first PUT creates the object dir (and any missing + // prefix dirs) whose entry in the bucket/prefix dir reliable_mkdir_all + // never fsynced. The fsync above persists the object dir's contents, + // not its own entry, so for a new inline object fsync the ancestor + // chain up to and including the bucket. Overwrites already have a + // durable object dir; the starts_with guard bounds the walk. + if let Some(admission) = file_sync_admission.as_ref() + && destination_was_absent + { + let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent()); + while let Some(ancestor_dir) = ancestor { + if !ancestor_dir.starts_with(&dst_volume_dir) { + break; + } + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std( + &dst_file_path, + rollback_data_dir, + local_rollback_path.as_deref(), + )?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); + if ancestor_dir == dst_volume_dir.as_path() { + break; + } + ancestor = ancestor_dir.parent(); + } + } + + Ok::<(), std::io::Error>(()) + } + .await; + + // The disk admission protects the durability chain, not staging + // cleanup or cache invalidation after that chain has completed. + drop(file_sync_admission.take()); + + // A post-commit rollback (for example, a commit-metadata fsync + // failure under strict durability) restores the old metadata; drop any + // descriptors cached during the committed window before propagating the + // error (rustfs/backlog#1177). Inline objects carry data in xl.meta, so + // this is mostly defensive and keeps both commit branches consistent. + if let Err(err) = post_commit { + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(DiskError::from(err)); + } + + // The commit no longer has a rollback path. Release the Windows + // object identity guard before best-effort staging cleanup. + #[cfg(windows)] + drop(rename_commit_guard); + + if let Some(backup_path) = local_rollback_path.as_deref() { + let _ = remove_file_if_exists(backup_path); + } + + // Cleanup + if let Some(ref cleanup) = cleanup_path { + let _ = self.delete_file(&dst_volume_dir, cleanup, true, false).await; + } else if let Some(parent) = src_file_path.parent() { + let _ = std::fs::remove_dir(parent); + } + + // Heal reuses a version's `data_dir` and lands the rebuilt shard on + // the SAME `//part.N` path. Without this, a cached + // descriptor would keep serving the pre-heal inode, defeating the heal + // and eroding read quorum (backlog#1145). + // + // The exact keys are derivable here, and this runs on every write, so + // use them rather than registering a predicate the read path would then + // have to evaluate. Readers build the same string + // (`{object}/{data_dir}/part.{n}`), and `fi.parts` enumerates every + // part of the version now at `dst_path` — any part path absent from it + // no longer exists for readers to ask for. + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + + Ok(RenameDataResp { + old_data_dir: cleanup_data_dir, + rollback_data_dir, + cleanup_data_dir, + sign: version_signature, + old_current_size, + }) + } + } + + pub(in crate::disk) async fn rename_data_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> super::RenameDataObservation { + let mut preflight_rejection = None; + let result = self + .rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection) + .await; + super::RenameDataObservation { + result, + preflight_rejection, + } + } + pub(crate) async fn rename_data_borrowed( &self, src_volume: &str, diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 7801274ef..c2f2c52b4 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -75,6 +75,25 @@ use time::OffsetDateTime; use tokio::io::{AsyncRead, AsyncWrite}; use uuid::Uuid; +/// Local preflight evidence stays outside DiskAPI and the RPC response format. +pub(crate) struct RenameDataObservation { + pub(crate) result: Result, + preflight_rejection: Option, +} + +impl RenameDataObservation { + fn unknown(result: Result) -> Self { + Self { + result, + preflight_rejection: None, + } + } + + pub(crate) fn rejected_before_publication(&self) -> bool { + self.result.is_err() && self.preflight_rejection.is_some() + } +} + const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/"; pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token"; @@ -711,6 +730,36 @@ impl Disk { .await } + pub(crate) async fn rename_data_borrowed_with_fence_observed( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + scanner_publication_lease_token: Option, + ) -> RenameDataObservation { + match self { + Disk::Local(local_disk) => { + local_disk + .rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None) + .await + } + Disk::Remote(remote_disk) => RenameDataObservation::unknown( + remote_disk + .rename_data_borrowed_with_fence( + src_volume, + src_path, + fi, + dst_volume, + dst_path, + scanner_publication_lease_token, + ) + .await, + ), + } + } + pub(crate) async fn rename_data_borrowed_with_fence( &self, src_volume: &str, diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 4d8315302..a7c52549d 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3841,9 +3841,17 @@ pub(in crate::set_disk) struct RenameTailOutcome { const EVENT_SET_DISK_RENAME_ROLLBACK: &str = "set_disk_rename_rollback"; +#[derive(Clone, Copy)] +enum RenameDispatchState { + NotDispatched, + RejectedBeforePublication, + MayHavePublished, +} + #[derive(Debug, Clone, PartialEq, Eq)] enum RenameRollbackOutcome { NotAttempted(DiskError), + RejectedBeforePublication(DiskError), Indeterminate(DiskError), Succeeded, Failed(DiskError), @@ -3855,6 +3863,7 @@ impl RenameRollbackOutcome { fn stage(&self) -> &'static str { match self { Self::NotAttempted(_) => "rename_not_dispatched", + Self::RejectedBeforePublication(_) => "rename_rejected_before_publication", Self::Indeterminate(_) => "rename_indeterminate", Self::Succeeded => "undo_succeeded", Self::Failed(_) => "undo_failed", @@ -3940,14 +3949,14 @@ async fn rollback_failed_rename( disks: &[Option], file_infos: Vec, errs: &[Option], - dispatched: &[bool], + dispatch_states: &[RenameDispatchState], rollback_dirs: &[Option], dst: (&str, &str), receipt: Option, ) { let owned_disks = disks.to_vec(); let owned_errs = errs.to_vec(); - let owned_dispatched = dispatched.to_vec(); + let owned_dispatch_states = dispatch_states.to_vec(); let owned_dirs = rollback_dirs.to_vec(); let owned_dst = (dst.0.to_string(), dst.1.to_string()); let coordinator_failure_receipt = receipt.clone(); @@ -3956,7 +3965,7 @@ async fn rollback_failed_rename( let rollback = tokio::spawn(async move { let disks = owned_disks.as_slice(); let errs = owned_errs.as_slice(); - let dispatched = owned_dispatched.as_slice(); + let dispatch_states = owned_dispatch_states.as_slice(); let rollback_dirs = owned_dirs.as_slice(); let dst = (owned_dst.0.as_str(), owned_dst.1.as_str()); let mut file_infos = file_infos; @@ -3967,8 +3976,13 @@ async fn rollback_failed_rename( for (disk_index, disk) in disks.iter().enumerate() { let rollback_dir = rollback_dirs[disk_index]; let outcome = match &errs[disk_index] { - Some(err) if dispatched[disk_index] => RenameRollbackOutcome::Indeterminate(err.clone()), - Some(err) => RenameRollbackOutcome::NotAttempted(err.clone()), + Some(err) => match dispatch_states[disk_index] { + RenameDispatchState::NotDispatched => RenameRollbackOutcome::NotAttempted(err.clone()), + RenameDispatchState::RejectedBeforePublication => { + RenameRollbackOutcome::RejectedBeforePublication(err.clone()) + } + RenameDispatchState::MayHavePublished => RenameRollbackOutcome::Indeterminate(err.clone()), + }, None => RenameRollbackOutcome::Failed(DiskError::DiskNotFound), }; outcomes.push(RenameRollbackDiskOutcome { @@ -4545,7 +4559,7 @@ impl SetDisks { let file_info = file_info.clone(); let successful_rename_completion_rank = successful_rename_completion_rank.clone(); tasks.spawn(async move { - let mut dispatched = false; + let mut dispatch_state = RenameDispatchState::NotDispatched; let result = std::panic::AssertUnwindSafe(async { #[allow(clippy::let_unit_value)] let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object); @@ -4571,9 +4585,9 @@ impl SetDisks { } let disk_wait_started = rustfs_io_metrics::put_stage_timer(); - dispatched = true; - let result = disk - .rename_data_borrowed_with_fence( + dispatch_state = RenameDispatchState::MayHavePublished; + let observed = disk + .rename_data_borrowed_with_fence_observed( &src_bucket, &src_object, &file_info, @@ -4582,6 +4596,8 @@ impl SetDisks { scanner_publication_lease_token, ) .await; + let rejected_before_publication = observed.rejected_before_publication(); + let result = observed.result; #[cfg(test)] if result.is_ok() { rollback_fault_injection::after_rename(&dst_object, i)?; @@ -4607,11 +4623,14 @@ impl SetDisks { }; rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms); } + if rejected_before_publication { + dispatch_state = RenameDispatchState::RejectedBeforePublication; + } result }) .catch_unwind() .await; - (i, dispatched, result) + (i, dispatch_state, result) }); } @@ -4622,7 +4641,7 @@ impl SetDisks { let mut results_seen = 0usize; let mut errs = vec![Some(DiskError::DiskNotFound); disk_count]; // Missing task results cannot prove that a disk mutation never ran. - let mut dispatched = vec![true; disk_count]; + let mut dispatch_states = vec![RenameDispatchState::MayHavePublished; disk_count]; let mut disk_versions = vec![None; disk_count]; let mut data_dirs = vec![None; disk_count]; let mut cleanup_data_dirs = vec![None; disk_count]; @@ -4633,8 +4652,8 @@ impl SetDisks { while let Some(joined) = tasks.join_next().await { results_seen += 1; match joined { - Ok((idx, was_dispatched, Ok(Ok(res)))) => { - dispatched[idx] = was_dispatched; + Ok((idx, dispatch_state, Ok(Ok(res)))) => { + dispatch_states[idx] = dispatch_state; data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir); cleanup_data_dirs[idx] = res.cleanup_data_dir; disk_versions[idx] = res.sign; @@ -4642,12 +4661,12 @@ impl SetDisks { errs[idx] = None; success_count += 1; } - Ok((idx, was_dispatched, Ok(Err(err)))) => { - dispatched[idx] = was_dispatched; + Ok((idx, dispatch_state, Ok(Err(err)))) => { + dispatch_states[idx] = dispatch_state; errs[idx] = Some(err); } - Ok((idx, was_dispatched, Err(_))) => { - dispatched[idx] = was_dispatched; + Ok((idx, dispatch_state, Err(_))) => { + dispatch_states[idx] = dispatch_state; errs[idx] = Some(DiskError::Unexpected); fanout_panic += 1; } @@ -4698,7 +4717,7 @@ impl SetDisks { &coordinator_disks, file_infos, &errs, - &dispatched, + &dispatch_states, &data_dirs, (&fanout_dst_bucket, &fanout_dst_object), rollback_receipt, @@ -4914,7 +4933,7 @@ impl SetDisks { let publication_scope = scanner_publication_commit_scope.clone(); async move { - let mut dispatched = false; + let mut dispatch_state = RenameDispatchState::NotDispatched; let result = std::panic::AssertUnwindSafe(async { // Test-only introspection guard: counts this operation as // in-flight for the whole body. Compiles to `()` in production. @@ -4956,9 +4975,9 @@ impl SetDisks { } let disk_wait_started = rustfs_io_metrics::put_stage_timer(); - dispatched = true; - let result = disk - .rename_data_borrowed_with_fence( + dispatch_state = RenameDispatchState::MayHavePublished; + let observed = disk + .rename_data_borrowed_with_fence_observed( &src_bucket, &src_object, file_info, @@ -4967,6 +4986,8 @@ impl SetDisks { scanner_publication_lease_token, ) .await; + let rejected_before_publication = observed.rejected_before_publication(); + let result = observed.result; #[cfg(test)] if result.is_ok() { rollback_fault_injection::after_rename(&dst_object, i)?; @@ -4992,11 +5013,14 @@ impl SetDisks { }; rustfs_io_metrics::record_put_rename_disk_wait_completion(position, duration_ms); } + if rejected_before_publication { + dispatch_state = RenameDispatchState::RejectedBeforePublication; + } result }) .catch_unwind() .await; - (dispatched, result) + (dispatch_state, result) } }); let results = join_all(futures).await; @@ -5043,9 +5067,9 @@ impl SetDisks { ); } - let mut dispatched = Vec::with_capacity(results.len()); - for (idx, (was_dispatched, result)) in results.iter().enumerate() { - dispatched.push(*was_dispatched); + let mut dispatch_states = Vec::with_capacity(results.len()); + for (idx, (dispatch_state, result)) in results.iter().enumerate() { + dispatch_states.push(*dispatch_state); match result { Ok(Ok(res)) => { data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir); @@ -5111,7 +5135,7 @@ impl SetDisks { disks, file_infos, &errs, - &dispatched, + &dispatch_states, &data_dirs, (&dst_bucket, &dst_object), rollback_receipt, @@ -7075,6 +7099,7 @@ pub(in crate::set_disk) mod rollback_fault_injection { Io, Panic, IoAfterRename, + VolumeNotFoundAfterRename, PanicAfterRename, CoordinatorPanic, } @@ -7123,6 +7148,7 @@ pub(in crate::set_disk) mod rollback_fault_injection { .copied(); match fault { Some((target, Fault::IoAfterRename)) if target == disk_index => Err(DiskError::FaultyDisk), + Some((target, Fault::VolumeNotFoundAfterRename)) if target == disk_index => Err(DiskError::VolumeNotFound), Some((target, Fault::PanicAfterRename)) if target == disk_index => panic!("injected panic after rename mutation"), _ => Ok(()), } @@ -10947,6 +10973,7 @@ mod tests { temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { for fault in [ rollback_fault_injection::Fault::IoAfterRename, + rollback_fault_injection::Fault::VolumeNotFoundAfterRename, rollback_fault_injection::Fault::PanicAfterRename, ] { let bucket = "rename-tail-unknown"; @@ -11014,6 +11041,7 @@ mod tests { for fault in [ rollback_fault_injection::Fault::Io, rollback_fault_injection::Fault::IoAfterRename, + rollback_fault_injection::Fault::VolumeNotFoundAfterRename, rollback_fault_injection::Fault::PanicAfterRename, rollback_fault_injection::Fault::CoordinatorPanic, ] { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index ce9f95d46..c76daf707 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -17505,27 +17505,69 @@ mod put_object_tmp_cleanup_tests { } #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] async fn put_object_failure_cleans_tmp_workspace_inline() { - let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await; + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] { + let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "tmp-clean-missing-bucket"; + let object = "orphan-object"; + let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace); + let writer = Arc::clone(&set_disks); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("missing-bucket PUT must stage before rename"); + let staged = non_trash_tmp_entries(&temp_dirs).await; + assert_eq!(staged.len(), 4, "every disk must have a staged workspace before rejection"); + for workspace in staged { + let mut entries = tokio::fs::read_dir(&workspace) + .await + .expect("staged workspace should be readable"); + let mut shards = 0; + while let Some(entry) = entries.next_entry().await.expect("staged data directory should be readable") { + if entry.file_type().await.expect("staged entry type").is_dir() { + let part = tokio::fs::metadata(entry.path().join("part.1")) + .await + .expect("staging must contain an actual erasure shard"); + assert!(part.len() > 0, "the shard must be written before the missing-bucket failure"); + shards += 1; + } + } + assert_eq!(shards, 1); + } + assert!(temp_dirs.iter().all(|dir| !dir.path().join(bucket).exists())); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("missing-bucket PUT must finish") + .expect("PUT task should join") + .expect_err("put_object into a missing bucket volume must fail"); + assert!(matches!(err, StorageError::VolumeNotFound), "original disk error expected: {err}"); - // The bucket volume is never created, so the shards are written into - // the tmp workspace and the commit fails at rename_data with a quorum - // error — exercising the failure-path cleanup. - let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]); - let err = set_disks - .put_object("tmp-clean-missing-bucket", "orphan-object", &mut reader, &ObjectOptions::default()) - .await - .expect_err("put_object into a missing bucket volume must fail"); - - // No polling: the failure path must clean the tmp workspace inline, - // before put_object returns (backlog#864 / backlog#898 hardening). - let leftovers = non_trash_tmp_entries(&temp_dirs).await; - assert!( - leftovers.is_empty(), - "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}" - ); - - drop(temp_dirs); + // No polling: known pre-publication rejection must clean staging + // inline, before PUT returns (backlog#864 / backlog#898). + let leftovers = non_trash_tmp_entries(&temp_dirs).await; + assert!( + leftovers.is_empty(), + "failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}" + ); + } + }) + .await; } #[tokio::test] @@ -18373,87 +18415,92 @@ mod put_object_tmp_cleanup_tests { temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] { - let (dirs, disks, set) = hermetic_set_disks(4).await; - let bucket = "put-incomplete-undo"; - let object = "incomplete-undo-object"; - make_completion_test_bucket(&disks, bucket).await; - let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]); - set.put_object( - bucket, - object, - &mut old_reader, - &ObjectOptions { - write_completion: WriteCompletion::TailDrained, - ..Default::default() - }, - ) - .await - .expect("old generation should be completely committed"); - wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await; - let old = disks[0] - .read_version("", bucket, object, "", &ReadOptions::default()) + for fault in [ + rollback_fault_injection::Fault::Io, + rollback_fault_injection::Fault::VolumeNotFoundAfterRename, + ] { + let (dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-incomplete-undo"; + let object = "incomplete-undo-object"; + make_completion_test_bucket(&disks, bucket).await; + let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]); + set.put_object( + bucket, + object, + &mut old_reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) .await - .expect("old metadata must be readable"); - let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory"); - let tasks = rename_fanout_barrier::observe_tasks(object); - let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); - let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); - let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io); - let writer = Arc::clone(&set); - let put = tokio::spawn(async move { - let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); - writer - .put_object( - bucket, - object, - &mut reader, - &ObjectOptions { - write_completion, - ..Default::default() - }, - ) + .expect("old generation should be completely committed"); + wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await; + let old = disks[0] + .read_version("", bucket, object, "", &ReadOptions::default()) .await - }); - tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) - .await - .expect("overwrite must enter the actual rename fan-out before failure injection"); - barrier.release(); - let err = tokio::time::timeout(Duration::from_secs(30), put) - .await - .expect("incomplete undo must return without hanging") - .expect("PUT task should join") - .expect_err("two renamed disks cannot satisfy write quorum three"); - assert!( - matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)), - "original quorum error expected: {err}" - ); - assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return"); - let leftovers = non_trash_tmp_entries(&dirs).await; - assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery"); - let backups = dirs - .iter() - .filter(|dir| { - dir.path() - .join(bucket) - .join(object) - .join(old_data_dir.to_string()) - .join(crate::disk::STORAGE_FORMAT_FILE_BACKUP) - .exists() - }) - .count(); - assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup"); - // The remaining three disks still serve the old generation; - // the failed minority must never become an acknowledged write. - let mut read = set - .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) - .await - .expect("old generation must remain readable after incomplete rollback"); - let mut body = Vec::new(); - read.stream - .read_to_end(&mut body) - .await - .expect("old generation should stream"); - assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]); + .expect("old metadata must be readable"); + let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory"); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); + let _undo_fault = rollback_fault_injection::arm(object, 0, fault); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + writer + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("overwrite must enter the actual rename fan-out before failure injection"); + barrier.release(); + let err = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("incomplete undo must return without hanging") + .expect("PUT task should join") + .expect_err("two renamed disks cannot satisfy write quorum three"); + assert!( + matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)), + "original quorum error expected: {err}" + ); + assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return"); + let leftovers = non_trash_tmp_entries(&dirs).await; + assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery"); + let backups = dirs + .iter() + .filter(|dir| { + dir.path() + .join(bucket) + .join(object) + .join(old_data_dir.to_string()) + .join(crate::disk::STORAGE_FORMAT_FILE_BACKUP) + .exists() + }) + .count(); + assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup"); + // The remaining three disks still serve the old generation; + // the failed minority must never become an acknowledged write. + let mut read = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("old generation must remain readable after incomplete rollback"); + let mut body = Vec::new(); + read.stream + .read_to_end(&mut body) + .await + .expect("old generation should stream"); + assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]); + } } }) .await; From c332ba3d41d4500ad55dc8f6a2e46451dd2a0c9d Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 12:21:08 +0800 Subject: [PATCH 12/12] test(ecstore): cover observed rename outer failures --- crates/ecstore/src/disk/local.rs | 190 +++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index d9f5956b3..b75967104 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -14140,6 +14140,196 @@ mod test { ); } + #[tokio::test] + async fn observed_rename_timeout_has_no_preflight_proof_and_retains_namespace_lease() { + use crate::disk::disk_store::LocalDiskWrapper; + use futures::FutureExt; + use std::sync::mpsc; + + temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let bucket = "observed-timeout-bucket"; + let object = "prefix/object"; + let tmp_object = "observed-timeout-stage"; + let data_dir = Uuid::new_v4(); + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + let staged_part = disk + .get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}/part.1")) + .expect("staged part path should resolve"); + fs::create_dir_all(staged_part.parent().expect("staged part should have a parent")) + .await + .expect("staged data directory should be created"); + fs::write(&staged_part, b"new-payload") + .await + .expect("staged part should be written"); + let staged_metadata = disk + .get_object_path_for_io(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{STORAGE_FORMAT_FILE}")) + .expect("staged metadata path should resolve"); + let destination = disk.io_get_object_path(bucket, object).expect("destination should resolve"); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + set_owned_file_write_before_open(&staged_metadata, move || { + entered_tx.send(()).expect("signal staged writer entry"); + // Dropping the sender also unblocks the syscall if the test fails. + let _ = release_rx.recv(); + }); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let operation = wrapper.clone(); + let fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None); + let rename = tokio::spawn(async move { + operation + .rename_data_observed(RUSTFS_META_TMP_BUCKET, tmp_object, &fi, bucket, object, None) + .await + }); + tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10))) + .await + .expect("staged writer waiter should run") + .expect("rename must enter the real staged metadata write"); + + // Advance only after the blocking syscall owns its lease and the wrapper's timer exists. + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(61)).await; + tokio::time::resume(); + let observed = tokio::time::timeout(Duration::from_secs(5), rename) + .await + .expect("wrapper timeout must not wait for the blocked syscall") + .expect("the wrapper waiter must not panic"); + assert!(!observed.rejected_before_publication(), "a timeout must carry no local preflight proof"); + assert!(matches!(observed.result, Err(DiskError::Timeout))); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1)); + assert_eq!(snapshot.total_errors_timeout, 1); + assert_eq!(snapshot.total_writes, 0); + assert_eq!(snapshot.total_waiting, 0); + let volume_lock = os::disk_volume_mutation_lock(&disk.root, bucket); + assert!( + Arc::clone(&volume_lock).try_write_owned().is_err(), + "the blocked syscall must retain its volume guard" + ); + assert!( + os::acquire_rename_data_mutation_lease(&disk.root, bucket, &destination) + .now_or_never() + .is_none(), + "a same-object mutation must still wait for the blocked syscall" + ); + assert_eq!(fs::read(&staged_part).await.expect("staged data must remain"), b"new-payload"); + assert!(!destination.join(STORAGE_FORMAT_FILE).exists()); + + release_tx.send(()).expect("release timed-out staged writer"); + let lease = tokio::time::timeout( + Duration::from_secs(5), + os::acquire_rename_data_mutation_lease(&disk.root, bucket, &destination), + ) + .await + .expect("the namespace lease must be released when the syscall drains"); + drop(lease); + let _exclusive = tokio::time::timeout(Duration::from_secs(5), volume_lock.write_owned()) + .await + .expect("the volume guard must be released when the syscall drains"); + assert!( + !destination.join(STORAGE_FORMAT_FILE).exists(), + "timed-out waiter must not publish metadata later" + ); + }) + .await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn observed_rename_owned_task_panic_has_no_preflight_proof_and_releases_guard() { + use crate::disk::disk_store::LocalDiskWrapper; + use std::sync::mpsc; + + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let bucket = "observed-panic-bucket"; + let object = "prefix/object"; + let tmp_object = "observed-panic-stage"; + let data_dir = Uuid::new_v4(); + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + let staged_part = disk + .get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}/part.1")) + .expect("staged part path should resolve"); + fs::create_dir_all(staged_part.parent().expect("staged part should have a parent")) + .await + .expect("staged data directory should be created"); + fs::write(&staged_part, b"new-payload") + .await + .expect("staged part should be written"); + let destination = disk.io_get_object_path(bucket, object).expect("destination should resolve"); + let published_part = destination.join(data_dir.to_string()).join("part.1"); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { + entered_tx.send(()).expect("signal data publication"); + let _ = release_rx.recv(); + // This hook runs in the owned async mutation, outside spawn_blocking. + panic!("injected observed rename owner panic after publication"); + }); + let external_guard = Arc::new(()); + let guard_probe = Arc::downgrade(&external_guard); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let operation = wrapper.clone(); + let fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None); + let rename = tokio::spawn(async move { + operation + .rename_data_observed(RUSTFS_META_TMP_BUCKET, tmp_object, &fi, bucket, object, Some(external_guard)) + .await + }); + tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10))) + .await + .expect("publication waiter should run") + .expect("rename must publish data before the injected owner panic"); + assert!(guard_probe.upgrade().is_some(), "the owned task must retain the publication guard"); + assert_eq!(fs::read(&published_part).await.expect("new data must be published"), b"new-payload"); + assert!(!staged_part.exists(), "the real data rename must have consumed staging"); + assert!(!destination.join(STORAGE_FORMAT_FILE).exists()); + let volume_lock = os::disk_volume_mutation_lock(&disk.root, bucket); + assert!( + Arc::clone(&volume_lock).try_write_owned().is_err(), + "the mutation must retain its volume guard" + ); + + release_tx.send(()).expect("release mutation owner into the injected panic"); + let observed = tokio::time::timeout(Duration::from_secs(5), rename) + .await + .expect("owned task panic must reach the wrapper") + .expect("the wrapper must convert the inner task panic into an error"); + assert!( + !observed.rejected_before_publication(), + "a join failure must carry no local preflight proof" + ); + assert!(matches!(observed.result, Err(DiskError::Io(error)) if error.to_string() == "owned mutation task failed")); + assert!( + guard_probe.upgrade().is_none(), + "the guard must be released after the mutation owner unwinds" + ); + let snapshot = wrapper.metrics_snapshot(); + assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1)); + assert_eq!(snapshot.total_writes, 0); + assert_eq!(snapshot.total_waiting, 0); + let lease = tokio::time::timeout( + Duration::from_secs(5), + os::acquire_rename_data_mutation_lease(&disk.root, bucket, &destination), + ) + .await + .expect("panic must release the namespace lease"); + drop(lease); + let _exclusive = tokio::time::timeout(Duration::from_secs(5), volume_lock.write_owned()) + .await + .expect("panic must release the volume guard"); + assert_eq!( + fs::read(&published_part).await.expect("published recovery data must remain"), + b"new-payload" + ); + assert!(!destination.join(STORAGE_FORMAT_FILE).exists()); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn windows_and_unix_cancelled_staged_metadata_write_serializes_same_object_retry() { use std::sync::{Arc, mpsc};