From 4166b9503f3cc84bed869a3828163952cd359ede Mon Sep 17 00:00:00 2001 From: Hauser Date: Mon, 12 Oct 2026 13:47:37 +0800 Subject: [PATCH] fix(ecstore): recover interrupted delete transactions (#8468) * fix(ecstore): recover crash-interrupted delete retries Retain set-quorum cleanup proof and restore metadata before dropping rollback markers. Reclaim eligible exact-key residue during explicit delete retries, with bucket identity, version, retention and namespace fencing guards. Preserve prefix children and snapshot ownership through batched physical cleanup, and cover interrupted transactions and failure boundaries. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): route recovery test imports through contracts Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): preserve missing-key deletes at quorum * feat(ecstore): add certified set-wide delete recovery Atomically prepare deletion journals and exact targets, publish durable set receipts, and recover through one fenced disk/RPC core used by startup and scoped administrator healing. Preserve returning generations, ordinary missing-key quorum semantics, and legacy rollback wire compatibility. Retain namespace and volume ownership through late physical operations, order related transactions causally, and keep proof until full convergence. Remove the superseded local-only recovery path and add storage, protocol, RPC, durability and isolated Docker regression coverage. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): reclaim completed multipart delete sidecars * fix(ecstore): recover multipart sidecars with verified proofs Recognize native multipart sidecars at the shared recovery boundary and restore them without treating metadata or proof files as shard payloads. Validate required proof generations and bind every present live/staged shard copy to its metadata-bound Merkle proof before restoring metadata. Cover protected native MPU rollback, split source layouts, missing or corrupt proofs, and payload corruption with recomputed bitrot checksums. Add real multipart Docker regressions in both integrity modes. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): settle superseded committed version deletes Allow a validated committed decision to finish its own data cleanup after a later decision removes current metadata. Prepared copies need an inspected same-transaction receipt matching immutable local intent before promotion, and retain their local journal without importing peer hashes. Reuse the authority matcher for dry-run and settlement. Cover chained explicit-version deletions, partial receipt publication, mismatch rejection and no-resurrection, while retaining the no-decision absence protection. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ecstore): clarify delete recovery case label Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): restore recovery CI guard coverage Keep production error sites visible to the ratchet by placing test modules last. Preserve native encoding errors, group preparation identity and ownership, and reduce RPC enum size without changing the wire payload. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(admin): preserve owned OIDC validation errors Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(ecstore): unify recovery authority and snapshot work Use one user-bucket legacy settlement boundary and one adoption source predicate for preview and execution. Reuse immutable snapshot digests, consume owned generation metadata, index version membership, and prepare empty witnesses through the canonical journal path. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): preserve delete recovery invariants --------- Co-authored-by: heihutu Co-authored-by: zhi22915 Co-authored-by: overtrue --- .../e2e_test/src/reliant/grpc_lock_server.rs | 7 + crates/ecstore/src/api/mod.rs | 11 +- crates/ecstore/src/cluster/rpc/client.rs | 102 + crates/ecstore/src/cluster/rpc/remote_disk.rs | 395 +- crates/ecstore/src/disk/disk_store.rs | 203 +- crates/ecstore/src/disk/local.rs | 1619 +++++++- .../ecstore/src/disk/local/delete_recovery.rs | 3646 +++++++++++++++++ crates/ecstore/src/disk/mod.rs | 653 +++ crates/ecstore/src/object_api/types.rs | 5 + crates/ecstore/src/set_disk/mod.rs | 1 + .../src/set_disk/ops/delete_recovery.rs | 1992 +++++++++ crates/ecstore/src/set_disk/ops/heal.rs | 183 +- crates/ecstore/src/set_disk/ops/mod.rs | 1 + crates/ecstore/src/set_disk/ops/object.rs | 1047 ++++- crates/ecstore/src/store/heal.rs | 128 + crates/ecstore/src/store/object.rs | 282 +- crates/heal/src/heal/storage.rs | 49 + crates/heal/src/heal/task/heal_bucket.rs | 36 + crates/heal/src/heal/task/heal_erasure_set.rs | 15 + crates/heal/src/heal/task/heal_object.rs | 48 +- .../src/generated/proto_gen/node_service.rs | 70 + crates/protos/src/lib.rs | 136 +- crates/protos/src/node.proto | 16 + docs/README.md | 3 + .../operations/delete-transaction-recovery.md | 137 + rustfs/src/server/http.rs | 45 + rustfs/src/storage/rpc/node_service.rs | 208 +- rustfs/src/storage/rpc/node_service/disk.rs | 179 +- rustfs/src/storage/storage_api.rs | 101 +- rustfs/src/storage_api.rs | 2 + scripts/test/delete_recovery_docker.py | 1257 ++++++ 31 files changed, 12239 insertions(+), 338 deletions(-) create mode 100644 crates/ecstore/src/disk/local/delete_recovery.rs create mode 100644 crates/ecstore/src/set_disk/ops/delete_recovery.rs create mode 100644 docs/operations/delete-transaction-recovery.md create mode 100644 scripts/test/delete_recovery_docker.py diff --git a/crates/e2e_test/src/reliant/grpc_lock_server.rs b/crates/e2e_test/src/reliant/grpc_lock_server.rs index ccc466d23..b97e96555 100644 --- a/crates/e2e_test/src/reliant/grpc_lock_server.rs +++ b/crates/e2e_test/src/reliant/grpc_lock_server.rs @@ -465,6 +465,13 @@ impl NodeService for MinimalLockNodeService { Err(Status::unimplemented("lock-only test server")) } + async fn recover_delete_transaction( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("delete recovery is not supported by the lock fixture")) + } + async fn settle_part_transaction( &self, _request: Request, diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e5e22995f..a43116a77 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -423,11 +423,12 @@ pub mod disk { pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage}; pub use crate::disk::{ BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, - CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, - DiskStore, FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, - PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, - validate_batch_read_version_item_count, + CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, + DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, + DeleteRecoveryRpcReply, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, + FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction, + RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, + SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count, }; pub use bytes::Bytes; pub use endpoint::Endpoint; diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 27bd0542f..75706254c 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -124,6 +124,22 @@ pub async fn node_service_time_out_client_for_class( .max_encoding_message_size(max_message_size)) } +pub(crate) async fn node_service_delete_recovery_client( + addr: &str, + interceptor: TonicInterceptor, + timeout: std::time::Duration, +) -> Result>, DiskError> { + let interceptor = interceptor.with_rpc_audience(addr).map_err(DiskError::from)?; + let channel = rustfs_protos::create_channel_with_timeout(addr, timeout) + .await + .map_err(|error| DiskError::RemoteClientUnavailable(error.to_string()))?; + let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); + let max_message_size = rustfs_protos::internode_rpc_max_message_size(); + Ok(NodeServiceClient::with_interceptor(channel, interceptor) + .max_decoding_message_size(max_message_size) + .max_encoding_message_size(max_message_size)) +} + pub async fn node_service_time_out_client_no_auth( addr: &String, ) -> Result>, Box> { @@ -528,6 +544,92 @@ mod tests { use tracing_opentelemetry::OpenTelemetrySpanExt; use tracing_subscriber::{Registry, layer::SubscriberExt}; + #[tokio::test] + async fn delete_recovery_channel_uses_its_scoped_deadline() { + use bytes::Bytes; + use http_body_util::StreamBody; + use hyper::{body::Frame, service::service_fn}; + use hyper_util::rt::{TokioExecutor, TokioIo}; + use rustfs_protos::proto_gen::node_service::{RecoverDeleteTransactionRequest, RecoverDeleteTransactionResponse}; + use tonic_prost::prost::Message; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind recovery peer"); + let addr = format!("http://{}", listener.local_addr().expect("peer address")); + let server = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.expect("accept recovery connection"); + tokio::spawn(async move { + let service = service_fn(|_request: HttpRequest| async { + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + let payload = RecoverDeleteTransactionResponse { + success: true, + protocol_version: 1, + result: Bytes::new(), + error: None, + } + .encode_to_vec(); + let mut data = vec![0]; + data.extend_from_slice(&u32::try_from(payload.len()).expect("small response").to_be_bytes()); + data.extend_from_slice(&payload); + let mut trailers = http::HeaderMap::new(); + trailers.insert("grpc-status", "0".parse().expect("status")); + let frames: Vec, std::convert::Infallible>> = + vec![Ok(Frame::data(data.into())), Ok(Frame::trailers(trailers))]; + let mut response = HttpResponse::new(StreamBody::new(futures::stream::iter(frames))); + response + .headers_mut() + .insert("content-type", "application/grpc".parse().expect("content type")); + Ok::<_, std::convert::Infallible>(response) + }); + let _ = hyper::server::conn::http2::Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + let request = || RecoverDeleteTransactionRequest { + disk: "disk".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: Bytes::new(), + }; + // Seed an existing short-lived control channel to reproduce the inherited deadline. + let control = tonic::transport::Endpoint::from_shared(addr.clone()) + .expect("test endpoint") + .timeout(std::time::Duration::from_millis(50)) + .connect() + .await + .expect("control connect"); + rustfs_common::cache_connection(addr.clone(), control).await; + let mut foreground = node_service_time_out_client(&addr, TonicInterceptor::NoOp(NoOpInterceptor)) + .await + .expect("control client"); + let error = foreground + .recover_delete_transaction(request()) + .await + .expect_err("short control deadline"); + assert!(matches!(error.code(), tonic::Code::Cancelled | tonic::Code::DeadlineExceeded)); + // The 2-second test override bypasses the production range parser, which rejects 2. + let mut recovery = node_service_delete_recovery_client( + &addr, + TonicInterceptor::NoOp(NoOpInterceptor), + std::time::Duration::from_secs(2), + ) + .await + .expect("scoped recovery client"); + assert!( + recovery + .recover_delete_transaction(request()) + .await + .expect("scoped deadline must survive control deadline") + .into_inner() + .success + ); + server.abort(); + rustfs_protos::evict_failed_connection(&addr).await; + } + #[tokio::test] async fn rpc_client_factories_do_not_reuse_https_channels_without_tls_generation() { let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve local address"); diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 0ba276567..e067609a0 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -14,7 +14,8 @@ use crate::cluster::rpc::client::{ AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, - node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth, + node_service_delete_recovery_client, node_service_time_out_client, node_service_time_out_client_for_class, + node_service_time_out_client_no_auth, }; use crate::cluster::rpc::internode_data_transport::{ InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest, @@ -22,10 +23,11 @@ use crate::cluster::rpc::internode_data_transport::{ }; use crate::disk::error::{Error, Result}; use crate::disk::{ - BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DeleteOptions, DiskAPI, DiskInfo, - DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, - ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, - batch_read_version_one_by_one, + BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION, + DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, + DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, + DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions, + RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one, disk_store::{ DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE, get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout, @@ -59,10 +61,10 @@ use rustfs_protos::proto_gen::node_service::{ CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, - ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, - SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, - SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, - node_service_client::NodeServiceClient, + ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest, + RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, + SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, node_service_client::NodeServiceClient, }; use serde::{Serialize, de::DeserializeOwned}; use std::{ @@ -142,6 +144,45 @@ fn decode_delete_versions_errors(response: DeleteVersionsResponse, expected_len: .collect() } +fn decode_delete_recovery_batch_errors(codes: Vec>, expected_len: usize) -> Result>> { + if codes.len() != expected_len { + return Err(Error::other("malformed recoverable batch delete response length")); + } + codes + .into_iter() + .map(|code| match code { + None => Ok(None), + Some(code) => DiskError::from_u32(code) + .map(Some) + .ok_or_else(|| Error::other("invalid recoverable batch delete error code")), + }) + .collect() +} + +fn validate_delete_recovery_inspection( + volume: &str, + object: &str, + transaction: Uuid, + inspection: &DeleteRecoveryInspection, +) -> Result<()> { + if inspection.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || inspection.id.object != object + || inspection.id.transaction != transaction + || transaction.is_nil() + { + return Err(Error::other("delete recovery inspection does not match request")); + } + if let Some(journal) = &inspection.journal + && (journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || journal.volume != volume + || journal.object != object + || journal.transaction != transaction) + { + return Err(Error::other("delete recovery journal does not match request")); + } + Ok(()) +} + fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result { if !response.success { return Err(response.error.unwrap_or_default().into()); @@ -858,6 +899,75 @@ fn spawn_control_channel_prewarm(addr: String) { } impl RemoteDisk { + async fn recover_delete_transaction( + &self, + volume: &str, + operation: DeleteRecoveryRpcOperation, + ) -> Result { + let payload = rmp_serde::to_vec_named(&operation)?; + if payload.len() > operation.max_wire_payload_bytes() { + return Err(Error::other("delete recovery operation exceeds payload limit")); + } + let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) { + Some(crate::disk::get_delete_recovery_timeout()?) + } else { + None + }; + self.execute_with_timeout( + || async { + let mut client = if let Some(timeout) = settle_timeout { + if let Some(error) = self.offline_bypass_error() { + return Err(error); + } + node_service_delete_recovery_client( + &self.addr, + TonicInterceptor::Signature(gen_tonic_signature_interceptor()), + timeout, + ) + .await? + } else { + self.get_client().await? + }; + let mut request = Request::new(RecoverDeleteTransactionRequest { + disk: self.endpoint.to_string(), + volume: volume.to_owned(), + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + operation: payload.into(), + }); + let canonical_body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()) + .map_err(|_| Error::other("recover_delete_transaction request length cannot be represented"))?; + crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?; + // UNIMPLEMENTED is deliberately propagated: older peers cannot settle transactions. + let response = client + .recover_delete_transaction(request) + .await + .map_err(|status| { + if status.code() == Code::Unimplemented { + DiskError::UnsupportedDisk + } else { + status.into() + } + })? + .into_inner(); + if response.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION { + return Err(DiskError::UnsupportedDisk); + } + if !response.success { + return Err(response + .error + .ok_or_else(|| Error::other("missing delete recovery error"))? + .into()); + } + if response.error.is_some() { + return Err(Error::other("malformed delete recovery success")); + } + rustfs_protos::decode_delete_recovery_payload(&response.result).map_err(Into::into) + }, + settle_timeout.unwrap_or_else(get_max_timeout_duration), + ) + .await + } + async fn rename_file_with_durability( &self, src_volume: &str, @@ -1920,6 +2030,12 @@ fn encode_msgpack(value: &T) -> Result> { encode_msgpack_with_capacity(value, MSGPACK_ENCODE_CAPACITY_HINT) } +// Optional fields are omitted by serde; named maps keep rollback flags and +// transaction UUIDs independent of field order across peer versions. +fn encode_delete_options_msgpack(value: &DeleteOptions) -> Result> { + Ok(rmp_serde::to_vec_named(value)?) +} + fn encode_file_info_msgpack(value: &FileInfo) -> Result> { encode_msgpack_with_capacity(value, FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT) } @@ -2566,7 +2682,7 @@ impl DiskAPI for RemoteDisk { // `_bin` support for DeleteVersion is new (grpc-optimization P2); always dual-write // JSON + msgpack until its fallback counter has read zero across a release window. let file_info_bin = encode_file_info_msgpack(&fi)?; - let opts_bin = encode_msgpack(&opts)?; + let opts_bin = encode_delete_options_msgpack(&opts)?; let conditional_marker = opts.expected_delete_marker.is_some(); let file_info = serde_json::to_string(&fi)?; let opts = serde_json::to_string(&opts)?; @@ -2638,7 +2754,7 @@ impl DiskAPI for RemoteDisk { // `_bin` support for DeleteVersions is new (grpc-optimization P2); always dual-write JSON + // msgpack until its fallback counter has read zero across a release window. - let opts_bin = match encode_msgpack(&opts) { + let opts_bin = match encode_delete_options_msgpack(&opts) { Ok(opts_bin) => opts_bin, Err(err) => { let mut errors = Vec::with_capacity(versions.len()); @@ -3617,6 +3733,138 @@ impl DiskAPI for RemoteDisk { } #[tracing::instrument(level = "trace", skip_all)] + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> Result<()> { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::ApplyVersion { + object: object.to_owned(), + file_info: Box::new(file_info), + force_del_marker, + opts, + }, + ) + .await? + { + DeleteRecoveryRpcReply::AppliedVersion => Ok(()), + _ => Err(Error::other("unexpected recoverable delete reply")), + } + } + + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + let expected_len = versions.len(); + let result = match self + .recover_delete_transaction(volume, DeleteRecoveryRpcOperation::ApplyVersions { versions, opts }) + .await + { + Ok(DeleteRecoveryRpcReply::AppliedVersions(codes)) => decode_delete_recovery_batch_errors(codes, expected_len), + Ok(_) => Err(Error::other("unexpected recoverable batch delete reply")), + Err(error) => Err(error), + }; + match result { + Ok(errors) => errors, + Err(error) => vec![Some(error); expected_len], + } + } + + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::Discover { + object: object.map(str::to_owned), + cursor, + limit, + }, + ) + .await? + { + DeleteRecoveryRpcReply::Discovered(page) => { + if object.is_some_and(|object| page.transactions.iter().any(|id| id.object != object)) { + return Err(Error::other("delete recovery discovery escaped requested object")); + } + if page.transactions.len() > usize::try_from(limit).map_err(|_| Error::other("invalid discovery limit"))? + || page + .transactions + .iter() + .any(|id| id.transaction.is_nil() || id.object.is_empty()) + { + return Err(Error::other("malformed delete discovery page")); + } + Ok(page) + } + _ => Err(Error::other("unexpected delete discovery reply")), + } + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::Inspect { + object: object.to_owned(), + transaction, + }, + ) + .await? + { + DeleteRecoveryRpcReply::Inspected(inspection) => { + validate_delete_recovery_inspection(volume, object, transaction, &inspection)?; + Ok(inspection) + } + _ => Err(Error::other("unexpected delete inspection reply")), + } + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> Result { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::Settle { + object: object.to_owned(), + transaction, + expected_fingerprint, + action, + }, + ) + .await? + { + DeleteRecoveryRpcReply::Settled(inspection) => { + validate_delete_recovery_inspection(volume, object, transaction, &inspection)?; + Ok(inspection) + } + _ => Err(Error::other("unexpected delete settlement reply")), + } + } + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> { self.execute_with_timeout( || async { @@ -4030,6 +4278,45 @@ mod tests { static INIT: Once = Once::new(); + #[test] + fn recoverable_batch_delete_rejects_invalid_error_codes_and_length() { + let errors = decode_delete_recovery_batch_errors(vec![None, Some(DiskError::FileNotFound.to_u32())], 2) + .expect("valid per-item errors"); + assert!(errors[0].is_none()); + assert_eq!(errors[1], Some(DiskError::FileNotFound)); + assert!(decode_delete_recovery_batch_errors(vec![None], 2).is_err()); + assert!(decode_delete_recovery_batch_errors(vec![Some(0)], 1).is_err()); + assert!(decode_delete_recovery_batch_errors(vec![Some(u32::MAX)], 1).is_err()); + } + + #[test] + fn delete_recovery_inspection_rejects_wrong_transaction_object_and_protocol() { + let transaction = Uuid::new_v4(); + let inspection = DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: crate::disk::DeleteTransactionId { + object: "object".into(), + transaction, + }, + state: crate::disk::DeleteRecoveryState::Absent, + journal: None, + current_digest: None, + current_versions: Vec::new(), + backup_digest: None, + rollback_data_complete: false, + commit_data_safe: false, + explicit_delete_safe: false, + fingerprint: [0; 32], + }; + validate_delete_recovery_inspection("bucket", "object", transaction, &inspection).expect("matching inspection"); + assert!(validate_delete_recovery_inspection("bucket", "other", transaction, &inspection).is_err()); + assert!(validate_delete_recovery_inspection("bucket", "object", Uuid::new_v4(), &inspection).is_err()); + assert!(validate_delete_recovery_inspection("bucket", "object", Uuid::nil(), &inspection).is_err()); + let mut incompatible = inspection; + incompatible.protocol_version += 1; + assert!(validate_delete_recovery_inspection("bucket", "object", transaction, &incompatible).is_err()); + } + #[test] fn compare_and_update_wire_outcome_fails_closed() { assert_eq!( @@ -4063,32 +4350,19 @@ mod tests { } #[test] - fn retired_marker_options_preserve_legacy_positional_wire_shape() { + fn delete_options_named_wire_preserves_rollback_and_legacy_marker_fields() { #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)] struct LegacyDeleteOptions { recursive: bool, immediate: bool, undo_write: bool, + #[serde(default)] undo_delete: bool, old_data_dir: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + expected_delete_marker: Option, } - let legacy = LegacyDeleteOptions { - recursive: false, - immediate: false, - undo_write: false, - undo_delete: false, - old_data_dir: None, - }; - let original = encode_msgpack(&legacy).unwrap(); - let current = encode_msgpack(&DeleteOptions::default()).unwrap(); - assert_eq!(current, original, "ordinary deletes must retain the older peer's positional payload"); - assert_eq!(rmp_serde::from_slice::(¤t).unwrap(), legacy); - assert!( - rmp_serde::from_slice::(&original) - .unwrap() - .expected_delete_marker - .is_none() - ); + let transaction = Uuid::new_v4(); let mut marker = FileInfo { deleted: true, version_id: Some(Uuid::new_v4()), @@ -4097,12 +4371,69 @@ mod tests { }; marker.set_delete_marker_incarnation(Uuid::new_v4()); let marker = rustfs_filemeta::MetaDeleteMarker::from(marker); + for expected_delete_marker in [None, Some(marker)] { + for (undo_write, undo_delete) in [(true, false), (false, true)] { + let legacy = LegacyDeleteOptions { + recursive: false, + immediate: false, + undo_write, + undo_delete, + old_data_dir: Some(transaction), + expected_delete_marker: expected_delete_marker.clone(), + }; + // Older senders use positional arrays. Their unchanged field prefix and + // missing optional fields must also decode on the named-map sender's peer. + for (encoding, payload) in [ + ("positional", rmp_serde::to_vec(&legacy).expect("encode legacy positional options")), + ("named", rmp_serde::to_vec_named(&legacy).expect("encode legacy named options")), + ] { + let decoded: DeleteOptions = rmp_serde::from_slice(&payload).expect("decode legacy options on current peer"); + assert_eq!(decoded.undo_write, undo_write, "{encoding} undo-write flag"); + assert_eq!(decoded.undo_delete, undo_delete, "{encoding} undo-delete flag"); + assert_eq!(decoded.old_data_dir, Some(transaction), "{encoding} transaction UUID"); + assert_eq!(decoded.expected_delete_marker, expected_delete_marker, "{encoding} marker precondition"); + assert!(decoded.delete_recovery_bucket_incarnation.is_none(), "{encoding} omitted incarnation"); + assert!(!decoded.recoverable_delete, "{encoding} omitted recovery flag"); + } + for incarnation in [None, Some(Uuid::new_v4())] { + let current = DeleteOptions { + undo_write, + undo_delete, + old_data_dir: Some(transaction), + expected_delete_marker: expected_delete_marker.clone(), + delete_recovery_bucket_incarnation: incarnation, + recoverable_delete: true, + ..Default::default() + }; + let payload = encode_delete_options_msgpack(¤t).expect("encode current named options"); + let decoded: DeleteOptions = rmp_serde::from_slice(&payload).expect("decode current options on current peer"); + assert_eq!(decoded.undo_write, undo_write); + assert_eq!(decoded.undo_delete, undo_delete); + assert_eq!(decoded.old_data_dir, Some(transaction)); + assert_eq!(decoded.expected_delete_marker, expected_delete_marker); + assert_eq!(decoded.delete_recovery_bucket_incarnation, incarnation); + assert!(decoded.recoverable_delete); + let decoded: LegacyDeleteOptions = + rmp_serde::from_slice(&payload).expect("decode current options on legacy peer"); + assert_eq!(decoded, legacy, "unknown recovery fields cannot shift rollback or marker values"); + } + } + } + } + + #[test] + fn malformed_nonempty_delete_options_do_not_fall_back_to_json() { let options = DeleteOptions { - expected_delete_marker: Some(marker.clone()), + undo_write: true, + old_data_dir: Some(Uuid::new_v4()), ..Default::default() }; - let decoded: DeleteOptions = rmp_serde::from_slice(&encode_msgpack(&options).unwrap()).unwrap(); - assert_eq!(decoded.expected_delete_marker, Some(marker)); + let json = serde_json::to_string(&options).expect("valid compatibility JSON"); + assert!(decode_msgpack_or_json::(&[0xc1], &json, "DeleteOptions").is_err()); + let decoded: DeleteOptions = + decode_msgpack_or_json(&[], &json, "DeleteOptions").expect("absent MessagePack permits JSON compatibility"); + assert!(decoded.undo_write); + assert_eq!(decoded.old_data_dir, options.old_data_dir); } #[test] diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 98de238f5..01091325b 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -466,12 +466,11 @@ impl LocalDiskWrapper { self.track_disk_health_mutation( "delete_version", DiskMetricMutation::Delete, - || async { + || { Box::pin( self.disk .delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner), ) - .await }, get_max_timeout_duration(), ) @@ -1816,6 +1815,8 @@ impl LocalDiskWrapper { undo_delete: false, old_data_dir: None, expected_delete_marker: None, + recoverable_delete: false, + delete_recovery_bucket_incarnation: None, }, ) .await?; @@ -2001,10 +2002,86 @@ impl LocalDiskWrapper { } /// Get the current disk ID + pub(crate) async fn delete_versions_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + let version_count = versions.len(); + self.metrics.record_operation_call("delete_versions"); + // Check if disk is faulty before proceeding + if self.health.is_faulty() { + self.metrics.record_availability_error(); + return vec![Some(DiskError::FaultyDisk); versions.len()]; + } + + // Check if disk is stale + if let Err(e) = self.check_disk_stale().await { + self.metrics.record_availability_error(); + return vec![Some(e); versions.len()]; + } + + let _io_permit = match self.acquire_disk_io_permit().await { + Ok(permit) => permit, + Err(error) => return vec![Some(error); version_count], + }; + + // Record operation start + self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed); + let waiting_guard = self.health.waiting_guard(); + let metric_waiting_guard = self.metrics.waiting_guard(); + let started = Instant::now(); + + // Execute the operation + let result = self + .disk + .delete_versions_with_namespace_owner(volume, versions, opts, namespace_owner) + .await; + self.metrics.record_operation_latency("delete_versions", started.elapsed()); + self.record_batch_delete_error_metrics(&result); + + drop(waiting_guard); + drop(metric_waiting_guard); + let has_err = result.iter().any(|e| e.is_some()); + if !has_err { + // Log success and decrement waiting counter + self.health.record_operation_success(&self.endpoint(), "operation_success"); + self.metrics.record_mutation_success(DiskMetricMutation::Delete); + } + + result + } + pub async fn get_current_disk_id(&self) -> Option { *self.disk_id.read().await } + pub(crate) async fn settle_delete_transaction_with_namespace_owner( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: crate::disk::DeleteRecoveryDigest, + action: crate::disk::DeleteRecoveryAction, + namespace_owner: Option>, + ) -> Result { + self.track_disk_health_mutation( + "settle_delete_transaction", + DiskMetricMutation::Delete, + || { + Box::pin(async { + self.disk + .delete_recovery_settle(volume, object, transaction, expected_fingerprint, action, namespace_owner) + .await + }) + }, + crate::disk::get_delete_recovery_timeout()?, + ) + .await + } + /// Track disk health for an operation. /// This method should wrap disk operations to ensure health checking. pub async fn track_disk_health(&self, operation: F, timeout_duration: Duration) -> Result @@ -2195,6 +2272,87 @@ impl LocalDiskWrapper { #[async_trait::async_trait] impl DiskAPI for LocalDiskWrapper { + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + mut opts: DeleteOptions, + ) -> Result<()> { + crate::disk::validate_delete_recovery_options(&opts)?; + if object.starts_with('/') { + return Err(DiskError::InvalidPath); + } + opts.recoverable_delete = true; + self.delete_version(volume, object, file_info, force_del_marker, opts).await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + mut opts: DeleteOptions, + ) -> Vec> { + if let Err(err) = crate::disk::validate_delete_recovery_options(&opts) { + return vec![Some(err); versions.len()]; + } + if versions.iter().any(|version| version.name.starts_with('/')) { + return vec![Some(DiskError::InvalidPath); versions.len()]; + } + opts.recoverable_delete = true; + self.delete_versions(volume, versions, opts).await + } + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + self.track_disk_health_with_op( + "discover_delete_transactions", + || async { + self.disk + .discover_delete_transactions(volume, object, cursor.clone(), limit) + .await + }, + get_max_timeout_duration(), + ) + .await + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + ) -> Result { + self.track_disk_health_with_op( + "inspect_delete_transaction", + || async { self.disk.inspect_delete_transaction(volume, object, transaction).await }, + get_max_timeout_duration(), + ) + .await + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: crate::disk::DeleteRecoveryDigest, + action: crate::disk::DeleteRecoveryAction, + ) -> Result { + self.track_disk_health_mutation( + "settle_delete_transaction", + DiskMetricMutation::Delete, + || async { + self.disk + .settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + }, + crate::disk::get_delete_recovery_timeout()?, + ) + .await + } fn has_replacement_mount_lease(&self) -> bool { self.disk.has_replacement_mount_lease() } @@ -2408,46 +2566,7 @@ impl DiskAPI for LocalDiskWrapper { } async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) -> Vec> { - let version_count = versions.len(); - self.metrics.record_operation_call("delete_versions"); - // Check if disk is faulty before proceeding - if self.health.is_faulty() { - self.metrics.record_availability_error(); - return vec![Some(DiskError::FaultyDisk); versions.len()]; - } - - // Check if disk is stale - if let Err(e) = self.check_disk_stale().await { - self.metrics.record_availability_error(); - return vec![Some(e); versions.len()]; - } - - let _io_permit = match self.acquire_disk_io_permit().await { - Ok(permit) => permit, - Err(error) => return vec![Some(error); version_count], - }; - - // Record operation start - self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed); - let waiting_guard = self.health.waiting_guard(); - let metric_waiting_guard = self.metrics.waiting_guard(); - let started = Instant::now(); - - // Execute the operation - let result = self.disk.delete_versions(volume, versions, opts).await; - self.metrics.record_operation_latency("delete_versions", started.elapsed()); - self.record_batch_delete_error_metrics(&result); - - drop(waiting_guard); - drop(metric_waiting_guard); - let has_err = result.iter().any(|e| e.is_some()); - if !has_err { - // Log success and decrement waiting counter - self.health.record_operation_success(&self.endpoint(), "operation_success"); - self.metrics.record_mutation_success(DiskMetricMutation::Delete); - } - - result + self.delete_versions_with_namespace_owner(volume, versions, opts, None).await } async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index be9497308..11134035f 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -17,6 +17,7 @@ pub(in crate::disk) use self::commit::LocalRenamePreflightRejection; use self::commit::lock_rename_commit_directories; mod commit; +pub(super) mod delete_recovery; #[cfg(all(unix, test))] #[path = "fd_cache_tests.rs"] mod fd_cache_tests; @@ -130,8 +131,10 @@ tokio::task_local! { const RUSTFS_META_TMP_OLD_BUCKET: &str = ".rustfs.sys/tmp-old"; const INLINE_METADATA_ROLLBACK_DIR_XOR: u128 = 0x7275737466735f696e6c696e655f7262; const DELETE_MARKER_ROLLBACK_FILE: &str = "xl.meta.delete-marker.rollback"; +pub(crate) const DELETE_QUORUM_COMMITTED_FILE: &str = "delete-quorum-committed"; pub(crate) const DELETE_DATA_DIR_MARKER_PREFIX: &str = "delete-data."; pub(crate) const RESERVED_DELETE_DATA_DIR_MARKER_PREFIX: &str = "reserve-delete-data."; +pub(crate) const RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX: &str = "recovery-delete-data."; /// Largest directory read the delete-residue probe issues before it must /// fall back to a complete read. Residue holds one or two data dirs, so an /// under-filled batch settles the common case without materializing large @@ -161,6 +164,16 @@ pub(crate) fn metadata_less_part_file(entry: &str) -> bool { .is_some_and(|part_number| part_number.parse::().is_ok_and(|part_number| part_number > 0)) } +/// Native multipart sidecars travel with their part during delete recovery. +/// Callers must additionally require a regular file inside a bound data dir. +fn delete_recovery_part_file(entry: &str) -> bool { + metadata_less_part_file(entry) + || entry.strip_suffix(".meta").is_some_and(metadata_less_part_file) + || entry.split_once(".integrity.").is_some_and(|(part, generation)| { + metadata_less_part_file(part) && Uuid::parse_str(generation).is_ok_and(|id| !id.is_nil()) + }) +} + fn delete_transaction_marker_id(entry: &str, prefix: &str) -> Option { entry .strip_prefix(prefix) @@ -177,10 +190,15 @@ fn is_delete_transaction_marker(entry: &str, prefix: &str) -> bool { /// an `xl.meta`, an unknown file) means the directory is not plain delete residue. fn is_metadata_less_data_dir_entry(entry: &str) -> bool { !entry.ends_with(SLASH_SEPARATOR) - && (metadata_less_part_file(entry) + && (delete_recovery_part_file(entry) || entry == STORAGE_FORMAT_FILE_BACKUP + || entry == DELETE_QUORUM_COMMITTED_FILE + || entry == delete_recovery::PREPARED + || entry == delete_recovery::TARGET + || entry == delete_recovery::FINISHED || is_delete_transaction_marker(entry, DELETE_DATA_DIR_MARKER_PREFIX) - || is_delete_transaction_marker(entry, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX)) + || is_delete_transaction_marker(entry, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX) + || is_delete_transaction_marker(entry, RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX)) } const STARTUP_CLEANUP_WAIT_TIMEOUT: Duration = Duration::from_secs(2); const ENV_BITROT_SIZE_MISMATCH_RETRY_COUNT: &str = "RUSTFS_BITROT_SIZE_MISMATCH_RETRY_COUNT"; @@ -276,10 +294,21 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r #[cfg(test)] async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> { - write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, data, None).await + write_delete_rollback_file( + object_dir.parent().unwrap_or(object_dir), + "test-backup", + object_dir, + rollback_dir, + STORAGE_FORMAT_FILE_BACKUP, + data, + None, + ) + .await } async fn write_delete_rollback_file( + io_root: &Path, + volume: &str, object_dir: &Path, rollback_dir: Uuid, name: &str, @@ -290,16 +319,33 @@ async fn write_delete_rollback_file( let path = backup_dir.join(name); let lease = os::acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await; let data = data.to_vec(); - let object_dir = object_dir.to_path_buf(); + let sync = effective_durability(volume).syncs_commit_metadata(); + let staging_root = io_root.join(RUSTFS_META_TMP_BUCKET); + let io_root = io_root.to_path_buf(); os::run_blocking_namespace_operation(lease, move || { + reject_local_disk_symlink_components(&io_root, &path).map_err(std::io::Error::from)?; + reject_local_disk_symlink_components(&io_root, &staging_root).map_err(std::io::Error::from)?; std::fs::create_dir_all(&backup_dir)?; + std::fs::create_dir_all(&staging_root)?; #[cfg(test)] run_owned_file_write_before_open(&path); - // Rollback must never consume a partially written backup. Stage outside - // the rollback directory so interrupted writes cannot become staged data. - let mut staged = tempfile::Builder::new().prefix(".delete-rollback-").tempfile_in(object_dir)?; + // Private directory staging is reclaimed by existing tmp cleanup after + // a crash and cannot leave a regular artifact in the user's bucket. + let staging_dir = tempfile::Builder::new() + .prefix(".delete-rollback-") + .tempdir_in(&staging_root)?; + let mut staged = tempfile::NamedTempFile::new_in(staging_dir.path())?; std::io::Write::write_all(&mut staged, &data)?; + if sync { + staged.as_file().sync_all()?; + } staged.persist(path).map_err(|err| err.error)?; + if sync { + os::fsync_dir_std(&backup_dir)?; + if let Some(parent) = backup_dir.parent() { + os::fsync_dir_std(parent)?; + } + } Ok(()) }) .await @@ -336,14 +382,35 @@ async fn restore_metadata_backup_with_namespace_owner( } async fn restore_delete_rollback_with_namespace_owner( + volume: &str, object_dir: &Path, xl_path: &Path, rollback_dir: Uuid, publication_root: &os::PublicationRoot, namespace_owner: Option>, ) -> Result<()> { - remove_version_delete_markers(object_dir, rollback_dir, namespace_owner.clone()).await?; let rollback_path = object_dir.join(rollback_dir.to_string()); + match fs::symlink_metadata(rollback_path.join(DELETE_QUORUM_COMMITTED_FILE)).await { + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + let recovery_journal = match fs::symlink_metadata(rollback_path.join(delete_recovery::PREPARED)).await { + Ok(metadata) if metadata.is_file() => { + let journal: super::DeleteRecoveryJournal = serde_json::from_slice( + &fs::read(rollback_path.join(delete_recovery::PREPARED)) + .await + .map_err(to_file_error)?, + )?; + if journal.volume != volume || journal.transaction != rollback_dir { + return Err(DiskError::FileCorrupt); + } + Some(journal) + } + Ok(_) => return Err(DiskError::InvalidPath), + Err(err) if err.kind() == ErrorKind::NotFound => None, + Err(err) => return Err(to_file_error(err).into()), + }; let mut staged_paths = Vec::new(); let mut remove_new_metadata = false; match fs::read_dir(&rollback_path).await { @@ -352,65 +419,166 @@ async fn restore_delete_rollback_with_namespace_owner( let name = entry.file_name(); if name == DELETE_MARKER_ROLLBACK_FILE { remove_new_metadata = true; - } else if name != STORAGE_FORMAT_FILE_BACKUP { + } else if name != STORAGE_FORMAT_FILE_BACKUP + && name != DELETE_QUORUM_COMMITTED_FILE + && name != delete_recovery::PREPARED + && name != delete_recovery::TARGET + && name != delete_recovery::FINISHED + { + if let Some(journal) = &recovery_journal { + let id = name + .to_str() + .and_then(|name| Uuid::parse_str(name).ok()) + .ok_or(DiskError::FileCorrupt)?; + if !journal.data_dirs.contains(&id) || !entry.file_type().await.map_err(to_file_error)?.is_dir() { + return Err(DiskError::FileCorrupt); + } + } staged_paths.push((entry.path(), object_dir.join(name))); } } } - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) if err.kind() == ErrorKind::NotFound => { + return remove_version_delete_markers(volume, object_dir, rollback_dir, namespace_owner).await; + } Err(err) => return Err(to_file_error(err).into()), } let had_staged_paths = !staged_paths.is_empty(); for (src, dst) in staged_paths { - os::rename_all_with_owner(&src, &dst, object_dir, publication_root, namespace_owner.clone()).await?; + if recovery_journal.is_some() { + let mut entries = fs::read_dir(&src).await.map_err(to_file_error)?; + let mut files = Vec::new(); + while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { + let name = entry.file_name(); + let name = name.to_str().ok_or(DiskError::FileCorrupt)?; + if !entry.file_type().await.map_err(to_file_error)?.is_file() + || (!delete_recovery_part_file(name) + && ![ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .any(|prefix| name == format!("{prefix}{rollback_dir}"))) + { + return Err(DiskError::FileCorrupt); + } + files.push((entry.path(), dst.join(name))); + } + os::create_dir_all_with_namespace_owner(&dst, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + for (source, destination) in files { + match fs::symlink_metadata(&destination).await { + Ok(metadata) if metadata.is_file() => { + let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, namespace_owner.clone()).await; + let identical = os::run_blocking_namespace_operation(lease, move || { + let source_meta = std::fs::symlink_metadata(&source)?; + let destination_meta = std::fs::symlink_metadata(&destination)?; + if !source_meta.is_file() + || !destination_meta.is_file() + || source_meta.len() != destination_meta.len() + { + return Ok(false); + } + let mut source_file = std::fs::File::open(&source)?; + let mut destination_file = std::fs::File::open(&destination)?; + let mut left = [0u8; 65536]; + let mut right = [0u8; 65536]; + let mut remaining = source_meta.len(); + while remaining > 0 { + let length = usize::try_from(remaining.min(left.len() as u64)).map_err(std::io::Error::other)?; + std::io::Read::read_exact(&mut source_file, &mut left[..length])?; + std::io::Read::read_exact(&mut destination_file, &mut right[..length])?; + if left[..length] != right[..length] { + return Ok(false); + } + remaining -= length as u64; + } + std::fs::remove_file(source)?; + Ok(true) + }) + .await + .map_err(to_file_error)?; + if !identical { + return Err(DiskError::FileCorrupt); + } + } + Ok(_) => return Err(DiskError::InvalidPath), + Err(err) if err.kind() == ErrorKind::NotFound => { + os::rename_all_with_owner(&source, &destination, object_dir, publication_root, namespace_owner.clone()) + .await?; + } + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&dst, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + os::fsync_dir_with_owner(&src, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } + let _ = os::remove_dir_with_owner(&src, namespace_owner.clone()).await; + } else { + os::rename_all_with_owner(&src, &dst, object_dir, publication_root, namespace_owner.clone()).await?; + } } let backup_path = rollback_path.join(STORAGE_FORMAT_FILE_BACKUP); match os::rename_all_with_owner(&backup_path, xl_path, object_dir, publication_root, namespace_owner.clone()).await { - Ok(()) => { - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - // A missing backup only means "remove the newly-created delete marker" - // when the marker proves there was no old metadata to restore. + Ok(()) => {} Err(DiskError::FileNotFound) if remove_new_metadata => { match os::remove_file_with_owner(xl_path, namespace_owner.clone()).await { - Ok(()) => { - let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()) - .await; - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - Err(err) if err.kind() == ErrorKind::NotFound => { - let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()) - .await; - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - Err(err) => Err(to_file_error(err).into()), + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), } } - Err(DiskError::FileNotFound) if had_staged_paths => Err(DiskError::FileNotFound), + Err(DiskError::FileNotFound) if had_staged_paths => return Err(DiskError::FileNotFound), Err(DiskError::FileNotFound) => match fs::metadata(xl_path).await { - Ok(_) => { - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - Err(err) if err.kind() == ErrorKind::NotFound => Err(DiskError::FileNotFound), - Err(err) => Err(to_file_error(err).into()), + Ok(_) => {} + Err(err) if err.kind() == ErrorKind::NotFound => return Err(DiskError::FileNotFound), + Err(err) => return Err(to_file_error(err).into()), }, - Err(err) => Err(err), + Err(err) => return Err(err), } + delete_recovery::restore_barrier(volume, object_dir, rollback_dir).await?; + // Persist both sides of the restore rename before discarding its evidence. + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(object_dir, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + os::fsync_dir_with_owner(&rollback_path, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } + remove_version_delete_markers(volume, object_dir, rollback_dir, namespace_owner.clone()).await?; + if remove_new_metadata { + let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()).await; + } + for name in [delete_recovery::TARGET, delete_recovery::PREPARED, delete_recovery::FINISHED] { + match os::remove_file_with_owner(rollback_path.join(name), namespace_owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner).await; + Ok(()) } async fn remove_version_delete_markers( + volume: &str, object_dir: &Path, rollback_dir: Uuid, namespace_owner: Option>, ) -> Result<()> { let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); let committed_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); + let recovery_name = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); let mut entries = match fs::read_dir(object_dir).await { Ok(entries) => entries, Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), @@ -422,13 +590,20 @@ async fn remove_version_delete_markers( { continue; } - for marker_name in [&reserved_name, &committed_name] { + for marker_name in [&reserved_name, &committed_name, &recovery_name] { match os::remove_file_with_owner(entry.path().join(marker_name), namespace_owner.clone()).await { Ok(()) => {} Err(err) if err.kind() == ErrorKind::NotFound => {} Err(err) => return Err(to_file_error(err).into()), } } + // Also sync on retry: a previous unlink may have succeeded before its + // directory sync failed. Keep the journal until those removals persist. + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&entry.path(), namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } } Ok(()) } @@ -464,7 +639,8 @@ async fn restore_delete_rollback_after_error( }; if let Err(restore_err) = - restore_delete_rollback_with_namespace_owner(object_dir, xl_path, rollback_dir, publication_root, namespace_owner).await + restore_delete_rollback_with_namespace_owner(volume, object_dir, xl_path, rollback_dir, publication_root, namespace_owner) + .await { warn!( volume, @@ -6300,6 +6476,32 @@ impl LocalDisk { result.map(|()| DataDirDeleteStatus::Deleted) } + pub(crate) async fn delete_versions_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + let mut errs = Vec::with_capacity(versions.len()); + for _ in 0..versions.len() { + errs.push(None); + } + + for (i, ver) in versions.iter().enumerate() { + if let Err(e) = self + .delete_versions_internal(volume, ver.name.as_str(), &ver.versions, &opts, namespace_owner.clone()) + .await + { + errs[i] = Some(e); + } else { + errs[i] = None; + } + } + + errs + } + async fn delete_version_inner(&self, volume: &str, path: &str, fi: FileInfo, mutation: DeleteVersionMutation) -> Result<()> { let DeleteVersionMutation { force_del_marker, @@ -6347,10 +6549,24 @@ impl LocalDisk { check_path_length(file_path.to_string_lossy().as_ref())?; let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]); - let namespace_owner: Option> = + let mut namespace_owner: Option> = Some(os::acquire_metadata_mutation_lease(&self.get_object_path(volume, path)?, namespace_owner).await); + if opts.recoverable_delete || (opts.undo_write && opts.undo_delete) { + let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await; + namespace_owner = Some(Arc::new((namespace_owner, volume_guard))); + } if opts.old_data_dir.is_some() && opts.undo_write { - return self.undo_write(file_path.as_path(), &fi, &opts, namespace_owner).await; + if opts.undo_delete + && let Some(transaction) = opts.old_data_dir + && self + .rollback_prepared_delete(volume, path, transaction, namespace_owner.clone()) + .await? + { + return Ok(()); + } + return self + .undo_write(volume, file_path.as_path(), &fi, &opts, namespace_owner) + .await; } let rollback_dir = opts.old_data_dir; @@ -6360,12 +6576,12 @@ impl LocalDisk { if err != DiskError::FileNotFound { return Err(err); } - self.finish_committed_delete_residue(volume, path, namespace_owner.clone()) + self.finish_internal_delete_residue(volume, path, namespace_owner.clone()) .await; if fi.deleted && force_del_marker { return self - .write_missing_delete_marker(volume, path, fi, file_path.as_path(), rollback_dir, namespace_owner.clone()) + .write_missing_delete_marker(volume, path, fi, file_path.as_path(), &opts, namespace_owner.clone()) .await; } @@ -6397,15 +6613,35 @@ impl LocalDisk { } let old_dir = meta.delete_version(&fi)?; let mut reserved_version_delete = false; + let mut recovery_target = None; if let Some(rollback_dir) = rollback_dir { - write_delete_rollback_file( - file_path.as_path(), - rollback_dir, - STORAGE_FORMAT_FILE_BACKUP, - &buf, - namespace_owner.clone(), - ) - .await?; + if opts.recoverable_delete { + recovery_target = self + .prepare_delete_recovery( + volume, + path, + Some(&buf), + std::slice::from_ref(&fi), + None, + delete_recovery::DeletePreparationContext { + transaction: rollback_dir, + bucket_incarnation: opts.delete_recovery_bucket_incarnation, + namespace_owner: namespace_owner.clone(), + }, + ) + .await?; + } else { + write_delete_rollback_file( + self.io_root(), + volume, + file_path.as_path(), + rollback_dir, + STORAGE_FORMAT_FILE_BACKUP, + &buf, + namespace_owner.clone(), + ) + .await?; + } } if let Some(uuid) = old_dir { @@ -6562,7 +6798,7 @@ impl LocalDisk { } let commit_result = if !meta.versions.is_empty() { - let buf = match meta.marshal_msg() { + let buf = match recovery_target.map(Ok).unwrap_or_else(|| meta.marshal_msg()) { Ok(buf) => buf, Err(err) => { let err: DiskError = err.into(); @@ -6642,6 +6878,10 @@ impl LocalDisk { .await); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } + if reserved_version_delete && let Some(rollback_dir) = rollback_dir && let Err(err) = self @@ -6698,6 +6938,7 @@ impl LocalDisk { async fn undo_write( &self, + volume: &str, file_path: &Path, fi: &FileInfo, opts: &DeleteOptions, @@ -6707,6 +6948,7 @@ impl LocalDisk { let xl_path = path_join(&[file_path, Path::new(STORAGE_FORMAT_FILE)]); if opts.undo_delete { restore_delete_rollback_with_namespace_owner( + volume, file_path, &xl_path, old_data_dir, @@ -6836,13 +7078,29 @@ impl LocalDisk { namespace_owner: Option>, ) -> Result<()> { crate::hp_guard!("LocalDisk::delete"); + let mut opt = opt; + let mut namespace_owner = namespace_owner; let handled_version_delete = if opt.recursive && opt.immediate && let Some((object, transaction_id)) = path.rsplit_once('/') && let Ok(transaction_id) = Uuid::parse_str(transaction_id) + && !transaction_id.is_nil() { - self.finish_version_delete(volume, object, transaction_id, namespace_owner.clone()) - .await? + if opt.old_data_dir == Some(transaction_id) { + let object_path = self.get_object_path(volume, object)?; + namespace_owner = Some(os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await); + let handled = self + .finish_delete_transaction_with_namespace_owner(volume, object, transaction_id, namespace_owner.clone()) + .await?; + // A key below the UUID prefix may be created concurrently with + // cleanup. Never recursively remove newly published descendants. + opt.recursive = false; + handled + } else { + self.finish_version_delete(volume, object, transaction_id, false, namespace_owner.clone()) + .await? + .0 + } } else { false }; @@ -6855,6 +7113,104 @@ impl LocalDisk { } } + // The caller owns the object metadata lease; retry recovery must not + // reacquire it while holding the same mutation domain. + async fn finish_delete_transaction_with_namespace_owner( + &self, + volume: &str, + object: &str, + transaction_id: Uuid, + namespace_owner: Option>, + ) -> Result { + let path = format!("{object}/{transaction_id}"); + let transaction_path = self.io_get_object_path(volume, &path)?; + let mut transaction_cleanup = false; + match fs::read_dir(&transaction_path).await { + Ok(mut entries) => { + while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { + let name = entry.file_name(); + if !entry.file_type().await.map_err(to_file_error)?.is_file() + || (name != STORAGE_FORMAT_FILE_BACKUP + && name != DELETE_MARKER_ROLLBACK_FILE + && name != DELETE_QUORUM_COMMITTED_FILE) + { + return Err(DiskError::FileCorrupt); + } + transaction_cleanup = true; + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + if transaction_cleanup { + // The set invokes transaction cleanup only after observing write + // quorum. Unlike data-dir markers, this receipt proves that + // decision and also covers inline objects with no data directory. + let receipt_path = transaction_path.join(DELETE_QUORUM_COMMITTED_FILE); + let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, namespace_owner.clone()).await; + let sync = effective_durability(volume).syncs_commit_metadata(); + os::run_blocking_namespace_operation(lease, move || { + let receipt = match std::fs::OpenOptions::new().write(true).create_new(true).open(&receipt_path) { + Ok(receipt) => receipt, + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + if !std::fs::symlink_metadata(&receipt_path)?.is_file() { + return Err(std::io::Error::new(ErrorKind::InvalidData, "delete receipt is not a regular file")); + } + std::fs::File::open(receipt_path)? + } + Err(err) => return Err(err), + }; + if sync { + receipt.sync_all()?; + os::fsync_dir_std(&transaction_path)?; + if let Some(parent) = transaction_path.parent() { + os::fsync_dir_std(parent)?; + } + } + Ok(()) + }) + .await + .map_err(to_file_error)?; + // Retain the receipt until both the backup and marked data are + // gone. Every interrupted cleanup remains recognizable. + for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] { + match os::remove_file_with_owner(self.io_get_object_path(volume, &path)?.join(name), namespace_owner.clone()) + .await + { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&self.io_get_object_path(volume, &path)?, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } + } + let (found, physically_deleted) = self + .finish_version_delete(volume, object, transaction_id, true, namespace_owner.clone()) + .await?; + if !physically_deleted { + // Snapshot readers still own these parts. Keep the quorum receipt + // until their release finishes the physical data removal. + return Err(DiskError::DiskOngoingReq); + } + if transaction_cleanup { + match os::remove_file_with_owner( + self.io_get_object_path(volume, &path)?.join(DELETE_QUORUM_COMMITTED_FILE), + namespace_owner, + ) + .await + { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + Ok(found || transaction_cleanup) + } + async fn delete_unleased(&self, volume: &str, path: &str, opt: &DeleteOptions) -> Result<()> { self.delete_unleased_with_namespace_owner(volume, path, opt, None).await } @@ -6875,7 +7231,94 @@ impl LocalDisk { let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; - self.delete_file_with_namespace_owner(&volume_dir, &file_path, opt.recursive, opt.immediate, namespace_owner) + let mut recursive = opt.recursive; + if recursive + && let Some(transaction) = opt.old_data_dir + && path + .rsplit_once('/') + .is_some_and(|(_, name)| Uuid::parse_str(name).is_ok_and(|id| id != transaction || opt.recoverable_delete)) + { + // Snapshot release replays these same options. Remove only this + // transaction's regular files; UUID-prefix child keys can appear + // after classification and must never be recursively removed. + let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner.clone()).await; + let data_path = file_path.clone(); + let sync = effective_durability(volume).syncs_commit_metadata(); + let keep_rollback_witnesses = opt.recoverable_delete; + os::run_blocking_namespace_operation(lease, move || { + let entries = match std::fs::read_dir(&data_path) { + Ok(entries) => entries, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(err), + }; + let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"); + let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"); + let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"); + let corrupt = + || std::io::Error::new(ErrorKind::InvalidData, "delete data directory contains unrecognized entries"); + let mut parts = Vec::new(); + let mut markers = Vec::new(); + let mut has_committed_marker = false; + for entry in entries { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_str().ok_or_else(corrupt)?; + if !entry.file_type()?.is_file() { + return Err(corrupt()); + } + if delete_recovery_part_file(name) { + parts.push(entry.path()); + } else if name == committed || name == recovery_committed || name == reserved { + has_committed_marker |= name == committed || name == recovery_committed; + markers.push(entry.path()); + } else if keep_rollback_witnesses + && [ + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + ] + .contains(&name) + { + // Explicit legacy deletion keeps rollback witnesses until + // every snapshot-protected payload has disappeared. + } else { + return Err(corrupt()); + } + } + if !parts.is_empty() && !has_committed_marker { + return Err(corrupt()); + } + for files in [parts, markers] { + for entry in files { + #[cfg(all(any(test, feature = "test-util"), not(windows)))] + os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &entry); + match std::fs::remove_file(entry) { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + } + if sync { + os::fsync_dir_std(&data_path)?; + } + } + match std::fs::remove_dir(data_path) { + Ok(()) => Ok(()), + Err(err) if is_benign_object_rmdir_error(&err) => Ok(()), + Err(err) => Err(err), + } + }) + .await + .map_err(|err| { + if err.kind() == ErrorKind::InvalidData { + DiskError::FileCorrupt + } else { + to_file_error(err).into() + } + })?; + recursive = false; + } + self.delete_file_with_namespace_owner(&volume_dir, &file_path, recursive, opt.immediate, namespace_owner) .await?; // A deleted shard must not remain readable through the io_uring fd cache. self.io_backend.invalidate_cached_fds_under(volume, path); @@ -7245,20 +7688,63 @@ impl LocalDisk { path: &str, fi: FileInfo, object_dir: &Path, - rollback_dir: Option, + opts: &DeleteOptions, namespace_owner: Option>, ) -> Result<()> { + let rollback_dir = opts.old_data_dir; + let incarnation = opts.delete_recovery_bucket_incarnation; let xl_path = object_dir.join(STORAGE_FORMAT_FILE); + let mut recovery_target = None; if let Some(rollback_dir) = rollback_dir { - write_delete_rollback_file(object_dir, rollback_dir, DELETE_MARKER_ROLLBACK_FILE, &[], namespace_owner.clone()) + if opts.recoverable_delete { + recovery_target = self + .prepare_delete_recovery( + volume, + path, + None, + std::slice::from_ref(&fi), + None, + delete_recovery::DeletePreparationContext { + transaction: rollback_dir, + bucket_incarnation: incarnation, + namespace_owner: namespace_owner.clone(), + }, + ) + .await?; + } else { + write_delete_rollback_file( + self.io_root(), + volume, + object_dir, + rollback_dir, + DELETE_MARKER_ROLLBACK_FILE, + &[], + namespace_owner.clone(), + ) .await?; + } } - if let Err(err) = self - .write_metadata_with_namespace_owner(volume, path, fi, namespace_owner.clone()) + let committed = if let Some(target) = recovery_target { + async { + fi.validate_for_metadata_read()?; + self.write_all_meta_with_namespace_owner( + volume, + &format!("{path}/{STORAGE_FORMAT_FILE}"), + &target, + true, + namespace_owner.clone(), + ) + .await + } .await - { + } else { + self.write_metadata_with_namespace_owner(volume, path, fi, namespace_owner.clone()) + .await + }; + if let Err(err) = committed { if let Some(rollback_dir) = rollback_dir && let Err(restore_err) = restore_delete_rollback_with_namespace_owner( + volume, object_dir, &xl_path, rollback_dir, @@ -7281,13 +7767,28 @@ impl LocalDisk { } return Err(err); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } Ok(()) } - async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> { + async fn delete_versions_internal( + &self, + volume: &str, + path: &str, + fis: &[FileInfo], + opts: &DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { let volume_dir = self.io_get_bucket_path(volume)?; let object_path = self.get_object_path(volume, path)?; - let namespace_owner: Option> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await); + let mut namespace_owner: Option> = + Some(os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await); + if opts.recoverable_delete || (opts.undo_write && opts.undo_delete) { + let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await; + namespace_owner = Some(Arc::new((namespace_owner, volume_guard))); + } let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?; let object_dir = xlpath .parent() @@ -7297,7 +7798,14 @@ impl LocalDisk { && opts.undo_write { if opts.undo_delete { + if self + .rollback_prepared_delete(volume, path, rollback_dir, namespace_owner.clone()) + .await? + { + return Ok(()); + } return restore_delete_rollback_with_namespace_owner( + volume, object_dir, &xlpath, rollback_dir, @@ -7323,7 +7831,7 @@ impl LocalDisk { { Ok(data) => data, Err(DiskError::FileNotFound) => { - self.finish_committed_delete_residue(volume, path, namespace_owner.clone()) + self.finish_internal_delete_residue(volume, path, namespace_owner.clone()) .await; // `deleted` alone can be an explicit marker purge; only // `mark_deleted` may create metadata that was not present. @@ -7331,14 +7839,7 @@ impl LocalDisk { return Err(DiskError::FileNotFound); }; return self - .write_missing_delete_marker( - volume, - path, - delete_marker, - object_dir, - opts.old_data_dir, - namespace_owner.clone(), - ) + .write_missing_delete_marker(volume, path, delete_marker, object_dir, opts, namespace_owner.clone()) .await; } Err(err) => return Err(err), @@ -7353,9 +7854,35 @@ impl LocalDisk { fm.unmarshal_msg(&data)?; let rollback_dir = opts.old_data_dir; let mut reserved_version_delete = false; + let mut recovery_target = None; if let Some(rollback_dir) = rollback_dir { - write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, &data, namespace_owner.clone()) + if opts.recoverable_delete { + recovery_target = self + .prepare_delete_recovery( + volume, + path, + Some(&data), + fis, + None, + delete_recovery::DeletePreparationContext { + transaction: rollback_dir, + bucket_incarnation: opts.delete_recovery_bucket_incarnation, + namespace_owner: namespace_owner.clone(), + }, + ) + .await?; + } else { + write_delete_rollback_file( + self.io_root(), + volume, + object_dir, + rollback_dir, + STORAGE_FORMAT_FILE_BACKUP, + &data, + namespace_owner.clone(), + ) .await?; + } } for fi in fis.iter() { @@ -7601,6 +8128,9 @@ impl LocalDisk { ) .await); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } if reserved_version_delete && let Some(rollback_dir) = rollback_dir && let Err(err) = self @@ -7629,7 +8159,7 @@ impl LocalDisk { // Update xl.meta atomically: a concurrent reader or crash mid-write must // never observe a truncated xl.meta for versions that were not deleted. - let buf = match fm.marshal_msg() { + let buf = match recovery_target.map(Ok).unwrap_or_else(|| fm.marshal_msg()) { Ok(buf) => buf, Err(err) => { let err: DiskError = err.into(); @@ -7706,6 +8236,10 @@ impl LocalDisk { .await); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } + if reserved_version_delete && let Some(rollback_dir) = rollback_dir && let Err(err) = self @@ -9237,7 +9771,16 @@ impl LocalDisk { Err(err) => return Err(to_file_error(err).into()), }; let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); - let committed_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); + let recovery_owned = + fs::try_exists(self.io_get_object_path(volume, &format!("{object}/{rollback_dir}/{}", delete_recovery::PREPARED))?) + .await + .map_err(to_file_error)?; + let prefix = if recovery_owned { + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX + } else { + DELETE_DATA_DIR_MARKER_PREFIX + }; + let committed_name = format!("{prefix}{rollback_dir}"); while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { if !entry.file_type().await.map_err(to_file_error)?.is_dir() || !entry.file_name().to_str().is_some_and(|name| Uuid::parse_str(name).is_ok()) @@ -9272,14 +9815,21 @@ impl LocalDisk { /// removed the object's metadata on this disk and committed, so the data /// it marked can no longer be read, healed or rolled back here. Finishing /// removes what `T`'s own cleanup pass would have removed. Reserved - /// markers, rollback backups without a committed marker, and object + /// transaction receipts also prove quorum commitment for inline objects. + /// Reserved markers, rollback backups without either proof, and object /// directories that hold `xl.meta` are left alone. - async fn finish_committed_delete_residue( + async fn finish_internal_delete_residue( &self, volume: &str, path: &str, namespace_owner: Option>, ) -> usize { + // RUSTFS_COMPAT_TODO(backlog#2807): retain legacy cleanup for internal + // maintenance until old system-bucket records drain. User-bucket + // residue requires the explicit set-wide recovery decision. + if !crate::bucket::utils::is_meta_bucketname(volume) { + return 0; + } let transactions = match self.committed_delete_transactions(volume, path).await { Ok(transactions) => transactions, Err(err) => { @@ -9289,7 +9839,7 @@ impl LocalDisk { subsystem = LOG_SUBSYSTEM_DISK_LOCAL, volume, path, - operation = "scan_committed_delete_residue", + operation = "scan_internal_delete_residue", error = ?err, "Disk local delete failed" ); @@ -9303,17 +9853,49 @@ impl LocalDisk { }; let mut finished = 0; for transaction in transactions { - // Unlike the delete's own pass, drop the rollback backup before the - // marked data: if this is cut short, the marker that proves the - // transaction is still in place for the next attempt. - let result = match self - .delete_unleased_with_namespace_owner(volume, &format!("{path}/{transaction}"), &opts, namespace_owner.clone()) - .await - { - Ok(()) | Err(DiskError::FileNotFound) => self - .finish_version_delete(volume, path, transaction, namespace_owner.clone()) - .await - .map(|_| ()), + let transaction_path = format!("{path}/{transaction}"); + let receipt = self + .io_get_object_path(volume, &transaction_path) + .map(|transaction_path| transaction_path.join(DELETE_QUORUM_COMMITTED_FILE)); + let result = match receipt { + Ok(receipt) => match fs::symlink_metadata(receipt).await { + Ok(metadata) if metadata.is_file() => { + match self + .finish_delete_transaction_with_namespace_owner(volume, path, transaction, namespace_owner.clone()) + .await + { + Ok(_) => { + self.delete_unleased_with_namespace_owner( + volume, + &transaction_path, + &DeleteOptions { + recursive: false, + ..opts.clone() + }, + namespace_owner.clone(), + ) + .await + } + Err(err) => Err(err), + } + } + Ok(_) => Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => { + // Preserve the legacy marker-only cleanup policy without + // upgrading disk-local evidence into a quorum receipt. + match self + .delete_unleased_with_namespace_owner(volume, &transaction_path, &opts, namespace_owner.clone()) + .await + { + Ok(()) | Err(DiskError::FileNotFound) => self + .finish_version_delete(volume, path, transaction, false, namespace_owner.clone()) + .await + .map(|_| ()), + Err(err) => Err(err), + } + } + Err(err) => Err(to_file_error(err).into()), + }, Err(err) => Err(err), }; match result { @@ -9325,7 +9907,7 @@ impl LocalDisk { volume, path, transaction = %transaction, - operation = "finish_committed_delete_residue", + operation = "finish_internal_delete_residue", error = ?err, "Disk local delete failed" ), @@ -9385,7 +9967,22 @@ impl LocalDisk { Err(err) if err.kind() == ErrorKind::NotFound => continue, Err(err) => return Err(to_file_error(err).into()), }; + let mut has_quorum_receipt = false; + let mut safe_receipt_dir = true; while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { + let name = entry.file_name(); + let file_type = entry.file_type().await.map_err(to_file_error)?; + if !file_type.is_file() + || (name != STORAGE_FORMAT_FILE_BACKUP + && name != DELETE_MARKER_ROLLBACK_FILE + && name != DELETE_QUORUM_COMMITTED_FILE) + { + safe_receipt_dir = false; + } + if entry.file_name() == DELETE_QUORUM_COMMITTED_FILE { + has_quorum_receipt = file_type.is_file(); + continue; + } let Some(transaction) = entry .file_name() .to_str() @@ -9402,8 +9999,31 @@ impl LocalDisk { Err(err) => return Err(to_file_error(err).into()), } } + if has_quorum_receipt + && safe_receipt_dir + && let Some(transaction) = data_dir + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| Uuid::parse_str(name).ok()) + { + transactions.insert(transaction); + } } - Ok(transactions) + let mut legacy = BTreeSet::new(); + for transaction in transactions { + let rollback = object_path.join(transaction.to_string()); + let mut recovery_owned = false; + for name in [delete_recovery::PREPARED, delete_recovery::TARGET, delete_recovery::FINISHED] { + recovery_owned |= fs::try_exists(rollback.join(name)).await.map_err(to_file_error)?; + } + if let Ok(metadata) = fs::symlink_metadata(rollback.join(DELETE_QUORUM_COMMITTED_FILE)).await { + recovery_owned |= metadata.len() > 0; + } + if !recovery_owned { + legacy.insert(transaction); + } + } + Ok(legacy) } async fn finish_version_delete( @@ -9411,17 +10031,30 @@ impl LocalDisk { volume: &str, object: &str, rollback_dir: Uuid, + preserve_prefix_children: bool, namespace_owner: Option>, - ) -> Result { + ) -> Result<(bool, bool)> { let object_path = self.io_get_object_path(volume, object)?; let mut entries = match fs::read_dir(object_path).await { Ok(entries) => entries, - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(false), + Err(err) if err.kind() == ErrorKind::NotFound => return Ok((false, true)), Err(err) => return Err(to_file_error(err).into()), }; - let marker_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); + let recovery_owned = preserve_prefix_children + && fs::try_exists( + self.io_get_object_path(volume, &format!("{object}/{rollback_dir}/{}", delete_recovery::PREPARED))?, + ) + .await + .map_err(to_file_error)?; + let prefix = if recovery_owned { + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX + } else { + DELETE_DATA_DIR_MARKER_PREFIX + }; + let marker_name = format!("{prefix}{rollback_dir}"); let mut first_err = None; let mut found = false; + let mut deferred = false; while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { let Some(data_dir) = entry.file_name().to_str().and_then(|data_dir| Uuid::parse_str(data_dir).ok()) else { continue; @@ -9432,25 +10065,27 @@ impl LocalDisk { Err(err) if err.kind() == ErrorKind::NotFound => continue, Err(err) => return Err(to_file_error(err).into()), } - if let Err(err) = self + match self .delete_data_dir_with_namespace_owner( volume, &format!("{object}/{data_dir}"), DeleteOptions { recursive: true, + old_data_dir: preserve_prefix_children.then_some(rollback_dir), ..Default::default() }, namespace_owner.clone(), ) .await - && first_err.is_none() - && err != DiskError::FileNotFound - && err != DiskError::VolumeNotFound { - first_err = Some(err); + Ok(DataDirDeleteStatus::Deferred) => deferred = true, + Ok(DataDirDeleteStatus::Deleted) => {} + Err(err) if err == DiskError::FileNotFound || err == DiskError::VolumeNotFound => {} + Err(err) if first_err.is_none() => first_err = Some(err), + Err(_) => {} } } - first_err.map_or(Ok(found), Err) + first_err.map_or(Ok((found, !deferred)), Err) } async fn abort_reserved_version_delete_with_failure( @@ -9560,6 +10195,64 @@ pub(crate) async fn batch_shard_pread(requests: Vec<(std::path::PathBuf, usize, #[async_trait::async_trait] impl DiskAPI for LocalDisk { + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + mut opts: DeleteOptions, + ) -> Result<()> { + super::validate_delete_recovery_options(&opts)?; + if object.starts_with('/') { + return Err(DiskError::InvalidPath); + } + opts.recoverable_delete = true; + self.delete_version(volume, object, file_info, force_del_marker, opts).await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + mut opts: DeleteOptions, + ) -> Vec> { + if let Err(err) = super::validate_delete_recovery_options(&opts) { + return vec![Some(err); versions.len()]; + } + if versions.iter().any(|version| version.name.starts_with('/')) { + return vec![Some(DiskError::InvalidPath); versions.len()]; + } + opts.recoverable_delete = true; + self.delete_versions(volume, versions, opts).await + } + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + self.delete_recovery_discover(volume, object, cursor, limit).await + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + self.delete_recovery_inspect(volume, object, transaction).await + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: super::DeleteRecoveryDigest, + action: super::DeleteRecoveryAction, + ) -> Result { + self.delete_recovery_settle(volume, object, transaction, expected_fingerprint, action, None) + .await + } fn to_string(&self) -> String { self.root.to_string_lossy().to_string() } @@ -11261,23 +11954,7 @@ impl DiskAPI for LocalDisk { } #[tracing::instrument(level = "trace", skip_all)] async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) -> Vec> { - let mut errs = Vec::with_capacity(versions.len()); - for _ in 0..versions.len() { - errs.push(None); - } - - for (i, ver) in versions.iter().enumerate() { - if let Err(e) = self - .delete_versions_internal(volume, ver.name.as_str(), &ver.versions, &opts) - .await - { - errs[i] = Some(e); - } else { - errs[i] = None; - } - } - - errs + self.delete_versions_with_namespace_owner(volume, versions, opts, None).await } #[tracing::instrument(level = "trace", skip_all)] @@ -11632,7 +12309,7 @@ mod test { } } - fn test_file_info(name: &str, version_id: Uuid, data_dir: Option, data: Option) -> FileInfo { + pub(super) fn test_file_info(name: &str, version_id: Uuid, data_dir: Option, data: Option) -> FileInfo { let size = data .as_ref() .map(|data| i64::try_from(data.len()).expect("test data length should fit i64")) @@ -11694,7 +12371,7 @@ mod test { assert!(!should_read_legacy_inline_part(&ec12, &config)); } - fn test_meta(fi: FileInfo) -> Vec { + pub(super) fn test_meta(fi: FileInfo) -> Vec { let mut meta = FileMeta::default(); meta.add_version(fi).expect("test metadata should accept file info"); meta.marshal_msg().expect("test metadata should encode") @@ -11958,7 +12635,7 @@ mod test { assert!(!child_path.exists(), "the exact locked directory should be removed"); } - async fn ensure_test_volume(disk: &LocalDisk, volume: &str) { + pub(super) async fn ensure_test_volume(disk: &LocalDisk, volume: &str) { match disk.make_volume(volume).await { Ok(()) | Err(DiskError::VolumeExists) => {} Err(err) => panic!("test volume should be available: {err:?}"), @@ -17890,7 +18567,7 @@ mod test { .await .expect("existing metadata should be written"); - disk.delete_versions_internal(bucket, object, &[missing_fi, existing_fi], &DeleteOptions::default()) + disk.delete_versions_internal(bucket, object, &[missing_fi, existing_fi], &DeleteOptions::default(), None) .await .expect("missing non-deleted version should not abort deletion"); @@ -18047,6 +18724,7 @@ mod test { old_data_dir: Some(rollback_dir), ..Default::default() }, + None, ) .await .expect("missing inline data dir should not fail batch deletion"); @@ -18260,8 +18938,7 @@ mod test { } } - /// Leave the state a delete reaches once it committed on this disk but - /// before its cleanup pass: what a cancelled request used to strand. + /// A legacy disk-local mutation without a structured set decision. async fn write_committed_delete_residue(disk: &LocalDisk, bucket: &str, object: &str) -> CommittedDeleteResidue { let version_id = Uuid::new_v4(); let data_dir = Uuid::new_v4(); @@ -18301,6 +18978,56 @@ mod test { residue } + #[tokio::test] + async fn delete_recovery_user_bucket_apply_preserves_ambiguous_legacy_residue() { + for batch in [false, true] { + for recoverable in [false, true] { + let dir = tempfile::tempdir().expect("residue fixture directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 endpoint")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "user-legacy-residue"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let original = fs::read(residue.backup()).await.expect("legacy metadata backup"); + let request = FileInfo { + name: object.to_owned(), + version_id: Some(Uuid::new_v4()), + ..Default::default() + }; + let opts = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + recoverable_delete: recoverable, + delete_recovery_bucket_incarnation: Some(Uuid::new_v4()), + ..Default::default() + }; + if batch { + let errors = disk + .delete_versions( + bucket, + vec![FileInfoVersions { + name: object.to_owned(), + versions: vec![request], + ..Default::default() + }], + opts, + ) + .await; + assert!(matches!(errors.as_slice(), [Some(DiskError::FileNotFound)]), "{errors:?}"); + } else { + assert!(matches!( + disk.delete_version(bucket, object, request, false, opts).await, + Err(DiskError::FileVersionNotFound) + )); + } + assert_eq!(fs::read(residue.backup()).await.expect("preserved backup"), original); + assert_eq!(fs::read(residue.part()).await.expect("preserved shard"), b"stale"); + assert!(residue.marker().is_file(), "a missing local apply cannot certify historical deletion"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); + } + } + } + fn residue_delete_request(object: &str) -> Vec { vec![FileInfoVersions { name: object.to_string(), @@ -18312,14 +19039,14 @@ mod test { }] } - // #6898: repeating the delete of a key whose earlier delete committed but - // never cleaned up reclaims the residue and still reports the key missing. + // Internal maintenance retains its legacy cleanup behavior while user + // buckets require the separate set-wide authorization boundary. #[tokio::test] async fn delete_versions_finishes_committed_delete_residue() { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "prefix/object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18339,7 +19066,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18359,14 +19086,13 @@ mod test { assert!(!residue.object_dir.exists(), "the committed residue must be reclaimed"); } - // The rollback backup may already be gone; the committed marker alone - // proves the delete. + // Internal maintenance can finish its legacy marker after backup cleanup. #[tokio::test] async fn delete_versions_finishes_committed_marker_without_rollback_backup() { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18382,15 +19108,14 @@ mod test { assert!(!residue.object_dir.exists(), "the committed marker alone must be enough"); } - // Only a committed marker on this disk proves the delete. A reserved marker, - // a rollback backup alone, or live metadata beside the marker is left for - // the owning transaction, rollback, or heal. + // Internal maintenance preserves reserved markers, backup-only states and + // live metadata for the owning transaction, rollback or heal. #[tokio::test] async fn committed_delete_residue_finish_leaves_unproven_state_alone() { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; ensure_test_volume(&disk, bucket).await; // A rollback backup without a committed marker, as an inline object leaves it. @@ -18419,7 +19144,7 @@ mod test { for object in ["backup-only", "reserved", "live"] { assert_eq!( - disk.finish_committed_delete_residue(bucket, object, None).await, + disk.finish_internal_delete_residue(bucket, object, None).await, 0, "{object} carries no committed proof" ); @@ -18446,7 +19171,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "logs"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18458,7 +19183,7 @@ mod test { .await .expect("nested metadata should be written"); - assert_eq!(disk.finish_committed_delete_residue(bucket, object, None).await, 0); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); assert!(residue.backup().exists() && residue.marker().exists() && residue.part().exists()); assert!(nested.exists()); } @@ -18474,7 +19199,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18492,7 +19217,7 @@ mod test { let _ = release_rx.recv(); }); - let mut finish = Box::pin(disk.finish_committed_delete_residue(bucket, object, None)); + let mut finish = Box::pin(disk.finish_internal_delete_residue(bucket, object, None)); tokio::time::timeout(Duration::from_secs(10), async { tokio::select! { entered = entered_rx => entered.expect("finish should reach the marked data dir"), @@ -18508,7 +19233,7 @@ mod test { // The abandoned rename is still parked; the next attempt recognizes the // residue by its marker and finishes it. - assert_eq!(disk.finish_committed_delete_residue(bucket, object, None).await, 1); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); assert!(!residue.object_dir.exists(), "the retried finish must reclaim the residue"); // Let the abandoned rename run into its missing source and wait until @@ -18530,7 +19255,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let committed = write_committed_delete_residue(&disk, bucket, object).await; @@ -18549,12 +19274,620 @@ mod test { .await .expect("unproven backup should be written"); - assert_eq!(disk.finish_committed_delete_residue(bucket, object, None).await, 1); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); assert!(!committed.object_dir.join(committed.data_dir.to_string()).exists()); assert!(unproven_backup.exists(), "a backup without its own committed marker must remain"); assert_eq!(recorder.counter_value(METRIC_DISK_DELETE_RESIDUE_FINISHED_TOTAL, &[]), 1); } + #[tokio::test] + async fn delete_quorum_receipt_finishes_inline_backup_after_interrupted_cleanup() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let object_dir = disk.io_get_object_path(bucket, object).expect("object path"); + let transaction = Uuid::new_v4(); + let rollback_path = object_dir.join(transaction.to_string()); + fs::create_dir_all(&rollback_path).await.expect("transaction directory"); + fs::write(rollback_path.join(STORAGE_FORMAT_FILE_BACKUP), b"inline metadata") + .await + .expect("inline rollback backup"); + // Backup alone remains ambiguous. Only the set-quorum receipt permits cleanup. + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); + fs::write(rollback_path.join(DELETE_QUORUM_COMMITTED_FILE), b"") + .await + .expect("quorum receipt"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!object_dir.exists(), "inline backup and receipt must be reclaimed"); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_cleanup_removes_backup_before_marked_data() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "quorum-cleanup-order"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let data_path = residue.part(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install_at(hooks::Stage::Remove, &data_path, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let transaction_path = format!("{object}/{}", residue.transaction); + let mut cleanup = Box::pin(disk.delete( + bucket, + &transaction_path, + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + )); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("data cleanup starts"), + result = cleanup.as_mut() => panic!("cleanup returned before data removal: {result:?}"), + } + }) + .await + .expect("cleanup reaches data removal"); + assert!(!residue.backup().exists(), "backup must disappear before data"); + assert!(residue.marker().exists(), "data proof must remain during removal"); + assert!( + residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE) + .is_file() + ); + drop(release); + tokio::time::timeout(Duration::from_secs(10), cleanup) + .await + .expect("cleanup completes") + .expect("cleanup succeeds"); + assert!(!residue.object_dir.exists(), "completed cleanup reclaims every transaction file"); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_rollback_keeps_markers_until_metadata_restore() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "rollback-marker-order"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let backup = residue.backup(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install_at(hooks::Stage::Rename, &backup, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let request = residue_delete_request(object); + let mut rollback = Box::pin(disk.delete_versions( + bucket, + request, + DeleteOptions { + undo_write: true, + undo_delete: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + )); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("metadata restore starts"), + result = rollback.as_mut() => panic!("rollback returned before restore: {result:?}"), + } + }) + .await + .expect("rollback reaches metadata restore"); + assert!(residue.marker().exists(), "rollback must retain its marker until metadata is restored"); + assert!(residue.backup().exists(), "backup must remain before restore rename"); + drop(release); + let errors = tokio::time::timeout(Duration::from_secs(10), rollback) + .await + .expect("rollback completes"); + assert!(errors.iter().all(Option::is_none), "{errors:?}"); + assert!(residue.object_dir.join(STORAGE_FORMAT_FILE).is_file()); + assert!(!residue.marker().exists()); + assert!(residue.part().exists()); + } + + #[tokio::test] + async fn delete_rollback_rejects_quorum_committed_transaction() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "quorum-rollback-rejection"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + fs::write( + residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE), + b"", + ) + .await + .expect("quorum receipt"); + let errors = disk + .delete_versions( + bucket, + residue_delete_request(object), + DeleteOptions { + undo_write: true, + undo_delete: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(matches!(errors.as_slice(), [Some(DiskError::FileCorrupt)]), "{errors:?}"); + assert!( + !residue.object_dir.join(STORAGE_FORMAT_FILE).exists(), + "a committed delete must not resurrect metadata" + ); + assert!(residue.backup().exists() && residue.marker().exists()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_receipt_sync_failure_preserves_rollback_backup() { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let rollback_path = residue.object_dir.join(residue.transaction.to_string()); + os::fsync_dir_recorder::set_failure(&rollback_path, ErrorKind::Other); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(result.is_err(), "failed receipt durability must stop cleanup"); + assert!(residue.backup().exists() && residue.part().exists() && residue.marker().exists()); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists(), "retry must complete the already-decided deletion"); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_quorum_backup_removal_sync_failure_retains_receipt() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let transaction_path = residue.object_dir.join(residue.transaction.to_string()); + let receipt = transaction_path.join(DELETE_QUORUM_COMMITTED_FILE); + let _hook = hooks::install_at(hooks::Stage::Remove, &residue.backup(), move || { + // Arm after the receipt's initial fsync to fail the ordering fence + // between backup unlink and eventual receipt removal. + os::fsync_dir_recorder::set_failure(&transaction_path, ErrorKind::Other); + }); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(result.is_err(), "backup unlink must become durable before removing commitment proof"); + assert!(receipt.is_file() && residue.part().is_file() && residue.marker().is_file()); + assert!(!residue.backup().exists()); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_quorum_payload_removal_sync_failure_retains_markers_and_receipt() { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + os::fsync_dir_recorder::set_failure(&residue.object_dir.join(residue.data_dir.to_string()), ErrorKind::Other); + let receipt = residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(result.is_err(), "payload unlink sync failure must stop marker cleanup"); + assert!(receipt.is_file() && residue.marker().is_file()); + assert!(!residue.part().exists()); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_generic_uuid_backup_directory_does_not_publish_quorum_receipt() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "generic-uuid-delete"; + ensure_test_volume(&disk, bucket).await; + let path = format!("object/{}", Uuid::new_v4()); + let directory = disk.io_get_object_path(bucket, &path).expect("UUID directory"); + fs::create_dir_all(&directory).await.expect("UUID directory"); + fs::write(directory.join("part.1"), b"data").await.expect("part"); + fs::write(directory.join(STORAGE_FORMAT_FILE_BACKUP), b"write transaction backup") + .await + .expect("non-delete rollback backup"); + let receipt = directory.join(DELETE_QUORUM_COMMITTED_FILE); + let (receipt_tx, receipt_rx) = tokio::sync::oneshot::channel(); + let _hook = hooks::install_at(hooks::Stage::Rename, &directory, move || { + let _ = receipt_tx.send(receipt.exists()); + }); + disk.delete( + bucket, + &path, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + .expect("generic directory deletion"); + assert!( + !receipt_rx.await.expect("inspect directory before deletion"), + "ordinary UUID data is not a transaction" + ); + } + + #[tokio::test] + async fn delete_quorum_receipt_preserves_uuid_prefix_child() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let transaction = Uuid::new_v4(); + let transaction_path = format!("{object}/{transaction}"); + let rollback = disk.io_get_object_path(bucket, &transaction_path).expect("transaction path"); + let child = rollback.join("child").join(STORAGE_FORMAT_FILE); + fs::create_dir_all(child.parent().expect("child directory")) + .await + .expect("child directory"); + fs::write(&child, b"live child metadata").await.expect("live child"); + fs::write(rollback.join(DELETE_QUORUM_COMMITTED_FILE), b"") + .await + .expect("receipt"); + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), b"old backup") + .await + .expect("backup"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); + let result = disk + .delete( + bucket, + &transaction_path, + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await; + assert!(matches!(result, Err(DiskError::FileCorrupt)), "{result:?}"); + assert_eq!(fs::read(&child).await.expect("preserved child"), b"live child metadata"); + assert!(rollback.join(STORAGE_FORMAT_FILE_BACKUP).is_file()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_cleanup_preserves_child_created_after_classification() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "receipt-prefix-race"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let child = residue + .object_dir + .join(residue.transaction.to_string()) + .join("child") + .join(STORAGE_FORMAT_FILE); + let published_child = child.clone(); + let _hook = hooks::install_at(hooks::Stage::Remove, &residue.part(), move || { + std::fs::create_dir_all(published_child.parent().expect("child directory")).expect("concurrent child directory"); + std::fs::write(&published_child, b"concurrent child").expect("concurrent child metadata"); + }); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + result.expect("cleanup preserves a concurrent child and removes only deleted payload"); + assert_eq!(fs::read(child).await.expect("preserved concurrent child"), b"concurrent child"); + assert!(!residue.part().exists(), "the original deleted payload must still be reclaimed"); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_rollback_backup_sync_failure_precedes_metadata_mutation() { + for mode in [DurabilityMode::Strict, DurabilityMode::None] { + let _mode = durability_mode_override::set(mode); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "backup-durability-gate"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let root = disk.io_get_object_path(bucket, object).expect("object path"); + let data_dir = Uuid::new_v4(); + let data_path = root.join(data_dir.to_string()); + fs::create_dir_all(&data_path).await.expect("data directory"); + fs::write(data_path.join("part.1"), b"original payload") + .await + .expect("payload"); + let original = test_meta(test_file_info(object, Uuid::nil(), Some(data_dir), None)); + fs::write(root.join(STORAGE_FORMAT_FILE), &original) + .await + .expect("original metadata"); + let transaction = Uuid::new_v4(); + let rollback = root.join(transaction.to_string()); + os::fsync_dir_recorder::set_failure(&rollback, ErrorKind::Other); + let errors = disk + .delete_versions( + bucket, + residue_delete_request(object), + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await; + if mode == DurabilityMode::Strict { + assert!(errors.iter().any(Option::is_some), "backup sync failure must stop deletion"); + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("intact metadata"), original); + assert_eq!(fs::read(data_path.join("part.1")).await.expect("intact payload"), b"original payload"); + } else { + assert!(errors.iter().all(Option::is_none), "{errors:?}"); + assert!( + !os::fsync_dir_recorder::was_fsynced(&rollback), + "none mode must skip backup directory sync" + ); + assert!( + os::fsync_dir_recorder::take_failure(&rollback).is_some(), + "none mode must not consume a sync failure" + ); + assert!(!root.join(STORAGE_FORMAT_FILE).exists(), "none mode still applies the requested delete"); + } + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_rollback_restore_sync_failure_retains_markers() { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "restore-durability-gate"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + os::fsync_dir_recorder::set_failure(&residue.object_dir, ErrorKind::Other); + let opts = DeleteOptions { + undo_write: true, + undo_delete: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }; + let errors = disk + .delete_versions(bucket, residue_delete_request(object), opts.clone()) + .await; + assert!(errors.iter().any(Option::is_some), "restore sync failure must remain visible"); + let bytes = fs::read(residue.object_dir.join(STORAGE_FORMAT_FILE)) + .await + .expect("restored metadata is readable"); + assert_eq!(FileMeta::load(&bytes).expect("valid restored metadata").versions.len(), 1); + assert!(residue.marker().exists(), "failed durability must retain transaction evidence"); + assert!(residue.part().exists(), "rollback must retain payload"); + let errors = disk.delete_versions(bucket, residue_delete_request(object), opts).await; + assert!(errors.iter().all(Option::is_none), "retry must finish rollback: {errors:?}"); + assert!(!residue.marker().exists()); + assert!(residue.object_dir.join(STORAGE_FORMAT_FILE).is_file()); + } + + #[tokio::test] + async fn delete_quorum_receipt_survives_deferred_snapshot_cleanup() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let data_path = format!("{object}/{}", residue.data_dir); + let snapshot = disk + .acquire_snapshot_lease(bucket, &data_path) + .await + .expect("snapshot protects deleted payload"); + let receipt = residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(matches!(result, Err(DiskError::DiskOngoingReq)), "{result:?}"); + assert!(receipt.is_file() && residue.marker().is_file() && residue.part().is_file()); + assert!(!residue.backup().exists(), "committed backup is no longer needed"); + disk.release_snapshot_lease(bucket, &data_path, snapshot) + .await + .expect("snapshot release finishes physical cleanup"); + assert!(!residue.part().exists()); + assert!(receipt.is_file(), "receipt remains until transaction cleanup observes physical removal"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_data_cleanup_preserves_concurrent_uuid_prefix_child() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "quorum-data-prefix-race"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let child = residue + .object_dir + .join(residue.data_dir.to_string()) + .join("child") + .join(STORAGE_FORMAT_FILE); + let published = child.clone(); + let _hook = hooks::install_at(hooks::Stage::Remove, &residue.part(), move || { + std::fs::create_dir_all(published.parent().expect("child directory")).expect("concurrent child directory"); + std::fs::write(published, b"new child").expect("concurrent child metadata"); + }); + disk.delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await + .expect("safe cleanup preserves new descendants"); + assert_eq!(fs::read(child).await.expect("preserved child"), b"new child"); + assert!(!residue.part().exists()); + assert!(!residue.marker().exists()); + } + + #[tokio::test] + async fn delete_quorum_snapshot_replay_preserves_live_uuid_prefix_child() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let data_path = format!("{object}/{}", residue.data_dir); + let snapshot = disk.acquire_snapshot_lease(bucket, &data_path).await.expect("snapshot lease"); + let receipt = residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE); + assert!(matches!( + disk.delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + } + ) + .await, + Err(DiskError::DiskOngoingReq) + )); + let child = residue + .object_dir + .join(residue.data_dir.to_string()) + .join("child") + .join(STORAGE_FORMAT_FILE); + fs::create_dir_all(child.parent().expect("child directory")) + .await + .expect("live child directory"); + fs::write(&child, b"live child").await.expect("live child metadata"); + assert!( + disk.release_snapshot_lease(bucket, &data_path, snapshot).await.is_err(), + "replay must reject mixed live contents" + ); + assert_eq!(fs::read(&child).await.expect("preserved child"), b"live child"); + assert!( + receipt.is_file() && residue.part().is_file(), + "blocked replay retains payload and commitment evidence" + ); + fs::remove_file(&child).await.expect("remove test child metadata"); + fs::remove_dir(child.parent().expect("child directory")) + .await + .expect("remove test child directory"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + #[tokio::test] async fn test_delete_version_error_after_staging_restores_data_dir() { use tempfile::tempdir; @@ -21825,6 +23158,8 @@ mod test { undo_delete: false, old_data_dir: None, expected_delete_marker: None, + delete_recovery_bucket_incarnation: None, + recoverable_delete: false, }; disk.delete("test-volume", "test-file.txt", delete_opts) .await diff --git a/crates/ecstore/src/disk/local/delete_recovery.rs b/crates/ecstore/src/disk/local/delete_recovery.rs new file mode 100644 index 000000000..c1b90219e --- /dev/null +++ b/crates/ecstore/src/disk/local/delete_recovery.rs @@ -0,0 +1,3646 @@ +use super::*; +use crate::disk::{ + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, + DeleteRecoveryJournal, DeleteRecoveryPage, DeleteRecoveryState, DeleteTransactionId, +}; +use sha2::{Digest, Sha256}; + +pub(super) const PREPARED: &str = "delete-prepared"; +pub(super) const TARGET: &str = "delete-target"; +pub(super) const FINISHED: &str = "delete-commit-finished"; +const MAX_METADATA: u64 = 64 * 1024 * 1024; +struct DiscoverySession { + disk_root: PathBuf, + volume: String, + object: Option, + root: PathBuf, + _mount: std::fs::File, + frames: Vec<(String, std::fs::ReadDir)>, + touched: Instant, +} + +static DISCOVERY_SESSIONS: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| ParkingLotMutex::new(HashMap::new())); + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct CommitReceipt { + journal: DeleteRecoveryJournal, + finished: bool, +} + +pub(super) struct DeletePreparationContext { + pub transaction: Uuid, + pub bucket_incarnation: Option, + pub namespace_owner: Option>, +} + +pub(super) fn digest(bytes: &[u8]) -> DeleteRecoveryDigest { + Sha256::digest(bytes).into() +} + +pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result { + canonical.data = None; + canonical.erasure.index = 0; + canonical.erasure.checksums.clear(); + canonical.is_latest = false; + canonical.num_versions = 0; + canonical.successor_mod_time = None; + rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut canonical.metadata); + // Only schema-declared string fields and the inline presence marker are + // aliases here. Unknown suffixes and opaque binary fields stay distinct. + use rustfs_utils::http::metadata_compat as compat; + for suffix in [ + compat::SUFFIX_INLINE_DATA, + compat::SUFFIX_COMPRESSION, + compat::SUFFIX_COMPRESSION_SIZE, + compat::SUFFIX_ACTUAL_SIZE, + compat::SUFFIX_ACTUAL_OBJECT_SIZE, + compat::SUFFIX_ACTUAL_OBJECT_SIZE_CAP, + compat::SUFFIX_TRANSITION_STATUS, + compat::SUFFIX_TRANSITIONED_OBJECTNAME, + compat::SUFFIX_TRANSITION_TIER, + compat::SUFFIX_TRANSITIONED_VERSION_STATE, + compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + compat::SUFFIX_PURGESTATUS, + compat::SUFFIX_REPLICA_STATUS, + compat::SUFFIX_REPLICATION_STATUS, + compat::SUFFIX_REPLICA_TIMESTAMP, + compat::SUFFIX_REPLICATION_TIMESTAMP, + compat::SUFFIX_TAGGING_TIMESTAMP, + compat::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, + compat::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, + ] { + if !compat::contains_key_str(&canonical.metadata, suffix) { + continue; + } + // The inline marker is a presence flag whose persisted value is empty. + // Compare both declared aliases explicitly before normalizing it; the + // general compatibility reader rejects empty strings by design. + let value = if suffix == compat::SUFFIX_INLINE_DATA { + let rustfs_key = compat::internal_key_rustfs(suffix); + let minio_key = format!("{}{suffix}", compat::MINIO_INTERNAL_PREFIX); + let mut value = None; + for (key, candidate) in &canonical.metadata { + if !key.eq_ignore_ascii_case(&rustfs_key) && !key.eq_ignore_ascii_case(&minio_key) { + continue; + } + if value.is_some_and(|current: &String| current != candidate) { + return Err(DiskError::FileCorrupt); + } + value = Some(candidate); + } + value.ok_or(DiskError::FileCorrupt)?.clone() + } else { + compat::get_consistent_str(&canonical.metadata, suffix) + .ok_or(DiskError::FileCorrupt)? + .to_owned() + }; + if value.is_empty() && suffix != compat::SUFFIX_INLINE_DATA { + continue; + } + compat::insert_str(&mut canonical.metadata, suffix, value); + } + fn ordered(value: serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let sorted: std::collections::BTreeMap<_, _> = map.into_iter().collect(); + serde_json::Value::Object(sorted.into_iter().map(|(key, value)| (key, ordered(value))).collect()) + } + serde_json::Value::Array(values) => serde_json::Value::Array(values.into_iter().map(ordered).collect()), + other => other, + } + } + Ok(crate::disk::DeleteRecoveryVersionIdentity { + version_id: canonical.version_id, + mod_time: canonical.mod_time, + data_dir: canonical.data_dir, + deleted: canonical.deleted, + metadata_digest: digest(&serde_json::to_vec(&ordered(serde_json::to_value(canonical)?))?), + }) +} + +fn safe_path(path: &str, empty: bool) -> bool { + (empty && path.is_empty()) + || (!path.is_empty() && !path.starts_with('/') && crate::bucket::utils::is_valid_object_prefix(path)) +} + +fn regular_metadata_bytes(path: &Path) -> Result>> { + match std::fs::symlink_metadata(path) { + Ok(meta) if meta.is_file() => Ok(Some(std::fs::read(path).map_err(to_file_error)?)), + Ok(_) => Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(err) => Err(to_file_error(err).into()), + } +} + +fn regular_bytes(path: &Path) -> Result>> { + if std::fs::symlink_metadata(path).is_ok_and(|meta| meta.len() > MAX_METADATA) { + return Err(DiskError::FileCorrupt); + } + regular_metadata_bytes(path) +} + +fn journal_bytes(journal: &DeleteRecoveryJournal) -> Result> { + let bytes = serde_json::to_vec(journal)?; + // The same journal is carried by adoption requests and decision receipts. + if bytes.len() > usize::try_from(MAX_METADATA).map_err(|_| DiskError::FileCorrupt)? - 128 { + return Err(DiskError::FileCorrupt); + } + let projected = crate::disk::DeleteRecoveryRpcReply::Inspected(DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: DeleteTransactionId { + object: journal.object.clone(), + transaction: journal.transaction, + }, + state: DeleteRecoveryState::NewPrepared, + journal: Some(journal.clone()), + current_digest: journal.original_digest, + current_versions: journal.original_versions.clone(), + backup_digest: journal.original_digest, + rollback_data_complete: true, + commit_data_safe: true, + explicit_delete_safe: false, + fingerprint: [0; 32], + }); + if rmp_serde::to_vec_named(&projected)?.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES - 4096 { + return Err(DiskError::FileCorrupt); + } + Ok(bytes) +} + +fn regular_marker(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(meta) if meta.is_file() && meta.len() == 0 => Ok(true), + Ok(_) => Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(false), + Err(err) => Err(to_file_error(err).into()), + } +} + +fn complete_part_sources(live: &Path, staged: &Path, info: &FileInfo) -> Result { + let layout = info.validate(ValidationMode::RequireErasure)?.ok_or(DiskError::FileCorrupt)?; + for part in &info.parts { + let expected = u64::try_from(layout.shard_file_size(part.size).ok_or(DiskError::FileCorrupt)?) + .map_err(|_| DiskError::FileCorrupt)?; + let name = format!("part.{}", part.number); + if ![live, staged] + .iter() + .any(|root| std::fs::symlink_metadata(root.join(&name)).is_ok_and(|meta| meta.is_file() && meta.len() >= expected)) + { + return Ok(false); + } + if let Some(integrity) = &part.integrity { + let name = integrity.file_name(); + let length = u64::try_from(integrity.index_size()?).map_err(|_| DiskError::FileCorrupt)?; + if ![live, staged] + .iter() + .any(|root| std::fs::symlink_metadata(root.join(&name)).is_ok_and(|meta| meta.is_file() && meta.len() == length)) + { + return Ok(false); + } + } + } + Ok(true) +} + +async fn verify_recovery_integrity_sources( + live: &Path, + staged: &Path, + integrity: &rustfs_filemeta::shard_integrity::PartIntegrity, + coding_index: usize, + algorithm: HashAlgorithm, + owner: Option>, +) -> Result<()> { + let lease = os::acquire_namespace_mutation_lease_with_owner(live, owner).await; + let roots = [live.to_path_buf(), staged.to_path_buf()]; + let integrity = integrity.clone(); + os::run_blocking_namespace_operation(lease, move || { + Ok((|| { + integrity.validate()?; + if coding_index >= integrity.layout.shards() + || !matches!(algorithm, HashAlgorithm::HighwayHash256S | HashAlgorithm::HighwayHash256SLegacy) + { + return Err(DiskError::FileCorrupt); + } + let codec = coding::Erasure::try_new_with_options( + usize::from(integrity.layout.data), + usize::from(integrity.layout.parity), + usize::try_from(integrity.layout.block_size).map_err(|_| DiskError::FileCorrupt)?, + integrity.layout.legacy, + ) + .map_err(DiskError::from)?; + let shard_size = codec.shard_size(); + let part_size = + usize::try_from(codec.shard_file_size(i64::try_from(integrity.size).map_err(|_| DiskError::FileCorrupt)?)) + .map_err(|_| DiskError::FileCorrupt)?; + let encoded_size = u64::try_from(coding::bitrot_shard_file_size(part_size, shard_size, algorithm.clone())) + .map_err(|_| DiskError::FileCorrupt)?; + let expected = u64::try_from(integrity.index_size()?).map_err(|_| DiskError::FileCorrupt)?; + let header = integrity.index_header(); + let mut checked = false; + for root in &roots { + let path = root.join(integrity.file_name()); + match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_file() && metadata.len() == expected => {} + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(to_file_error(err).into()), + } + let mut file = std::fs::File::open(path).map_err(to_file_error)?; + let mut actual = [0u8; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE]; + std::io::Read::read_exact(&mut file, &mut actual).map_err(to_file_error)?; + if actual != header { + return Err(DiskError::FileCorrupt); + } + // An interrupted restore can split payload and proof between + // live and staged roots. Verify every present payload copy + // against the declared proof, independently of its location. + let mut shards = Vec::new(); + for root in &roots { + let path = root.join(format!("part.{}", integrity.number)); + match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_file() && metadata.len() == encoded_size => { + shards.push(std::fs::File::open(path).map_err(to_file_error)?); + } + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if shards.is_empty() { + return Err(DiskError::FileCorrupt); + } + let mut record = vec![0; integrity.record_size()]; + let mut payload = vec![0; shard_size]; + let mut bitrot_hash = vec![0; algorithm.size()]; + let mut remaining = part_size; + for stripe in 0..integrity.stripes { + std::io::Read::read_exact(&mut file, &mut record).map_err(to_file_error)?; + integrity.verify_record(stripe, &record)?; + let length = remaining.min(shard_size); + if length == 0 { + return Err(DiskError::FileCorrupt); + } + for shard in &mut shards { + std::io::Read::read_exact(shard, &mut bitrot_hash).map_err(to_file_error)?; + std::io::Read::read_exact(shard, &mut payload[..length]).map_err(to_file_error)?; + integrity.verify_shard(stripe, coding_index, &payload[..length], &record)?; + } + remaining -= length; + } + if remaining != 0 { + return Err(DiskError::FileCorrupt); + } + if integrity.stripes == 0 && integrity.root != integrity.root_digest(&integrity.padding_digest(0)) { + return Err(DiskError::FileCorrupt); + } + checked = true; + } + if !checked { + return Err(DiskError::FileCorrupt); + } + Ok(()) + })()) + }) + .await + .map_err(to_file_error)? +} + +struct ExplicitDeletePlan { + data_dirs: Vec, + proof_files: Vec, +} + +fn explicit_delete_plan( + root: &Path, + transaction: Uuid, + original: Option<&FileMeta>, + journal: Option<&DeleteRecoveryJournal>, + current_missing: bool, +) -> Result> { + if !current_missing + || journal + .is_some_and(|journal| journal.target_digest.is_some() || (journal.original_digest.is_some() && original.is_none())) + { + return Ok(None); + } + let mut referenced = HashSet::new(); + if let Some(metadata) = original { + if metadata.versions.len() != 1 || metadata.versions[0].header.version_type != rustfs_filemeta::VersionType::Object { + return Ok(None); + } + let info = metadata.versions[0].into_fileinfo("", "", false)?; + if info.version_id.is_some_and(|id| !id.is_nil()) + || info.deleted + || info.is_remote() + || info.metadata.keys().any(|key| { + let key = key.to_ascii_lowercase(); + key.starts_with("x-amz-object-lock-") + || rustfs_utils::http::metadata_compat::internal_key_strip_suffix_prefix(&key, "objectlock-").is_some() + }) + { + return Ok(None); + } + referenced.extend(metadata.get_data_dirs()?.into_iter().flatten().filter(|id| !id.is_nil())); + } + let rollback = root.join(transaction.to_string()); + let marker_names = [ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .map(|prefix| format!("{prefix}{transaction}")); + let mut data_dirs = Vec::new(); + let mut proof_files = Vec::new(); + let mut candidates = Vec::new(); + match std::fs::read_dir(&rollback) { + Ok(entries) => { + for entry in entries { + let entry = entry.map_err(to_file_error)?; + let kind = entry.file_type().map_err(to_file_error)?; + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + if kind.is_dir() && Uuid::parse_str(&name).is_ok_and(|id| referenced.contains(&id)) { + candidates.push(entry.path()); + } else if kind.is_file() + && [ + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + PREPARED, + TARGET, + FINISHED, + ] + .contains(&name.as_str()) + { + proof_files.push(entry.path()); + } else if kind.is_file() + && (marker_names.contains(&name) || (delete_recovery_part_file(&name) && referenced.contains(&transaction))) + { + candidates.push(rollback.clone()); + } else { + return Ok(None); + } + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + match std::fs::read_dir(root) { + Ok(entries) => { + for entry in entries { + let entry = entry.map_err(to_file_error)?; + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + let Some(id) = Uuid::parse_str(&name).ok().filter(|id| !id.is_nil()) else { + continue; + }; + if !entry.file_type().map_err(to_file_error)?.is_dir() { + continue; + } + let own_marker = marker_names + .iter() + .any(|name| std::fs::symlink_metadata(entry.path().join(name)).is_ok_and(|metadata| metadata.is_file())); + if referenced.contains(&id) || own_marker { + candidates.push(entry.path()); + } + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + candidates.sort(); + candidates.dedup(); + for dir in candidates { + let mut has_parts = false; + for entry in std::fs::read_dir(&dir).map_err(to_file_error)? { + let entry = entry.map_err(to_file_error)?; + if !entry.file_type().map_err(to_file_error)?.is_file() { + return Ok(None); + } + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + if delete_recovery_part_file(&name) { + has_parts = true; + } else if !(marker_names.contains(&name) + || (dir == rollback + && [ + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + PREPARED, + TARGET, + FINISHED, + ] + .contains(&name.as_str()))) + { + return Ok(None); + } + } + if has_parts && original.is_none() { + return Ok(None); + } + data_dirs.push(dir); + } + data_dirs.sort_by_key(|path| std::cmp::Reverse(path.components().count())); + Ok(Some(ExplicitDeletePlan { data_dirs, proof_files })) +} + +fn inspection(root: &Path, volume: &str, object: &str, transaction: Uuid) -> Result { + let rollback = root.join(transaction.to_string()); + let before = regular_metadata_bytes(&rollback.join(STORAGE_FORMAT_FILE_BACKUP))?; + let current = regular_metadata_bytes(&root.join(STORAGE_FORMAT_FILE))?; + let prepared = regular_bytes(&rollback.join(PREPARED))?; + let target = regular_metadata_bytes(&rollback.join(TARGET))?; + let receipt_bytes = regular_bytes(&rollback.join(DELETE_QUORUM_COMMITTED_FILE))?; + let receipt: Option = receipt_bytes + .as_deref() + .filter(|bytes| !bytes.is_empty()) + .map(serde_json::from_slice) + .transpose()?; + let committed = receipt.is_some(); + let finished = regular_marker(&rollback.join(FINISHED))? || receipt.as_ref().is_some_and(|receipt| receipt.finished); + let journal: Option = prepared + .as_deref() + .map(serde_json::from_slice) + .transpose()? + .or_else(|| receipt.as_ref().map(|receipt| receipt.journal.clone())); + if receipt + .as_ref() + .is_some_and(|receipt| journal.as_ref() != Some(&receipt.journal)) + { + return Err(DiskError::FileCorrupt); + } + if let Some(journal) = &journal + && (journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || journal.volume != volume + || journal.object != object + || journal.transaction != transaction + || journal.transaction.is_nil() + || journal.data_dirs.iter().any(|id| id.is_nil() || *id == transaction)) + { + return Err(DiskError::FileCorrupt); + } + let backup_digest = before.as_deref().map(digest); + let current_digest = current.as_deref().map(digest); + let prepared_digest = prepared.as_deref().map(digest); + let target_digest = target.as_deref().map(digest); + let receipt_digest = receipt_bytes.as_deref().map(digest); + if let Some(journal) = &journal { + if (!finished + && journal.target_digest != target_digest + && !(target.is_none() + && !committed + && before.is_none() + && journal.original_digest.is_some() + && current_digest == journal.original_digest)) + || target_digest.is_some_and(|hash| journal.target_digest != Some(hash)) + || backup_digest.is_some_and(|hash| journal.original_digest != Some(hash)) + { + return Err(DiskError::FileCorrupt); + } + if let Some(bytes) = &target { + FileMeta::load(bytes)?; + } + } + let current_meta = current.as_deref().map(FileMeta::load).transpose()?; + let before_meta = before.as_deref().map(FileMeta::load).transpose()?; + let mut fingerprint = Sha256::new(); + for hash in [prepared_digest, target_digest, backup_digest, current_digest, receipt_digest] { + fingerprint.update([u8::from(hash.is_some())]); + if let Some(hash) = hash { + fingerprint.update(hash); + } + } + fingerprint.update([u8::from(committed), u8::from(finished)]); + let mut any_marker = false; + let mut descriptions = Vec::new(); + let entries = match std::fs::read_dir(root) { + Ok(entries) => Some(entries), + Err(err) if err.kind() == ErrorKind::NotFound => None, + Err(err) => return Err(to_file_error(err).into()), + }; + if let Some(entries) = entries { + for entry in entries { + let entry = entry.map_err(to_file_error)?; + let kind = entry.file_type().map_err(to_file_error)?; + if !kind.is_file() && !kind.is_dir() { + return Err(DiskError::InvalidPath); + } + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + descriptions.push((name.clone(), kind.is_dir(), entry.metadata().map_err(to_file_error)?.len())); + if kind.is_dir() && Uuid::parse_str(&name).is_ok_and(|id| !id.is_nil()) { + for child in std::fs::read_dir(entry.path()).map_err(to_file_error)? { + let child = child.map_err(to_file_error)?; + let kind = child.file_type().map_err(to_file_error)?; + if !kind.is_dir() && !kind.is_file() { + return Err(DiskError::InvalidPath); + } + let child_name = child.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + let metadata = child.metadata().map_err(to_file_error)?; + descriptions.push((format!("{name}/{child_name}"), kind.is_dir(), metadata.len())); + if kind.is_file() + && [ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .any(|prefix| child_name == format!("{prefix}{transaction}")) + { + any_marker = true; + } + let staged_id = Uuid::parse_str(&child_name).ok(); + let known_staged = kind.is_dir() + && staged_id.is_some_and(|dir| { + journal.as_ref().is_some_and(|journal| journal.data_dirs.contains(&dir)) + || before_meta + .as_ref() + .is_some_and(|metadata| metadata.get_data_dirs().is_ok_and(|dirs| dirs.contains(&Some(dir)))) + }); + if name == transaction.to_string() + && !(kind.is_dir() && known_staged) + && !(journal.is_none() + && kind.is_file() + && (delete_recovery_part_file(&child_name) + || [DELETE_DATA_DIR_MARKER_PREFIX, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX] + .iter() + .any(|prefix| child_name == format!("{prefix}{transaction}")))) + && (kind.is_dir() + || ![ + PREPARED, + TARGET, + FINISHED, + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + ] + .contains(&child_name.as_str())) + { + return Err(DiskError::FileCorrupt); + } + if name == transaction.to_string() && kind.is_dir() { + for file in std::fs::read_dir(child.path()).map_err(to_file_error)? { + let file = file.map_err(to_file_error)?; + let kind = file.file_type().map_err(to_file_error)?; + let file_name = file.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + if !kind.is_file() + || (!delete_recovery_part_file(&file_name) + && ![ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .any(|prefix| file_name == format!("{prefix}{transaction}"))) + { + return Err(DiskError::FileCorrupt); + } + descriptions.push(( + format!("{name}/{child_name}/{file_name}"), + false, + file.metadata().map_err(to_file_error)?.len(), + )); + } + } + } + } + } + } + descriptions.sort(); + fingerprint.update(serde_json::to_vec(&descriptions)?); + let mut rollback_data_complete = false; + let mut commit_data_safe = false; + if let Some(journal) = &journal { + rollback_data_complete = (journal.original_digest == backup_digest || current_digest == journal.original_digest) + && (current_digest == journal.original_digest || current_digest == journal.target_digest); + if let Some(meta) = before_meta + .as_ref() + .or_else(|| current_meta.as_ref().filter(|_| current_digest == journal.original_digest)) + { + let inline: HashMap<&str, &[u8]> = if meta.data.as_slice().is_empty() { + HashMap::new() + } else { + if !meta.data.version_ok() { + return Err(DiskError::FileCorrupt); + } + rmp_serde::from_slice(meta.data.after_version())? + }; + for version in &meta.versions { + if version.header.version_type != rustfs_filemeta::VersionType::Object { + continue; + } + let mut info = version.into_fileinfo(volume, object, true)?; + info.hydrate_shard_integrity()?; + if info.is_remote() { + continue; + } + if info.inline_data() { + let id = info.version_id.unwrap_or_default(); + rollback_data_complete &= + inline.contains_key(id.to_string().as_str()) || (id.is_nil() && inline.contains_key("null")); + continue; + } + let Some(data_dir) = info.data_dir.filter(|id| !id.is_nil()) else { + rollback_data_complete = false; + continue; + }; + rollback_data_complete &= + complete_part_sources(&root.join(data_dir.to_string()), &rollback.join(data_dir.to_string()), &info)?; + } + } + commit_data_safe = current_digest == journal.original_digest || current_digest == journal.target_digest; + if current_digest != journal.original_digest && current_digest != journal.target_digest { + // A later certified delete can remove the remaining metadata of + // this committed version delete. Absence has no live references, + // but cannot itself authorize an undecided transaction. + commit_data_safe = (committed && current.is_none()) + || current_meta.as_ref().is_some_and(|meta| { + meta.get_data_dirs() + .is_ok_and(|dirs| !dirs.into_iter().flatten().any(|dir| journal.data_dirs.contains(&dir))) + && !meta.versions.iter().any(|version| { + version + .header + .version_id + .is_some_and(|id| !id.is_nil() && journal.deleted_versions.contains(&id)) + }) + }); + } + } + // In-place shard repair leaves a backup in the live data directory. + // Preserve it without interpreting it as an unfinished delete. + let live_write_backup = journal.is_none() + && !any_marker + && !regular_marker(&rollback.join(DELETE_MARKER_ROLLBACK_FILE))? + && prepared.is_none() + && target.is_none() + && receipt_bytes.is_none() + && !finished + && current_meta.as_ref().is_some_and(|meta| { + meta.get_data_dirs() + .is_ok_and(|dirs| dirs.into_iter().flatten().any(|dir| dir == transaction)) + }) + && before_meta.as_ref().is_some_and(|meta| { + meta.get_data_dirs() + .is_ok_and(|dirs| dirs.into_iter().flatten().any(|dir| dir == transaction)) + }); + let state = if live_write_backup { + DeleteRecoveryState::Absent + } else if journal.is_some() { + if committed { + if finished { + DeleteRecoveryState::CommitFinished + } else { + DeleteRecoveryState::QuorumCommitted + } + } else { + DeleteRecoveryState::NewPrepared + } + } else if before.is_some() || any_marker || receipt_bytes.is_some() || prepared.is_some() || target.is_some() { + DeleteRecoveryState::LegacyAmbiguous + } else { + DeleteRecoveryState::Absent + }; + let explicit_delete_safe = + explicit_delete_plan(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none())?.is_some(); + Ok(DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: DeleteTransactionId { + object: object.to_owned(), + transaction, + }, + state, + journal, + current_digest, + backup_digest, + rollback_data_complete, + commit_data_safe, + explicit_delete_safe, + current_versions: current_meta + .as_ref() + .map(|meta| { + meta.versions + .iter() + .map(|version| generation(version.into_fileinfo(volume, object, true)?)) + .collect::>>() + }) + .transpose()? + .unwrap_or_default(), + fingerprint: fingerprint.finalize().into(), + }) +} + +impl LocalDisk { + async fn delete_recovery_predecessors( + &self, + volume: &str, + object: &str, + transaction: Uuid, + incarnation: Option, + owner: Option>, + ) -> Result> { + let root = self.io_get_object_path(volume, object)?; + let volume = volume.to_owned(); + let object = object.to_owned(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await; + os::run_blocking_namespace_operation(lease, move || { + Ok((|| { + let entries = match std::fs::read_dir(&root) { + Ok(entries) => entries, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Vec::new()), + Err(err) => return Err(to_file_error(err).into()), + }; + let mut predecessors = BTreeSet::new(); + let mut visited = 0usize; + for entry in entries { + visited += 1; + if visited > 65536 { + return Err(DiskError::DiskOngoingReq); + } + let entry = entry.map_err(to_file_error)?; + let Some(id) = entry + .file_name() + .to_str() + .and_then(|name| Uuid::parse_str(name).ok()) + .filter(|id| !id.is_nil() && *id != transaction) + else { + continue; + }; + if !entry.file_type().map_err(to_file_error)?.is_dir() { + return Err(DiskError::InvalidPath); + } + let prepared = regular_bytes(&entry.path().join(PREPARED))?; + let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?; + let mut records = Vec::new(); + if let Some(bytes) = prepared { + records.push(serde_json::from_slice::(&bytes)?); + } + if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) { + records.push(serde_json::from_slice::(&bytes)?.journal); + } + for record in records { + if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || record.volume != volume + || record.object != object + || record.transaction != id + || (incarnation.is_some() + && record.bucket_incarnation.is_some() + && record.bucket_incarnation != incarnation) + || record.predecessors.iter().any(|before| before.is_nil() || *before == id) + { + return Err(DiskError::FileCorrupt); + } + predecessors.insert(id); + } + } + Ok(predecessors.into_iter().collect()) + })()) + }) + .await + .map_err(to_file_error)? + } + pub(super) async fn delete_recovery_discover( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + if !(1..=256).contains(&limit) + || !safe_path(volume, false) + || volume.contains('/') + || object.is_some_and(|object| !safe_path(object, false)) + { + return Err(DiskError::InvalidPath); + } + let bucket_path = match object { + Some(object) => self.io_get_object_path(volume, object)?, + None => self.io_get_bucket_path(volume)?, + }; + let disk_root = self.io_root().to_path_buf(); + let (token, mut session) = if let Some(cursor) = cursor { + let session = DISCOVERY_SESSIONS + .lock() + .remove(&cursor.token) + .ok_or(DiskError::FileNotFound)?; + if session.disk_root != disk_root + || session.volume != volume + || session.object.as_deref() != object + || session.touched.elapsed() > Duration::from_secs(300) + { + return Err(DiskError::InvalidPath); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let before = session._mount.metadata().map_err(to_file_error)?; + #[cfg(target_os = "linux")] + let current = self.mount_lease.metadata().map_err(to_file_error)?; + #[cfg(not(target_os = "linux"))] + let current = std::fs::metadata(self.io_root()).map_err(to_file_error)?; + if before.dev() != current.dev() || before.ino() != current.ino() { + return Err(DiskError::InvalidPath); + } + } + (cursor.token, session) + } else { + #[cfg(target_os = "linux")] + let (mount, root) = { + use std::os::fd::AsRawFd; + let mount = self.mount_lease.try_clone().map_err(to_file_error)?; + let root = PathBuf::from(format!("/proc/self/fd/{}/.", mount.as_raw_fd())).join(volume); + let root = match object { + Some(object) => root.join(object), + None => root, + }; + (mount, root) + }; + #[cfg(windows)] + let (mount, root) = { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_FLAG_BACKUP_SEMANTICS, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_TRAVERSE, + }; + // Match the existing publication-root directory open contract. + let mount = std::fs::OpenOptions::new() + .access_mode(FILE_TRAVERSE | FILE_READ_ATTRIBUTES) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(self.io_root()) + .map_err(to_file_error)?; + (mount, bucket_path.clone()) + }; + #[cfg(all(not(target_os = "linux"), not(windows)))] + let (mount, root) = (std::fs::File::open(self.io_root()).map_err(to_file_error)?, bucket_path.clone()); + ( + Uuid::new_v4(), + DiscoverySession { + disk_root, + volume: volume.to_owned(), + object: object.map(str::to_owned), + root, + _mount: mount, + frames: Vec::new(), + touched: Instant::now(), + }, + ) + }; + let lease = os::acquire_namespace_mutation_lease_with_owner(&bucket_path, None).await; + os::run_blocking_namespace_operation(lease, move || { + if session.frames.is_empty() { + match std::fs::read_dir(&session.root) { + Ok(entries) => session.frames.push((String::new(), entries)), + Err(err) if err.kind() == ErrorKind::NotFound => { + return Ok(DeleteRecoveryPage { + transactions: Vec::new(), + next: None, + }); + } + Err(err) => return Err(err), + } + } + let mut transactions = BTreeSet::new(); + let mut visited = 0usize; + while !session.frames.is_empty() + && transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))? + && visited < 4096 + { + let (parent, entries) = session + .frames + .last_mut() + .ok_or_else(|| std::io::Error::from(ErrorKind::InvalidData))?; + let entry = match entries.next() { + Some(entry) => entry?, + None => { + session.frames.pop(); + continue; + } + }; + let name = entry + .file_name() + .into_string() + .map_err(|_| std::io::Error::from(ErrorKind::InvalidData))?; + if !safe_path(&name, false) || name.contains('/') { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + let relative = if parent.is_empty() { + name.clone() + } else { + format!("{parent}/{name}") + }; + let path = session.root.join(&relative); + reject_local_disk_symlink_components(&session.root, &path).map_err(std::io::Error::from)?; + let kind = match std::fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; + if !kind.is_dir() && !kind.is_file() { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + visited += 1; + if kind.is_file() { + let bound = match session.object.as_deref() { + Some(object) if !parent.is_empty() && !parent.contains('/') => Some((object, parent.as_str())), + Some(_) => None, + None => parent.rsplit_once('/'), + }; + if let Some((object, id)) = bound { + let transaction = + if [PREPARED, STORAGE_FORMAT_FILE_BACKUP, DELETE_QUORUM_COMMITTED_FILE].contains(&name.as_str()) { + Uuid::parse_str(id).ok().filter(|id| !id.is_nil()) + } else { + [ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .find_map(|prefix| delete_transaction_marker_id(&name, prefix)) + }; + if let Some(transaction) = transaction { + let physical = match session.object.as_ref() { + Some(_) => session.root.clone(), + None => session.root.join(object), + }; + let proof_root = physical.join(transaction.to_string()); + let prepared = regular_bytes(&proof_root.join(PREPARED)).map_err(std::io::Error::from)?; + let receipt = + regular_bytes(&proof_root.join(DELETE_QUORUM_COMMITTED_FILE)).map_err(std::io::Error::from)?; + let record = match prepared { + Some(bytes) => { + Some(serde_json::from_slice::(&bytes).map_err(std::io::Error::other)?) + } + None => receipt + .filter(|bytes| !bytes.is_empty()) + .map(|bytes| serde_json::from_slice::(&bytes).map(|receipt| receipt.journal)) + .transpose() + .map_err(std::io::Error::other)?, + }; + let logical = if let Some(journal) = record { + let bound_path = match session.object.as_ref() { + Some(scope) if scope == &journal.object => physical.clone(), + Some(_) => return Err(std::io::Error::from(ErrorKind::InvalidData)), + None => build_local_disk_object_path(&session.root, "", &journal.object).1, + }; + if journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || journal.volume != session.volume + || journal.transaction != transaction + || !safe_path(&journal.object, false) + || bound_path != physical + { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + journal.object + } else { + object.to_owned() + }; + transactions.insert((logical, transaction)); + } + } + } else { + if session.object.is_some() && (!parent.is_empty() || !Uuid::parse_str(&name).is_ok_and(|id| !id.is_nil())) { + continue; + } + // A valid 1024-byte key can exceed 256 one-byte path + // components; allow its transaction/data directories too. + if session.frames.len() >= 1024 { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + session.frames.push((relative, std::fs::read_dir(path)?)); + } + } + let next = if session.frames.is_empty() { + None + } else { + session.touched = Instant::now(); + let mut sessions = DISCOVERY_SESSIONS.lock(); + sessions.retain(|_, session| session.touched.elapsed() <= Duration::from_secs(300)); + if sessions.len() >= 128 { + return Err(std::io::Error::from(ErrorKind::WouldBlock)); + } + sessions.insert(token, session); + Some(DeleteRecoveryCursor { token }) + }; + Ok(DeleteRecoveryPage { + transactions: transactions + .into_iter() + .map(|(object, transaction)| DeleteTransactionId { object, transaction }) + .collect(), + next, + }) + }) + .await + .map_err(|err| to_file_error(err).into()) + } + + pub(super) async fn delete_recovery_inspect( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + if !safe_path(object, false) || transaction.is_nil() { + return Err(DiskError::InvalidPath); + } + let object_path = self.get_object_path(volume, object)?; + let owner: Option> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await); + self.delete_recovery_inspect_owned(volume, object, transaction, owner).await + } + + pub(super) async fn delete_recovery_inspect_owned( + &self, + volume: &str, + object: &str, + transaction: Uuid, + owner: Option>, + ) -> Result { + let root = self.io_get_object_path(volume, object)?; + self.io_get_object_path(volume, &format!("{object}/{transaction}"))?; + let volume = volume.to_owned(); + let object = object.to_owned(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await; + os::run_blocking_namespace_operation(lease, move || Ok(inspection(&root, &volume, &object, transaction))) + .await + .map_err(to_file_error)? + } + + async fn activate_delete_preparation( + &self, + journal: &DeleteRecoveryJournal, + original: Option<&[u8]>, + target: Option>, + owner: Option>, + ) -> Result>> { + let volume = journal.volume.as_str(); + let object = journal.object.as_str(); + let transaction = journal.transaction; + let journal = journal_bytes(journal)?; + self.wait_for_startup_cleanup().await; + let temporary = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?; + let root = self.io_get_object_path(volume, object)?; + let destination = root.join(transaction.to_string()); + let original = original.map(<[u8]>::to_vec); + let sync = effective_durability(volume).syncs_commit_metadata(); + #[cfg(test)] + let record_destination = destination.clone(); + let staging_root = temporary.clone(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, owner.clone()).await; + let (staged, target) = os::run_blocking_namespace_operation(lease, move || { + std::fs::create_dir_all(&staging_root)?; + let staged = Arc::new( + tempfile::Builder::new() + .prefix("delete-preparation-") + .tempdir_in(&staging_root)?, + ); + let missing_marker = original.is_none() && target.is_some(); + for (name, bytes) in [ + (STORAGE_FORMAT_FILE_BACKUP, original.as_deref()), + (TARGET, target.as_deref()), + (PREPARED, Some(journal.as_slice())), + (DELETE_MARKER_ROLLBACK_FILE, missing_marker.then_some(&[][..])), + ] { + let Some(bytes) = bytes else { + continue; + }; + #[cfg(test)] + run_owned_file_write_before_open(&record_destination.join(name)); + let mut file = std::fs::File::create(staged.path().join(name))?; + std::io::Write::write_all(&mut file, bytes)?; + if sync { + file.sync_all()?; + } + } + if sync { + os::fsync_dir_std(staged.path())?; + } + Ok((staged, target)) + }) + .await + .map_err(to_file_error)?; + self.delete_recovery_barrier(volume, object, transaction, "PREACTIVATION") + .await?; + match fs::symlink_metadata(&destination).await { + Ok(_) => return Err(DiskError::OutdatedXLMeta), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + // The directory owner accompanies the blocking rename, so cancelling + // its waiter cannot delete staged files while activation is running. + let publication_owner: Option> = Some(Arc::new((owner.clone(), staged.clone()))); + os::rename_all_with_owner( + staged.path(), + &destination, + &self.io_get_bucket_path(volume)?, + &self.publication_root, + publication_owner, + ) + .await?; + if sync { + let bucket = self.io_get_bucket_path(volume)?; + let mut path = root.as_path(); + loop { + os::fsync_dir_with_owner(path, owner.clone()).await.map_err(to_file_error)?; + if path == bucket { + break; + } + path = path.parent().ok_or(DiskError::InvalidPath)?; + } + os::fsync_dir_with_owner(&temporary, owner.clone()) + .await + .map_err(to_file_error)?; + } + self.delete_recovery_barrier(volume, object, transaction, "PREPARED").await?; + Ok(target) + } + + pub(super) async fn prepare_delete_recovery( + &self, + volume: &str, + object: &str, + original: Option<&[u8]>, + requests: &[FileInfo], + authority: Option<&DeleteRecoveryJournal>, + context: DeletePreparationContext, + ) -> Result>> { + let DeletePreparationContext { + transaction, + bucket_incarnation: incarnation, + namespace_owner: owner, + } = context; + let mut plan = original.map(FileMeta::load).transpose()?.unwrap_or_else(FileMeta::new); + let mut deleted_headers: Vec<_> = plan + .versions + .iter() + .map(|version| (version.header.version_id, std::mem::discriminant(&version.header.version_type))) + .collect(); + let mut original_versions = Vec::new(); + for version in &plan.versions { + original_versions.push(generation(version.into_fileinfo(volume, object, true)?)?); + } + let mut data_dirs = Vec::with_capacity(requests.len()); + for request in requests { + if original.is_none() { + plan.add_version(request.clone())?; + continue; + } + match plan.delete_version(request) { + Ok(Some(dir)) => { + let _ = plan.data.remove(vec![request.version_id.unwrap_or_default(), dir]); + data_dirs.push(dir); + } + Ok(None) => {} + Err(err) => { + let err: DiskError = err.into(); + if request.deleted || (err != DiskError::FileVersionNotFound && err != DiskError::FileNotFound) { + return Err(err); + } + } + } + } + if data_dirs.iter().any(|dir| *dir == transaction || dir.is_nil()) { + return Err(DiskError::FileCorrupt); + } + let target = if plan.versions.is_empty() { + None + } else { + Some(plan.marshal_msg()?) + }; + let predecessors = match authority { + Some(authority) => authority.predecessors.clone(), + None => { + self.delete_recovery_predecessors(volume, object, transaction, incarnation, owner.clone()) + .await? + } + }; + let surviving: HashSet<_> = plan + .versions + .iter() + .map(|version| (version.header.version_id, std::mem::discriminant(&version.header.version_type))) + .collect(); + deleted_headers.retain(|header| !surviving.contains(header)); + let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect(); + let requests = requests + .iter() + .map(|request| { + let mut sparse = request.clone(); + sparse.data = None; + sparse + }) + .collect(); + let mut journal = DeleteRecoveryJournal { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + volume: volume.to_owned(), + object: object.to_owned(), + transaction, + bucket_incarnation: incarnation, + original_digest: original.map(digest), + target_digest: target.as_deref().map(digest), + data_dirs, + deleted_versions, + requests, + original_versions, + predecessors, + }; + if let Some(authority) = authority { + if authority.volume != volume + || authority.object != object + || authority.transaction != transaction + || authority.bucket_incarnation != incarnation + { + return Err(DiskError::OutdatedXLMeta); + } + journal.original_versions.clone_from(&authority.original_versions); + journal.deleted_versions.clone_from(&authority.deleted_versions); + } + drop(plan); + self.activate_delete_preparation(&journal, original, target, owner).await + } + + pub(crate) async fn delete_recovery_settle( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + namespace_owner: Option>, + ) -> Result { + if !safe_path(object, false) || transaction.is_nil() { + return Err(DiskError::InvalidPath); + } + let object_path = self.get_object_path(volume, object)?; + let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await; + let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await; + let owner: Option> = Some(Arc::new((metadata_owner, volume_guard))); + self.delete_recovery_settle_owned(volume, object, transaction, expected_fingerprint, action, owner) + .await + } + + pub(super) async fn rollback_prepared_delete( + &self, + volume: &str, + object: &str, + transaction: Uuid, + owner: Option>, + ) -> Result { + let prepared = self.io_get_object_path(volume, &format!("{object}/{transaction}/{PREPARED}"))?; + if regular_bytes(&prepared)?.is_none() { + return Ok(false); + } + let state = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + self.delete_recovery_settle_owned(volume, object, transaction, state.fingerprint, DeleteRecoveryAction::Rollback, owner) + .await?; + Ok(true) + } + + async fn delete_recovery_settle_owned( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + owner: Option>, + ) -> Result { + let observed = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + if observed.fingerprint != expected_fingerprint { + return Err(DiskError::OutdatedXLMeta); + } + if let DeleteRecoveryAction::AdoptCommit { authority } = &action { + if authority.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || authority.volume != volume + || authority.object != object + || authority.transaction != transaction + { + return Err(DiskError::OutdatedXLMeta); + } + if observed.state == DeleteRecoveryState::NewPrepared { + let local = observed.journal.as_ref().ok_or(DiskError::FileCorrupt)?; + if observed.current_digest.is_some() || !local.matches_prepared_authority(authority) { + return Err(DiskError::OutdatedXLMeta); + } + // The coordinator supplies an inspected same-transaction + // receipt. Retain this drive's immutable hashes, data dirs and + // causal history when completing its partial publication. + let root = self.io_get_object_path(volume, object)?; + let receipt = CommitReceipt { + journal: local.clone(), + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + if observed.state != DeleteRecoveryState::Absent { + return Err(DiskError::OutdatedXLMeta); + } + if observed.current_digest.is_none() { + // A replay witness inherits the original decision's causal + // history; observing later records cannot backdate new edges. + let mut witness = authority.as_ref().clone(); + witness.original_digest = None; + witness.target_digest = None; + witness.data_dirs.clear(); + let root = self.io_get_object_path(volume, object)?; + let _ = self.activate_delete_preparation(&witness, None, None, owner.clone()).await?; + let receipt = CommitReceipt { + journal: witness, + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + if !authority.matches_adoption_source(&observed.current_versions) { + return Err(DiskError::OutdatedXLMeta); + } + let root = self.io_get_object_path(volume, object)?; + let bytes = self.read_all(volume, &format!("{object}/{STORAGE_FORMAT_FILE}")).await?; + let _ = self + .prepare_delete_recovery( + volume, + object, + Some(&bytes), + &authority.requests, + Some(authority.as_ref()), + DeletePreparationContext { + transaction, + bucket_incarnation: authority.bucket_incarnation, + namespace_owner: owner.clone(), + }, + ) + .await?; + let adopted = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + let adopted_journal = adopted.journal.ok_or(DiskError::FileCorrupt)?; + let receipt = CommitReceipt { + journal: adopted_journal, + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + if observed.state == DeleteRecoveryState::Absent { + if action == DeleteRecoveryAction::PublishCommit { + if observed.current_digest.is_some() { + return Err(DiskError::OutdatedXLMeta); + } + self.prepare_delete_recovery( + volume, + object, + None, + &[], + None, + DeletePreparationContext { + transaction, + bucket_incarnation: None, + namespace_owner: owner.clone(), + }, + ) + .await?; + } else { + return Ok(observed); + } + } + let observed = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + if action == DeleteRecoveryAction::DiscardLegacy { + if observed.state != DeleteRecoveryState::LegacyAmbiguous || !observed.explicit_delete_safe { + return Err(DiskError::FileCorrupt); + } + let root = self.io_get_object_path(volume, object)?; + let rollback = root.join(transaction.to_string()); + let plan_root = root.clone(); + let missing = observed.current_digest.is_none(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await; + let plan = os::run_blocking_namespace_operation(lease, move || { + Ok((|| { + let before = + regular_metadata_bytes(&plan_root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))? + .as_deref() + .map(FileMeta::load) + .transpose()?; + explicit_delete_plan(&plan_root, transaction, before.as_ref(), None, missing)?.ok_or(DiskError::FileCorrupt) + })()) + }) + .await + .map_err(to_file_error)??; + for dir in plan.data_dirs { + let relative = dir.strip_prefix(&root).map_err(|_| DiskError::InvalidPath)?; + let data_dir = relative + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| Uuid::parse_str(name).ok()) + .ok_or(DiskError::FileCorrupt)?; + let parent = relative.parent().filter(|parent| !parent.as_os_str().is_empty()); + let staged_object = + parent.map_or_else(|| object.to_owned(), |parent| format!("{object}/{}", parent.to_string_lossy())); + self.reserve_version_delete_with_namespace_owner(volume, &staged_object, data_dir, transaction, owner.clone()) + .await?; + let reserved = dir.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + let private = dir.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + match os::rename_with_namespace_owner(&reserved, &private, owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&dir, owner.clone()).await.map_err(to_file_error)?; + } + if self + .delete_data_dir_with_namespace_owner( + volume, + &format!("{object}/{}", relative.to_string_lossy()), + DeleteOptions { + recursive: true, + old_data_dir: Some(transaction), + recoverable_delete: true, + ..Default::default() + }, + owner.clone(), + ) + .await? + == DataDirDeleteStatus::Deferred + { + return Err(DiskError::DiskOngoingReq); + } + } + for file in plan.proof_files { + match os::remove_file_with_owner(file, owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() && fs::try_exists(&rollback).await.map_err(to_file_error)? { + os::fsync_dir_with_owner(&rollback, owner.clone()) + .await + .map_err(to_file_error)?; + } + self.delete_unleased_with_namespace_owner( + volume, + &format!("{object}/{transaction}"), + &DeleteOptions { + immediate: true, + ..Default::default() + }, + owner.clone(), + ) + .await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + let journal = observed.journal.as_ref().ok_or(DiskError::FileCorrupt)?; + let root = self.io_get_object_path(volume, object)?; + let rollback = root.join(transaction.to_string()); + match action { + DeleteRecoveryAction::PublishCommit => { + if !observed.commit_data_safe { + return Err(DiskError::OutdatedXLMeta); + } + if matches!(observed.state, DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished) { + return Ok(observed); + } + let receipt = CommitReceipt { + journal: journal.clone(), + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + } + DeleteRecoveryAction::FinishCommit => { + if !matches!(observed.state, DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished) + || !observed.commit_data_safe + { + return Err(DiskError::OutdatedXLMeta); + } + if observed.state == DeleteRecoveryState::CommitFinished { + return Ok(observed); + } + for dir in &journal.data_dirs { + self.reserve_version_delete_with_namespace_owner(volume, object, *dir, transaction, owner.clone()) + .await?; + } + if observed.current_digest == journal.original_digest { + if journal.target_digest.is_some() { + let target = self.read_all(volume, &format!("{object}/{transaction}/{TARGET}")).await?; + self.write_all_meta_with_namespace_owner( + volume, + &format!("{object}/{STORAGE_FORMAT_FILE}"), + &target, + true, + owner.clone(), + ) + .await?; + } else { + self.delete_file_with_namespace_owner( + &self.io_get_bucket_path(volume)?, + &root.join(STORAGE_FORMAT_FILE), + false, + false, + owner.clone(), + ) + .await?; + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&root, owner.clone()).await.map_err(to_file_error)?; + } + } + } + self.commit_reserved_version_delete_with_namespace_owner(volume, object, transaction, owner.clone()) + .await?; + for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] { + match os::remove_file_with_owner(rollback.join(name), owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&rollback, owner.clone()) + .await + .map_err(to_file_error)?; + } + let (_, complete) = self + .finish_version_delete(volume, object, transaction, true, owner.clone()) + .await?; + if !complete { + return Err(DiskError::DiskOngoingReq); + } + for dir in &journal.data_dirs { + let staged = rollback.join(dir.to_string()); + match fs::symlink_metadata(&staged).await { + Ok(metadata) if metadata.is_dir() => { + let staged_object = format!("{object}/{transaction}"); + self.reserve_version_delete_with_namespace_owner( + volume, + &staged_object, + *dir, + transaction, + owner.clone(), + ) + .await?; + let source = staged.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + let destination = staged.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + match os::rename_with_namespace_owner(&source, &destination, owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&staged, owner.clone()) + .await + .map_err(to_file_error)?; + } + if self + .delete_data_dir_with_namespace_owner( + volume, + &format!("{staged_object}/{dir}"), + DeleteOptions { + recursive: true, + old_data_dir: Some(transaction), + ..Default::default() + }, + owner.clone(), + ) + .await? + == DataDirDeleteStatus::Deferred + { + return Err(DiskError::DiskOngoingReq); + } + } + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + write_delete_rollback_file(self.io_root(), volume, &root, transaction, FINISHED, &[], owner.clone()).await?; + let receipt = CommitReceipt { + journal: journal.clone(), + finished: true, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + } + DeleteRecoveryAction::FinalizeCommit => { + if observed.state != DeleteRecoveryState::CommitFinished { + return Err(DiskError::DiskOngoingReq); + } + for name in [TARGET, PREPARED, FINISHED] { + match os::remove_file_with_owner(rollback.join(name), owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&rollback, owner.clone()) + .await + .map_err(to_file_error)?; + } + os::remove_file_with_owner(rollback.join(DELETE_QUORUM_COMMITTED_FILE), owner.clone()) + .await + .map_err(to_file_error)?; + self.delete_unleased_with_namespace_owner( + volume, + &format!("{object}/{transaction}"), + &DeleteOptions { + immediate: true, + ..Default::default() + }, + owner.clone(), + ) + .await?; + } + DeleteRecoveryAction::Rollback => { + if observed.state != DeleteRecoveryState::NewPrepared || !observed.rollback_data_complete { + return Err(DiskError::FileCorrupt); + } + let source = if observed.backup_digest.is_some() { + format!("{object}/{transaction}/{STORAGE_FORMAT_FILE_BACKUP}") + } else { + format!("{object}/{STORAGE_FORMAT_FILE}") + }; + if journal.original_digest.is_some() { + let bytes = self.read_all(volume, &source).await?; + let metadata = FileMeta::load(&bytes)?; + for version in &metadata.versions { + if version.header.version_type != rustfs_filemeta::VersionType::Object { + continue; + } + let mut info = version.into_fileinfo(volume, object, true)?; + info.hydrate_shard_integrity()?; + if info.inline_data() || info.is_remote() { + continue; + } + let dir = info.data_dir.filter(|id| !id.is_nil()).ok_or(DiskError::FileCorrupt)?; + let codec = coding::Erasure::try_new_with_options( + info.erasure.data_blocks, + info.erasure.parity_blocks, + info.erasure.block_size, + info.uses_legacy_checksum, + ) + .map_err(DiskError::from)?; + let live = self.io_get_object_path(volume, &format!("{object}/{dir}"))?; + let staged = self.io_get_object_path(volume, &format!("{object}/{transaction}/{dir}"))?; + for part in &info.parts { + let name = format!("part.{}", part.number); + let checksum = info.erasure.get_checksum_info(part.number); + let algo = if info.uses_legacy_checksum && checksum.algorithm == HashAlgorithm::HighwayHash256S { + HashAlgorithm::HighwayHash256SLegacy + } else { + checksum.algorithm + }; + if let Some(integrity) = &part.integrity { + verify_recovery_integrity_sources( + &live, + &staged, + integrity, + info.erasure.index.checked_sub(1).ok_or(DiskError::FileCorrupt)?, + algo.clone(), + owner.clone(), + ) + .await?; + } + let mut checked = false; + for part_root in [&live, &staged] { + let path = part_root.join(&name); + match fs::symlink_metadata(&path).await { + Ok(metadata) if metadata.is_file() => {} + Ok(_) => return Err(DiskError::InvalidPath), + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(to_file_error(err).into()), + } + self.bitrot_verify( + &path, + usize::try_from( + codec.shard_file_size(i64::try_from(part.size).map_err(|_| DiskError::FileCorrupt)?), + ) + .map_err(|_| DiskError::FileCorrupt)?, + algo.clone(), + codec.shard_size(), + ) + .await?; + checked = true; + } + if !checked { + return Err(DiskError::FileCorrupt); + } + } + } + } + if journal.original_digest.is_none() && journal.target_digest.is_none() { + for name in [TARGET, PREPARED, FINISHED] { + match os::remove_file_with_owner(rollback.join(name), owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + self.delete_unleased_with_namespace_owner( + volume, + &format!("{object}/{transaction}"), + &DeleteOptions { + immediate: true, + ..Default::default() + }, + owner.clone(), + ) + .await?; + } else { + restore_delete_rollback_with_namespace_owner( + volume, + &root, + &root.join(STORAGE_FORMAT_FILE), + transaction, + &self.publication_root, + owner.clone(), + ) + .await?; + } + } + DeleteRecoveryAction::AdoptCommit { .. } => return Err(DiskError::MethodNotAllowed), + DeleteRecoveryAction::DiscardLegacy => return Err(DiskError::MethodNotAllowed), + } + self.delete_recovery_inspect_owned(volume, object, transaction, owner).await + } + + pub(super) async fn delete_recovery_barrier(&self, volume: &str, object: &str, transaction: Uuid, stage: &str) -> Result<()> { + barrier(volume, object, transaction, stage, &self.root).await + } +} + +#[cfg(not(feature = "e2e-test-hooks"))] +async fn barrier(_volume: &str, _object: &str, _transaction: Uuid, _stage: &str, _drive: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(feature = "e2e-test-hooks")] +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct BarrierArm { + nonce: String, + bucket: String, + object: String, + stage: String, + #[serde(default)] + disk: Option, +} + +#[cfg(feature = "e2e-test-hooks")] +async fn barrier(volume: &str, object: &str, transaction: Uuid, stage: &str, drive: &Path) -> Result<()> { + let Some(directory) = std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").map(PathBuf::from) else { + return Ok(()); + }; + let bytes = match fs::read(directory.join("arm.json")).await { + Ok(bytes) => bytes, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(to_file_error(err).into()), + }; + let arm: BarrierArm = serde_json::from_slice(&bytes)?; + if arm.bucket != volume + || arm.object != object + || arm.stage != stage + || arm.disk.as_deref().is_some_and(|disk| Path::new(disk) != drive) + { + return Ok(()); + } + if arm.nonce.is_empty() + || arm.nonce.len() > 64 + || !arm + .nonce + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') + { + return Err(DiskError::InvalidPath); + } + let drive_hash = rustfs_utils::crypto::hex(digest(drive.to_string_lossy().as_bytes())); + let observation = serde_json::json!({"transaction": transaction, "bucket": volume, "object": object, "stage": stage, "pid": std::process::id(), "drive": drive}); + fs::write( + directory.join(format!("reached.{}.{}.{}.json", arm.nonce, std::process::id(), &drive_hash[..16])), + serde_json::to_vec(&observation)?, + ) + .await + .map_err(to_file_error)?; + let deadline = Instant::now() + Duration::from_secs(60); + while !fs::try_exists(directory.join(format!("release.{}", arm.nonce))) + .await + .map_err(to_file_error)? + { + if Instant::now() >= deadline { + return Err(DiskError::Timeout); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + if fs::read(directory.join(format!("release.{}", arm.nonce))) + .await + .map_err(to_file_error)? + == b"io_error" + { + return Err(DiskError::Io(std::io::Error::new( + ErrorKind::Other, + "injected delete recovery barrier failure", + ))); + } + Ok(()) +} + +pub(super) async fn restore_barrier(volume: &str, object_dir: &Path, transaction: Uuid) -> Result<()> { + #[cfg(feature = "e2e-test-hooks")] + if let Some(directory) = std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").map(PathBuf::from) { + let bytes = match fs::read(directory.join("arm.json")).await { + Ok(bytes) => bytes, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(to_file_error(err).into()), + }; + let arm: BarrierArm = serde_json::from_slice(&bytes)?; + if arm.bucket == volume && arm.stage == "RESTORE" && object_dir.ends_with(Path::new(volume).join(&arm.object)) { + let suffix = Path::new(volume).join(&arm.object); + let canonical = fs::canonicalize(object_dir).await.map_err(to_file_error)?; + let mut drive = canonical; + for _ in suffix.components() { + drive.pop(); + } + return barrier(volume, &arm.object, transaction, "RESTORE", &drive).await; + } + } + let _ = (volume, object_dir, transaction); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::disk::local::test::{ensure_test_volume, test_file_info, test_meta}; + use tokio::io::AsyncReadExt; + + #[test] + fn delete_recovery_native_sidecar_classifier_is_bounded() { + assert!(delete_recovery_part_file("part.1")); + assert!(delete_recovery_part_file("part.1.meta")); + assert!(delete_recovery_part_file(&format!("part.1.integrity.{}", Uuid::new_v4()))); + for name in [ + "part.0.meta", + "part.1.meta.extra", + "part.1.integrity.invalid", + "xl.meta", + "child/", + ] { + assert!(!delete_recovery_part_file(name), "unknown sidecar {name} must stay protected"); + } + assert!(!delete_recovery_part_file(&format!("part.1.integrity.{}", Uuid::nil()))); + assert!(!metadata_less_part_file("part.1.meta"), "the heal shard classifier remains unchanged"); + } + + #[tokio::test] + #[serial_test::serial(shard_integrity_rollout)] + async fn delete_recovery_multipart_rollback_moves_native_sidecars_and_preserves_proofs() { + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use crate::storage_api_contracts::multipart::{CompletePart, MultipartOperations}; + use crate::storage_api_contracts::object::ObjectIO; + use http::HeaderMap; + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some("true")), + (rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, Some("true")), + ], + async { + for damage in [None, Some("missing"), Some("corrupt"), Some("payload")] { + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "recovery-multipart"; + let object = "object"; + store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap(); + let upload = store + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .unwrap(); + let payload = vec![63; 6 * 1024 * 1024]; + let mut reader = PutObjReader::from_vec(payload.clone()); + let part = store + .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + store + .clone() + .complete_multipart_upload( + bucket, + object, + &upload.upload_id, + vec![CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + .unwrap(); + let disks = store.pools[0].disk_set[0].disks.read().await.clone(); + let transaction = Uuid::new_v4(); + let mut originals = Vec::new(); + for (dir, disk) in dirs.iter().zip(&disks) { + let root = dir.path().join(bucket).join(object); + let original = fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let metadata = FileMeta::load(&original).unwrap(); + let mut info = metadata.versions[0].into_fileinfo(bucket, object, true).unwrap(); + info.hydrate_shard_integrity().unwrap(); + let data = info.data_dir.unwrap(); + let proof_name = info.parts[0] + .integrity + .as_ref() + .expect("real protected MPU proof") + .file_name(); + assert!(root.join(data.to_string()).join("part.1.meta").is_file()); + disk.as_ref() + .unwrap() + .delete_version_recoverable( + bucket, + object, + FileInfo { + name: object.to_owned(), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .unwrap(); + let staged = root.join(transaction.to_string()).join(data.to_string()); + fs::rename(root.join(data.to_string()), &staged).await.unwrap(); + originals.push((root, data, proof_name, original)); + } + let (root, data, proof, _) = &originals[0]; + let proof_path = root.join(transaction.to_string()).join(data.to_string()).join(proof); + if let Some(damage) = damage { + if damage == "missing" { + fs::remove_file(&proof_path).await.unwrap(); + } else if damage == "corrupt" { + let mut bytes = fs::read(&proof_path).await.unwrap(); + bytes[rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE] ^= 1; + fs::write(&proof_path, bytes).await.unwrap(); + } else { + let metadata = FileMeta::load(&originals[0].3).unwrap(); + let info = metadata.versions[0].into_fileinfo(bucket, object, true).unwrap(); + let codec = coding::Erasure::try_new_with_options( + info.erasure.data_blocks, + info.erasure.parity_blocks, + info.erasure.block_size, + info.uses_legacy_checksum, + ) + .unwrap(); + let algorithm = info.erasure.get_checksum_info(1).algorithm; + let path = proof_path.parent().unwrap().join("part.1"); + let mut bytes = fs::read(&path).await.unwrap(); + let hash_size = algorithm.size(); + let part_size = usize::try_from(codec.shard_file_size(info.parts[0].size as i64)).unwrap(); + let length = part_size.min(codec.shard_size()); + bytes[hash_size] ^= 1; + let rewritten = algorithm.hash_encode(&bytes[hash_size..hash_size + length]).as_ref().to_vec(); + bytes[..hash_size].copy_from_slice(&rewritten); + coding::bitrot_verify( + std::io::Cursor::new(bytes.clone()), + bytes.len(), + part_size, + algorithm, + codec.shard_size(), + ) + .await + .expect("recomputed self-described bitrot checksum passes the former rollback gate"); + fs::write(path, bytes).await.unwrap(); + } + let disk = disks[0].as_ref().unwrap(); + let observed = disk.inspect_delete_transaction(bucket, object, transaction).await.unwrap(); + let backup = fs::read(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP)) + .await + .unwrap(); + assert!(matches!( + disk.settle_delete_transaction( + bucket, + object, + transaction, + observed.fingerprint, + DeleteRecoveryAction::Rollback + ) + .await, + Err(DiskError::FileCorrupt) + )); + assert!(!root.join(STORAGE_FORMAT_FILE).exists()); + assert!(root.join(transaction.to_string()).join(PREPARED).is_file()); + assert_eq!( + fs::read(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP)) + .await + .unwrap(), + backup + ); + continue; + } + // Both directions of a partially restored native layout: + // proof still staged with live payload, and the inverse. + for (index, (root, data, proof, _)) in originals.iter().take(2).enumerate() { + let live = root.join(data.to_string()); + fs::create_dir(&live).await.unwrap(); + let name = if index == 0 { "part.1" } else { proof.as_str() }; + fs::rename(root.join(transaction.to_string()).join(data.to_string()).join(name), live.join(name)) + .await + .unwrap(); + } + for (disk, (root, data, proof, original)) in disks.iter().zip(originals) { + let disk = disk.as_ref().unwrap(); + let observed = disk.inspect_delete_transaction(bucket, object, transaction).await.unwrap(); + assert!(observed.rollback_data_complete); + disk.settle_delete_transaction( + bucket, + object, + transaction, + observed.fingerprint, + DeleteRecoveryAction::Rollback, + ) + .await + .expect("rollback restores native multipart sidecars"); + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), original); + assert!(root.join(data.to_string()).join("part.1.meta").is_file()); + assert!(root.join(data.to_string()).join(proof).is_file()); + assert!(!root.join(transaction.to_string()).exists()); + } + let mut reader = store.pools[0].disk_set[0] + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .unwrap(); + let mut restored = Vec::new(); + reader.stream.read_to_end(&mut restored).await.unwrap(); + assert_eq!(restored, payload, "restored protected MPU remains readable through integrity checks"); + } + }, + ) + .await; + } + + async fn fixture() -> (tempfile::TempDir, LocalDisk, FileInfo, Vec) { + let temp = tempfile::tempdir().expect("temporary disk"); + let endpoint = Endpoint::try_from(temp.path().to_str().expect("UTF-8 disk path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + ensure_test_volume(&disk, "recovery").await; + let mut info = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"complete original inline payload"))); + info.set_inline_data(); + let bytes = test_meta(info.clone()); + let path = disk.io_get_object_path("recovery", "object/xl.meta").expect("metadata path"); + fs::create_dir_all(path.parent().expect("object directory")) + .await + .expect("object directory"); + fs::write(path, &bytes).await.expect("original metadata"); + (temp, disk, info, bytes) + } + + fn request() -> FileInfo { + FileInfo { + name: "object".to_owned(), + version_id: Some(Uuid::nil()), + ..Default::default() + } + } + + #[test] + fn delete_recovery_generation_known_aliases_are_equal_unknowns_stay_distinct() { + use rustfs_utils::http::metadata_compat as compat; + let mut legacy = request(); + legacy + .metadata + .insert(format!("{}{}", compat::MINIO_INTERNAL_PREFIX, compat::SUFFIX_INLINE_DATA), String::new()); + legacy + .metadata + .insert(format!("{}{}", compat::MINIO_INTERNAL_PREFIX, compat::SUFFIX_COMPRESSION), "gzip".into()); + let mut modern = legacy.clone(); + compat::insert_str(&mut modern.metadata, compat::SUFFIX_INLINE_DATA, String::new()); + compat::insert_str(&mut modern.metadata, compat::SUFFIX_COMPRESSION, "gzip".into()); + assert_eq!( + generation(legacy.clone()).expect("legacy identity"), + generation(modern.clone()).expect("modern identity") + ); + modern.metadata.insert( + format!("{}{}", compat::RUSTFS_INTERNAL_PREFIX, compat::SUFFIX_COMPRESSION), + "different".into(), + ); + assert!(matches!(generation(modern), Err(DiskError::FileCorrupt))); + let mut unknown = legacy.clone(); + legacy + .metadata + .insert("x-minio-internal-unknown-extension".into(), "same".into()); + unknown + .metadata + .insert("x-rustfs-internal-unknown-extension".into(), "same".into()); + assert_ne!( + generation(legacy).expect("legacy unknown field"), + generation(unknown).expect("modern unknown field") + ); + } + + #[tokio::test] + async fn delete_recovery_inactive_preparation_artifacts_use_existing_tmp_cleanup() { + let (_temp, disk, _, original) = fixture().await; + let temporary = disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).unwrap(); + fs::create_dir_all(temporary.join("delete-preparation-interrupted")) + .await + .unwrap(); + fs::write(temporary.join("delete-preparation-interrupted").join(TARGET), b"unfinished target") + .await + .unwrap(); + fs::create_dir_all(temporary.join(".delete-rollback-interrupted")) + .await + .unwrap(); + fs::write( + temporary.join(".delete-rollback-interrupted").join("metadata"), + b"unpublished replacement", + ) + .await + .unwrap(); + assert!( + disk.discover_delete_transactions("recovery", Some("object"), None, 16) + .await + .unwrap() + .transactions + .is_empty() + ); + let ready = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let notify = Arc::new(Notify::new()); + LocalDisk::cleanup_tmp_on_startup(disk.io_root(), &disk.publication_root, ready.clone(), notify) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + while ready.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!temporary.join("delete-preparation-interrupted").exists()); + assert!(!temporary.join(".delete-rollback-interrupted").exists()); + assert!(!disk.io_root().join(RUSTFS_META_TMP_OLD_BUCKET).exists()); + assert!(temporary.join(".trash").is_dir(), "startup recreates the active trash directory"); + assert_eq!(disk.read_all("recovery", "object/xl.meta").await.unwrap().as_ref(), original); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_activation_is_complete_before_metadata_mutation_even_after_cancel() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + use futures::FutureExt; + let (_temp, disk, _, original) = fixture().await; + let disk = Arc::new(disk); + let transaction = Uuid::new_v4(); + let destination = disk.io_get_object_path("recovery", &format!("object/{transaction}")).unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Rename, &destination, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let deleting = disk.clone(); + let mut request = request(); + request.expire_restored = true; + let task = tokio::spawn(async move { + deleting + .delete_version_recoverable( + "recovery", + "object", + request, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + }); + let observations = std::panic::AssertUnwindSafe(async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .unwrap() + .unwrap(); + assert!(!destination.exists()); + assert_eq!(disk.read_all("recovery", "object/xl.meta").await.unwrap().as_ref(), original); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + }) + .catch_unwind() + .await; + let _ = release_tx.send(()); + drop(hook); + tokio::time::timeout(Duration::from_secs(5), async { + while !destination.exists() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + for name in [PREPARED, TARGET, STORAGE_FORMAT_FILE_BACKUP] { + assert!(destination.join(name).is_file()); + } + let state = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(state.state, DeleteRecoveryState::NewPrepared); + assert_eq!(state.current_digest, Some(digest(&original))); + if let Err(panic) = observations { + std::panic::resume_unwind(panic); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_receiver_volume_guard_outlives_cancelled_restore() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + use futures::FutureExt; + let (_temp, disk, _, _) = fixture().await; + let disk = Arc::new(disk); + let transaction = Uuid::new_v4(); + let state = apply(&disk, transaction).await; + let metadata = disk.io_get_object_path("recovery", "object/xl.meta").unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Rename, &metadata, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let restoring = disk.clone(); + let task = tokio::spawn(async move { + restoring + .settle_delete_transaction("recovery", "object", transaction, state.fingerprint, DeleteRecoveryAction::Rollback) + .await + }); + let mut deletion = None; + let observations = std::panic::AssertUnwindSafe(async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .unwrap() + .unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let removing = disk.clone(); + deletion = Some(tokio::spawn(async move { removing.delete_volume("recovery", true).await })); + assert!( + tokio::time::timeout(Duration::from_millis(100), deletion.as_mut().unwrap()) + .await + .is_err() + ); + }) + .catch_unwind() + .await; + let _ = release_tx.send(()); + drop(hook); + if let Some(deletion) = deletion { + tokio::time::timeout(Duration::from_secs(5), deletion) + .await + .unwrap() + .unwrap() + .unwrap(); + } + disk.make_volume("recovery").await.unwrap(); + assert!(!metadata.exists(), "late restore must finish before volume recreation"); + if let Err(panic) = observations { + std::panic::resume_unwind(panic); + } + } + + async fn apply(disk: &LocalDisk, transaction: Uuid) -> DeleteRecoveryInspection { + disk.delete_version_recoverable( + "recovery", + "object", + request(), + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .expect("prepared metadata application"); + let inspected = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("inspect applied transaction"); + assert_eq!( + inspected.current_digest, + inspected.journal.as_ref().unwrap().target_digest, + "planned target must match the unchanged mutation loop" + ); + inspected + } + + #[tokio::test] + async fn delete_recovery_uncommitted_inline_rolls_back_exact_bytes() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let observed = apply(&disk, transaction).await; + assert_eq!(observed.state, DeleteRecoveryState::NewPrepared); + assert!(observed.rollback_data_complete); + let restored = disk + .settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("rollback intact prepared metadata"); + assert_eq!(restored.state, DeleteRecoveryState::Absent); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("readable original metadata") + .as_ref(), + original + ); + } + + #[tokio::test] + async fn delete_recovery_preparation_records_existing_decision_before_new_generation() { + let (_temp, disk, info, original) = fixture().await; + let first = Uuid::new_v4(); + let applied = apply(&disk, first).await; + disk.settle_delete_transaction("recovery", "object", first, applied.fingerprint, DeleteRecoveryAction::PublishCommit) + .await + .expect("persist first decision"); + let mut newer = info; + newer.mod_time = newer.mod_time.map(|time| time + time::Duration::seconds(1)); + let newer_bytes = test_meta(newer); + assert_ne!(newer_bytes, original); + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").unwrap(), &newer_bytes) + .await + .unwrap(); + let second = Uuid::new_v4(); + let second_state = apply(&disk, second).await; + assert_eq!(second_state.journal.unwrap().predecessors, vec![first]); + } + + #[tokio::test] + async fn delete_recovery_large_intent_fails_before_metadata_application() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let mut large = request(); + large + .metadata + .insert("oversized-intent".to_owned(), "x".repeat(usize::try_from(MAX_METADATA).unwrap())); + assert!( + disk.delete_version_recoverable( + "recovery", + "object", + large, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + } + ) + .await + .is_err() + ); + assert_eq!(disk.read_all("recovery", "object/xl.meta").await.unwrap().as_ref(), original); + assert!( + !disk + .io_get_object_path("recovery", &format!("object/{transaction}/{PREPARED}")) + .unwrap() + .exists() + ); + } + + #[tokio::test] + async fn delete_recovery_metadata_updates_publish_exact_prepared_target_bytes() { + for batch in [false, true] { + for transition in [false, true] { + let (_temp, disk, mut info, _) = fixture().await; + info.metadata.insert("x-amz-meta-first".into(), "retained-a".into()); + info.metadata.insert("x-amz-meta-second".into(), "retained-b".into()); + info.metadata + .insert(rustfs_filemeta::metadata_keys::RESTORE.into(), "ongoing-request=\"false\"".into()); + let original = test_meta(info); + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").unwrap(), &original) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + let mut change = request(); + if transition { + change.transition_status = "complete".into(); + change.transition_tier = "test-tier".into(); + change.transitioned_objname = "remote/object".into(); + } else { + change.expire_restored = true; + } + let opts = DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }; + if batch { + let errors = disk + .delete_versions_recoverable( + "recovery", + vec![FileInfoVersions { + name: "object".into(), + volume: "recovery".into(), + versions: vec![change], + ..Default::default() + }], + opts, + ) + .await; + assert_eq!(errors, vec![None]); + } else { + disk.delete_version_recoverable("recovery", "object", change, false, opts) + .await + .unwrap(); + } + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(observed.current_digest, observed.journal.unwrap().target_digest); + let applied = disk.read_all("recovery", "object/xl.meta").await.unwrap(); + let prepared = disk + .read_all("recovery", &format!("object/{transaction}/{TARGET}")) + .await + .unwrap(); + assert_eq!(applied, prepared, "batch={batch}, transition={transition}"); + assert_ne!(applied.as_ref(), original); + } + } + } + + async fn legacy_fixture(disk: &LocalDisk, original: &[u8], transaction: Uuid) -> PathBuf { + let root = disk.io_get_object_path("recovery", "object").unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + fs::create_dir(root.join(transaction.to_string())).await.unwrap(); + fs::write(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP), original) + .await + .unwrap(); + root + } + + #[tokio::test] + async fn delete_recovery_live_repair_backup_preserves_legacy_delete_guards() { + for shape in ["live", "rollback-marker", "missing-current", "foreign-backup"] { + let (_temp, disk, mut info, _) = fixture().await; + let transaction = Uuid::new_v4(); + info.data_dir = Some(transaction); + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let current = test_meta(info.clone()); + fs::write(root.join(STORAGE_FORMAT_FILE), ¤t).await.unwrap(); + let rollback = root.join(transaction.to_string()); + fs::create_dir(&rollback).await.unwrap(); + if shape == "foreign-backup" { + info.data_dir = Some(Uuid::new_v4()); + } + let backup = test_meta(info); + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), &backup).await.unwrap(); + if shape == "rollback-marker" { + fs::write(rollback.join(DELETE_MARKER_ROLLBACK_FILE), []).await.unwrap(); + } else if shape == "missing-current" { + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + } + let inspection = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!( + inspection.state, + if shape == "live" { + DeleteRecoveryState::Absent + } else { + DeleteRecoveryState::LegacyAmbiguous + }, + "{shape}" + ); + assert_eq!(fs::read(rollback.join(STORAGE_FORMAT_FILE_BACKUP)).await.unwrap(), backup); + if shape != "missing-current" { + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), current); + } + } + } + + #[tokio::test] + async fn delete_recovery_explicit_legacy_discard_accepts_old_empty_receipt() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let root = legacy_fixture(&disk, &original, transaction).await; + fs::write(root.join(transaction.to_string()).join(DELETE_QUORUM_COMMITTED_FILE), []) + .await + .unwrap(); + let state = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(state.state, DeleteRecoveryState::LegacyAmbiguous); + assert!(state.explicit_delete_safe); + let cleared = disk + .settle_delete_transaction("recovery", "object", transaction, state.fingerprint, DeleteRecoveryAction::DiscardLegacy) + .await + .unwrap(); + assert_eq!(cleared.state, DeleteRecoveryState::Absent); + assert!(!root.join(transaction.to_string()).exists()); + } + + #[tokio::test] + async fn delete_recovery_prepared_explicit_delete_checks_original_header() { + for protected in ["none", "version", "lock", "internal-lock"] { + let (_temp, disk, mut info, _) = fixture().await; + match protected { + "version" => info.version_id = Some(Uuid::new_v4()), + "lock" => { + info.metadata.insert("x-amz-object-lock-mode".into(), "COMPLIANCE".into()); + } + "internal-lock" => rustfs_utils::http::metadata_compat::insert_str( + &mut info.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, + "known-lock".to_owned(), + ), + _ => {} + } + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").unwrap(), test_meta(info.clone())) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + disk.delete_version_recoverable( + "recovery", + "object", + FileInfo { + version_id: info.version_id, + ..request() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .unwrap(); + let state = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(state.state, DeleteRecoveryState::NewPrepared); + assert_eq!(state.explicit_delete_safe, protected == "none", "original header {protected}"); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_cancelled_restore_retains_actual_key_and_bucket_guards() { + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::disk::os::prepared_publication_test_hooks as hooks; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use crate::storage_api_contracts::object::ObjectIO; + use futures::FutureExt; + use rustfs_heal_contracts::heal_channel::HealOpts; + + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "recover-owned-timeout"; + store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap(); + store + .put_object(bucket, "object", &mut PutObjReader::from_vec(vec![61; 4096]), &ObjectOptions::default()) + .await + .unwrap(); + let disks = store.pools[0].disk_set[0].disk_inventory().await; + let transaction = Uuid::new_v4(); + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); + for disk in disks.iter().flatten() { + disk.delete_version_recoverable( + bucket, + "object", + request(), + false, + DeleteOptions { + old_data_dir: Some(transaction), + delete_recovery_bucket_incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .unwrap(); + } + let metadata = disks[0] + .as_ref() + .unwrap() + .get_object_path_for_io_if_local(bucket, "object/xl.meta") + .unwrap() + .unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Rename, &metadata, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let before = store.ctx.namespace_commit_generation(); + let recovering = store.clone(); + let mut recovery = + tokio::spawn(async move { recovering.recover_delete_transactions(bucket, &HealOpts::default()).await }); + let mut writer = None; + let mut bucket_writer = None; + let observations = std::panic::AssertUnwindSafe(async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .unwrap() + .unwrap(); + assert!(tokio::time::timeout(Duration::from_millis(100), &mut recovery).await.is_err()); + assert!(store.ctx.namespace_commits_pending()); + assert_eq!(store.ctx.namespace_commit_generation(), before + 1); + recovery.abort(); + assert!(recovery.await.unwrap_err().is_cancelled()); + assert!(store.ctx.namespace_commits_pending(), "late restore must retain namespace accounting"); + let writing = store.clone(); + writer = Some(tokio::spawn(async move { + writing + .put_object(bucket, "object", &mut PutObjReader::from_vec(vec![62; 4096]), &ObjectOptions::default()) + .await + })); + assert!( + tokio::time::timeout(Duration::from_millis(100), writer.as_mut().unwrap()) + .await + .is_err(), + "same-key PUT must wait for the late restore" + ); + let removing = store.clone(); + bucket_writer = Some(tokio::spawn(async move { removing.acquire_bucket_lifecycle_write_lock(bucket).await })); + assert!( + tokio::time::timeout(Duration::from_millis(100), bucket_writer.as_mut().unwrap()) + .await + .is_err(), + "bucket incarnation mutation must wait for the late restore" + ); + }) + .catch_unwind() + .await; + let _ = release_tx.send(()); + drop(hook); + if let Some(writer) = writer { + tokio::time::timeout(Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap() + .unwrap(); + } + if let Some(writer) = bucket_writer { + drop( + tokio::time::timeout(Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap() + .unwrap(), + ); + } + if let Err(panic) = observations { + std::panic::resume_unwind(panic); + } + } + + #[tokio::test] + async fn delete_recovery_explicit_legacy_discard_preserves_versions_locks_and_live_children() { + for protected in ["version", "lock", "internal-lock", "child", "unknown"] { + let (_temp, disk, mut info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let root = legacy_fixture(&disk, &original, transaction).await; + let rollback = root.join(transaction.to_string()); + match protected { + "version" => { + info.version_id = Some(Uuid::new_v4()); + } + "lock" => { + info.metadata.insert("X-Amz-Object-Lock-Legal-Hold".into(), "ON".into()); + } + "internal-lock" => { + rustfs_utils::http::metadata_compat::insert_str( + &mut info.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, + "old-lock-state".to_owned(), + ); + } + "child" => { + fs::create_dir(rollback.join("live-child")).await.unwrap(); + } + _ => { + fs::write(rollback.join("unknown"), b"preserve").await.unwrap(); + } + } + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), test_meta(info)) + .await + .unwrap(); + match disk.inspect_delete_transaction("recovery", "object", transaction).await { + Ok(state) => { + assert!(!state.explicit_delete_safe, "protected shape {protected}"); + assert!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + state.fingerprint, + DeleteRecoveryAction::DiscardLegacy + ) + .await + .is_err() + ); + } + Err(error) => assert!(matches!(error, DiskError::FileCorrupt | DiskError::InvalidPath)), + } + assert!(rollback.join(STORAGE_FORMAT_FILE_BACKUP).exists()); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_explicit_legacy_discard_rejects_selected_symlink() { + let (temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let root = legacy_fixture(&disk, &original, transaction).await; + std::os::unix::fs::symlink(temp.path(), root.join(transaction.to_string()).join("linked")).unwrap(); + assert_eq!( + disk.inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap_err(), + DiskError::InvalidPath + ); + assert!(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP).exists()); + } + + #[tokio::test] + async fn delete_recovery_commit_retains_proof_until_explicit_finalization() { + let (_temp, disk, _info, _original) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + let committed = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .expect("durable set decision"); + let finished = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + committed.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("physical cleanup"); + assert_eq!(finished.state, DeleteRecoveryState::CommitFinished); + assert_eq!( + disk.finish_internal_delete_residue("recovery", "object", None).await, + 0, + "legacy retry must not GC new decision proof" + ); + let receipt = disk + .io_get_object_path("recovery", &format!("object/{transaction}/{DELETE_QUORUM_COMMITTED_FILE}")) + .expect("receipt path"); + assert!(receipt.is_file()); + let settled = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + finished.fingerprint, + DeleteRecoveryAction::FinalizeCommit, + ) + .await + .expect("finalize after coordinator convergence proof"); + assert_eq!(settled.state, DeleteRecoveryState::Absent); + assert!(!disk.io_get_object_path("recovery", "object").expect("object path").exists()); + } + + #[tokio::test] + async fn delete_recovery_fingerprint_rejects_new_generation() { + let (_temp, disk, mut info, _original) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + info.mod_time = Some(OffsetDateTime::now_utc()); + info.data = Some(Bytes::from_static(b"new generation")); + let newer = test_meta(info); + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").expect("metadata path"), &newer) + .await + .expect("concurrent new generation"); + assert!(matches!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("new generation remains") + .as_ref(), + newer + ); + } + + #[tokio::test] + async fn delete_recovery_untouched_peer_adopts_only_matching_generation() { + let (_temp, source, info, original) = fixture().await; + let (_peer_temp, peer, _peer_info, _) = fixture().await; + fs::write(peer.io_get_object_path("recovery", "object/xl.meta").expect("peer metadata"), &original) + .await + .expect("same original generation"); + let transaction = Uuid::new_v4(); + let applied = apply(&source, transaction).await; + let committed = source + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .expect("source durable decision"); + let authority = committed.journal.expect("source intent"); + let untouched = peer + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("untouched peer"); + let adopted = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + untouched.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority.clone()), + }, + ) + .await + .expect("generation-bound adoption"); + assert_eq!(adopted.state, DeleteRecoveryState::QuorumCommitted); + let finished = peer + .settle_delete_transaction("recovery", "object", transaction, adopted.fingerprint, DeleteRecoveryAction::FinishCommit) + .await + .expect("converge untouched peer"); + assert_eq!(finished.state, DeleteRecoveryState::CommitFinished); + assert!(finished.current_digest.is_none()); + let (_foreign_temp, foreign, _, _) = fixture().await; + let mut replacement = info; + replacement.mod_time = Some(OffsetDateTime::now_utc()); + let newer = test_meta(replacement); + fs::write( + foreign + .io_get_object_path("recovery", "object/xl.meta") + .expect("foreign metadata"), + &newer, + ) + .await + .expect("foreign generation"); + let observed = foreign + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("inspect foreign generation"); + assert!(matches!( + foreign + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority) + } + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert_eq!( + foreign + .read_all("recovery", "object/xl.meta") + .await + .expect("protected foreign generation") + .as_ref(), + newer + ); + } + + #[tokio::test] + async fn delete_recovery_adoption_preserves_unrelated_new_version() { + let (_source_temp, source, original_info, original) = fixture().await; + let (_peer_temp, peer, _, _) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&source, transaction).await; + let committed = source + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + let mut unrelated = original_info; + unrelated.version_id = Some(Uuid::new_v4()); + unrelated.mod_time = unrelated.mod_time.map(|time| time + time::Duration::seconds(1)); + let mut peer_meta = FileMeta::load(&original).unwrap(); + peer_meta.add_version(unrelated.clone()).unwrap(); + fs::write( + peer.io_get_object_path("recovery", "object/xl.meta").unwrap(), + peer_meta.marshal_msg().unwrap(), + ) + .await + .unwrap(); + let observed = peer + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + let adopted = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(committed.journal.unwrap()), + }, + ) + .await + .unwrap(); + peer.settle_delete_transaction( + "recovery", + "object", + transaction, + adopted.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .unwrap(); + let remaining = FileMeta::load(&peer.read_all("recovery", "object/xl.meta").await.unwrap()).unwrap(); + assert_eq!(remaining.versions.len(), 1); + assert_eq!(remaining.versions[0].header.version_id, unrelated.version_id); + } + + #[tokio::test] + async fn delete_recovery_absent_replay_witness_cannot_acquire_future_predecessors() { + let (_source_temp, source, _, _) = fixture().await; + let (_peer_temp, peer, _, _) = fixture().await; + let transaction = Uuid::new_v4(); + let source_state = apply(&source, transaction).await; + let committed = source + .settle_delete_transaction( + "recovery", + "object", + transaction, + source_state.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + let authority = committed.journal.unwrap(); + assert!(authority.predecessors.is_empty()); + let future = Uuid::new_v4(); + apply(&peer, future).await; + let absent = peer + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(absent.state, DeleteRecoveryState::Absent); + assert!(absent.current_digest.is_none()); + let adopted = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + absent.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority.clone()), + }, + ) + .await + .unwrap(); + let witness = adopted.journal.unwrap(); + assert_eq!(witness.predecessors, authority.predecessors); + assert!(!witness.predecessors.contains(&future)); + assert_eq!(witness.original_versions, authority.original_versions); + assert_eq!(witness.deleted_versions, authority.deleted_versions); + assert_eq!(witness.bucket_incarnation, authority.bucket_incarnation); + assert!(witness.original_digest.is_none() && witness.target_digest.is_none() && witness.data_dirs.is_empty()); + } + + #[tokio::test] + async fn delete_recovery_raw_discovery_paginates_and_binds_exact_key_scope() { + let (temp, disk, _info, _original) = fixture().await; + for index in 0..512 { + let path = temp + .path() + .join("recovery") + .join(format!("key-{index:04}")) + .join(Uuid::new_v4().to_string()); + fs::create_dir_all(&path).await.expect("hidden transaction directory"); + fs::write(path.join(STORAGE_FORMAT_FILE_BACKUP), b"legacy backup") + .await + .expect("hidden backup"); + } + let mut cursor = None; + let mut found = BTreeSet::new(); + loop { + let page = disk + .discover_delete_transactions("recovery", None, cursor, 13) + .await + .expect("bounded raw discovery"); + found.extend(page.transactions.into_iter().map(|id| id.object)); + cursor = page.next; + if cursor.is_none() { + break; + } + } + assert_eq!(found.len(), 512, "every hidden legacy transaction must be discovered across pages"); + let scoped = disk + .discover_delete_transactions("recovery", Some("key-0000"), None, 1) + .await + .expect("shallow exact-key discovery"); + assert_eq!(scoped.transactions.len(), 1); + assert_eq!(scoped.transactions[0].object, "key-0000"); + if let Some(cursor) = scoped.next { + assert!( + disk.discover_delete_transactions("recovery", Some("key-0001"), Some(cursor), 1) + .await + .is_err(), + "a cursor cannot cross object scopes" + ); + } + } + + #[tokio::test] + async fn delete_recovery_gc_interruption_replays_finished_receipt_without_journal() { + let (_temp, disk, _info, _) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + let committed = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .expect("record decision"); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + committed.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("finish physical cleanup"); + let root = disk + .io_get_object_path("recovery", &format!("object/{transaction}")) + .expect("transaction root"); + fs::remove_file(root.join(PREPARED)) + .await + .expect("simulate partially completed proof GC"); + fs::remove_file(root.join(FINISHED)) + .await + .expect("simulate removed completion marker"); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("receipt preserves completed intent"); + assert_eq!(observed.state, DeleteRecoveryState::CommitFinished); + let repeated = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("finish is idempotent after interrupted GC"); + let finalized = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + repeated.fingerprint, + DeleteRecoveryAction::FinalizeCommit, + ) + .await + .expect("finish proof GC"); + assert_eq!(finalized.state, DeleteRecoveryState::Absent); + } + + #[tokio::test] + async fn delete_recovery_completed_restore_without_backup_can_finish_rollback() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + apply(&disk, transaction).await; + let root = disk.io_get_object_path("recovery", "object").expect("object root"); + fs::rename( + root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP), + root.join(STORAGE_FORMAT_FILE), + ) + .await + .expect("simulate crash after rollback metadata rename"); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("already restored generation"); + assert_eq!(observed.state, DeleteRecoveryState::NewPrepared); + assert!( + observed.rollback_data_complete, + "restored original replaces consumed backup as rollback evidence" + ); + disk.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("finish interrupted rollback"); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("original remains readable") + .as_ref(), + original + ); + } + + #[tokio::test] + async fn delete_recovery_completed_restore_without_target_can_finish_versioned_rollback() { + let (_temp, disk, info, original) = fixture().await; + let mut remaining = info; + remaining.version_id = Some(Uuid::new_v4()); + remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1)); + let mut metadata = FileMeta::load(&original).unwrap(); + metadata.add_version(remaining).unwrap(); + let original = metadata.marshal_msg().unwrap(); + let root = disk.io_get_object_path("recovery", "object").unwrap(); + fs::write(root.join(STORAGE_FORMAT_FILE), &original).await.unwrap(); + let transaction = Uuid::new_v4(); + let observed = apply(&disk, transaction).await; + assert!(observed.journal.as_ref().unwrap().target_digest.is_some()); + let rollback = root.join(transaction.to_string()); + let target = fs::read(rollback.join(TARGET)).await.unwrap(); + fs::remove_file(rollback.join(TARGET)).await.unwrap(); + assert!(matches!( + disk.inspect_delete_transaction("recovery", "object", transaction).await, + Err(DiskError::FileCorrupt) + )); + fs::write(rollback.join(TARGET), target).await.unwrap(); + fs::rename(rollback.join(STORAGE_FORMAT_FILE_BACKUP), root.join(STORAGE_FORMAT_FILE)) + .await + .unwrap(); + fs::remove_file(rollback.join(TARGET)).await.unwrap(); + let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE); + fs::write( + &receipt_path, + serde_json::to_vec(&CommitReceipt { + journal: observed.journal.unwrap(), + finished: false, + }) + .unwrap(), + ) + .await + .unwrap(); + assert!(matches!( + disk.inspect_delete_transaction("recovery", "object", transaction).await, + Err(DiskError::FileCorrupt) + )); + fs::remove_file(receipt_path).await.unwrap(); + let retry = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(retry.state, DeleteRecoveryState::NewPrepared); + assert!(retry.rollback_data_complete); + disk.settle_delete_transaction("recovery", "object", transaction, retry.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("finish cleanup of an exactly restored versioned generation"); + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), original); + assert!(!rollback.exists()); + } + + #[tokio::test] + async fn delete_recovery_absent_remaining_metadata_requires_existing_commit_receipt() { + let (_temp, disk, info, original) = fixture().await; + let mut remaining = info; + remaining.version_id = Some(Uuid::new_v4()); + remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1)); + let mut metadata = FileMeta::load(&original).unwrap(); + metadata.add_version(remaining).unwrap(); + let root = disk.io_get_object_path("recovery", "object").unwrap(); + fs::write(root.join(STORAGE_FORMAT_FILE), metadata.marshal_msg().unwrap()) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + assert!(applied.journal.as_ref().unwrap().target_digest.is_some()); + let rollback = root.join(transaction.to_string()); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let undecided = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(undecided.state, DeleteRecoveryState::NewPrepared); + assert!( + !undecided.commit_data_safe, + "absence cannot authorize a target-Some undecided transaction" + ); + assert!(matches!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + undecided.fingerprint, + DeleteRecoveryAction::PublishCommit + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert!(rollback.join(PREPARED).is_file()); + assert!(rollback.join(STORAGE_FORMAT_FILE_BACKUP).is_file()); + assert!(!rollback.join(DELETE_QUORUM_COMMITTED_FILE).exists()); + fs::copy(rollback.join(TARGET), root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let applied = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let certified = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(certified.state, DeleteRecoveryState::QuorumCommitted); + assert!(certified.commit_data_safe); + let finished = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + certified.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("certified obsolete version cleanup has no live metadata references"); + assert_eq!(finished.state, DeleteRecoveryState::CommitFinished); + assert!( + !root.join(STORAGE_FORMAT_FILE).exists(), + "cleanup must never republish the old remaining version" + ); + assert!( + rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file(), + "set-wide GC still owns the final witness" + ); + } + + #[tokio::test] + async fn delete_recovery_partial_publication_adopts_same_intent_without_replacing_local_journal() { + let (_temp, disk, mut original, _) = fixture().await; + let (_peer_temp, peer, _, _) = fixture().await; + let mut remaining = original.clone(); + remaining.version_id = Some(Uuid::new_v4()); + remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1)); + for (index, target) in [(1, &disk), (2, &peer)] { + original.erasure.index = index; + remaining.erasure.index = index; + let mut metadata = FileMeta::load(&test_meta(original.clone())).unwrap(); + metadata.add_version(remaining.clone()).unwrap(); + fs::write( + target.io_get_object_path("recovery", "object/xl.meta").unwrap(), + metadata.marshal_msg().unwrap(), + ) + .await + .unwrap(); + } + let transaction = Uuid::new_v4(); + let local = apply(&disk, transaction).await; + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let rollback = root.join(transaction.to_string()); + let prepared = fs::read(rollback.join(PREPARED)).await.unwrap(); + let target = fs::read(rollback.join(TARGET)).await.unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let applied = apply(&peer, transaction).await; + let committed = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + let authority = committed.journal.unwrap(); + assert_ne!( + local.journal.as_ref().unwrap().original_digest, + authority.original_digest, + "each drive owns distinct raw metadata" + ); + let mut invalid = Vec::new(); + let mut changed = authority.clone(); + changed.bucket_incarnation = Some(Uuid::new_v4()); + invalid.push(changed); + let mut changed = authority.clone(); + changed.deleted_versions.push(Uuid::new_v4()); + invalid.push(changed); + let mut changed = authority.clone(); + changed.requests[0].version_id = Some(Uuid::new_v4()); + invalid.push(changed); + let mut changed = authority.clone(); + changed + .original_versions + .iter_mut() + .find(|version| version.version_id.unwrap_or_default().is_nil()) + .unwrap() + .metadata_digest[0] ^= 1; + invalid.push(changed); + for authority in invalid { + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert!(matches!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority) + } + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert!(!rollback.join(DELETE_QUORUM_COMMITTED_FILE).exists()); + } + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert!(!observed.commit_data_safe); + let adopted = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority), + }, + ) + .await + .expect("same-T certified intent completes partial publication"); + assert_eq!( + adopted.journal, local.journal, + "receipt preserves local hashes and data-directory ownership" + ); + assert_eq!(fs::read(rollback.join(PREPARED)).await.unwrap(), prepared); + assert_eq!(fs::read(rollback.join(TARGET)).await.unwrap(), target); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + adopted.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .unwrap(); + assert!(!root.join(STORAGE_FORMAT_FILE).exists()); + assert!(rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file()); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_rollback_marker_sync_failure_retains_journal() { + for mode in [DurabilityMode::Strict, DurabilityMode::None] { + let _mode = durability_mode_override::set(mode); + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + apply(&disk, transaction).await; + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let data = root.join(Uuid::new_v4().to_string()); + fs::create_dir(&data).await.unwrap(); + let marker = data.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + fs::write(&marker, []).await.unwrap(); + let prepared = root.join(transaction.to_string()).join(PREPARED); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + os::fsync_dir_recorder::set_failure(&data, ErrorKind::Other); + let result = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::Rollback, + ) + .await; + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), original); + assert!(!marker.exists(), "the unlink precedes the durability barrier"); + if mode == DurabilityMode::Strict { + assert!(result.is_err(), "a failed marker-directory sync must stop proof removal"); + assert!(prepared.exists(), "rollback evidence survives the sync failure"); + let retry = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + retry.fingerprint, + DeleteRecoveryAction::Rollback, + ) + .await + .expect("retry syncs the directory even though its marker is already absent"); + assert!(os::fsync_dir_recorder::was_fsynced(&data)); + } else { + result.expect("none mode does not add a marker durability barrier"); + assert!(!os::fsync_dir_recorder::was_fsynced(&data)); + assert!(os::fsync_dir_recorder::take_failure(&data).is_some()); + } + assert!(!prepared.exists()); + } + } + + #[tokio::test] + async fn delete_recovery_validates_staged_parts_before_restoring_metadata() { + use crate::erasure::coding::BitrotWriter; + let (_temp, disk, _info, _) = fixture().await; + let data_dir = Uuid::new_v4(); + let payload = Bytes::from_static(b"complete staged external payload"); + let mut info = test_file_info("object", Uuid::nil(), Some(data_dir), None); + info.size = i64::try_from(payload.len()).expect("payload length"); + info.erasure.block_size = payload.len(); + info.parts[0].size = payload.len(); + info.parts[0].actual_size = info.size; + let mut second = info.parts[0].clone(); + second.number = 2; + info.parts.push(second); + info.size *= 2; + let original = test_meta(info.clone()); + let root = disk.io_get_object_path("recovery", "object").expect("object root"); + fs::write(root.join(STORAGE_FORMAT_FILE), &original) + .await + .expect("external original metadata"); + let mut writer = BitrotWriter::new( + std::io::Cursor::new(Vec::new()), + info.erasure.shard_size(), + HashAlgorithm::HighwayHash256S, + ); + assert_eq!(writer.write(&payload).await.expect("encode intact external shard"), payload.len()); + writer.shutdown().await.expect("flush shard encoding"); + let encoded = writer.into_inner().into_inner(); + fs::create_dir_all(root.join(data_dir.to_string())) + .await + .expect("external data directory"); + fs::write(root.join(data_dir.to_string()).join("part.1"), &encoded) + .await + .expect("intact external part"); + fs::write(root.join(data_dir.to_string()).join("part.2"), &encoded) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + let _ = disk + .prepare_delete_recovery( + "recovery", + "object", + Some(&original), + &[request()], + None, + DeletePreparationContext { + transaction, + bucket_incarnation: None, + namespace_owner: None, + }, + ) + .await + .expect("prepared intent"); + fs::rename( + root.join(data_dir.to_string()), + root.join(transaction.to_string()).join(data_dir.to_string()), + ) + .await + .expect("simulate staged legacy-compatible data"); + fs::create_dir(root.join(data_dir.to_string())).await.unwrap(); + fs::rename( + root.join(transaction.to_string()).join(data_dir.to_string()).join("part.1"), + root.join(data_dir.to_string()).join("part.1"), + ) + .await + .unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)) + .await + .expect("simulate applied metadata removal"); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("inspect complete staged data"); + assert!(observed.rollback_data_complete); + disk.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("verified staged rollback"); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("restored metadata") + .as_ref(), + original + ); + assert_eq!( + disk.read_all("recovery", &format!("object/{data_dir}/part.1")) + .await + .expect("restored shard") + .as_ref(), + encoded + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_raw_discovery_rejects_symlink_and_forged_cursor() { + use std::os::unix::fs::symlink; + let (temp, disk, _info, _) = fixture().await; + let outside = tempfile::tempdir().expect("outside sentinel"); + fs::write(outside.path().join("sentinel"), b"outside bytes") + .await + .expect("outside sentinel"); + symlink(outside.path(), temp.path().join("recovery").join("untrusted-prefix")).expect("untrusted symlink"); + assert!(disk.discover_delete_transactions("recovery", None, None, 16).await.is_err()); + assert!( + disk.discover_delete_transactions("recovery", None, Some(DeleteRecoveryCursor { token: Uuid::new_v4() }), 16) + .await + .is_err() + ); + assert_eq!( + fs::read(outside.path().join("sentinel")) + .await + .expect("outside remains intact"), + b"outside bytes" + ); + } +} diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 89fa4c76e..c1166f405 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -230,6 +230,105 @@ pub enum Disk { } impl Disk { + pub(crate) async fn delete_versions_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + match self { + Self::Local(disk) => { + disk.delete_versions_with_namespace_owner(volume, versions, opts, namespace_owner) + .await + } + Self::Remote(disk) => { + let _owner = namespace_owner; + disk.delete_versions(volume, versions, opts).await + } + } + } + + pub(crate) async fn delete_versions_recoverable_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + mut opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + if let Err(error) = validate_delete_recovery_options(&opts) { + return vec![Some(error); versions.len()]; + } + if versions.iter().any(|version| version.name.starts_with('/')) { + return vec![Some(DiskError::InvalidPath); versions.len()]; + } + opts.recoverable_delete = true; + match self { + Self::Local(disk) => { + disk.delete_versions_with_namespace_owner(volume, versions, opts, namespace_owner) + .await + } + Self::Remote(disk) => { + let _owner = namespace_owner; + disk.delete_versions_recoverable(volume, versions, opts).await + } + } + } + + pub(crate) async fn settle_delete_transaction_with_namespace_owner( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + namespace_owner: Option>, + ) -> Result { + match self { + Self::Local(disk) => { + disk.settle_delete_transaction_with_namespace_owner( + volume, + object, + transaction, + expected_fingerprint, + action, + namespace_owner, + ) + .await + } + Self::Remote(disk) => { + let _owner = namespace_owner; + disk.settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + } + } + } + + pub(crate) async fn delete_version_recoverable_with_namespace_owner( + &self, + volume: &str, + path: &str, + fi: FileInfo, + force_del_marker: bool, + mut opts: DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { + validate_delete_recovery_options(&opts)?; + if path.starts_with('/') { + return Err(DiskError::InvalidPath); + } + opts.recoverable_delete = true; + match self { + Self::Local(disk) => { + disk.delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner) + .await + } + Self::Remote(disk) => { + disk.delete_version_recoverable(volume, path, fi, force_del_marker, opts) + .await + } + } + } pub(crate) async fn set_disk_id_state(&self, id: Option) -> Result<()> { match self { Disk::Local(local_disk) => { @@ -257,6 +356,70 @@ impl Disk { #[async_trait::async_trait] impl DiskAPI for Disk { + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> Result<()> { + self.delete_version_recoverable_with_namespace_owner(volume, object, file_info, force_del_marker, opts, None) + .await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + match self { + Self::Local(disk) => disk.delete_versions_recoverable(volume, versions, opts).await, + Self::Remote(disk) => disk.delete_versions_recoverable(volume, versions, opts).await, + } + } + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + match self { + Self::Local(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await, + Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await, + } + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + match self { + Self::Local(disk) => disk.inspect_delete_transaction(volume, object, transaction).await, + Self::Remote(disk) => disk.inspect_delete_transaction(volume, object, transaction).await, + } + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> Result { + match self { + Self::Local(disk) => { + disk.settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + } + Self::Remote(disk) => { + disk.settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + } + } + } fn to_string(&self) -> String { match self { Disk::Local(local_disk) => local_disk.to_string(), @@ -1246,6 +1409,51 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { Err(DiskError::MethodNotAllowed) } async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; + async fn discover_delete_transactions( + &self, + _volume: &str, + _object: Option<&str>, + _cursor: Option, + _limit: u32, + ) -> Result { + Err(DiskError::MethodNotAllowed) + } + async fn delete_version_recoverable( + &self, + _volume: &str, + _object: &str, + _file_info: FileInfo, + _force_del_marker: bool, + _opts: DeleteOptions, + ) -> Result<()> { + Err(DiskError::MethodNotAllowed) + } + async fn delete_versions_recoverable( + &self, + _volume: &str, + versions: Vec, + _opts: DeleteOptions, + ) -> Vec> { + vec![Some(DiskError::MethodNotAllowed); versions.len()] + } + async fn inspect_delete_transaction( + &self, + _volume: &str, + _object: &str, + _transaction: Uuid, + ) -> Result { + Err(DiskError::MethodNotAllowed) + } + async fn settle_delete_transaction( + &self, + _volume: &str, + _object: &str, + _transaction: Uuid, + _expected_fingerprint: DeleteRecoveryDigest, + _action: DeleteRecoveryAction, + ) -> Result { + Err(DiskError::MethodNotAllowed) + } // VerifyFile async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result; // CheckParts @@ -1582,6 +1790,269 @@ pub struct DeleteOptions { /// Remote calls carrying it must use DeleteRetiredMarker, never DeleteVersion. #[serde(default, skip_serializing_if = "Option::is_none")] pub expected_delete_marker: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete_recovery_bucket_incarnation: Option, + #[serde(default)] + pub recoverable_delete: bool, +} + +pub const DELETE_RECOVERY_PROTOCOL_VERSION: u32 = 1; +pub(crate) fn get_delete_recovery_timeout() -> Result { + let value = std::env::var("RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS"); + match value { + Ok(value) => parse_delete_recovery_timeout(Some(&value)), + Err(std::env::VarError::NotPresent) => parse_delete_recovery_timeout(None), + Err(std::env::VarError::NotUnicode(_)) => Err(DiskError::other("invalid delete recovery timeout configuration")), + } +} + +fn parse_delete_recovery_timeout(value: Option<&str>) -> Result { + let seconds = match value { + None => 900, + Some(value) => value + .parse::() + .ok() + .filter(|seconds| (30..=86400).contains(seconds)) + .ok_or_else(|| DiskError::other("delete recovery timeout must be an integer between 30 and 86400 seconds"))?, + }; + Ok(std::time::Duration::from_secs(seconds)) +} + +pub(crate) fn validate_delete_recovery_options(opts: &DeleteOptions) -> Result<()> { + if opts.old_data_dir.is_none_or(|id| id.is_nil()) + || opts.undo_write + || opts.undo_delete + || opts.recursive + || opts.immediate + || opts.expected_delete_marker.is_some() + || opts.delete_recovery_bucket_incarnation.is_some_and(|id| id.is_nil()) + { + return Err(DiskError::FileCorrupt); + } + Ok(()) +} +pub type DeleteRecoveryDigest = [u8; 32]; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteTransactionId { + pub object: String, + pub transaction: Uuid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryCursor { + pub token: Uuid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryPage { + pub transactions: Vec, + pub next: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryJournal { + pub protocol_version: u32, + pub volume: String, + pub object: String, + pub transaction: Uuid, + pub bucket_incarnation: Option, + pub original_digest: Option, + pub target_digest: Option, + pub data_dirs: Vec, + pub deleted_versions: Vec, + pub requests: Vec, + pub original_versions: Vec, + #[serde(default)] + pub predecessors: Vec, +} + +impl DeleteRecoveryJournal { + /// Checks source generations only; the coordinator must supply a certified + /// decision and the receiver must retain its namespace/incarnation fences. + pub(crate) fn matches_adoption_source(&self, current: &[DeleteRecoveryVersionIdentity]) -> bool { + if self.requests.is_empty() { + return false; + } + let mut originals = std::collections::HashMap::with_capacity(self.original_versions.len()); + for version in &self.original_versions { + originals.entry(version.version_id.unwrap_or_default()).or_insert(version); + } + let mut locals = std::collections::HashMap::with_capacity(current.len()); + for version in current { + locals.entry(version.version_id.unwrap_or_default()).or_insert(version); + } + self.requests.iter().all(|request| { + if request.deleted && request.mark_deleted { + return current.first() == self.original_versions.first(); + } + let id = request.version_id.unwrap_or_default(); + let Some(original) = originals.get(&id) else { + return false; + }; + match locals.get(&id) { + Some(local) => local == original, + None => !id.is_nil(), + } + }) + } + + /// Matches immutable local preparation intent against an independently + /// inspected same-transaction commit receipt. This match is not authority + /// by itself; the coordinator must supply the certified receipt. + pub(crate) fn matches_prepared_authority(&self, authority: &Self) -> bool { + if self.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || authority.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || self.transaction.is_nil() + || self.transaction != authority.transaction + || self.volume != authority.volume + || self.object != authority.object + || self.bucket_incarnation != authority.bucket_incarnation + || self.bucket_incarnation.is_some_and(|id| id.is_nil()) + || self.requests.is_empty() + || self.requests != authority.requests + || self.deleted_versions.iter().collect::>() + != authority.deleted_versions.iter().collect::>() + { + return false; + } + let sources: std::collections::HashMap<_, _> = self + .original_versions + .iter() + .map(|version| (version.version_id.unwrap_or_default(), version)) + .collect(); + let certified: std::collections::HashMap<_, _> = authority + .original_versions + .iter() + .map(|version| (version.version_id.unwrap_or_default(), version)) + .collect(); + if sources.len() != self.original_versions.len() || certified.len() != authority.original_versions.len() { + return false; + } + self.requests.iter().all(|request| { + let (local, remote) = if request.deleted && request.mark_deleted { + (self.original_versions.first(), authority.original_versions.first()) + } else { + let id = request.version_id.unwrap_or_default(); + (sources.get(&id).copied(), certified.get(&id).copied()) + }; + match (local, remote) { + (Some(local), Some(remote)) => local == remote, + (None, None) => self.original_digest.is_none() && authority.original_digest.is_none(), + _ => false, + } + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryVersionIdentity { + pub version_id: Option, + pub mod_time: Option, + pub data_dir: Option, + pub deleted: bool, + pub metadata_digest: DeleteRecoveryDigest, +} + +impl DeleteRecoveryVersionIdentity { + pub(crate) fn from_file_info(info: FileInfo) -> error::Result { + local::delete_recovery::generation(info) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum DeleteRecoveryState { + Absent, + NewPrepared, + QuorumCommitted, + CommitFinished, + LegacyAmbiguous, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryInspection { + pub protocol_version: u32, + pub id: DeleteTransactionId, + pub state: DeleteRecoveryState, + pub journal: Option, + pub current_digest: Option, + pub current_versions: Vec, + pub backup_digest: Option, + pub rollback_data_complete: bool, + pub commit_data_safe: bool, + pub explicit_delete_safe: bool, + pub fingerprint: DeleteRecoveryDigest, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum DeleteRecoveryAction { + PublishCommit, + FinishCommit, + FinalizeCommit, + Rollback, + DiscardLegacy, + AdoptCommit { authority: Box }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum DeleteRecoveryRpcOperation { + Discover { + object: Option, + cursor: Option, + limit: u32, + }, + Inspect { + object: String, + transaction: Uuid, + }, + Settle { + object: String, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + }, + ApplyVersion { + object: String, + file_info: Box, + force_del_marker: bool, + opts: DeleteOptions, + }, + ApplyVersions { + versions: Vec, + opts: DeleteOptions, + }, +} + +impl DeleteRecoveryRpcOperation { + /// Full metadata application and adoption share the existing mutation envelope. + pub fn max_wire_payload_bytes(&self) -> usize { + match self { + Self::ApplyVersion { .. } + | Self::ApplyVersions { .. } + | Self::Settle { + action: DeleteRecoveryAction::AdoptCommit { .. }, + .. + } => rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES, + _ => rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum DeleteRecoveryRpcReply { + Discovered(DeleteRecoveryPage), + Inspected(DeleteRecoveryInspection), + Settled(DeleteRecoveryInspection), + AppliedVersion, + AppliedVersions(Vec>), } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -1872,6 +2343,8 @@ mod tests { undo_delete: false, old_data_dir: Some(Uuid::new_v4()), expected_delete_marker: None, + delete_recovery_bucket_incarnation: None, + recoverable_delete: false, }; assert!(opts.recursive); @@ -2296,3 +2769,183 @@ mod tests { } } } + +#[cfg(test)] +mod delete_recovery_timeout_tests { + use super::*; + #[test] + fn recovery_timeout_defaults_and_validates_range() { + assert_eq!( + parse_delete_recovery_timeout(None).expect("default timeout"), + std::time::Duration::from_secs(900) + ); + for seconds in ["30", "900", "86400"] { + assert!(parse_delete_recovery_timeout(Some(seconds)).is_ok()); + } + for value in ["", "invalid", "0", "2", "29", "86401", "-1", "18446744073709551616"] { + assert!(parse_delete_recovery_timeout(Some(value)).is_err()); + } + } +} + +#[cfg(test)] +mod delete_recovery_wire_limit_tests { + use super::*; + + #[test] + fn boxed_apply_version_preserves_existing_wire_payload() { + #[derive(Serialize)] + enum PriorOperation<'a> { + ApplyVersion { + object: &'a str, + file_info: &'a FileInfo, + force_del_marker: bool, + opts: &'a DeleteOptions, + }, + } + let file_info = FileInfo { + name: "object".into(), + version_id: Some(Uuid::new_v4()), + metadata: std::collections::HashMap::from([("metadata".into(), "value".into())]), + ..Default::default() + }; + let opts = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + recoverable_delete: true, + delete_recovery_bucket_incarnation: Some(Uuid::new_v4()), + ..Default::default() + }; + let prior = PriorOperation::ApplyVersion { + object: "object", + file_info: &file_info, + force_del_marker: true, + opts: &opts, + }; + let encoded = rmp_serde::to_vec_named(&prior).expect("encode the previous unboxed request shape"); + let operation = DeleteRecoveryRpcOperation::ApplyVersion { + object: "object".into(), + file_info: Box::new(file_info.clone()), + force_del_marker: true, + opts: opts.clone(), + }; + assert_eq!(rmp_serde::to_vec_named(&operation).expect("encode boxed request"), encoded); + assert_eq!( + serde_json::to_value(&operation).expect("boxed request JSON"), + serde_json::to_value(prior).expect("previous request JSON") + ); + let decoded: DeleteRecoveryRpcOperation = + rustfs_protos::decode_delete_recovery_payload(&encoded).expect("decode the previous payload"); + assert_eq!(rmp_serde::to_vec_named(&decoded).expect("re-encode decoded prior request"), encoded); + } + #[test] + fn adoption_and_inspection_reply_support_large_version_journals() { + let transaction = Uuid::new_v4(); + let request = FileInfo { + metadata: std::collections::HashMap::from([("metadata".into(), "x".repeat(2048))]), + ..Default::default() + }; + let journal = DeleteRecoveryJournal { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + volume: "bucket".into(), + object: "object".into(), + transaction, + bucket_incarnation: None, + original_digest: None, + target_digest: None, + data_dirs: Vec::new(), + deleted_versions: Vec::new(), + requests: vec![request; 1000], + original_versions: Vec::new(), + predecessors: Vec::new(), + }; + let operation = DeleteRecoveryRpcOperation::Settle { + object: "object".into(), + transaction, + expected_fingerprint: [0; 32], + action: DeleteRecoveryAction::AdoptCommit { + authority: Box::new(journal.clone()), + }, + }; + let encoded = rmp_serde::to_vec_named(&operation).expect("encode large adoption"); + assert!(encoded.len() > rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES); + assert!(encoded.len() < operation.max_wire_payload_bytes()); + let decoded: DeleteRecoveryRpcOperation = + rustfs_protos::decode_delete_recovery_payload(&encoded).expect("large adoption can decode"); + assert_eq!(decoded.max_wire_payload_bytes(), rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES); + let reply = DeleteRecoveryRpcReply::Inspected(DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: DeleteTransactionId { + object: "object".into(), + transaction, + }, + state: DeleteRecoveryState::QuorumCommitted, + journal: Some(journal), + current_digest: None, + current_versions: Vec::new(), + backup_digest: None, + rollback_data_complete: false, + commit_data_safe: true, + explicit_delete_safe: false, + fingerprint: [0; 32], + }); + let encoded = rmp_serde::to_vec_named(&reply).expect("encode large inspection"); + assert!(encoded.len() > rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES); + assert!(rustfs_protos::decode_delete_recovery_payload::(&encoded).is_ok()); + let inspect = DeleteRecoveryRpcOperation::Inspect { + object: "object".into(), + transaction, + }; + assert_eq!(inspect.max_wire_payload_bytes(), rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES); + } +} + +#[cfg(test)] +mod delete_recovery_adoption_tests { + use super::*; + + #[test] + fn source_matching_preserves_non_null_absence_and_rejects_foreign_or_missing_null() { + let id = Uuid::new_v4(); + let source = DeleteRecoveryVersionIdentity { + version_id: Some(id), + mod_time: None, + data_dir: None, + deleted: false, + metadata_digest: [1; 32], + }; + let mut journal = DeleteRecoveryJournal { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + volume: "bucket".into(), + object: "object".into(), + transaction: Uuid::new_v4(), + bucket_incarnation: None, + original_digest: None, + target_digest: None, + data_dirs: Vec::new(), + deleted_versions: vec![id], + requests: vec![FileInfo { + version_id: Some(id), + ..Default::default() + }], + original_versions: vec![source.clone()], + predecessors: Vec::new(), + }; + assert!(journal.matches_adoption_source(&[]), "certified non-null removal can already be absent"); + assert!(journal.matches_adoption_source(std::slice::from_ref(&source))); + let mut foreign = source.clone(); + foreign.metadata_digest = [2; 32]; + assert!(!journal.matches_adoption_source(std::slice::from_ref(&foreign))); + journal.original_versions.push(foreign.clone()); + assert!( + journal.matches_adoption_source(&[source.clone(), foreign.clone()]), + "retain original first-match semantics" + ); + assert!(!journal.matches_adoption_source(&[foreign, source])); + journal.requests[0].version_id = None; + journal.original_versions[0].version_id = None; + assert!(!journal.matches_adoption_source(&[]), "null absence does not establish its generation"); + assert!(journal.matches_adoption_source(std::slice::from_ref(&journal.original_versions[0]))); + journal.requests.clear(); + assert!(!journal.matches_adoption_source(&journal.original_versions)); + } +} diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 78f8df7a7..1b9048bba 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -938,6 +938,10 @@ pub struct ObjectOptions { /// A genuine target write guard; loss fences alone do not authorize a borrowed write. #[doc(hidden)] pub write_lock_context: Option, + /// In-process delete ownership for late filesystem completion. Never + /// serialized or copied into object metadata. + #[doc(hidden)] + pub delete_mutation_owner: Option>, /// Internal read-only inspection must not enqueue metadata or payload repairs. pub suppress_read_repair: bool, /// Control-plane writers that immediately read or CAS the same namespace @@ -1158,6 +1162,7 @@ impl std::fmt::Debug for ObjectOptions { .field("http_preconditions", &self.http_preconditions.is_some()) .field("delete_replication", &self.delete_replication.is_some()) .field("delete_replication_config_snapshot", &self.delete_replication_config_snapshot) + .field("delete_mutation_owner", &self.delete_mutation_owner.is_some()) .field("namespace_lock_fence", &self.namespace_lock_fence.is_some()) .field("bucket_lifecycle_lock_fence", &self.bucket_lifecycle_lock_fence.is_some()) .field("replication_request", &self.replication_request) diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index bf4bf7a3f..c63244ea8 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -874,6 +874,7 @@ mod ctx; mod metadata; mod ops; pub(crate) use ops::bucket::BucketInfoQuorum; +pub(crate) use ops::delete_recovery::{DeleteRecoveryContext, DeleteRecoverySummary}; pub(crate) use ops::heal::AbsenceProofRequest; #[cfg(test)] pub(crate) use ops::heal::DanglingDeleteFailure; diff --git a/crates/ecstore/src/set_disk/ops/delete_recovery.rs b/crates/ecstore/src/set_disk/ops/delete_recovery.rs new file mode 100644 index 000000000..71d76f8b8 --- /dev/null +++ b/crates/ecstore/src/set_disk/ops/delete_recovery.rs @@ -0,0 +1,1992 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Set-wide delete decisions and replay. Decisions remain until every +//! participant and related transaction for the exact key converges. + +use crate::disk::error::{DiskError, Result, RpcStatusError}; +use crate::disk::{ + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteRecoveryAction, DeleteRecoveryInspection, DeleteRecoveryState, DeleteTransactionId, + DiskAPI, DiskStore, +}; +use crate::set_disk::SetDisks; +use futures::future::join_all; +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::sync::Arc; +use uuid::Uuid; + +const DISCOVERY_PAGE_SIZE: u32 = 128; +const MAX_DISCOVERED_TRANSACTIONS: usize = 1_000_000; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DeleteRecoverySummary { + pub discovered: usize, + pub committed: usize, + pub rolled_back: usize, + pub blocked: usize, + pub pending: usize, + pub reclaimed: usize, + /// Read-only evidence for ordinary missing-key S3 semantics. This never + /// authorizes settlement while a configured participant is unavailable. + pub no_transactions_at_quorum: bool, +} + +#[derive(Clone)] +pub(crate) struct DeleteRecoveryContext<'a> { + pub incarnation: Option, + pub dry_run: bool, + pub explicit_delete: bool, + /// Only the locked heal path may repair a live marker-creation source. + pub repair_marker_source: bool, + pub namespace_owner: Option>, + pub can_commit: &'a (dyn Fn() -> bool + Sync), +} + +impl<'a> DeleteRecoveryContext<'a> { + pub(crate) fn new(incarnation: Option, can_commit: &'a (dyn Fn() -> bool + Sync)) -> Self { + Self { + incarnation, + dry_run: false, + explicit_delete: false, + repair_marker_source: false, + namespace_owner: None, + can_commit, + } + } +} + +impl DeleteRecoverySummary { + pub(crate) fn add(&mut self, other: Self) { + self.discovered += other.discovered; + self.committed += other.committed; + self.rolled_back += other.rolled_back; + self.blocked += other.blocked; + self.pending += other.pending; + self.reclaimed += other.reclaimed; + } +} + +pub(in crate::set_disk) fn unsupported_recovery(error: &DiskError) -> bool { + matches!(error, DiskError::UnsupportedDisk) + || matches!(error, DiskError::Io(source) if source.get_ref() + .and_then(|source| source.downcast_ref::()) + .is_some_and(|status| status.status().code() == tonic::Code::Unimplemented)) +} + +/// Set authority needs generation identities, not the potentially large +/// per-version apply requests carried by returning-drive adoption. +struct DecisionEvidence { + transaction: Uuid, + deleted_versions: HashSet, + original_versions: HashSet, + predecessors: HashSet, + target_absent: bool, +} + +fn receipt(state: DeleteRecoveryState) -> bool { + matches!(state, DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished) +} + +fn validate(inspection: &DeleteRecoveryInspection, object: &str, transaction: Uuid) -> Result<()> { + if inspection.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || inspection.id.object != object + || inspection.id.transaction != transaction + { + return Err(DiskError::FileCorrupt); + } + Ok(()) +} + +async fn inspect( + disks: &[Option], + bucket: &str, + object: &str, + transaction: Uuid, +) -> Vec> { + join_all(disks.iter().map(|disk| async move { + let state = disk + .as_ref() + .ok_or(DiskError::DiskNotFound)? + .inspect_delete_transaction(bucket, object, transaction) + .await?; + validate(&state, object, transaction)?; + Ok(state) + })) + .await +} + +async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Result> { + let mut cursor = None; + let mut ids = BTreeSet::new(); + loop { + let page = disk + .discover_delete_transactions(bucket, object, cursor, DISCOVERY_PAGE_SIZE) + .await?; + if page.transactions.len() > usize::try_from(DISCOVERY_PAGE_SIZE).map_err(|_| DiskError::FileCorrupt)? { + return Err(DiskError::FileCorrupt); + } + for id in page.transactions { + if id.transaction.is_nil() || object.is_some_and(|object| object != id.object) { + return Err(DiskError::FileCorrupt); + } + ids.insert((id.object, id.transaction)); + if ids.len() > MAX_DISCOVERED_TRANSACTIONS { + return Err(DiskError::other("delete recovery transaction limit exceeded")); + } + } + cursor = page.next; + if cursor.is_none() { + break; + } + } + Ok(ids + .into_iter() + .map(|(object, transaction)| DeleteTransactionId { object, transaction }) + .collect()) +} + +async fn settle( + disk: &DiskStore, + bucket: &str, + state: &DeleteRecoveryInspection, + action: DeleteRecoveryAction, + can_commit: &(impl Fn() -> bool + Sync + ?Sized), + namespace_owner: Option<&Arc>, +) -> Result { + if !can_commit() { + return Err(DiskError::other("delete recovery namespace fence was lost")); + } + let result = disk + .settle_delete_transaction_with_namespace_owner( + bucket, + &state.id.object, + state.id.transaction, + state.fingerprint, + action, + namespace_owner.cloned(), + ) + .await; + let next = result?; + validate(&next, &state.id.object, state.id.transaction)?; + Ok(next) +} + +/// Each phase fans out across independent drives, and awaits all started work +/// before evaluating the next decision or cleanup barrier. +async fn settle_group( + disks: &[Option], + bucket: &str, + copies: &mut [DeleteRecoveryInspection], + action: impl Fn(&DeleteRecoveryInspection) -> Option + Sync, + can_commit: &(impl Fn() -> bool + Sync + ?Sized), + namespace_owner: Option<&Arc>, +) -> Result { + let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| { + let action = action(copy); + async move { + match action { + Some(action) => settle( + disk.as_ref().ok_or(DiskError::DiskNotFound)?, + bucket, + copy, + action, + can_commit, + namespace_owner, + ) + .await + .map(Some), + None => Ok(None), + } + } + })) + .await; + let mut pending = false; + for (copy, result) in copies.iter_mut().zip(results) { + match result { + Ok(Some(next)) => *copy = next, + Ok(None) => {} + Err(DiskError::DiskOngoingReq) => pending = true, + Err(error) => return Err(error), + } + } + Ok(pending) +} + +/// A later decision may require settling its prepared predecessor before an +/// untouched participant can adopt that decision. Ordering provides no commit +/// authority; reject contradictory causal records before any settlement. +fn causal_transaction_order(states: &BTreeMap>) -> Result> { + let mut indegrees: BTreeMap<_, usize> = states.keys().map(|id| (*id, 0)).collect(); + let mut successors: BTreeMap> = BTreeMap::new(); + for (transaction, copies) in states { + for predecessor in copies + .iter() + .filter_map(|copy| copy.journal.as_ref()) + .flat_map(|journal| &journal.predecessors) + { + if states.contains_key(predecessor) && successors.entry(*predecessor).or_default().insert(*transaction) { + let degree = indegrees.get_mut(transaction).ok_or(DiskError::FileCorrupt)?; + *degree = degree.checked_add(1).ok_or(DiskError::FileCorrupt)?; + } + } + } + let mut ready: BTreeSet<_> = indegrees + .iter() + .filter_map(|(id, degree)| (*degree == 0).then_some(*id)) + .collect(); + let mut order = Vec::with_capacity(states.len()); + while let Some(transaction) = ready.pop_first() { + order.push(transaction); + if let Some(children) = successors.get(&transaction) { + for child in children { + let degree = indegrees.get_mut(child).ok_or(DiskError::FileCorrupt)?; + *degree = degree.checked_sub(1).ok_or(DiskError::FileCorrupt)?; + if *degree == 0 { + ready.insert(*child); + } + } + } + } + if order.len() != states.len() { + return Err(DiskError::FileCorrupt); + } + Ok(order) +} + +impl SetDisks { + /// Publication follows the ordinary metadata quorum. Once a publication is + /// attempted, response loss must never trigger blind rollback. + pub(in crate::set_disk) async fn publish_delete_commit( + &self, + bucket: &str, + object: &str, + transaction: Uuid, + disks: &[Option], + write_quorum: usize, + mut context: DeleteRecoveryContext<'_>, + ) -> Result<()> { + if context.namespace_owner.is_none() { + context.namespace_owner = Some(self.ctx.begin_namespace_commit()); + } + let can_commit = context.can_commit; + let states = inspect(disks, bucket, object, transaction).await; + let untouched_absence = |state: &Result| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none()); + if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum + && states.iter().all(|state| { + untouched_absence(state) || matches!(state, Err(error) if unavailable(error) && !unsupported_recovery(error)) + }) + { + // An ordinary missing-key no-op requires no recovery decision. + // Do not manufacture witness transactions while a drive is offline. + return Ok(()); + } + let published = join_all(disks.iter().zip(&states).map(|(disk, state)| async { + let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?; + let state = state.as_ref().map_err(Clone::clone)?; + if state.state == DeleteRecoveryState::LegacyAmbiguous { + return Err(DiskError::FileCorrupt); + } + settle( + disk, + bucket, + state, + DeleteRecoveryAction::PublishCommit, + can_commit, + context.namespace_owner.as_ref(), + ) + .await + })) + .await; + if published + .iter() + .filter(|state| matches!(state, Ok(state) if receipt(state.state))) + .count() + < write_quorum + { + return Err(DiskError::ErasureWriteQuorum); + } + // Physical cleanup is retryable after the certified decision. An + // unavailable participant retains the decision for its eventual return. + let _ = self.recover_delete_key(bucket, object, context).await; + Ok(()) + } + + pub(crate) async fn recover_delete_bucket( + &self, + bucket: &str, + context: DeleteRecoveryContext<'_>, + ) -> Result { + self.recover_delete_prefix(bucket, "", context).await + } + + pub(crate) async fn recover_delete_prefix( + &self, + bucket: &str, + prefix: &str, + context: DeleteRecoveryContext<'_>, + ) -> Result { + let disks = self.disk_inventory().await; + if disks.len() != self.set_drive_count || disks.iter().any(Option::is_none) { + return Ok(DeleteRecoverySummary { + pending: 1, + ..Default::default() + }); + } + let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, None))).await; + let mut keys = BTreeSet::new(); + for page in pages { + match page { + Ok(ids) => { + keys.extend( + ids.into_iter() + .map(|id| id.object) + .filter(|object| object.starts_with(prefix)), + ); + } + Err(error) if unavailable(&error) => { + return Ok(DeleteRecoverySummary { + pending: 1, + ..Default::default() + }); + } + Err(error) => return Err(error), + } + } + let mut summary = DeleteRecoverySummary::default(); + // Lock order: caller's bucket lifecycle fence, then one exact key. + for object in keys { + summary.add(self.recover_delete_object(bucket, &object, context.clone()).await?); + } + Ok(summary) + } + + pub(crate) async fn recover_delete_object( + &self, + bucket: &str, + object: &str, + mut context: DeleteRecoveryContext<'_>, + ) -> Result { + // Lock order: caller's bucket lifecycle fence, then one exact key. + let guard = Arc::new( + self.acquire_write_lock_diag("recover_delete", bucket, object) + .await + .map_err(DiskError::other)?, + ); + if !context.dry_run { + let namespace_owner = context + .namespace_owner + .clone() + .unwrap_or_else(|| self.ctx.begin_namespace_commit()); + context.namespace_owner = Some(Arc::new((namespace_owner, guard.clone()))); + } + let can_commit = context.can_commit; + let fence = || can_commit() && !guard.is_lock_lost(); + context.can_commit = &fence; + self.recover_delete_key(bucket, object, context).await + } + + /// Caller retains the exact key's write lock and its lifecycle owner. + pub(in crate::set_disk) async fn recover_delete_key( + &self, + bucket: &str, + object: &str, + context: DeleteRecoveryContext<'_>, + ) -> Result { + let DeleteRecoveryContext { + incarnation, + dry_run, + explicit_delete, + repair_marker_source, + namespace_owner, + can_commit, + } = context; + let disks = self.disk_inventory().await; + let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some); + let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, Some(object)))).await; + let mut transactions = BTreeSet::new(); + let mut inspected = 0; + let mut offline = !complete_inventory; + let mut unsupported = false; + for page in pages { + match page { + Ok(ids) => { + inspected += 1; + transactions.extend(ids.into_iter().map(|id| id.transaction)); + } + Err(error) if unavailable(&error) => { + offline = true; + unsupported |= unsupported_recovery(&error); + } + Err(error) => return Err(error), + } + } + if offline { + return Ok(DeleteRecoverySummary { + discovered: transactions.len(), + pending: 1, + no_transactions_at_quorum: transactions.is_empty() && !unsupported && inspected > self.set_drive_count / 2, + ..Default::default() + }); + } + let mut summary = DeleteRecoverySummary { + discovered: transactions.len(), + ..Default::default() + }; + let mut all_states = BTreeMap::new(); + for transaction in transactions { + let mut complete = Vec::with_capacity(disks.len()); + for state in inspect(&disks, bucket, object, transaction).await { + match state { + Ok(state) => { + if let Some(journal) = &state.journal { + let witness = journal.original_digest.is_none() + && journal.target_digest.is_none() + && journal.requests.is_empty(); + if incarnation.is_some_and(|expected| journal.bucket_incarnation != Some(expected) && !witness) { + summary.blocked += 1; + return Ok(summary); + } + } + complete.push(state); + } + Err(error) if unavailable(&error) => { + summary.pending += 1; + return Ok(summary); + } + Err(error) => return Err(error), + } + } + all_states.insert(transaction, complete); + } + let prerequisite = if repair_marker_source && !dry_run { + let mut active = all_states + .iter() + .filter(|(_, copies)| copies.iter().any(|copy| copy.state != DeleteRecoveryState::Absent)); + let first = active.next(); + if active.next().is_none() { + first.and_then(|(&transaction, copies)| { + marker_source_to_repair(copies, incarnation).map(|source| (transaction, source)) + }) + } else { + None + } + } else { + None + }; + if let Some((transaction, source)) = prerequisite { + Box::pin(self.heal_delete_recovery_source(bucket, object, &source, can_commit)).await?; + let refreshed = inspect(&disks, bucket, object, transaction) + .await + .into_iter() + .collect::>>()?; + if !can_commit() { + return Err(DiskError::DiskOngoingReq); + } + if refreshed.iter().any(|copy| { + copy.journal + .as_ref() + .is_some_and(|journal| journal.bucket_incarnation != incarnation) + }) { + return Err(DiskError::OutdatedXLMeta); + } + all_states.insert(transaction, refreshed); + } + let transaction_order = causal_transaction_order(&all_states)?; + // Legacy snapshots contain no set decision. A fresh authorized removal + // applies only after the entire key is absent and every raw shape is safe. + let legacy_only = explicit_delete + && all_states.values().flatten().all(|copy| { + copy.current_digest.is_none() + && matches!(copy.state, DeleteRecoveryState::Absent | DeleteRecoveryState::LegacyAmbiguous) + && (copy.state != DeleteRecoveryState::LegacyAmbiguous || copy.explicit_delete_safe) + }); + // This is a fresh authorized delete of an absent unversioned key, + // rather than evidence about the outcome of a historical transaction. + let fresh_absence_decision = explicit_delete + && all_states.values().flatten().all(|copy| { + copy.current_digest.is_none() + && match copy.state { + DeleteRecoveryState::Absent => true, + DeleteRecoveryState::NewPrepared => copy.explicit_delete_safe, + DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished => { + copy.journal.as_ref().is_some_and(|journal| journal.target_digest.is_none()) + } + DeleteRecoveryState::LegacyAmbiguous => false, + } + }); + // Receipt cleanup alone does not mutate xl.meta. Returning peers and + // undecided preparations still need fences before and after recovery. + let metadata_may_change = all_states.values().any(|copies| { + !copies.iter().all(|copy| copy.state == DeleteRecoveryState::Absent) + && copies.iter().any(|copy| { + copy.state == DeleteRecoveryState::NewPrepared + || (copy.state == DeleteRecoveryState::Absent && copy.current_digest.is_some()) + || copy + .journal + .as_ref() + .is_some_and(|journal| copy.current_digest != journal.target_digest) + }) + }); + let namespace_owner = if dry_run { + None + } else { + Some(namespace_owner.unwrap_or_else(|| self.ctx.begin_namespace_commit())) + }; + if !dry_run && metadata_may_change { + self.invalidate_get_object_metadata_cache(bucket, object).await; + } + let authorities = all_states + .values() + .flatten() + .filter(|state| receipt(state.state)) + .filter_map(|state| state.journal.as_ref()) + .map(|journal| DecisionEvidence { + transaction: journal.transaction, + deleted_versions: journal.deleted_versions.iter().copied().collect(), + original_versions: journal.original_versions.iter().cloned().collect(), + predecessors: journal.predecessors.iter().copied().collect(), + target_absent: journal.target_digest.is_none(), + }) + .collect::>(); + let mut unresolved = false; + let mut mutated = false; + for transaction in transaction_order { + let copies = all_states.get_mut(&transaction).ok_or(DiskError::FileCorrupt)?; + if mutated { + // A sibling may have restored metadata or reclaimed shared + // evidence. Refresh fingerprints before acting on this group. + *copies = inspect(&disks, bucket, object, transaction) + .await + .into_iter() + .collect::>>()?; + } + if copies.iter().all(|state| state.state == DeleteRecoveryState::Absent) { + continue; + } + if copies.iter().any(|state| state.state == DeleteRecoveryState::LegacyAmbiguous) { + if legacy_only { + if dry_run { + summary.reclaimed += 1; + continue; + } + mutated = true; + if settle_group( + &disks, + bucket, + copies, + |copy| { + (copy.state == DeleteRecoveryState::LegacyAmbiguous).then_some(DeleteRecoveryAction::DiscardLegacy) + }, + can_commit, + namespace_owner.as_ref(), + ) + .await? + { + summary.pending += 1; + unresolved = true; + } else { + summary.reclaimed += 1; + } + } else { + summary.blocked += 1; + unresolved = true; + } + continue; + } + let authority = copies + .iter() + .filter(|state| receipt(state.state)) + .filter_map(|state| state.journal.as_ref()) + .max_by_key(|journal| !journal.requests.is_empty()) + .cloned(); + let prepared = copies.iter().find_map(|state| state.journal.as_ref()); + let covered = prepared + .filter(|journal| !journal.deleted_versions.is_empty()) + .is_some_and(|journal| { + authorities.iter().any(|authority| { + journal.deleted_versions.iter().all(|version| { + authority.deleted_versions.contains(version) + && (!version.is_nil() + || journal + .original_versions + .iter() + .filter(|source| source.version_id.is_none_or(|id| id.is_nil())) + .all(|source| authority.original_versions.contains(source)) + || (authority.predecessors.contains(&journal.transaction) + && journal.target_digest.is_none() + && authority.target_absent + && copies.iter().all(|copy| { + copy.current_digest.is_none() || copy.current_versions == journal.original_versions + }))) + }) + }) + }); + let original_quorum = prepared.is_some_and(|journal| { + copies + .iter() + .filter(|copy| copy.current_versions == journal.original_versions) + .count() + > self.set_drive_count / 2 + }); + let all_decisions_precede = !authorities.is_empty() + && authorities.iter().all(|authority| { + copies + .iter() + .filter_map(|copy| copy.journal.as_ref()) + .any(|journal| journal.predecessors.contains(&authority.transaction)) + }); + if authority.is_some() || covered || fresh_absence_decision { + let certified_same_transaction = authority.is_some(); + let reference = authority + .or_else(|| copies.iter().find_map(|state| state.journal.clone())) + .ok_or(DiskError::FileCorrupt)?; + if dry_run { + if copies.iter().any(|copy| { + if certified_same_transaction + && copy.state == DeleteRecoveryState::NewPrepared + && copy.current_digest.is_none() + && !copy.commit_data_safe + { + copy.journal + .as_ref() + .is_none_or(|local| !local.matches_prepared_authority(&reference)) + } else if copy.state == DeleteRecoveryState::Absent { + copy.current_digest.is_some() && !reference.matches_adoption_source(©.current_versions) + } else { + !copy.commit_data_safe + } + }) { + summary.blocked += 1; + } else { + summary.committed += 1; + } + continue; + } + mutated = true; + let mut pending = settle_group( + &disks, + bucket, + copies, + |copy| { + Some( + if copy.state == DeleteRecoveryState::Absent + || (certified_same_transaction + && copy.state == DeleteRecoveryState::NewPrepared + && copy.current_digest.is_none() + && !copy.commit_data_safe) + { + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(reference.clone()), + } + } else { + DeleteRecoveryAction::PublishCommit + }, + ) + }, + can_commit, + namespace_owner.as_ref(), + ) + .await?; + if !pending { + pending = settle_group( + &disks, + bucket, + copies, + |_| Some(DeleteRecoveryAction::FinishCommit), + can_commit, + namespace_owner.as_ref(), + ) + .await?; + } + if pending { + summary.pending += 1; + unresolved = true; + } else { + summary.committed += 1; + } + } else { + // A sibling decision or foreign generation can supersede this + // snapshot. Never guess by restoring the whole old backup. + if (!authorities.is_empty() && !original_quorum && !all_decisions_precede) + || copies + .iter() + .any(|state| state.state != DeleteRecoveryState::Absent && !state.rollback_data_complete) + { + summary.blocked += 1; + unresolved = true; + continue; + } + let originals = copies + .iter() + .filter_map(|state| state.journal.as_ref()) + .flat_map(|journal| journal.original_versions.iter()) + .collect::>(); + if copies.iter().any(|state| { + state.state == DeleteRecoveryState::Absent + && state.current_versions.iter().any(|version| !originals.contains(version)) + }) { + summary.blocked += 1; + unresolved = true; + continue; + } + if dry_run { + summary.rolled_back += 1; + continue; + } + mutated = true; + if settle_group( + &disks, + bucket, + copies, + |copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback), + can_commit, + namespace_owner.as_ref(), + ) + .await? + { + summary.pending += 1; + unresolved = true; + } else { + summary.rolled_back += 1; + } + } + } + if !unresolved && !dry_run { + // GC is a key-wide barrier: every predecessor must have settled + // before the last surviving commit witness can disappear. + let mut settled = Vec::with_capacity(all_states.len()); + for transaction in all_states.keys() { + let fresh = inspect(&disks, bucket, object, *transaction).await; + if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) { + summary.pending += 1; + return Ok(summary); + } + settled.push(fresh.into_iter().collect::>>()?); + } + for (index, mut copies) in settled.into_iter().enumerate() { + if index > 0 { + copies = inspect(&disks, bucket, object, copies[0].id.transaction) + .await + .into_iter() + .collect::>>()?; + if copies + .iter() + .any(|copy| !matches!(copy.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished)) + { + summary.pending += 1; + return Ok(summary); + } + } + if settle_group( + &disks, + bucket, + &mut copies, + |copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit), + can_commit, + namespace_owner.as_ref(), + ) + .await? + { + summary.pending += 1; + } + } + } + + if !dry_run && metadata_may_change { + self.invalidate_get_object_metadata_cache(bucket, object).await; + } + Ok(summary) + } +} + +// A lagging member may have the complete older suffix of a marker creation's +// live source. Repair that source before applying the original strict matcher. +fn marker_source_to_repair( + copies: &[DeleteRecoveryInspection], + incarnation: Option, +) -> Option { + let expected = incarnation.filter(|id| !id.is_nil())?; + let authority = copies + .iter() + .filter(|copy| receipt(copy.state)) + .find_map(|copy| copy.journal.as_ref())?; + let source = authority.original_versions.first()?; + let request = authority.requests.first()?; + if authority.bucket_incarnation != Some(expected) + || authority.requests.len() != 1 + || !request.deleted + || !request.mark_deleted + || request.version_id.is_none_or(|id| id.is_nil()) + || authority + .original_versions + .iter() + .any(|version| version.version_id == request.version_id) + || !authority.deleted_versions.is_empty() + || !authority.data_dirs.is_empty() + || authority.target_digest.is_none() + || source.version_id.is_none_or(|id| id.is_nil()) + || source.deleted + { + return None; + } + let mut missing_source = false; + for copy in copies { + match copy.state { + DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished => { + let journal = copy.journal.as_ref()?; + if journal.bucket_incarnation != Some(expected) + || journal.original_versions != authority.original_versions + || !journal.matches_prepared_authority(authority) + || copy.current_digest != journal.target_digest + { + return None; + } + } + DeleteRecoveryState::Absent => { + if copy.journal.is_some() + || copy.current_versions.is_empty() + || !authority.original_versions.ends_with(©.current_versions) + { + return None; + } + missing_source |= copy.current_versions.first() != Some(source); + } + _ => return None, + } + } + missing_source.then(|| source.clone()) +} + +fn unavailable(error: &DiskError) -> bool { + if unsupported_recovery(error) { + return true; + } + match error { + DiskError::DiskNotFound + | DiskError::FaultyDisk + | DiskError::DiskOngoingReq + | DiskError::Timeout + | DiskError::RemoteClientUnavailable(_) => true, + DiskError::Io(source) => { + if let Some(status) = crate::cluster::rpc::client::embedded_tonic_status(source) { + return crate::cluster::rpc::client::is_network_like_status(status); + } + matches!( + source.kind(), + std::io::ErrorKind::TimedOut + | std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::NotConnected + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::UnexpectedEof + ) + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::metadata_sys::{self, init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::disk::{DeleteOptions, ReadOptions, STORAGE_FORMAT_FILE}; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions}; + use crate::storage_api_contracts::object::{ObjectIO, ObjectOperations}; + use crate::store::ECStore; + use rustfs_filemeta::FileInfo; + use rustfs_heal_contracts::heal_channel::HealOpts; + use std::sync::Arc; + use tokio::io::AsyncReadExt; + + async fn fixture(bucket: &str) -> (Vec, Arc, Vec) { + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative recovery bucket"); + let disks = store.pools[0].disk_set[0] + .disk_inventory() + .await + .into_iter() + .map(|disk| disk.expect("fixture drive is online")) + .collect(); + (dirs, store, disks) + } + + async fn put(store: &Arc, bucket: &str, body: &[u8], versioned: bool) -> Option { + let mut reader = PutObjReader::from_vec(body.to_vec()); + store + .put_object( + bucket, + "object", + &mut reader, + &ObjectOptions { + versioned, + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("put recovery source") + .version_id + } + + async fn applied(store: &Arc, disks: &[DiskStore], bucket: &str, version: Option, count: usize) -> Uuid { + applied_at(store, disks, bucket, version, count, Uuid::new_v4()).await + } + + async fn applied_at( + store: &Arc, + disks: &[DiskStore], + bucket: &str, + version: Option, + count: usize, + transaction: Uuid, + ) -> Uuid { + let incarnation = store + .bucket_incarnation_id_from_disk(bucket) + .await + .expect("source bucket identity"); + for disk in disks.iter().take(count) { + disk.delete_version_recoverable( + bucket, + "object", + FileInfo { + name: "object".to_string(), + version_id: version, + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + delete_recovery_bucket_incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("apply prepared delete without publishing a set decision"); + } + transaction + } + + async fn publish(disk: &DiskStore, bucket: &str, transaction: Uuid) { + let state = disk + .inspect_delete_transaction(bucket, "object", transaction) + .await + .expect("inspect prepared source"); + disk.settle_delete_transaction(bucket, "object", transaction, state.fingerprint, DeleteRecoveryAction::PublishCommit) + .await + .expect("persist a decision after the simulated metadata quorum"); + } + + async fn body(store: &Arc, bucket: &str, version: Option) -> Vec { + let opts = ObjectOptions { + version_id: version.map(|version| version.to_string()), + ..Default::default() + }; + let mut reader = store + .get_object_reader(bucket, "object", None, http::HeaderMap::new(), &opts) + .await + .expect("open recovered object"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("read exact recovered bytes"); + body + } + + #[tokio::test] + async fn delete_recovery_set_rolls_back_undecided_inline_bytes() { + let bucket = "recover-undecided"; + let (_dirs, store, disks) = fixture(bucket).await; + let original = vec![21; 4096]; + put(&store, bucket, &original, false).await; + applied(&store, &disks, bucket, None, disks.len()).await; + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("recover undisputed prepared delete"); + assert!( + report.detail.contains("rolled_back=1"), + "actual rollback must be reported: {}", + report.detail + ); + assert_eq!(body(&store, bucket, None).await, original); + } + + #[tokio::test] + async fn delete_recovery_marker_source_heal_preserves_foreign_history_and_damaged_data() { + fn snapshot(root: &std::path::Path) -> BTreeMap> { + let mut files = BTreeMap::new(); + let mut pending = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + pending.push(path); + } else { + files.insert(path.strip_prefix(root).unwrap().to_path_buf(), std::fs::read(path).unwrap()); + } + } + } + files + } + + for shape in ["live-backup", "foreign", "partial-history", "damaged-data"] { + let bucket = "recover-marker-source-guards"; + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket( + bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); + let set = &store.pools[0].disk_set[0]; + let disks: Vec<_> = set.disk_inventory().await.into_iter().map(Option::unwrap).collect(); + let original = vec![71; 1024 * 1024]; + let oldest = put(&store, bucket, &original, true).await.unwrap(); + let first = put(&store, bucket, &original, true).await.unwrap(); + set.disks.write().await[3] = None; + let payload = vec![72; 1024 * 1024]; + let source = put(&store, bucket, &payload, true).await.unwrap(); + store + .delete_object( + bucket, + "object", + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("publish marker while one member lacks the source version"); + set.disks.write().await[3] = Some(disks[3].clone()); + let mut first_info = disks[3] + .read_version("", bucket, "object", &first.to_string(), &ReadOptions::default()) + .await + .unwrap(); + match shape { + "live-backup" => { + let root = dirs[3].path().join(bucket).join("object"); + let backup = root.join(first_info.data_dir.unwrap().to_string()).join("xl.meta.bkp"); + tokio::fs::write(backup, tokio::fs::read(root.join("xl.meta")).await.unwrap()) + .await + .unwrap(); + } + "foreign" => { + first_info.version_id = Some(Uuid::new_v4()); + first_info.mod_time = Some(time::OffsetDateTime::now_utc()); + disks[3].write_metadata("", bucket, "object", first_info).await.unwrap(); + } + "partial-history" => { + disks[3] + .delete_version( + bucket, + "object", + FileInfo { + name: "object".into(), + version_id: Some(oldest), + ..Default::default() + }, + false, + DeleteOptions::default(), + ) + .await + .unwrap(); + } + "damaged-data" => { + // Three metadata sources meet write quorum, but corrupting + // two shards leaves fewer than EC2+2's two required readers. + for (index, disk) in disks.iter().take(2).enumerate() { + let info = disk + .read_version("", bucket, "object", &source.to_string(), &ReadOptions::default()) + .await + .unwrap(); + let part = dirs[index] + .path() + .join(bucket) + .join("object") + .join(info.data_dir.unwrap().to_string()) + .join("part.1"); + let mut bytes = tokio::fs::read(&part).await.unwrap(); + *bytes.last_mut().unwrap() ^= 1; + tokio::fs::write(part, bytes).await.unwrap(); + } + } + _ => unreachable!(), + } + let before: Vec<_> = dirs + .iter() + .map(|dir| snapshot(&dir.path().join(bucket).join("object"))) + .collect(); + let result = store + .heal_object_at_incarnation( + bucket, + "object", + &source.to_string(), + incarnation, + &HealOpts { + scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep, + ..Default::default() + }, + ) + .await; + if shape == "live-backup" { + assert!(result.unwrap().error.is_none(), "plain live backups cannot block source repair"); + assert_eq!(body(&store, bucket, Some(source)).await, payload); + assert_eq!(body(&store, bucket, Some(first)).await, original); + } else { + assert!(result.is_err() || result.unwrap().error.is_some(), "{shape} must fail safely"); + for (dir, expected) in dirs.iter().zip(before) { + assert_eq!(snapshot(&dir.path().join(bucket).join("object")), expected, "{shape}"); + } + } + } + } + + #[tokio::test] + async fn delete_recovery_admin_prefix_excludes_other_keys_and_restores_hidden_match() { + let bucket = "recover-prefix-scope"; + let (dirs, store, disks) = fixture(bucket).await; + let original = vec![27; 4096]; + put(&store, bucket, &original, false).await; + let transaction = applied(&store, &disks, bucket, None, disks.len()).await; + let outside = store + .recover_delete_prefix_at_incarnation(bucket, "other/", None, &HealOpts::default()) + .await + .expect("inspect unrelated prefix"); + assert!(outside.is_none()); + for dir in &dirs { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(transaction.to_string()) + .join("xl.meta.bkp") + .exists() + ); + } + let matching = store + .recover_delete_prefix_at_incarnation(bucket, "obj", None, &HealOpts::default()) + .await + .expect("recover matching hidden key") + .expect("report matching transaction"); + assert!(matching.detail.contains("rolled_back=1")); + assert_eq!(body(&store, bucket, None).await, original); + } + + #[tokio::test] + async fn delete_recovery_set_commits_receipt_and_unblocks_bucket() { + let bucket = "recover-committed"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![22; 1024 * 1024], false).await; + let transaction = applied(&store, &disks, bucket, None, disks.len()).await; + publish(&disks[0], bucket, transaction).await; + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("complete certified delete"); + assert!( + report.detail.contains("committed=1"), + "certified replay must be reported: {}", + report.detail + ); + assert!( + store + .get_object_info(bucket, "object", &ObjectOptions::default()) + .await + .is_err() + ); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("non-force bucket deletion must succeed after proof GC"); + } + + #[tokio::test] + async fn delete_recovery_set_handles_prepared_subset() { + let bucket = "recover-subset"; + let (_dirs, store, disks) = fixture(bucket).await; + let original = vec![23; 4096]; + put(&store, bucket, &original, false).await; + applied(&store, &disks, bucket, None, 1).await; + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("untouched peers remain valid evidence"); + assert!(report.detail.contains("rolled_back=1")); + assert_eq!(body(&store, bucket, None).await, original); + } + + #[tokio::test] + async fn delete_recovery_set_adopts_untouched_returning_peer() { + let bucket = "recover-untouched"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![24; 1024 * 1024], false).await; + let transaction = applied(&store, &disks, bucket, None, 3).await; + for disk in disks.iter().take(3) { + publish(disk, bucket, transaction).await; + } + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("matching untouched generation must converge"); + assert!(report.detail.contains("committed=1")); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("returning old replica must not leave bucket residue"); + } + + #[tokio::test] + async fn delete_recovery_set_retains_decisions_until_offline_peer_returns() { + let bucket = "recover-offline"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![25; 4096], false).await; + let transaction = applied(&store, &disks, bucket, None, 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let available = set.disk_inventory().await; + set.publish_delete_commit(bucket, "object", transaction, &available, 3, DeleteRecoveryContext::new(None, &|| true)) + .await + .expect("commit quorum remains available"); + for dir in dirs.iter().take(3) { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(transaction.to_string()) + .join(crate::disk::local::DELETE_QUORUM_COMMITTED_FILE) + .exists(), + "decision must survive an unavailable participant" + ); + } + set.disks.write().await[3] = Some(disks[3].clone()); + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("returning prepared peer must roll forward"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("proof collection follows complete convergence"); + } + + #[tokio::test] + async fn delete_recovery_set_never_resurrects_prepared_old_null_after_new_delete() { + let bucket = "recover-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![26; 4096], false).await; + applied(&store, &disks, bucket, None, 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + put(&store, bucket, &vec![27; 4096], false).await; + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("later visible generation can commit with quorum"); + set.disks.write().await[3] = Some(disks[3].clone()); + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("later deletion must supersede hidden old null residue"); + assert!( + store + .get_object_info(bucket, "object", &ObjectOptions::default()) + .await + .is_err(), + "old bytes must never become visible again" + ); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("both generations' transactions converge"); + } + + #[tokio::test] + async fn delete_recovery_set_missing_version_does_not_ack_hidden_rollback() { + let bucket = "recover-version-history"; + let (_dirs, store, disks) = fixture(bucket).await; + metadata_sys::update_in(&store.ctx, bucket, crate::bucket::metadata::BUCKET_VERSIONING_CONFIG, + br#"Enabled"#.to_vec()) + .await.expect("enable versioning"); + let first = put(&store, bucket, &vec![28; 4096], true).await.expect("first version"); + let survivor = vec![29; 4096]; + let second = put(&store, bucket, &survivor, true).await.expect("second version"); + applied(&store, &disks, bucket, Some(first), 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let opts = ObjectOptions { + version_id: Some(first.to_string()), + versioned: true, + ..Default::default() + }; + assert!( + store.delete_object(bucket, "object", opts.clone()).await.is_err(), + "hidden prepared version cannot receive an idempotent success while a peer is unavailable" + ); + set.disks.write().await[3] = Some(disks[3].clone()); + store + .delete_object(bucket, "object", opts) + .await + .expect("recover then execute the currently authorized exact-version delete"); + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(first.to_string()), + ..Default::default() + } + ) + .await + .is_err() + ); + assert_eq!(body(&store, bucket, Some(second)).await, survivor); + } + + #[tokio::test] + async fn delete_recovery_set_dry_run_preserves_every_file() { + let bucket = "recover-dry-run"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![30; 4096], false).await; + let transaction = applied(&store, &disks, bucket, None, 4).await; + let before = disks[0] + .inspect_delete_transaction(bucket, "object", transaction) + .await + .expect("before inspection"); + let report = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect("dry-run report"); + assert!(report.detail.contains("dry run")); + let after = disks[0] + .inspect_delete_transaction(bucket, "object", transaction) + .await + .expect("after inspection"); + assert_eq!(before.fingerprint, after.fingerprint, "dry run may not modify recovery evidence"); + for dir in dirs { + assert!(!dir.path().join(bucket).join("object").join(STORAGE_FORMAT_FILE).exists()); + } + } + #[tokio::test] + async fn delete_recovery_set_admin_object_restores_hidden_metadata() { + let bucket = "recover-admin-object"; + let (_dirs, store, disks) = fixture(bucket).await; + let payload = vec![41; 4096]; + put(&store, bucket, &payload, false).await; + applied(&store, &disks, bucket, None, 4).await; + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("bucket identity"); + let result = store + .heal_object_at_incarnation(bucket, "object", "", incarnation, &HealOpts::default()) + .await + .expect("admin object recovery"); + assert!(result.error.is_none(), "recovered object should heal cleanly: {:?}", result.error); + assert_eq!(body(&store, bucket, None).await, payload); + } + + #[tokio::test] + async fn delete_recovery_current_version_delete_preserves_legacy_on_missing_participant() { + use crate::storage_api_contracts::object::ObjectToDelete; + for batch in [false, true] { + let bucket = "recover-current-with-legacy"; + let (dirs, store, disks) = fixture(bucket).await; + let original = vec![44; 1024 * 1024]; + let old_version = put(&store, bucket, &original, true).await.expect("historical version"); + let legacy = Uuid::new_v4(); + disks[3] + .delete_version( + bucket, + "object", + FileInfo { + name: "object".into(), + version_id: Some(old_version), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(legacy), + ..Default::default() + }, + ) + .await + .expect("historical disk-local mutation without a set decision"); + let backup = dirs[3] + .path() + .join(bucket) + .join("object") + .join(legacy.to_string()) + .join("xl.meta.bkp"); + let before = tokio::fs::read(&backup).await.expect("historical backup"); + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let current = put(&store, bucket, &vec![45; 4096], true) + .await + .expect("current version at quorum"); + set.disks.write().await[3] = Some(disks[3].clone()); + if batch { + let (_, errors) = store + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: "object".into(), + version_id: Some(current), + ..Default::default() + }], + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await; + assert!(errors.iter().all(Option::is_none), "current deletion at quorum: {errors:?}"); + } else { + store + .delete_object( + bucket, + "object", + ObjectOptions { + versioned: true, + version_id: Some(current.to_string()), + ..Default::default() + }, + ) + .await + .expect("current deletion at quorum"); + } + assert_eq!(tokio::fs::read(&backup).await.expect("legacy backup retained"), before); + let state = disks[3] + .inspect_delete_transaction(bucket, "object", legacy) + .await + .expect("legacy inspection"); + assert_eq!(state.state, DeleteRecoveryState::LegacyAmbiguous); + assert_eq!(body(&store, bucket, Some(old_version)).await, original); + } + } + + #[tokio::test] + async fn delete_recovery_dry_run_matches_adoption_of_already_absent_non_null_version() { + let bucket = "recover-preview-non-null"; + let (_dirs, store, disks) = fixture(bucket).await; + let first = put(&store, bucket, &vec![46; 4096], true).await.expect("first version"); + let remaining = vec![47; 4096]; + let second = put(&store, bucket, &remaining, true).await.expect("remaining version"); + disks[3] + .delete_version( + bucket, + "object", + FileInfo { + name: "object".into(), + version_id: Some(first), + ..Default::default() + }, + false, + DeleteOptions::default(), + ) + .await + .expect("returning peer already lacks the explicit version"); + let transaction = applied(&store, &disks, bucket, Some(first), 3).await; + publish(&disks[0], bucket, transaction).await; + let preview = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect("preview certified recovery"); + assert!( + preview.detail.contains("committed=1") && preview.detail.contains("blocked=0"), + "{}", + preview.detail + ); + let actual = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("recover certified removal"); + assert!( + actual.detail.contains("committed=1") && actual.detail.contains("blocked=0"), + "{}", + actual.detail + ); + assert_eq!(body(&store, bucket, Some(second)).await, remaining); + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(first.to_string()), + ..Default::default() + } + ) + .await + .is_err() + ); + } + + #[tokio::test] + async fn delete_recovery_set_legacy_requires_explicit_removal() { + let bucket = "recover-legacy-remove"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![42; 4096], false).await; + let transaction = Uuid::new_v4(); + for disk in &disks { + disk.delete_version( + bucket, + "object", + FileInfo { + name: "object".to_owned(), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .expect("create undecided historical delete shape"); + } + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("automatic report"); + assert!(report.detail.contains("blocked=1")); + for dir in &dirs { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(transaction.to_string()) + .join("xl.meta.bkp") + .exists() + ); + } + let dry = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + remove: true, + ..Default::default() + }, + ) + .await + .expect("authorized-removal dry run"); + assert!(dry.detail.contains("reclaimed=1")); + for dir in &dirs { + assert!(dir.path().join(bucket).join("object").exists()); + } + let report = store + .recover_delete_transactions( + bucket, + &HealOpts { + remove: true, + ..Default::default() + }, + ) + .await + .expect("explicit admin removal"); + assert!(report.detail.contains("reclaimed=1")); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("non-force bucket deletion"); + } + + #[tokio::test] + async fn delete_recovery_set_old_null_decision_cannot_commit_new_undecided_generation() { + let bucket = "recover-inverse-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![51; 4096], false).await; + let old_transaction = applied(&store, &disks, bucket, None, 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + set.publish_delete_commit( + bucket, + "object", + old_transaction, + &set.disk_inventory().await, + 3, + DeleteRecoveryContext::new(None, &|| true), + ) + .await + .expect("old generation decision while peer offline"); + let survivor = vec![52; 4096]; + put(&store, bucket, &survivor, false).await; + let new_transaction = applied(&store, &disks, bucket, None, 3).await; + set.disks.write().await[3] = Some(disks[3].clone()); + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("causal recovery across old receipt"); + assert!(report.detail.contains("rolled_back=1")); + assert_eq!( + body(&store, bucket, None).await, + survivor, + "an old null-version receipt cannot authorize deleting a later generation" + ); + for disk in &disks { + assert_eq!( + disk.inspect_delete_transaction(bucket, "object", new_transaction) + .await + .expect("new transaction settled") + .state, + DeleteRecoveryState::Absent + ); + } + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("delete recovered generation explicitly"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("all proof is eventually collected"); + } + + #[tokio::test] + async fn delete_recovery_set_ordinary_missing_single_and_batch_keep_quorum_semantics() { + let bucket = "recover-missing-quorum"; + let (_dirs, store, disks) = fixture(bucket).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let error = store + .delete_object(bucket, "single-missing", ObjectOptions::default()) + .await + .expect_err("ordinary missing storage result"); + assert!( + crate::error::is_err_object_not_found(&error), + "S3 may translate missing key into idempotent204; not a bucket/quorum error: {error}" + ); + let objects = ["batch-one", "batch-two"] + .into_iter() + .map(|name| crate::storage_api_contracts::object::ObjectToDelete { + object_name: name.to_owned(), + ..Default::default() + }) + .collect(); + let (deleted, errors) = store.delete_objects(bucket, objects, ObjectOptions::default()).await; + assert!( + errors.iter().all(Option::is_none), + "ordinary batch no-op must succeed at available quorum: {errors:?}" + ); + assert_eq!(deleted.len(), 2); + for disk in disks.iter().take(3) { + for key in ["single-missing", "batch-one", "batch-two"] { + assert!( + disk.discover_delete_transactions(bucket, Some(key), None, 128) + .await + .expect("no-op leaves no recovery journals") + .transactions + .is_empty() + ); + } + } + set.disks.write().await[3] = Some(disks[3].clone()); + let payload = vec![53; 4096]; + put(&store, bucket, &payload, false).await; + assert_eq!(body(&store, bucket, None).await, payload); + } + #[tokio::test] + async fn delete_recovery_set_fresh_delete_resolves_unknown_null_predecessor() { + let bucket = "recover-unknown-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![54; 4096], false).await; + let transaction = Uuid::new_v4(); + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("bucket identity"); + disks[3] + .delete_version_recoverable( + bucket, + "object", + FileInfo { + name: "object".to_owned(), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + delete_recovery_bucket_incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("only future-offline peer prepared old delete"); + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + put(&store, bucket, &vec![55; 4096], false).await; + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("later generation deleted at quorum"); + set.disks.write().await[3] = Some(disks[3].clone()); + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("unknown causal history is reported"); + assert!( + report.detail.contains("blocked=1"), + "automatic recovery cannot invent causality: {}", + report.detail + ); + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("fresh authorized delete certifies current absence"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("fresh intent drains all proven unprotected null history"); + } + #[test] + fn delete_recovery_network_errors_do_not_mask_application_corruption() { + assert!(unavailable(&DiskError::RemoteClientUnavailable( + "transport error: connection refused".to_owned() + ))); + assert!(unavailable(&DiskError::from(tonic::Status::unavailable("peer disconnected")))); + assert!( + unavailable(&DiskError::FaultyDisk), + "a drive disabled by health cannot vote in recovery inspection" + ); + assert!(!unavailable(&DiskError::FileCorrupt)); + assert!(!unavailable(&DiskError::other("invalid journal containing connection refused"))); + assert!(!unavailable(&DiskError::from(tonic::Status::internal("connection refused")))); + assert!(!unavailable(&DiskError::other("bucket unavailable-logs has corrupt metadata"))); + } + + #[tokio::test] + async fn delete_recovery_chained_committed_versions_finish_after_current_metadata_disappears() { + let bucket = "recover-chained-versions"; + let (_dirs, store, disks) = fixture(bucket).await; + let first = put(&store, bucket, &vec![61; 1024 * 1024], true) + .await + .expect("first version"); + let second = put(&store, bucket, &vec![62; 4096], true).await.expect("second version"); + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + for version in [first, second] { + store + .delete_object( + bucket, + "object", + ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), + ..Default::default() + }, + ) + .await + .expect("commit each explicit version at quorum while peer is absent"); + } + assert!( + store + .get_object_info(bucket, "object", &ObjectOptions::default()) + .await + .is_err() + ); + set.disks.write().await[3] = Some(disks[3].clone()); + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("older certified target may be superseded by a later certified absence"); + assert!( + report.detail.contains("committed=2"), + "both retained decisions must converge: {}", + report.detail + ); + for version in [first, second] { + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(version.to_string()), + ..Default::default() + } + ) + .await + .is_err(), + "certified cleanup must not resurrect either deleted version" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("no retained version decision may keep the empty bucket nonempty"); + } + + #[tokio::test] + async fn delete_recovery_partial_receipt_converges_after_later_version_absence() { + let bucket = "recover-partial-chained-versions"; + let (dirs, store, disks) = fixture(bucket).await; + let first = put(&store, bucket, &vec![63; 1024 * 1024], true) + .await + .expect("first version"); + let second = put(&store, bucket, &vec![64; 4096], true).await.expect("second version"); + let transaction = applied(&store, &disks, bucket, Some(first), 4).await; + // A crash after the first durable decision leaves other copies prepared. + publish(&disks[0], bucket, transaction).await; + let later = applied(&store, &disks, bucket, Some(second), 4).await; + publish(&disks[0], bucket, later).await; + let dry = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect("dry-run uses the same certified prepared-copy authority"); + assert!( + dry.detail.contains("committed=2") && dry.detail.contains("blocked=0"), + "accurate dry-run: {}", + dry.detail + ); + for dir in &dirs { + for id in [transaction, later] { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(id.to_string()) + .join("delete-prepared") + .exists(), + "dry-run must retain every preparation record" + ); + } + } + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("same-transaction receipt certifies each prepared absent copy"); + for dir in &dirs { + assert!(!dir.path().join(bucket).join("object").exists(), "all related proof must converge"); + } + for version in [first, second] { + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(version.to_string()), + ..Default::default() + } + ) + .await + .is_err(), + "no erased version may be recreated by receipt adoption" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("non-force bucket removal"); + } + + #[tokio::test] + async fn delete_recovery_causal_cycle_rejects_before_mutation_and_retains_proof() { + let bucket = "recover-causal-cycle"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![58; 4096], false).await; + let first = applied(&store, &disks, bucket, None, 4).await; + put(&store, bucket, &vec![59; 4096], false).await; + let second = applied(&store, &disks, bucket, None, 4).await; + let mut evidence = Vec::new(); + for (dir, disk) in dirs.iter().zip(&disks) { + let root = dir.path().join(bucket).join("object"); + let mut journal = disk + .inspect_delete_transaction(bucket, "object", first) + .await + .expect("inspect first preparation") + .journal + .expect("original journal"); + journal.predecessors.push(second); + let path = root.join(first.to_string()).join("delete-prepared"); + std::fs::write(&path, serde_json::to_vec(&journal).expect("encode cyclic fixture")).expect("write cyclic edge"); + for transaction in [first, second] { + for file in ["delete-prepared", "xl.meta.bkp"] { + let path = root.join(transaction.to_string()).join(file); + evidence.push((path.clone(), std::fs::read(path).expect("capture untouched proof"))); + } + } + } + let error = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect_err("cyclic history cannot settle"); + assert!( + matches!(error, crate::error::StorageError::FileCorrupt), + "typed corruption must reject cyclic evidence: {error:?}" + ); + for (path, bytes) in evidence { + assert_eq!(std::fs::read(path).expect("all proof retained"), bytes); + } + for dir in &dirs { + assert!( + !dir.path().join(bucket).join("object").join(STORAGE_FORMAT_FILE).exists(), + "cycle rejection must not restore or publish metadata" + ); + } + } + + #[tokio::test] + async fn delete_recovery_set_later_null_decision_covers_unapplied_original_peer() { + let bucket = "recover-unapplied-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![56; 1024 * 1024], false).await; + // Every generated v4 UUID sorts before MAX, so the former UUID order + // deterministically tried the newer decision before its predecessor. + applied_at(&store, &disks, bucket, None, 3, Uuid::from_u128(u128::MAX)).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + put(&store, bucket, &vec![57; 1024 * 1024], false).await; + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("later null decision"); + set.disks.write().await[3] = Some(disks[3].clone()); + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("causal older original can be removed"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("no stale old body or receipts remain"); + } +} diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 07df7f69c..200c13690 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -25,11 +25,14 @@ use super::super::{ use crate::bucket::retirement::MarkerRetirementContext; use crate::diagnostics::object_lock::ObjectLockAttempt; use crate::set_disk::{ObjectLockDiagGuard, is_object_lock_diag_enabled}; +use std::sync::Arc; #[derive(Clone, Copy)] struct ExplicitVersionHeal<'a> { opts: &'a HealOpts, allow_regeneration: bool, + preserve_unrepairable: bool, + commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>, retirement: Option<&'a MarkerRetirementContext<'a>>, } @@ -770,6 +773,8 @@ impl SetDisks { ExplicitVersionHeal { opts, allow_regeneration: true, + preserve_unrepairable: false, + commit_fence: None, retirement: None, }, &mut None, @@ -907,6 +912,8 @@ impl SetDisks { let ExplicitVersionHeal { opts, allow_regeneration: allow_explicit_version_regen, + preserve_unrepairable, + commit_fence, retirement, } = run; trace!( @@ -925,6 +932,9 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + return Err(DiskError::DiskOngoingReq); + } let mut result = HealResultItem { heal_item_type: HealItemType::Object.to_string(), @@ -1248,7 +1258,7 @@ impl SetDisks { // return here and never reach the post-heal reclaim tail // below. Sweep the strays on this path too (issues #3231, // #3191). Skipped on dry-run, like every mutating step. - if !opts.dry_run { + if !opts.dry_run && !preserve_unrepairable { self.reclaim_orphan_data_dirs_best_effort(bucket, object).await; } return Ok((result, None)); @@ -1328,7 +1338,7 @@ impl SetDisks { ); } - if read_repair_uses_shared_lock { + if read_repair_uses_shared_lock || preserve_unrepairable { return Ok((result, Some(cannot_heal_err))); } @@ -1527,6 +1537,9 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + return Err(DiskError::DiskOngoingReq); + } let disk = outdated_disk.as_ref().expect("checked above").clone(); let endpoint = disk.endpoint().to_string(); match disk.make_volume(bucket).await { @@ -1562,6 +1575,9 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + return Err(DiskError::DiskOngoingReq); + } if disks_to_heal_count == 0 { return Ok((result, volume_creation_error.or(Some(DiskError::ErasureWriteQuorum)))); } @@ -1873,6 +1889,10 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; + return Err(DiskError::DiskOngoingReq); + } rename_attempts += 1; // record the index of the updated disks parts_metadata[index].erasure.index = index + 1; @@ -2004,7 +2024,9 @@ impl SetDisks { // by pre-#3510 unversioned overwrites, which the dangling paths // above never touch (issues #3231, #3191). Best effort — a // failure must not fail the heal. - self.reclaim_orphan_data_dirs_best_effort(bucket, object).await; + if !preserve_unrepairable { + self.reclaim_orphan_data_dirs_best_effort(bucket, object).await; + } Ok((result, None)) } @@ -2070,7 +2092,7 @@ impl SetDisks { .await; } - if opts.dry_run { + if opts.dry_run || preserve_unrepairable { return Ok(( self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id) .await, @@ -3347,6 +3369,74 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { } impl SetDisks { + /// The recovery coordinator retains the key write lock and lifecycle fence. + /// Only current live metadata can supply this exact source; backups and + /// dangling cleanup cannot authorize prerequisite repair. + pub(in crate::set_disk) async fn heal_delete_recovery_source( + &self, + bucket: &str, + object: &str, + source: &crate::disk::DeleteRecoveryVersionIdentity, + can_commit: &(dyn Fn() -> bool + Sync), + ) -> disk::error::Result<()> { + if !can_commit() { + return Err(DiskError::DiskOngoingReq); + } + let version = source.version_id.ok_or(DiskError::OutdatedXLMeta)?.to_string(); + let disks = self.get_disks_internal().await; + let (metadata, errors, _) = Self::read_metadata(&disks, "", bucket, object, &version, true, false, false) + .await? + .into_legacy(); + let (read_quorum, write_quorum) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?; + let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?; + let write_quorum = usize::try_from(write_quorum).map_err(|_| DiskError::ErasureWriteQuorum)?; + let live_sources = metadata + .iter() + .zip(&errors) + .filter(|(info, error)| { + error.is_none() + && !info.deleted + && !info.is_remote() + && crate::disk::DeleteRecoveryVersionIdentity::from_file_info((*info).clone()) + .is_ok_and(|identity| identity == *source) + }) + .count(); + if live_sources < write_quorum || !can_commit() { + return Err(DiskError::OutdatedXLMeta); + } + let (_, mod_time, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum); + let selected = Self::pick_valid_fileinfo(&metadata, mod_time, etag, read_quorum)?; + if crate::disk::DeleteRecoveryVersionIdentity::from_file_info(selected)? != *source { + return Err(DiskError::OutdatedXLMeta); + } + let opts = HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }; + let (_, error) = Box::pin(self.heal_object_with_explicit_version_regen( + bucket, + object, + &version, + ExplicitVersionHeal { + opts: &opts, + allow_regeneration: false, + preserve_unrepairable: true, + commit_fence: Some(can_commit), + retirement: None, + }, + &mut None, + )) + .await?; + if let Some(error) = error { + return Err(error); + } + if !can_commit() { + return Err(DiskError::DiskOngoingReq); + } + Ok(()) + } + pub(crate) async fn heal_object_with_absence( &self, bucket: &str, @@ -3375,11 +3465,11 @@ impl SetDisks { .new_ns_lock(bucket, object) .await .map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?; - Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| { + Some(Arc::new(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| { self.map_namespace_lock_error(bucket, object, "write", e) .narrow_to_disk() .unwrap_or_else(DiskError::other) - })?) + })?)) } else { None }; @@ -3389,6 +3479,75 @@ impl SetDisks { return Ok((result, err.map(|e| e.into()))); } + let mut recovery_removed_version = false; + if !opts.read_repair + && _write_lock_guard.is_some() + && let Some(scope) = crate::store::bucket_heal_scope_for_object(bucket, object) + && !crate::bucket::utils::is_meta_bucketname(bucket) + { + let requested = Uuid::parse_str(version_id).ok(); + if !opts.dry_run && (version_id.is_empty() || requested.is_some()) { + let disks = self.get_disks_internal().await; + let before = Self::read_metadata(&disks, "", bucket, object, version_id, false, false, false) + .await + .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; + recovery_removed_version = before + .slots + .iter() + .filter_map(|slot| slot.file_info()) + .any(|info| version_id.is_empty() || Some(info.version_id.unwrap_or_default()) == requested); + } + let explicit_delete = opts.remove + && version_id.is_empty() + && crate::bucket::metadata_sys::has_authoritative_never_versioned_state_in(&self.ctx, bucket).await? + && matches!( + crate::bucket::metadata_sys::get_object_lock_config_state_in(&self.ctx, bucket).await?, + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent + ); + let recovery = self + .recover_delete_key( + bucket, + object, + crate::set_disk::DeleteRecoveryContext { + dry_run: opts.dry_run, + explicit_delete, + repair_marker_source: true, + namespace_owner: (!opts.dry_run).then(|| { + Arc::new((scope.clone(), self.ctx.begin_namespace_commit(), _write_lock_guard.clone())) + as Arc + }), + ..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &|| { + scope.check().is_ok() && !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()) + }) + }, + ) + .await?; + if opts.dry_run && recovery.discovered > 0 { + return Ok(( + HealResultItem { + heal_item_type: HealItemType::Object.to_string(), + bucket: bucket.to_owned(), + object: object.to_owned(), + version_id: version_id.to_owned(), + detail: format!( + "delete recovery (dry run): discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}, pending={}", + recovery.discovered, + recovery.committed, + recovery.rolled_back, + recovery.reclaimed, + recovery.blocked, + recovery.pending + ), + ..Default::default() + }, + None, + )); + } + if recovery.pending > 0 || recovery.blocked > 0 { + return Err(StorageError::SlowDown); + } + } + // The inner heal and missing-object report read the registry again; // release this snapshot guard before a topology writer can queue between reads. let disks = self.get_disks_internal().await; @@ -3440,12 +3599,12 @@ impl SetDisks { && errs .iter() .all(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound))) - && !_write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + && !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()) { *absence = Some(HealedObjectAbsence { pool_index: self.pool_index, set_index: self.set_index, - removed: false, + removed: recovery_removed_version, }); } return Ok((result, Some(err))); @@ -3463,6 +3622,8 @@ impl SetDisks { ExplicitVersionHeal { opts: &inner_opts, allow_regeneration: true, + preserve_unrepairable: false, + commit_fence: None, retirement: proof.retirement, }, absence, @@ -3483,6 +3644,8 @@ impl SetDisks { ExplicitVersionHeal { opts: &inner_opts, allow_regeneration: true, + preserve_unrepairable: false, + commit_fence: None, retirement: proof.retirement, }, absence, @@ -3493,7 +3656,7 @@ impl SetDisks { result, err.map(Into::into), absence, - _write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()), + _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()), bucket, object, )); @@ -3505,7 +3668,7 @@ impl SetDisks { result, err.map(Into::into), absence, - _write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()), + _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()), bucket, object, )) diff --git a/crates/ecstore/src/set_disk/ops/mod.rs b/crates/ecstore/src/set_disk/ops/mod.rs index 4cea63793..5dfaa66b5 100644 --- a/crates/ecstore/src/set_disk/ops/mod.rs +++ b/crates/ecstore/src/set_disk/ops/mod.rs @@ -19,6 +19,7 @@ pub(crate) mod bitrot_self_verify; pub(crate) mod bucket; +pub(crate) mod delete_recovery; pub(crate) mod heal; pub(crate) mod heal_walk; pub(crate) mod list; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 791873248..b35eb4cb7 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -64,6 +64,34 @@ use super::super::{ take_prepared_get_object_metadata, to_object_err, try_read_inline_data_shards_direct, warn, }; use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards}; +use super::delete_recovery::DeleteRecoveryContext; + +#[derive(Default)] +struct DeleteCommitContext<'a> { + incarnation: Option, + options: Option<&'a ObjectOptions>, + guard: Option>, +} + +impl<'a> DeleteCommitContext<'a> { + fn request(options: &'a ObjectOptions, guard: Option<&Arc>) -> Self { + Self { + incarnation: options.expected_bucket_incarnation_id, + options: Some(options), + guard: guard.cloned(), + } + } + + fn can_commit(&self, bucket: &str, object: &str) -> bool { + self.options.is_none_or(|options| { + ensure_delete_commit_locks_held(self.guard.as_deref(), bucket, object, options).is_ok() + && options + .scanner_publication_commit_scope + .as_ref() + .is_none_or(|scope| scope.can_commit()) + }) + } +} fn delete_marker_purge_candidate( file_info: &FileInfo, @@ -338,7 +366,7 @@ use crate::storage_api_contracts::object::HTTPPreconditions; use crate::store::ECStore; use crate::store::utils::clean_metadata; use crate::store::{RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence}; -use futures::FutureExt as _; +use futures::{FutureExt as _, StreamExt as _}; use http::HeaderValue; #[cfg(test)] use rustfs_filemeta::FileMeta; @@ -3308,6 +3336,76 @@ pub(in crate::set_disk) fn merge_replication_metadata_lww( } impl SetDisks { + async fn reclaim_missing_delete_residue( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + lock_guard: Option<&Arc>, + ) -> Result { + if is_meta_bucketname(bucket) + || opts.version_id.is_some() + || opts.versioned + || opts.version_suspended + || opts.http_preconditions.is_some() + || opts.data_movement + || opts.transition.expire_restored + || opts.replication_request + || opts.lifecycle_delete_all.is_some() + || opts.delete_prefix + || opts.expected_current_version_id.is_some() + || !metadata_sys::has_authoritative_never_versioned_state_in(&self.ctx, bucket) + .await + .unwrap_or(false) + { + return Ok(false); + } + let lock_state = match opts.object_lock_config_snapshot.as_ref() { + Some(snapshot) => Some(snapshot.state().clone()), + None => metadata_sys::get_object_lock_config_state_in(&self.ctx, bucket).await.ok(), + }; + if !matches!(lock_state, Some(ObjectLockConfigState::ConfirmedAbsent)) { + return Ok(false); + } + if let Some(expected) = opts.expected_bucket_incarnation_id { + self.validate_bucket_incarnation(bucket, expected, opts).await?; + } + ensure_delete_commit_locks_held(lock_guard.map(Arc::as_ref), bucket, object, opts)?; + begin_scanner_publication_delete_mutation(opts.scanner_publication_commit_scope.as_ref())?; + let result = self + .recover_delete_key( + bucket, + object, + DeleteRecoveryContext { + explicit_delete: true, + namespace_owner: Some(Arc::new(( + self.ctx.begin_namespace_commit(), + lock_guard.cloned(), + opts.object_lock_config_snapshot.clone(), + opts.scanner_publication_commit_scope.clone(), + opts.write_lock_context.clone(), + opts.delete_mutation_owner.clone(), + ))), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(lock_guard.map(Arc::as_ref), bucket, object, opts).is_ok() + && opts + .scanner_publication_commit_scope + .as_ref() + .is_none_or(|scope| scope.can_commit()) + }) + }, + ) + .await; + ensure_delete_commit_locks_held(lock_guard.map(Arc::as_ref), bucket, object, opts)?; + let recovered = result.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?; + if recovered.pending > 0 && !recovered.no_transactions_at_quorum { + return Err(StorageError::SlowDown); + } + if recovered.blocked > 0 { + return Err(StorageError::SlowDown); + } + Ok(recovered.reclaimed > 0 || recovered.committed > 0) + } pub(in crate::set_disk) async fn cleanup_rename_tail( &self, targets: Vec, @@ -7612,7 +7710,7 @@ impl SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(opts, None), ) .await?; if let Some((_, deleted_object)) = replication_delete { @@ -7686,7 +7784,7 @@ impl SetDisks { Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended)) } - #[tracing::instrument(skip(self, delete_marker_purge))] + #[tracing::instrument(skip(self, delete_marker_purge, commit_context))] async fn delete_object_version_with_purge( &self, bucket: &str, @@ -7694,12 +7792,28 @@ impl SetDisks { fi: &FileInfo, force_del_marker: bool, delete_marker_purge: Option, - source_bucket_incarnation_id: Option, + commit_context: DeleteCommitContext<'_>, ) -> Result<()> { + let source_bucket_incarnation_id = commit_context.incarnation; let transported = delete_file_info_with_replication_transport_metadata(fi); let fi = &transported; let disks = self.disk_inventory().await; - let namespace_owner = (!is_meta_bucketname(bucket)).then(|| self.ctx.begin_namespace_commit()); + let namespace_owner = (!is_meta_bucketname(bucket)).then(|| { + Arc::new(( + self.ctx.begin_namespace_commit(), + commit_context.guard.clone(), + commit_context + .options + .and_then(|options| options.object_lock_config_snapshot.clone()), + commit_context + .options + .and_then(|options| options.scanner_publication_commit_scope.clone()), + commit_context.options.and_then(|options| options.write_lock_context.clone()), + commit_context + .options + .and_then(|options| options.delete_mutation_owner.clone()), + )) as Arc + }); // Quorum is a property of the configured set, not of the current // online snapshot. A decommission/offline refresh may temporarily // shorten the snapshot; deriving quorum from it would turn a @@ -7714,20 +7828,33 @@ impl SetDisks { let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc); futures.push(async move { if let Some(disk) = disk { - match disk - .delete_version_with_namespace_owner( + let options = DeleteOptions { + old_data_dir: Some(rollback_dir), + delete_recovery_bucket_incarnation: source_bucket_incarnation_id, + ..Default::default() + }; + let result = if is_meta_bucketname(bucket) { + disk.delete_version_with_namespace_owner( bucket, object, fi.clone(), force_del_marker, - DeleteOptions { - old_data_dir: Some(rollback_dir), - ..Default::default() - }, + options, disk_namespace_owner, ) .await - { + } else { + disk.delete_version_recoverable_with_namespace_owner( + bucket, + object, + fi.clone(), + force_del_marker, + options, + disk_namespace_owner, + ) + .await + }; + match result { Ok(r) => Ok(r), Err(e) => Err(e), } @@ -7749,12 +7876,34 @@ impl SetDisks { } } - let quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object); + let mut quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object); + if !is_meta_bucketname(bucket) && errs.iter().flatten().any(super::delete_recovery::unsupported_recovery) { + quorum_result = Err(DiskError::UnsupportedDisk.into()); + } let should_rollback = quorum_result.is_err(); + let structured_cleanup = !is_meta_bucketname(bucket) && !should_rollback; + if structured_cleanup { + self.publish_delete_commit( + bucket, + object, + rollback_dir, + &disks, + write_quorum, + DeleteRecoveryContext { + namespace_owner: namespace_owner.clone().map(|owner| owner as Arc), + ..DeleteRecoveryContext::new(source_bucket_incarnation_id, &|| commit_context.can_commit(bucket, object)) + }, + ) + .await + .map_err(StorageError::from)?; + } #[cfg(test)] pause_delete_before_cleanup(bucket).await; let mut rollback_futures = Vec::new(); for (index, err) in errs.iter().enumerate() { + if structured_cleanup { + break; + } // backlog#1158: when rolling back, fan the idempotent undo out to every // online disk (each self-decides from its staged backup: restore if the // rollback dir is present, no-op otherwise). This covers a disk that @@ -7810,6 +7959,7 @@ impl SetDisks { DeleteOptions { recursive: true, immediate: true, + old_data_dir: Some(rollback_dir), ..Default::default() }, disk_namespace_owner, @@ -8192,8 +8342,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } else { self.bucket_incarnation_id_from_disk(bucket).await.ok() }; - self.delete_object_version_with_purge(bucket, object, fi, force_del_marker, None, source_bucket_incarnation_id) - .await + self.delete_object_version_with_purge( + bucket, + object, + fi, + force_del_marker, + None, + DeleteCommitContext { + incarnation: source_bucket_incarnation_id, + ..Default::default() + }, + ) + .await } #[tracing::instrument(skip(self, objects, opts))] @@ -8313,6 +8473,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let mut tier_reference_leases: Vec<(usize, String, Option)> = Vec::new(); let mut tier_free_version_receipt_candidates: HashMap = HashMap::new(); let mut delete_marker_purge_candidates = vec![None; objects.len()]; + let mut missing_delete_retry_indices = Vec::new(); + let mut deferred_recovery_indices = Vec::new(); for (i, dobj) in objects.iter().enumerate() { if del_errs[i].is_some() { @@ -8343,7 +8505,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id; let replication_needs_source = replicate_delete && (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name)); - let (goi, authoritative_file_info, gerr) = if object_lock_check_required + let (mut goi, mut authoritative_file_info, mut gerr) = if object_lock_check_required || replication_needs_source || opts.tier_delete_journal_api.is_some() || dobj.expected_identity.is_some() @@ -8356,9 +8518,74 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } else { (ObjectInfo::default(), FileInfo::default(), None) }; + if gerr + .as_ref() + .is_some_and(|error| is_err_object_not_found(error) || is_err_version_not_found(error)) + && !is_meta_bucketname(bucket) + && !opts.data_movement + && !opts.replication_request + && opts.http_preconditions.is_none() + { + let recovery = self + .recover_delete_key( + bucket, + &dobj.object_name, + DeleteRecoveryContext { + dry_run: !opts.no_lock, + namespace_owner: opts.delete_mutation_owner.clone(), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(None, bucket, &dobj.object_name, &opts).is_ok() + }) + }, + ) + .await; + match recovery { + Ok(recovery) if !opts.no_lock && recovery.discovered > 0 && recovery.pending == 0 => { + deferred_recovery_indices.push(i); + continue; + } + Ok(recovery) + if (recovery.pending > 0 + && !(recovery.no_transactions_at_quorum + && dobj.version_id.is_none() + && !versioned + && !version_suspended)) + || (recovery.blocked > 0 && (dobj.version_id.is_some() || versioned || version_suspended)) => + { + del_errs[i] = Some(if recovery.pending > 0 { + StorageError::InsufficientWriteQuorum(bucket.to_owned(), dobj.object_name.clone()) + } else { + StorageError::SlowDown + }); + continue; + } + Ok(recovery) if recovery.rolled_back > 0 || recovery.committed > 0 => { + let (fresh, info, _, error) = self + .get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts) + .await; + goi = fresh; + authoritative_file_info = info; + gerr = error; + } + Ok(_) => {} + Err(error) => { + del_errs[i] = Some(to_object_err(error.into(), vec![bucket, &dobj.object_name])); + continue; + } + } + } let source_missing = gerr .as_ref() .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)); + if source_missing + && dobj.version_id.is_none() + && dobj.expected_identity.is_none() + && !versioned + && !version_suspended + && opts.http_preconditions.is_none() + { + missing_delete_retry_indices.push(i); + } // A pool-local miss keeps the existing idempotent batch-delete // semantics. Exact-version deletes fan out to every pool, and a // retry may also arrive after the intended generation is gone. @@ -8596,10 +8823,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } let rollback_dir = Uuid::new_v4(); + let namespace_owner = (!is_meta_bucketname(bucket)).then(|| { + Arc::new(( + self.ctx.begin_namespace_commit(), + opts.delete_mutation_owner.clone(), + opts.object_lock_config_snapshot.clone(), + opts.scanner_publication_commit_scope.clone(), + opts.write_lock_context.clone(), + )) as Arc + }); - let disks = self.disks.read().await; - - let disks = disks.clone(); + let disks = self.disks.read().await.clone(); if opts .namespace_lock_fence @@ -8668,6 +8902,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // let mut errors = Vec::with_capacity(disks.len()); for disk in disks.iter() { + let disk_namespace_owner = namespace_owner.clone(); let vers = vers.clone(); let namespace_lock_fence = opts.namespace_lock_fence.clone(); let lock_lost_during_commit = Arc::clone(&lock_lost_during_commit); @@ -8677,15 +8912,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { return (0..vers.len()).map(|_| Some(DiskError::DiskOngoingReq)).collect(); } if let Some(disk) = disk { - disk.delete_versions( - bucket, - vers, - DeleteOptions { - old_data_dir: Some(rollback_dir), - ..Default::default() - }, - ) - .await + let options = DeleteOptions { + old_data_dir: Some(rollback_dir), + delete_recovery_bucket_incarnation: opts.expected_bucket_incarnation_id, + ..Default::default() + }; + if is_meta_bucketname(bucket) { + disk.delete_versions(bucket, vers, options).await + } else { + disk.delete_versions_recoverable_with_namespace_owner(bucket, vers, options, disk_namespace_owner) + .await + } } else { let mut errs = Vec::with_capacity(vers.len()); for _ in 0..vers.len() { @@ -8721,7 +8958,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } - let mut has_err = reduce_write_quorum_errs(&disk_err, OBJECT_OP_IGNORED_ERRS, disks.len() / 2 + 1); + let mut has_err = reduce_write_quorum_errs(&disk_err, OBJECT_OP_IGNORED_ERRS, self.set_drive_count / 2 + 1); + if !is_meta_bucketname(bucket) && disk_err.iter().flatten().any(super::delete_recovery::unsupported_recovery) { + has_err = Some(DiskError::UnsupportedDisk); + } if let Some(err) = has_err.clone() { let er = err.into(); if (is_err_object_not_found(&er) || is_err_version_not_found(&er)) && !del_objects[obj_idx].delete_marker { @@ -8765,10 +9005,67 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { #[cfg(test)] pause_delete_before_cleanup(bucket).await; + let mut published_groups = HashSet::new(); + if !is_meta_bucketname(bucket) { + let groups = vers.iter().enumerate().filter_map(|(index, group)| { + group + .versions + .iter() + .all(|version| del_errs[version.idx].is_none()) + .then_some(index) + }); + // Mark all attempted groups before starting publication. A lost + // response can hide a persisted decision and must never roll back. + let groups = groups.collect::>(); + published_groups.extend(groups.iter().map(|index| vers[*index].name.clone())); + let publications = futures::stream::iter(groups.into_iter().map(|index: usize| { + let group = &vers[index]; + let namespace_owner = namespace_owner.clone(); + let disks = &disks; + let opts = &opts; + async move { + let can_commit = || { + ensure_delete_commit_locks_held(None, bucket, &group.name, opts).is_ok() + && opts + .scanner_publication_commit_scope + .as_ref() + .is_none_or(|scope| scope.can_commit()) + }; + let result = self + .publish_delete_commit( + bucket, + &group.name, + rollback_dir, + disks, + self.set_drive_count / 2 + 1, + DeleteRecoveryContext { + namespace_owner, + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &can_commit) + }, + ) + .await; + (index, result) + } + })) + .buffer_unordered(16) + .collect::>() + .await; + for (index, result) in publications { + let group = &vers[index]; + if let Err(error) = result { + for version in &group.versions { + del_errs[version.idx] = Some(to_object_err(error.clone().into(), vec![bucket, &group.name])); + } + } + } + } let mut rollback_futures = Vec::new(); let committed_receipt_indices = committed_tier_free_version_receipt_indices(&vers, &del_errs, &tier_free_version_receipt_candidates); for fi_vers in &vers { + if published_groups.contains(&fi_vers.name) { + continue; + } // delete_versions commits one xl.meta per object group, so rollback must use the same boundary. let should_rollback = fi_vers.versions.iter().any(|fi| del_errs[fi.idx].is_some()); for (disk_idx, disk) in disks.iter().enumerate() { @@ -8788,10 +9085,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let bucket = bucket.to_string(); let object = fi_vers.name.clone(); let versions = fi_vers.clone(); + let disk_namespace_owner = namespace_owner.clone(); rollback_futures.push(async move { if should_rollback { let errs = disk - .delete_versions( + .delete_versions_with_namespace_owner( &bucket, vec![versions], DeleteOptions { @@ -8800,6 +9098,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { old_data_dir: Some(rollback_dir), ..Default::default() }, + disk_namespace_owner, ) .await; if let Some(err) = errs.into_iter().flatten().next() { @@ -8820,6 +9119,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { DeleteOptions { recursive: true, immediate: true, + old_data_dir: Some(rollback_dir), ..Default::default() }, ) @@ -8856,12 +9156,70 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { ); } + // A new explicit DELETE authorizes removal of the exact missing key's + // legacy residue; it does not infer whether the interrupted delete committed. + for index in missing_delete_retry_indices { + if del_errs[index].is_none() { + let object = &objects[index].object_name; + let result = match ensure_delete_commit_locks_held(None, bucket, object, &opts) { + Ok(()) => self.reclaim_missing_delete_residue(bucket, object, &opts, None).await, + Err(err) => Err(err), + }; + if let Err(err) = result { + del_errs[index] = Some(err); + } + } + } + // TODO(backlog): support partial object deletion for multi-part objects if dist_erasure { self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await; } + // Cold missing-key replay uses the same batch contract after the fast + // batch lock is released; renewable key ownership survives settlement. + for index in deferred_recovery_indices { + let object = &objects[index]; + let explicit_delete = object.version_id.is_none() + && !object.synthetic_version_id + && !opts.versioned + && !opts.version_suspended + && opts.http_preconditions.is_none() + && metadata_sys::has_authoritative_never_versioned_state_in(&self.ctx, bucket) + .await + .unwrap_or(false) + && matches!(object_lock_config.state(), ObjectLockConfigState::ConfirmedAbsent); + let recovery = self + .recover_delete_object( + bucket, + &object.object_name, + DeleteRecoveryContext { + explicit_delete, + namespace_owner: Some(Arc::new(( + self.ctx.begin_namespace_commit(), + opts.delete_mutation_owner.clone(), + opts.object_lock_config_snapshot.clone(), + ))), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(None, bucket, &object.object_name, &opts).is_ok() + }) + }, + ) + .await; + match recovery { + Ok(recovery) if recovery.pending == 0 && recovery.blocked == 0 => { + let (mut deleted, mut errors, mut receipts) = + Box::pin(self.delete_objects_with_accounting(bucket, vec![object.clone()], opts.clone())).await; + del_objects[index] = deleted.remove(0); + del_errs[index] = errors.remove(0); + accounting[index] = receipts.remove(0); + } + Ok(_) => del_errs[index] = Some(StorageError::SlowDown), + Err(error) => del_errs[index] = Some(to_object_err(error.into(), vec![bucket, &object.object_name])), + } + } + let purge_submissions = delete_marker_purge_candidates .into_iter() .enumerate() @@ -8918,7 +9276,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // Guard lock for single object delete let _lock_guard = if (!opts.delete_prefix || opts.delete_prefix_object) && !opts.no_lock { - Some(self.acquire_write_lock_diag("delete_object", bucket, object).await?) + Some(Arc::new(self.acquire_write_lock_diag("delete_object", bucket, object).await?)) } else { None }; @@ -8988,7 +9346,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } }; for version in plan { - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; // The lease closes the gap between the durable // reference proof and publishing either a committed // v6 journal or the legacy free-version fallback. @@ -9022,7 +9380,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await?; if let Some((_, deleted_object)) = replication_delete { @@ -9031,7 +9389,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } else { for version in &versions.versions { - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; let mut delete_request = FileInfo { name: object.to_string(), version_id: version.version_id, @@ -9045,12 +9403,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await?; } for version in &versions.free_versions { - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; let mut delete_request = FileInfo { name: object.to_string(), version_id: version.version_id, @@ -9065,7 +9423,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await?; } @@ -9081,7 +9439,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { self.validate_bucket_incarnation(bucket, expected_incarnation_id, &opts) .await?; } - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; self.delete_prefix_with_scanner_publication_lease( bucket, @@ -9132,14 +9490,73 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let mut version_found = true; // delete_object_version below derives its own majority quorum from the // disk array, so the object-derived quorum here is unused. - let (mut goi, authoritative_file_info, _write_quorum, gerr) = + let (mut goi, mut authoritative_file_info, _write_quorum, mut gerr) = self.get_object_info_fileinfo_and_quorum(bucket, object, &opts).await; + if gerr + .as_ref() + .is_some_and(|error| is_err_object_not_found(error) || is_err_version_not_found(error)) + && !is_meta_bucketname(bucket) + && !opts.data_movement + && !opts.replication_request + && opts.http_preconditions.is_none() + { + if let Some(expected) = opts.expected_bucket_incarnation_id { + self.validate_bucket_incarnation(bucket, expected, &opts).await?; + } + let recovery = self + .recover_delete_key( + bucket, + object, + DeleteRecoveryContext { + namespace_owner: Some(Arc::new(( + self.ctx.begin_namespace_commit(), + _lock_guard.clone(), + opts.object_lock_config_snapshot.clone(), + opts.scanner_publication_commit_scope.clone(), + opts.write_lock_context.clone(), + opts.delete_mutation_owner.clone(), + ))), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts).is_ok() + }) + }, + ) + .await?; + if (recovery.pending > 0 + && !(recovery.no_transactions_at_quorum + && opts.version_id.is_none() + && !opts.versioned + && !opts.version_suspended)) + || (recovery.blocked > 0 && (opts.version_id.is_some() || opts.versioned || opts.version_suspended)) + { + return Err(if recovery.pending > 0 { + StorageError::InsufficientWriteQuorum(bucket.to_owned(), object.to_owned()) + } else { + StorageError::SlowDown + }); + } + if recovery.rolled_back > 0 || recovery.committed > 0 { + (goi, authoritative_file_info, _, gerr) = self.get_object_info_fileinfo_and_quorum(bucket, object, &opts).await; + } + } if let Some(err) = &gerr && goi.name.is_empty() { if should_force_delete_marker_for_missing_version(&opts) { version_found = false; } else { + if is_err_object_not_found(err) { + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; + if self + .reclaim_missing_delete_residue(bucket, object, &opts, _lock_guard.as_ref()) + .await? + { + if let Some(scope) = scanner_publication_commit_scope.as_ref() { + let _ = scope.mark_committed(); + } + return Ok(ObjectInfo::default()); + } + } return Err(err.clone()); } } @@ -9173,11 +9590,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { expire_restored: true, ..Default::default() }; - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; - self.delete_object_version_with_purge(bucket, object, &dfi, false, None, opts.expected_bucket_incarnation_id) - .await - .map_err(|e| to_object_err(e, vec![bucket, object]))?; + self.delete_object_version_with_purge( + bucket, + object, + &dfi, + false, + None, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), + ) + .await + .map_err(|e| to_object_err(e, vec![bucket, object]))?; self.invalidate_get_object_metadata_cache(bucket, object).await; if let Some(scope) = scanner_publication_commit_scope.as_ref() { let _ = scope.mark_committed(); @@ -9257,7 +9681,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { None }; - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; let _tier_delete_lease = acquire_single_tier_delete_lease(&opts, &goi).await?; if opts.skip_free_version { @@ -9269,7 +9693,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &fi, should_force_delete_marker_for_missing_version(&opts), delete_marker_purge.clone(), - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -9316,7 +9740,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { dfi.set_delete_marker_incarnation(incarnation); } - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; let _tier_delete_lease = acquire_single_tier_delete_lease(&opts, &goi).await?; if opts.skip_free_version { @@ -9328,7 +9752,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &dfi, opts.delete_marker, delete_marker_purge, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -9959,7 +10383,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { pause_transition_transaction_at(bucket, object, TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit).await; upload_cleanup.disarm(); if let Err(err) = self - .delete_object_version_with_purge(bucket, object, &fi, false, None, opts.expected_bucket_incarnation_id) + .delete_object_version_with_purge(bucket, object, &fi, false, None, DeleteCommitContext::request(opts, None)) .await { warn!( @@ -22994,9 +23418,19 @@ mod marker_purge_receipt_tests { .acquire_write_lock_diag("marker_purge_receipt_test", bucket, object) .await .expect("own the deleted object namespace"); - set.delete_object_version_with_purge(bucket, object, &marker, true, None, Some(incarnation)) - .await - .expect("seed the real marker on every disk"); + set.delete_object_version_with_purge( + bucket, + object, + &marker, + true, + None, + DeleteCommitContext { + incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the real marker on every disk"); let marker = disks[0] .read_version("", bucket, object, &version.to_string(), &ReadOptions::default()) .await @@ -23014,7 +23448,17 @@ mod marker_purge_receipt_tests { set.disks.write().await[0] = None; tokio::time::timeout( Duration::from_secs(10), - set.delete_object_version_with_purge(bucket, object, &request, false, Some(purge.clone()), Some(incarnation)), + set.delete_object_version_with_purge( + bucket, + object, + &request, + false, + Some(purge.clone()), + DeleteCommitContext { + incarnation: Some(incarnation), + ..Default::default() + }, + ), ) .await .expect("the receipt must not reacquire the deleted object's owner") @@ -23202,6 +23646,54 @@ mod single_delete_namespace_owner_tests { ); } + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn batch_delete_uses_one_namespace_owner_across_object_groups() { + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "batch-delete-namespace"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let mut objects = Vec::new(); + for key in ["inline", "external"] { + let version = Uuid::new_v4(); + let body = vec![0x41; if key == "inline" { 4096 } else { 256 * 1024 }]; + seed_version(&set, bucket, key, version, &body).await; + objects.push(ObjectToDelete { + object_name: key.to_owned(), + version_id: Some(version), + ..Default::default() + }); + } + let before = set.ctx.namespace_commit_generation(); + assert!(!set.ctx.namespace_commits_pending()); + let (_, errors, _) = set + .delete_objects_with_accounting( + bucket, + objects, + ObjectOptions { + versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await; + assert!(errors.iter().all(Option::is_none), "batch groups must commit: {errors:?}"); + for dir in &dirs { + for key in ["inline", "external"] { + assert!(!dir.path().join(bucket).join(key).exists(), "all group evidence must converge"); + } + } + assert!(!set.ctx.namespace_commits_pending()); + assert_eq!( + set.ctx.namespace_commit_generation(), + before + 2, + "all batch groups share one complete owner lifetime" + ); + } + #[cfg(not(windows))] async fn assert_single_delete_physical_owner(case: &'static str) { use crate::disk::os::prepared_publication_test_hooks as hooks; @@ -23280,9 +23772,14 @@ mod single_delete_namespace_owner_tests { for (index, path) in metadata_paths.iter().enumerate() { let tx = entered_tx.clone(); let (release, rx) = std::sync::mpsc::channel::<()>(); - if last || cleanup { - let source = if cleanup { &cleanup_sources[index] } else { path }; - destination_guards.push(hooks::observe_rename_destination(source, move |destination| { + if cleanup { + let source = cleanup_sources[index].clone(); + guards.push(hooks::install_at(hooks::Stage::Remove, &source.join("part.1"), move || { + let _ = tx.send((index, source)); + let _ = rx.recv(); + })); + } else if last { + destination_guards.push(hooks::observe_rename_destination(path, move |destination| { let _ = tx.send((index, destination.to_path_buf())); let _ = rx.recv(); })); @@ -23355,6 +23852,12 @@ mod single_delete_namespace_owner_tests { } } } + if cleanup { + for (index, source) in cleanup_sources.iter().enumerate() { + assert_eq!(std::fs::read(source.join("part.1")).expect("paused original shard"), cleanup_parts[index], + "the paused physical cleanup must still own the exact original shard"); + } + } if rollback { tokio::time::pause(); tokio::time::advance(Duration::from_secs(61)).await; @@ -23431,11 +23934,7 @@ mod single_delete_namespace_owner_tests { if cleanup { assert!(!path.exists(), "metadata must remain deleted after cleanup cancellation"); assert!(!cleanup_sources[index].exists(), "physical cleanup must remove the shard directory"); - assert_eq!( - std::fs::read(physical_keys[&index].join("part.1")).expect("actual trashed shard"), - cleanup_parts[index], - "trash must contain the exact original shard" - ); + assert!(!physical_keys[&index].exists(), "the owned physical cleanup must remove its original shard directory"); continue; } if last { @@ -23545,3 +24044,431 @@ mod single_delete_namespace_owner_tests { assert_single_delete_physical_owner("rollback").await; } } + +#[cfg(test)] +mod crash_delete_retry_tests { + use super::*; + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::disk::STORAGE_FORMAT_FILE_BACKUP; + use crate::storage_api_contracts::bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions}; + + async fn stage_interrupted_delete(store: &Arc, dirs: &[tempfile::TempDir], bucket: &str, object: &str, size: usize) { + let mut reader = PutObjReader::from_vec(vec![37; size]); + store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("write the object before simulating an interrupted delete"); + let transaction = Uuid::new_v4(); + for dir in dirs { + let root = dir.path().join(bucket).join(object); + let xl = root.join(STORAGE_FORMAT_FILE); + let bytes = tokio::fs::read(&xl).await.expect("read original object metadata"); + let mut metadata = FileMeta::default(); + metadata.unmarshal_msg(&bytes).expect("decode original object metadata"); + let info = metadata.versions[0] + .into_fileinfo(bucket, object, false) + .expect("read original version"); + let backup = root.join(transaction.to_string()); + tokio::fs::create_dir_all(&backup) + .await + .expect("create transaction directory"); + tokio::fs::write(backup.join(STORAGE_FORMAT_FILE_BACKUP), bytes) + .await + .expect("stage intact rollback backup"); + if let Some(data_dir) = info.data_dir { + let parts = root.join(data_dir.to_string()); + if parts.is_dir() { + tokio::fs::write( + parts.join(format!("{}{}", crate::disk::local::RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, transaction)), + [], + ) + .await + .expect("reserve external deletion before its local commit"); + } + } + tokio::fs::remove_file(&xl) + .await + .expect("simulate crash after removing metadata"); + } + } + + #[tokio::test] + async fn crash_delete_retry_single_reclaims_inline_backup_only() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-single"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + store + .delete_object(bucket, "inline", ObjectOptions::default()) + .await + .expect("retry should reclaim the exact missing key"); + for dir in &dirs { + assert!( + !dir.path().join(bucket).join("inline").exists(), + "inline backup residue must be removed on every disk" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("the empty bucket must be deletable without force"); + } + + #[tokio::test] + async fn crash_delete_retry_batch_reclaims_inline_and_reserved_external() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-batch"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + stage_interrupted_delete(&store, &dirs, bucket, "external", 1024 * 1024).await; + let objects = ["inline", "external"] + .into_iter() + .map(|name| ObjectToDelete { + object_name: name.to_string(), + ..Default::default() + }) + .collect(); + let (_, errors) = store.delete_objects(bucket, objects, ObjectOptions::default()).await; + assert!( + errors.iter().all(Option::is_none), + "retry must complete for both crash shapes: {errors:?}" + ); + for dir in &dirs { + for object in ["inline", "external"] { + assert!(!dir.path().join(bucket).join(object).exists(), "retry must remove {object} on every disk"); + } + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("batch retry must unblock non-force bucket deletion"); + } + + #[tokio::test] + async fn crash_delete_retry_preserves_conditional_and_versioned_residue() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-gates"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + for opts in [ + ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_match: Some("old-etag".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ObjectOptions { + version_id: Some(Uuid::new_v4().to_string()), + ..Default::default() + }, + ] { + let _ = store.delete_object(bucket, "inline", opts).await; + for dir in &dirs { + assert!( + dir.path().join(bucket).join("inline").exists(), + "conditional/version-specific deletes must retain ambiguous backups" + ); + } + } + } + #[tokio::test] + async fn crash_delete_retry_preserves_suspended_bucket_null_backup() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-suspended"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + metadata_sys::update_in(&store.ctx, bucket, crate::bucket::metadata::BUCKET_VERSIONING_CONFIG, + b"Suspended".to_vec()) + .await.expect("persist suspended versioning state"); + let _ = store.delete_object(bucket, "inline", ObjectOptions::default()).await; + for dir in &dirs { + assert!( + dir.path().join(bucket).join("inline").exists(), + "authoritative suspended state must protect a null-version backup" + ); + } + } + + #[tokio::test] + async fn crash_delete_retry_rejects_stale_bucket_incarnation_at_set_boundary() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-incarnation"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + let error = store.pools[0].disk_set[0] + .delete_object( + bucket, + "inline", + ObjectOptions { + expected_bucket_incarnation_id: Some(Uuid::new_v4()), + ..Default::default() + }, + ) + .await + .expect_err("the set must reject a stale authorized bucket identity"); + assert!( + matches!(error, StorageError::BucketNotFound(_)), + "identity mismatch must retain its typed error: {error:?}" + ); + for dir in &dirs { + assert!( + dir.path().join(bucket).join("inline").exists(), + "a stale request must leave every backup intact" + ); + } + } + #[tokio::test] + #[serial_test::serial(shard_integrity_rollout)] + async fn batch_delete_completed_multipart_reclaims_native_part_sidecars() { + assert_completed_multipart_delete_reclaims_sidecars(false).await; + } + + #[tokio::test] + #[serial_test::serial(shard_integrity_rollout)] + async fn batch_delete_completed_multipart_reclaims_protected_part_sidecars() { + assert_completed_multipart_delete_reclaims_sidecars(true).await; + } + + async fn assert_completed_multipart_delete_reclaims_sidecars(protected: bool) { + temp_env::async_with_vars( + [ + (rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some(if protected { "true" } else { "false" })), + ( + rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, + Some(if protected { "true" } else { "false" }), + ), + ], + assert_completed_multipart_delete_reclaims_sidecars_inner(protected), + ) + .await; + } + + async fn assert_completed_multipart_delete_reclaims_sidecars_inner(protected: bool) { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "multipart-delete-cleanup"; + let object = "completed-object"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let upload = store + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("create native multipart upload"); + let mut reader = PutObjReader::from_vec(vec![37; 6 * 1024 * 1024]); + let part = store + .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .expect("write native part"); + store + .clone() + .complete_multipart_upload( + bucket, + object, + &upload.upload_id, + vec![CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + .expect("complete native multipart upload"); + for dir in &dirs { + let bytes = tokio::fs::read(dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE)) + .await + .expect("completed metadata"); + let mut metadata = FileMeta::default(); + metadata.unmarshal_msg(&bytes).expect("decode completed metadata"); + let mut info = metadata.versions[0] + .into_fileinfo(bucket, object, true) + .expect("completed version"); + info.hydrate_shard_integrity() + .expect("decode native completed integrity descriptor"); + let path = dir + .path() + .join(bucket) + .join(object) + .join(info.data_dir.expect("external multipart directory").to_string()); + assert!(path.join("part.1.meta").exists(), "real multipart completion retains its native sidecar"); + let integrity = info.parts[0].integrity.as_ref(); + assert_eq!(integrity.is_some(), protected, "completion must retain the selected native format"); + if let Some(integrity) = integrity { + assert!( + path.join(integrity.file_name()).exists(), + "real protected completion retains its integrity proof" + ); + } + } + let (_, errors) = store + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await; + assert!(errors.iter().all(Option::is_none), "native batch delete must succeed: {errors:?}"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("successful multipart delete must permit ordinary bucket removal"); + for dir in &dirs { + assert!( + !dir.path().join(bucket).exists(), + "bucket and completed part sidecars must be removed on every disk" + ); + } + } +} + +#[cfg(test)] +mod delete_recovery_inventory_tests { + use super::*; + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use std::future::Future; + use std::task::Poll; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial] + async fn batch_delete_completes_with_queued_disk_renewal() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let set = store.pools[0].disk_set[0].clone(); + let disks = set.disk_inventory().await.into_iter().flatten().collect::>(); + let bucket = "batch-delete-renew"; + let object = "existing-inline-version"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative real bucket"); + store + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x73; 4096]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed actual version"); + let endpoint = set.set_endpoints[0].clone(); + let ctx = crate::runtime::global::current_ctx(); + let was_dist = ctx.is_dist_erasure().await; + if was_dist { + ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await; + } + let map = ctx.local_disk_map(); + let previous = map.read().await.get(&endpoint.to_string()).cloned(); + + let barrier = DeleteCleanupBarrier::install(bucket); + let delete_store = store.clone(); + let mut deletion = tokio::spawn(async move { + delete_store + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: object.to_owned(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await + }); + barrier.wait_until_paused().await; + let extra_gate = set.disks.read().await; + let renewal = set.renew_disk(&endpoint); + tokio::pin!(renewal); + tokio::time::timeout( + Duration::from_secs(5), + futures::future::poll_fn(|cx| { + assert!( + std::pin::pin!(tokio::task::unconstrained(renewal.as_mut())) + .poll(cx) + .is_pending(), + "real renewal must reach inventory write" + ); + if set.disks.try_read().is_err() { + Poll::Ready(()) + } else { + Poll::Pending + } + }), + ) + .await + .expect("queue real renewal writer"); + let connected = map + .read() + .await + .get(&endpoint.to_string()) + .cloned() + .flatten() + .expect("renewal registered new handle"); + assert!(!Arc::ptr_eq(&connected, &disks[0])); + drop(extra_gate); + barrier.release(); + let outcome = tokio::time::timeout(Duration::from_secs(3), async { tokio::join!(renewal.as_mut(), &mut deletion) }).await; + if outcome.is_err() { + deletion.abort(); + } + drop(renewal); + if outcome.is_err() { + let _ = deletion.await; + } + { + let mut map = map.write().await; + match previous { + Some(old) => { + map.insert(endpoint.to_string(), old); + } + None => { + map.remove(&endpoint.to_string()); + } + } + } + if was_dist { + ctx.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure) + .await; + } + let (_, result) = + outcome.expect("batch DELETE and real queued disk renewal must finish without nested inventory-read deadlock"); + let (_, errors) = result.expect("delete task must finish"); + assert!(errors.iter().all(Option::is_none), "DELETE result must remain successful: {errors:?}"); + } +} diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 8e20fa184..fa50a54b4 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -270,6 +270,134 @@ impl ECStore { .map_err(|error| Error::other_with_context("bucket heal owner task failed", error))? } + /// Recover hidden delete journals through the lifecycle-owned heal boundary. + pub async fn recover_delete_transactions(self: &Arc, bucket: &str, opts: &HealOpts) -> Result { + Ok(self + .recover_delete_transactions_at_incarnation(bucket, None, None, opts) + .await? + .unwrap_or_else(|| HealResultItem { + heal_item_type: HealItemType::Bucket.to_string(), + bucket: bucket.to_owned(), + detail: "delete recovery: discovered=0".to_owned(), + ..Default::default() + })) + } + + pub async fn recover_delete_transactions_at_incarnation( + self: &Arc, + bucket: &str, + object: Option<&str>, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.recover_delete_transactions_in_scope(bucket, object, None, expected, opts) + .await + } + + pub async fn recover_delete_prefix_at_incarnation( + self: &Arc, + bucket: &str, + prefix: &str, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.recover_delete_transactions_in_scope(bucket, None, Some(prefix), expected, opts) + .await + } + + async fn recover_delete_transactions_in_scope( + self: &Arc, + bucket: &str, + object: Option<&str>, + prefix: Option<&str>, + expected: Option, + opts: &HealOpts, + ) -> Result> { + if opts.no_lock && !opts.dry_run { + return Err(StorageError::InvalidArgument( + "heal".to_string(), + "no_lock".to_string(), + "delete recovery requires namespace locking".to_string(), + )); + } + let expected = match expected { + Some(expected) => expected, + None => self.bucket_incarnation_id_from_disk(bucket).await?, + }; + let object = object.map(str::to_owned); + let prefix = prefix.map(str::to_owned); + self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move { + let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?; + let explicit_delete = opts.remove + && metadata_sys::has_authoritative_never_versioned_state_in(&store.ctx, &bucket).await? + && matches!( + metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket).await?, + metadata_sys::ObjectLockConfigState::ConfirmedAbsent + ); + let movement_gate = store.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.read().await; + let can_commit = || scope.check().is_ok(); + let recovery_context = crate::set_disk::DeleteRecoveryContext { + dry_run: opts.dry_run, + explicit_delete, + namespace_owner: (!opts.dry_run) + .then(|| Arc::new((scope.clone(), store.ctx.begin_namespace_commit())) as Arc), + ..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &can_commit) + }; + let mut summary = crate::set_disk::DeleteRecoverySummary::default(); + for pool in store.get_pools_for_heal_object(&opts)? { + if store.is_suspended(pool.pool_idx).await || store.is_pool_rebalancing(pool.pool_idx).await { + return Err(StorageError::SlowDown); + } + for (set_index, set) in pool.disk_set.iter().enumerate() { + if opts.set.is_some_and(|requested| requested != set_index) { + continue; + } + if let Some(object) = &object { + if !Arc::ptr_eq(set, &pool.get_disks_by_key(object)) { + continue; + } + summary.add(set.recover_delete_object(&bucket, object, recovery_context.clone()).await?); + } else if let Some(prefix) = &prefix { + summary.add(set.recover_delete_prefix(&bucket, prefix, recovery_context.clone()).await?); + } else { + summary.add(set.recover_delete_bucket(&bucket, recovery_context.clone()).await?); + } + } + } + if summary.pending > 0 { + return Err(StorageError::ErasureReadQuorum); + } + if object.is_some() && !opts.dry_run && summary.blocked > 0 { + return Err(StorageError::SlowDown); + } + if summary.discovered == 0 { + return Ok(None); + } + Ok(Some(HealResultItem { + heal_item_type: if object.is_some() { + HealItemType::Object + } else { + HealItemType::Bucket + } + .to_string(), + object: object.or(prefix).unwrap_or_default(), + bucket, + detail: format!( + "delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}", + if opts.dry_run { " (dry run)" } else { "" }, + summary.discovered, + summary.committed, + summary.rolled_back, + summary.reclaimed, + summary.blocked + ), + ..Default::default() + })) + }) + .await + } + pub async fn heal_bucket_at_incarnation( self: &Arc, bucket: &str, diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 4d287a702..9e16d5b90 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -4849,7 +4849,9 @@ impl ECStore { opts: ObjectOptions, guards: impl IntoIterator, ) -> Result { - let guards: Vec<_> = guards.into_iter().collect(); + let guards = Arc::new(guards.into_iter().collect::>()); + let mut opts = opts; + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), guards.clone()))); let store = Arc::clone(self); let bucket = bucket.to_owned(); let object = object.to_owned(); @@ -4861,7 +4863,7 @@ impl ECStore { if result.is_ok() { list_objects::observe_list_objects_mutation(&store, &bucket).await; } - // Keep: the only use of `guards` here, it moves them into the task so the locks outlive a dropped caller. + // Transient options also retain these guards through late disk I/O. drop(guards); result }) @@ -4930,7 +4932,7 @@ impl ECStore { let mut opts = opts; let receipt_sink = install_tier_free_version_receipt_sink(&mut opts); let result = self - .handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api) + .handle_delete_object_with_journal_inner_boxed(bucket, object, opts, tier_journal_api) .await; enqueue_recorded_tier_free_versions(self, receipt_sink).await; result @@ -4938,6 +4940,16 @@ impl ECStore { .await } + fn handle_delete_object_with_journal_inner_boxed<'a>( + &'a self, + bucket: &'a str, + object: &'a str, + opts: ObjectOptions, + tier_journal_api: Option>, + ) -> impl std::future::Future> + Send + 'a { + Box::pin(self.handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api)) + } + async fn handle_delete_object_with_journal_inner( &self, bucket: &str, @@ -4955,9 +4967,9 @@ impl ECStore { if is_meta_bucketname(bucket) || (opts.delete_prefix && opts.bucket_lifecycle_lock_fence.is_some()) { None } else if opts.delete_prefix { - Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?) + Some(Arc::new(self.acquire_bucket_lifecycle_write_lock(bucket).await?)) } else { - Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?) + Some(Arc::new(self.acquire_bucket_lifecycle_read_lock(bucket).await?)) }; let object = if opts.delete_prefix && !opts.delete_prefix_object { object.to_owned() @@ -4975,6 +4987,7 @@ impl ECStore { opts.tier_delete_journal_api = tier_journal_api.clone(); if let Some(guard) = _bucket_lifecycle_guard.as_ref() { opts.add_bucket_lifecycle_lock_guard(guard); + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), guard.clone()))); } if !is_meta_bucketname(bucket) { @@ -5227,6 +5240,40 @@ impl ECStore { return Ok(obj); } Err(err) => { + if (is_err_object_not_found(&err) || is_err_version_not_found(&err)) + && (opts.version_id.is_some() || (!opts.versioned && !opts.version_suspended)) + && opts.http_preconditions.is_none() + && !opts.replication_request + { + let mut reclaimed = None; + for pool in &self.pools { + if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await { + return Err(StorageError::SlowDown); + } + let pool_idx = pool.pool_idx; + let pool = pool.clone(); + match self + .run_external_decommission_capacity_object_delete( + pool_idx, + bucket, + object, + object, + opts.clone(), + |opts| async move { pool.delete_object(bucket, object, opts).await }, + ) + .await + { + Ok(result) => reclaimed = Some(result), + Err(missing) if is_err_object_not_found(&missing) || is_err_version_not_found(&missing) => {} + Err(error) => return Err(error), + } + } + if let Some(mut result) = reclaimed { + result.bucket = bucket.to_owned(); + result.name = decode_dir_object(object); + return Ok(result); + } + } // A folder key (`prefix/`) with no object metadata may still exist on // disk as an orphan empty-directory tree (issue #4189): listings show // it as a common prefix, but no regular delete path can remove it. @@ -5422,12 +5469,13 @@ impl ECStore { None } else { match self.acquire_bucket_lifecycle_read_lock(bucket).await { - Ok(guard) => Some(guard), + Ok(guard) => Some(Arc::new(guard)), Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), } }; if let Some(guard) = _bucket_lifecycle_guard.as_ref() { opts.add_bucket_lifecycle_lock_guard(guard); + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), guard.clone()))); } if opts.delete_replication_config_snapshot.is_none() { match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await { @@ -5493,10 +5541,11 @@ impl ECStore { if current_bucket_incarnation_id.is_some() { pause_delete_after_object_lock_snapshot(bucket).await; } - let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await { + let _object_lock_guards = Arc::new(match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await { Ok(guards) => guards, Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), - }; + }); + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), _object_lock_guards.clone()))); #[cfg(any(test, feature = "test-util"))] if !_object_lock_guards.is_empty() { notify_delete_namespace_acquired(bucket); @@ -9241,8 +9290,31 @@ mod tests { .expect("read original marker"); assert_eq!(info.delete_marker_incarnation(), Some(old_incarnation)); } - // Taking four actual members offline preserves their existing xl.meta. - // All deletion/recreation work below goes through the normal store API. + // Save each stale member's inline versions before converging the old + // bucket's delete receipts, then reintroduce them after recreation. + let mut stale_metadata = Vec::new(); + for disk in before.iter().skip(12).flatten() { + for version in &historical { + let info = disk + .read_version( + "", + bucket, + "history.bin", + &version.to_string(), + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read stale inline data version"); + assert!(info.inline_data() && info.data.is_some(), "fixture payload must be preserved in xl.meta"); + } + stale_metadata.push(( + disk.read_all(bucket, "history.bin/xl.meta").await.unwrap(), + disk.read_all(bucket, &format!("{object}/xl.meta")).await.unwrap(), + )); + } for slot in 12..16 { set.disks.write().await[slot] = None; } @@ -9272,6 +9344,29 @@ mod tests { ) .await .expect("delete exact old marker with four drives offline"); + let pending = store + .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) + .await + .expect_err("offline participants require retained delete receipts"); + assert!(matches!( + pending, + StorageError::BucketNotEmpty(_) | StorageError::BucketNotEmptyWithDetails { .. } + )); + assert!( + !crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation) + .await + .unwrap() + ); + *set.disks.write().await = before.clone(); + store + .recover_delete_transactions_at_incarnation( + bucket, + None, + Some(old_incarnation), + &rustfs_heal_contracts::heal_channel::HealOpts::default(), + ) + .await + .expect("converge old bucket deletes on all participants"); store .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) .await @@ -9281,6 +9376,9 @@ mod tests { .await .expect("durable retirement") ); + for slot in 12..16 { + set.disks.write().await[slot] = None; + } store .handle_make_bucket(bucket, &create) .await @@ -9306,6 +9404,13 @@ mod tests { .expect("write new generation"); new_versions.push((key, info.version_id.unwrap(), payload)); } + for (slot, (history, marker)) in (12..16).zip(stale_metadata) { + for (key, bytes) in [("history.bin", history), (object, marker)] { + let root = dirs[slot].path().join(bucket).join(key); + tokio::fs::create_dir_all(&root).await.unwrap(); + tokio::fs::write(root.join("xl.meta"), bytes).await.unwrap(); + } + } *set.disks.write().await = before.clone(); let heal = rustfs_heal_contracts::heal_channel::HealOpts { remove: true, @@ -11477,3 +11582,160 @@ mod tests { .await; } } + +#[cfg(test)] +mod missing_key_delete_quorum_tests { + use super::*; + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::storage_api_contracts::bucket::{BucketOperations, BucketOptions, MakeBucketOptions}; + + async fn degraded_store(bucket: &str) -> (Vec, Arc) { + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative never-versioned bucket"); + store.pools[0].disk_set[0].disks.write().await[0] = None; + let info = store + .get_bucket_info(bucket, &BucketOptions::default()) + .await + .expect("existing bucket must remain readable at quorum"); + assert_eq!(info.name, bucket); + let lookup = store + .get_object_info_for_delete(bucket, "never-existed", &ObjectOptions::default()) + .await; + assert!( + lookup.as_ref().is_err_and(is_err_object_not_found), + "remaining disks must establish absence through production lookup: {lookup:?}" + ); + (dirs, store) + } + + #[tokio::test] + async fn single_delete_missing_key_preserves_quorum_noop() { + let (_dirs, store) = degraded_store("missing-key-quorum-single").await; + let result = store + .delete_object_with_tier_delete_journal("missing-key-quorum-single", "never-existed", ObjectOptions::default()) + .await; + assert!( + result.as_ref().is_err_and(is_err_object_not_found), + "single missing-key delete must preserve its idempotent not-found outcome at quorum: {result:?}" + ); + } + + #[tokio::test] + async fn batch_delete_missing_key_preserves_quorum_noop() { + let (_dirs, store) = degraded_store("missing-key-quorum-batch").await; + let (_, errors, _) = store + .delete_objects_with_tier_delete_journal_and_accounting( + "missing-key-quorum-batch", + vec![ObjectToDelete { + object_name: "never-existed".to_owned(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await; + assert_eq!(errors.len(), 1, "batch DELETE must return one result per key"); + assert!( + errors.iter().all(Option::is_none), + "batch missing-key delete must preserve its idempotent success at quorum: {errors:?}" + ); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn batch_delete_timeout_retains_physical_key_guard() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + use crate::disk::{DiskAPI, ReadOptions}; + use crate::storage_api_contracts::object::ObjectIO; + temp_env::async_with_vars([("RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS", Some("30"))], async { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "review-batch-late-owner"; + let key = "object"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("fixture bucket"); + let mut reader = PutObjReader::from_vec(vec![0x51; 1024 * 1024]); + store + .put_object( + bucket, + key, + &mut reader, + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed real external object"); + let disks = store.pools[0].disk_set[0].disk_inventory().await; + let disk = disks[0].as_ref().expect("online drive"); + let info = disk + .read_version("", bucket, key, "", &ReadOptions::default()) + .await + .expect("seed metadata"); + assert!(!info.inline_data()); + let crate::disk::Disk::Local(local) = disk.as_ref() else { + panic!("local disk"); + }; + let part = local + .get_disk() + .get_object_path_for_io(bucket, key) + .expect("leased key") + .join(info.data_dir.expect("external data directory").to_string()) + .join("part.1"); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Remove, &part, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let deleting = store.clone(); + let delete = tokio::spawn(async move { + deleting + .delete_objects_with_tier_delete_journal_and_accounting( + bucket, + vec![ObjectToDelete { + object_name: key.to_owned(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .expect("physical cleanup entry") + .expect("entry hook"); + assert!(part.exists(), "payload unlink has not run"); + let (_, errors, _) = tokio::time::timeout(Duration::from_secs(40), delete) + .await + .expect("real settlement timeout must return") + .expect("delete task"); + assert!(errors.iter().all(Option::is_none), "certified batch response: {errors:?}"); + assert!( + store.ctx.namespace_commits_pending(), + "background physical operation must retain its owner" + ); + let ns = store.handle_new_ns_lock(bucket, key).await.expect("same production key lock"); + let probe = ns.get_write_lock(Duration::from_millis(300)).await; + let acquired_while_pending = probe.is_ok(); + drop(probe); + drop(release); + drop(hook); + assert!( + !acquired_while_pending, + "batch released the exact-key guard while physical cleanup was still paused after its real recovery timeout" + ); + let _released = ns + .get_write_lock(Duration::from_secs(10)) + .await + .expect("physical cleanup must eventually release the exact-key guard"); + }) + .await; + } +} diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index 051159c0e..35facd16b 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -574,6 +574,29 @@ pub trait HealStorageAPI: Send + Sync { /// Heal bucket using ecstore async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result; + /// Recover transactions omitted by ordinary listings. Alternate providers + /// may explicitly expose no transaction-recovery capability. + async fn recover_delete_transactions( + &self, + _bucket: &str, + _object: Option<&str>, + _expected: Option, + _opts: &HealOpts, + ) -> Result> { + Ok(None) + } + + /// Recover hidden transactions only within the requested object prefix. + async fn recover_delete_prefix_transactions( + &self, + _bucket: &str, + _prefix: &str, + _expected: Option, + _opts: &HealOpts, + ) -> Result> { + Ok(None) + } + /// Heal format using ecstore async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)>; @@ -1498,6 +1521,32 @@ impl HealStorageAPI for ECStoreHealStorage { .await) } + async fn recover_delete_transactions( + &self, + bucket: &str, + object: Option<&str>, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.ecstore + .recover_delete_transactions_at_incarnation(bucket, object, expected, opts) + .await + .map_err(Error::Storage) + } + + async fn recover_delete_prefix_transactions( + &self, + bucket: &str, + prefix: &str, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.ecstore + .recover_delete_prefix_at_incarnation(bucket, prefix, expected, opts) + .await + .map_err(Error::Storage) + } + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { debug!( target: "rustfs::heal::storage", diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index 73bc20291..cf6b4786f 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -220,6 +220,22 @@ impl HealTask { set: self.options.set_index, }; + let recovery_opts = HealOpts { + remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted, + ..heal_opts + }; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_transactions( + bucket, + None, + self.bucket_incarnation_id, + &recovery_opts, + )) + .await? + { + self.record_result_item(result).await; + } + let heal_result = match self.bucket_incarnation_id { Some(expected) => { self.await_with_control(self.storage.heal_bucket_at_incarnation(bucket, expected, &heal_opts)) @@ -446,6 +462,26 @@ impl HealTask { "Heal prefix started" ); + let opts = HealOpts { + dry_run: self.options.dry_run, + remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted, + no_lock: self.options.no_lock, + pool: self.options.pool_index, + set: self.options.set_index, + ..Default::default() + }; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_prefix_transactions( + bucket, + prefix, + self.bucket_incarnation_id, + &opts, + )) + .await? + { + self.record_result_item(result).await; + } + self.heal_bucket_objects(bucket, prefix).await } diff --git a/crates/heal/src/heal/task/heal_erasure_set.rs b/crates/heal/src/heal/task/heal_erasure_set.rs index df9b852e9..376fc5314 100644 --- a/crates/heal/src/heal/task/heal_erasure_set.rs +++ b/crates/heal/src/heal/task/heal_erasure_set.rs @@ -460,6 +460,21 @@ impl HealTask { set: self.options.set_index, }; + let recovery_opts = HealOpts { + remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted, + ..bucket_heal_opts + }; + // Include metadata-less transactions for both startup and admin heals. + for bucket in &buckets { + self.check_control_flags().await?; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_transactions(bucket, None, None, &recovery_opts)) + .await? + { + self.record_result_item(result).await; + } + } + for bucket in buckets.iter().filter(|_| self.source != HealRequestSource::Admin) { // Check control flags before starting each bucket heal self.check_control_flags().await?; diff --git a/crates/heal/src/heal/task/heal_object.rs b/crates/heal/src/heal/task/heal_object.rs index 0dc4ec298..e20d4f315 100644 --- a/crates/heal/src/heal/task/heal_object.rs +++ b/crates/heal/src/heal/task/heal_object.rs @@ -99,6 +99,40 @@ impl HealTask { // recreated bucket after the replay anchor was captured. return self.heal_mrf_partial_write_object(bucket, object, version_id).await; } + let heal_opts = HealOpts { + recursive: self.options.recursive, + dry_run: self.options.dry_run, + remove: self.options.remove_corrupted, + recreate: self.options.recreate_missing, + scan_mode: self.options.scan_mode, + update_parity: self.options.update_parity, + no_lock: self.options.no_lock, + read_repair: self.source == HealRequestSource::ReadRepair, + pool: self.options.pool_index, + set: self.options.set_index, + }; + let mut recovery_handled = false; + if matches!(self.source, HealRequestSource::Admin | HealRequestSource::Internal) { + let recovery_opts = HealOpts { + remove: heal_opts.remove && version_id.is_none_or(str::is_empty), + ..heal_opts + }; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_transactions( + bucket, + Some(object), + self.bucket_incarnation_id, + &recovery_opts, + )) + .await? + { + recovery_handled = true; + self.record_result_item(result).await; + if self.options.dry_run { + return Ok(()); + } + } + } let mut object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await { Ok(exists) => exists, Err(err @ Error::TransientSkip { .. }) => { @@ -129,7 +163,7 @@ impl HealTask { object }; - if !object_exists { + if !object_exists && !recovery_handled { // Background loops (scanner/MRF/autoheal/read-repair) routinely // race object deletion, so a missing target is per-object noise // for them; only foreground admin/internal requests keep the warn. @@ -197,18 +231,6 @@ impl HealTask { update_parity = self.options.update_parity, "Heal object stage entered" ); - let heal_opts = HealOpts { - recursive: self.options.recursive, - dry_run: self.options.dry_run, - remove: self.options.remove_corrupted, - recreate: self.options.recreate_missing, - scan_mode: self.options.scan_mode, - update_parity: self.options.update_parity, - no_lock: self.options.no_lock, - read_repair: self.source == HealRequestSource::ReadRepair, - pool: self.options.pool_index, - set: self.options.set_index, - }; let mut expected_identity = self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index); expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?; diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 309f485d3..f5c12e5ba 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -527,6 +527,29 @@ pub struct DeletePathsResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +/// Protocol 1 is additive and has no fallback to generic file mutation RPCs. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RecoverDeleteTransactionRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(uint32, tag = "3")] + pub protocol_version: u32, + #[prost(bytes = "bytes", tag = "4")] + pub operation: ::prost::bytes::Bytes, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RecoverDeleteTransactionResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(uint32, tag = "2")] + pub protocol_version: u32, + #[prost(bytes = "bytes", tag = "3")] + pub result: ::prost::bytes::Bytes, + #[prost(message, optional, tag = "4")] + pub error: ::core::option::Option, +} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnapshotLeaseRequest { #[prost(string, tag = "1")] @@ -2082,6 +2105,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "DeleteAtIncarnation")); self.inner.unary(req, path, codec).await } + pub async fn recover_delete_transaction( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RecoverDeleteTransaction"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "RecoverDeleteTransaction")); + self.inner.unary(req, path, codec).await + } pub async fn acquire_snapshot_lease( &mut self, request: impl tonic::IntoRequest, @@ -3463,6 +3501,10 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn recover_delete_transaction( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn acquire_snapshot_lease( &self, request: tonic::Request, @@ -4237,6 +4279,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/RecoverDeleteTransaction" => { + #[allow(non_camel_case_types)] + struct RecoverDeleteTransactionSvc(pub Arc); + impl tonic::server::UnaryService for RecoverDeleteTransactionSvc { + type Response = super::RecoverDeleteTransactionResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::recover_delete_transaction(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RecoverDeleteTransactionSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/AcquireSnapshotLease" => { #[allow(non_camel_case_types)] struct AcquireSnapshotLeaseSvc(pub Arc); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 451afa21a..5b69f697f 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -1259,6 +1259,37 @@ pub fn canonical_prepare_part_transaction_request_body( Ok(body.finish()) } +/// Recovery messages are bounded and must contain exactly one MessagePack value. +pub const DELETE_RECOVERY_MAX_PAYLOAD_BYTES: usize = DEFAULT_GRPC_SERVER_MESSAGE_LEN; +pub const DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES: usize = 1024 * 1024; + +pub fn decode_delete_recovery_payload(payload: &[u8]) -> std::io::Result { + if payload.is_empty() || payload.len() > DELETE_RECOVERY_MAX_PAYLOAD_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid delete recovery payload length", + )); + } + let mut cursor = std::io::Cursor::new(payload); + let value = rmp_serde::from_read(&mut cursor) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid delete recovery payload"))?; + if cursor.position() != u64::try_from(payload.len()).map_err(std::io::Error::other)? { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "trailing delete recovery payload")); + } + Ok(value) +} + +pub fn canonical_recover_delete_transaction_request_body( + request: &proto_gen::node_service::RecoverDeleteTransactionRequest, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-recover-delete-transaction-request-v1\0"); + body.push_str(&request.disk)?; + body.push_str(&request.volume)?; + body.push_u32(request.protocol_version); + body.push_bytes(&request.operation)?; + Ok(body.finish()) +} + pub fn canonical_settle_part_transaction_request_body( request: &proto_gen::node_service::SettlePartTransactionRequest, ) -> Result, std::num::TryFromIntError> { @@ -1313,6 +1344,40 @@ mod disk_mutation_canonical_tests { }; use super::*; + #[test] + fn delete_recovery_canonical_body_binds_all_wire_fields() { + let request = proto_gen::node_service::RecoverDeleteTransactionRequest { + disk: "disk".into(), + volume: "volume".into(), + protocol_version: 1, + operation: vec![0x80].into(), + }; + let mut bodies = vec![canonical_recover_delete_transaction_request_body(&request).expect("canonical body")]; + for mutate in [ + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.disk = "other".into(), + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.volume = "other".into(), + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.protocol_version = 2, + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.operation = vec![0x81].into(), + ] { + let mut changed = request.clone(); + mutate(&mut changed); + bodies.push(canonical_recover_delete_transaction_request_body(&changed).expect("changed canonical body")); + } + assert_all_distinct(&bodies); + } + + #[test] + fn delete_recovery_payload_rejects_trailing_malformed_and_oversized_values() { + let payload = rmp_serde::to_vec_named(&vec![1_u32, 2]).expect("encode test payload"); + assert_eq!(decode_delete_recovery_payload::>(&payload).expect("one payload"), vec![1, 2]); + let mut trailing = payload; + trailing.push(0xc0); + assert!(decode_delete_recovery_payload::>(&trailing).is_err()); + assert!(decode_delete_recovery_payload::>(&[]).is_err()); + assert!(decode_delete_recovery_payload::>(&[0xc1]).is_err()); + assert!(decode_delete_recovery_payload::>(&vec![0xc0; DELETE_RECOVERY_MAX_PAYLOAD_BYTES + 1]).is_err()); + } + fn assert_all_distinct(bodies: &[Vec]) { for (i, a) in bodies.iter().enumerate() { for (j, b) in bodies.iter().enumerate() { @@ -2905,7 +2970,7 @@ pub async fn get_channel_for_class(addr: &str, class: ChannelClass) -> Result Result Result> { /// channels to the same peer be cached independently. async fn build_channel(dial_addr: &str, cache_key: &str) -> Result> { let deadline = tokio::time::Instant::now() + internode_connect_timeout(); - build_channel_until(dial_addr, cache_key, deadline).await + build_channel_until(dial_addr, cache_key, deadline, internode_rpc_timeout()).await +} + +/// A separate channel deadline for slow authenticated recovery work. The cache +/// key keeps this deadline from changing foreground lock, read, and write RPCs. +#[derive(Debug)] +pub enum DeleteRecoveryChannelError { + Connection(Box), +} + +impl std::fmt::Display for DeleteRecoveryChannelError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Connection(error) => write!(formatter, "delete recovery channel connection failed: {error}"), + } + } +} + +impl Error for DeleteRecoveryChannelError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Connection(error) => Some(error.as_ref()), + } + } +} + +pub async fn create_channel_with_timeout(addr: &str, rpc_timeout: Duration) -> Result { + let cache_key = format!("delete-recovery:{}:{addr}", rpc_timeout.as_millis()); + let deadline = tokio::time::Instant::now() + internode_connect_timeout(); + build_channel_until(addr, &cache_key, deadline, rpc_timeout) + .await + .map_err(DeleteRecoveryChannelError::Connection) } async fn build_channel_until( dial_addr: &str, cache_key: &str, deadline: tokio::time::Instant, + rpc_timeout: Duration, ) -> Result> { tokio::time::timeout_at(deadline, async { if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { @@ -2962,7 +3059,7 @@ async fn build_channel_until( if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { return Ok(channel); } - let (channel, generation) = dial_channel(dial_addr, cache_key).await?; + let (channel, generation) = dial_channel(dial_addr, cache_key, rpc_timeout).await?; if cache_dialed_channel(dial_addr, cache_key, &channel, generation).await && let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { @@ -2980,15 +3077,13 @@ async fn build_channel_until( .map_err(channel_acquisition_timeout)? } -async fn dial_channel(dial_addr: &str, cache_key: &str) -> Result<(Channel, Option), Box> { +async fn dial_channel(dial_addr: &str, cache_key: &str, rpc_timeout: Duration) -> Result<(Channel, Option), Box> { debug!("Creating new gRPC channel to: {} (cache key: {})", dial_addr, cache_key); let dial_started_at = Instant::now(); let connect_timeout = internode_connect_timeout(); let tcp_keepalive = internode_tcp_keepalive(); let http2_keepalive_interval = internode_http2_keep_alive_interval(); let http2_keepalive_timeout = internode_http2_keep_alive_timeout(); - let rpc_timeout = internode_rpc_timeout(); - let mut connector = Endpoint::from_shared(dial_addr.to_string())? // Fast connection timeout for dead peer detection .connect_timeout(connect_timeout) @@ -3291,6 +3386,31 @@ mod tests { } } + #[tokio::test] + async fn tls02_delete_recovery_channels_reject_unbound_and_stale_https_cache() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve recovery peer address"); + let addr = format!("https://{}", listener.local_addr().expect("recovery peer address")); + drop(listener); + let rpc_timeout = Duration::from_secs(20); + let cache_key = format!("delete-recovery:{}:{addr}", rpc_timeout.as_millis()); + let generation = runtime_sources::outbound_tls_generation(); + for cached_generation in [None, Some(generation.checked_add(1).unwrap_or(0))] { + cache_connection(cache_key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + if let Some(generation) = cached_generation { + TLS_GENERATION_CACHE.lock().await.insert(cache_key.clone(), generation); + } else { + TLS_GENERATION_CACHE.lock().await.remove(&cache_key); + } + assert!( + create_channel_with_timeout(&addr, rpc_timeout).await.is_err(), + "recovery must dial the unavailable HTTPS peer instead of returning an unbound or stale cached channel" + ); + assert!(!rustfs_common::has_cached_connection(&cache_key).await); + assert!(!TLS_GENERATION_CACHE.lock().await.contains_key(&cache_key)); + } + } + #[tokio::test] async fn tls02_current_https_control_and_bulk_cache_entries_are_reused() { let _test_guard = TLS02_TEST_LOCK.lock().await; @@ -3444,7 +3564,7 @@ mod tests { for _ in 0..8 { tasks.push(tokio::spawn(async move { let deadline = tokio::time::Instant::now() + Duration::from_millis(40); - let result = build_channel_until(addr, addr, deadline).await; + let result = build_channel_until(addr, addr, deadline, internode_rpc_timeout()).await; result.err().is_some_and(|error| { error .downcast_ref::() diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index a99f73185..fd56c543c 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -391,6 +391,21 @@ message DeletePathsResponse { optional Error error = 2; } +// Protocol 1 is additive and has no fallback to generic file mutation RPCs. +message RecoverDeleteTransactionRequest { + string disk = 1; + string volume = 2; + uint32 protocol_version = 3; + bytes operation = 4; +} + +message RecoverDeleteTransactionResponse { + bool success = 1; + uint32 protocol_version = 2; + bytes result = 3; + optional Error error = 4; +} + message SnapshotLeaseRequest { string disk = 1; string volume = 2; @@ -1218,6 +1233,7 @@ service NodeService { rpc CompareAndUpdateFile(CompareAndUpdateFileRequest) returns (CompareAndUpdateFileResponse) {}; // auth-policy: body-bound rpc Delete(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound rpc DeleteAtIncarnation(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound + rpc RecoverDeleteTransaction(RecoverDeleteTransactionRequest) returns (RecoverDeleteTransactionResponse) {}; // auth-policy: body-bound rpc AcquireSnapshotLease(SnapshotLeaseRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound rpc RenewSnapshotLease(SnapshotLeaseRenewRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound rpc ReleaseSnapshotLease(SnapshotLeaseReleaseRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound diff --git a/docs/README.md b/docs/README.md index 2b89a015d..d30e3758a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -47,6 +47,9 @@ For concurrent bucket creation and deletion, see For persisted administrator bucket tasks and bucket recreation, see [Bucket heal recovery](operations/bucket-heal-recovery.md). +For interrupted object deletions and returning drives, see +[Delete transaction recovery](operations/delete-transaction-recovery.md). + For disk replacement across VM restarts and schema 5/6 maintenance migration, see [Replacement generation recovery](operations/replacement-generation-recovery.md). diff --git a/docs/operations/delete-transaction-recovery.md b/docs/operations/delete-transaction-recovery.md new file mode 100644 index 000000000..175eb84f1 --- /dev/null +++ b/docs/operations/delete-transaction-recovery.md @@ -0,0 +1,137 @@ +# Delete transaction recovery + +An interrupted object deletion can leave metadata backups, reserved shards, or +an unpublished target. Normal S3 listings cannot discover all of these files. +Recovery therefore inspects raw transaction records on every configured drive +in the owning erasure set under the bucket incarnation fence and exact-key +write lock. It does not modify GET or PUT behavior. + +## Decisions and replay + +New deletions assemble the backup, versioned prepared journal, and exact target +in the private system temporary area, then atomically publish the complete +transaction directory before modifying metadata. Interrupted preparation cannot +leave an undiscoverable target-only record or temporary file in the object key. After the metadata write quorum, the coordinator publishes durable +commit receipts to a write quorum before returning success. A failed or lost +publication response is indeterminate: recovery must inspect the persisted +records instead of immediately restoring the backup. + +| Evidence | Action | +| --- | --- | +| New prepared transaction, no decision, intact original shards and matching generation | Restore and sync the original metadata before removing preparation evidence. | +| Structured commit receipt | Publish the certified target to matching returning participants and finish deletion. | +| Missing, unavailable, or unsupported participant | Preserve records and retry when every configured participant can be inspected. | +| Newer generation or partially overlapping version history | Preserve records and report blocked recovery. | +| Historical backup or disk-local marker without a structured set decision | Preserve records during automatic recovery. | + +Missing local metadata does not authorize user-bucket residue cleanup, even +when the current object exists on a metadata quorum. Legacy local cleanup is +restricted to internal system-bucket maintenance. User-bucket legacy residue +uses only the guarded set-wide removal described below. Dry-run and execution +share the same source-generation check; an already-absent explicit non-null +version can converge without touching unrelated live versions. + +Rollback preserves native multipart metadata and integrity sidecars. Protected +parts require their declared proof generation; recovery verifies both the proof +records and actual shard payloads before restoring metadata or removing +recovery evidence. Missing or corrupt required proofs leave the transaction +blocked with its backup intact. + +A disk-local `delete-data.` marker does not prove an erasure-set decision. +Returning drives cannot restore stale data after a certified deletion. Commit +receipts remain until every participant and related transaction for the key +has converged. For consecutive certified version deletions, an older decision +can finish its own data cleanup after later deletion removes current metadata. +A prepared copy needs a matching same-transaction decision before promotion; +absence alone cannot certify it. Cross-transaction null-version supersession also requires causal +predecessor evidence; timestamps alone cannot authorize it. Cleanup respects active snapshots and removes only validated +transaction files, with directory removal performed nonrecursively. + +## Entry points + +Unclean-shutdown erasure-set healing performs raw transaction recovery before +ordinary object listing. Administrator bucket, set, prefix, and object heal use the +same recovery core. Existing heal selection, lifecycle admission, namespace +locking, and movement guards still apply. Prefix recovery settles only matching +keys, including transactions hidden from listings. Dry-run inspects and reports without +settling transactions. + +An unconditional DELETE of a missing exact key, or an administrator heal with +`remove` enabled, can authorize historical residue removal only when the +bucket is authoritatively never-versioned and has no Object Lock. Every +configured drive must be inspected, the current object must be absent, and all +residue must match the guarded legacy layout. Versioned/suspended buckets, +retained versions, conditions, unknown entries, symlinks, and prefix children +are not covered by that authorization. The same fresh intent can certify the +absence of an unprotected new prepared null-version transaction when every +configured drive has no current metadata. This is a new authorized decision; +it does not infer an old commit from missing causal evidence. Mixed or +unrecognized historical layouts remain blocked. + +A missing specific-version DELETE first recovers hidden transactions and then +reads the current version again. It must not acknowledge an unresolved hidden +version as an idempotent no-op. + +## Upgrade and failure handling + +Upgrade all participating nodes before relying on the new protocol. Recovery +uses one authenticated additive RPC; unsupported peers cannot silently accept +new flags through the legacy delete endpoint. An online older participant +causes the modern operation to fail before decision publication. Modern nodes +also reject legacy forward transactional user-bucket deletes. A cluster still +controlled by an older majority retains the older behavior. Do not downgrade +while recovery records remain; old binaries cannot interpret the new protocol. + +Keep unavailable drives and transaction directories intact. Bring the full +set online, run a scoped heal in dry-run mode, inspect its recovery result, +and then run the same scoped heal without dry-run. Use `remove` for ambiguous +historical residue only after verifying that the exact objects are intended +to be deleted and the bucket meets the restrictions above. Do not recursively +remove transaction directories by hand to make a bucket appear empty. + +Recovery reports discovered, committed, rolled-back, reclaimed historical, +and blocked transaction counts. Unavailable participants return a retryable +quorum error; blocked evidence requires operator investigation. A successful +S3 delete response does not imply immediate physical cleanup while snapshots +or returning participants still retain necessary proof. + +## Settlement deadline and durability + +`RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS` controls the settlement RPC deadline, +including full shard verification during rollback. Key and bucket lifecycle +ownership stay attached to late local filesystem operations after cancellation +or timeout. The default is 900 seconds; +valid values are 30 through 86400 seconds. Invalid values fail explicitly. +Increase it when intact large objects need more verification time. It does not +change normal object read or delete-apply deadlines. A timeout can conceal a +completed mutation, so rerun inspection and recovery rather than guessing the +outcome from the RPC error. + +The records and restores follow the configured durability policy. Strict mode +syncs the required metadata and directory changes. Process SIGKILL testing +checks restart behavior; it does not establish power-loss behavior on every +filesystem or storage device. Mode `none` intentionally offers weaker sync +guarantees. Do not treat a throughput comparison as a durability substitute. + +## Reproduce the Docker regression suite + +Build the candidate with the test-only barriers enabled: + +```sh +cargo build --locked --release -p rustfs --bin rustfs --no-default-features --features e2e-test-hooks +python3 scripts/test/delete_recovery_docker.py --help +``` + +Pass a native Linux binary with `--binary`, a Linux image containing Python and +curl with `--image`, and a unique Docker namespace. The optional +`--baseline-binary` enables the mixed-version case. The harness uses synthetic +S3 objects, localhost ports, labeled isolated volumes, and a non-root server. +It stops only its own containers and keeps JSON results and logs in the chosen +artifact directory. Production builds must not enable `e2e-test-hooks`. + +Run the preactivation/prepare/apply/decision process-stop cases, interrupted restore, +returning and untouched peers, both generation orderings, version/retention +and prefix protection, administrator dry-run/removal, response-loss, +permission/corruption, and mixed-version cases. Preserve the source identity +of both binaries with the results. A passing process-stop matrix does not +cover hardware power cuts, Windows runtime behavior, or DELETE throughput. diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index c93f7bdd1..f57c4ac6e 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -4167,6 +4167,51 @@ mod tests { ); } + #[tokio::test] + #[serial_test::serial] + async fn cold_delete_recovery_discovery_uses_production_body_auth() { + use storage::DiskAPI; + temp_env::async_with_vars([("RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT", Some("false"))], async { + let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string()); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind recovery RPC peer"); + let addr = listener.local_addr().expect("recovery RPC address"); + let previous_node_name = rustfs_common::get_global_local_node_name().await; + rustfs_common::set_global_local_node_name(&addr.to_string()).await; + let node = InterceptedService::new(NodeServiceServer::new(make_server()), check_auth); + let service = RpcRequestPathService::new(Routes::new(node).prepare()); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.expect("accept recovery RPC connection"); + ConnBuilder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(socket), TowerToHyperService::new(service)) + .await + .expect("serve recovery RPC connection"); + }); + let mut endpoint = + Endpoint::try_from(format!("http://{addr}/cold-recovery-disk").as_str()).expect("remote recovery endpoint"); + endpoint.is_local = false; + let disk = storage::new_disk( + &endpoint, + &storage::DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("production remote disk client"); + // Recovery may race a heal-control peer-state refresh, before another + // authenticated response has repopulated the shared peer state. + let result = disk.discover_delete_transactions("cold-recovery-bucket", None, None, 1).await; + server.abort(); + let _ = server.await; + rustfs_common::set_global_local_node_name(&previous_node_name).await; + assert!( + matches!(result, Err(crate::storage_api::error::DiskError::DiskNotFound)), + "the cold request must pass production authentication and reach the missing-disk handler: {result:?}" + ); + }) + .await; + } + #[tokio::test] #[serial_test::serial] async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() { diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 54f2315b4..255bf1044 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -1536,6 +1536,13 @@ impl Node for NodeService { self.handle_rename_part(request).await } + async fn recover_delete_transaction( + &self, + request: Request, + ) -> Result, Status> { + self.handle_recover_delete_transaction(request).await + } + async fn settle_part_transaction( &self, request: Request, @@ -3073,13 +3080,14 @@ mod tests { LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, - ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, - RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest, - ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, - SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, - StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, - TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, - UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest, + ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest, ReloadPoolMetaRequest, + ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, ScannerActivityRequest, + ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, + ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest, + SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, + StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, TierMutationPeerState, + TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, WriteRequest, heal_control_service_client::HealControlServiceClient, heal_control_service_server::{HealControlService as _, HealControlServiceServer}, node_service_client::NodeServiceClient, @@ -3097,7 +3105,7 @@ mod tests { use tonic::{Request, Response, Status}; use uuid::Uuid; - const DISK_MUTATION_RPC_METHODS: [&str; 24] = [ + const DISK_MUTATION_RPC_METHODS: [&str; 25] = [ "renamedata", "renamedataatincarnation", "writemetadataatincarnation", @@ -3116,6 +3124,7 @@ mod tests { "renamepart", "prepareparttransaction", "settleparttransaction", + "recoverdeletetransaction", "deletevolume", "makevolume", "makevolumes", @@ -4386,6 +4395,177 @@ mod tests { assert!(!signed.into_inner().success, "the unknown test disk cannot resolve"); } + #[tokio::test] + async fn delete_recovery_apply_rejects_excessive_batch_cardinality_before_lookup() { + use crate::storage::storage_api::{DeleteOptions, DeleteRecoveryRpcOperation, FileInfoVersions}; + let service = make_server(); + for versions in [ + vec![FileInfoVersions::default(); 1001], + vec![FileInfoVersions { + name: "object".into(), + versions: vec![Default::default(); 1001], + ..Default::default() + }], + ] { + let operation = DeleteRecoveryRpcOperation::ApplyVersions { + versions, + opts: DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + ..Default::default() + }, + }; + let mut request = Request::new(RecoverDeleteTransactionRequest { + disk: "unknown".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: rmp_serde::to_vec_named(&operation).expect("encode batch").into(), + }); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical batch"); + set_tonic_canonical_body_digest(&mut request, &body).expect("batch digest"); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("oversized batch must reject before disk lookup"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + } + } + + #[tokio::test] + async fn delete_recovery_apply_rejects_legacy_cleanup_options_and_absolute_keys() { + use crate::storage::storage_api::{DeleteOptions, DeleteRecoveryRpcOperation}; + let service = make_server(); + let valid = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + ..Default::default() + }; + let mut invalid = vec![DeleteOptions::default()]; + for mutate in [ + |opts: &mut DeleteOptions| opts.old_data_dir = Some(Uuid::nil()), + |opts: &mut DeleteOptions| opts.undo_delete = true, + |opts: &mut DeleteOptions| opts.undo_write = true, + |opts: &mut DeleteOptions| opts.recursive = true, + |opts: &mut DeleteOptions| opts.immediate = true, + ] { + let mut opts = valid.clone(); + mutate(&mut opts); + invalid.push(opts); + } + let mut operations = Vec::new(); + for opts in invalid { + operations.push(DeleteRecoveryRpcOperation::ApplyVersion { + object: "object".into(), + file_info: Default::default(), + force_del_marker: false, + opts: opts.clone(), + }); + operations.push(DeleteRecoveryRpcOperation::ApplyVersions { + versions: Vec::new(), + opts, + }); + } + operations.push(DeleteRecoveryRpcOperation::ApplyVersion { + object: "/object".into(), + file_info: Default::default(), + force_del_marker: false, + opts: valid.clone(), + }); + operations.push(DeleteRecoveryRpcOperation::ApplyVersions { + versions: vec![crate::storage::storage_api::FileInfoVersions { + name: "/object".into(), + ..Default::default() + }], + opts: valid, + }); + for operation in operations { + let mut request = Request::new(RecoverDeleteTransactionRequest { + disk: "unknown".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: rmp_serde::to_vec_named(&operation).expect("encode apply operation").into(), + }); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical apply"); + set_tonic_canonical_body_digest(&mut request, &body).expect("apply digest"); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("invalid apply must reject before disk lookup"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + } + } + + #[tokio::test] + async fn delete_recovery_requires_digest_and_valid_protocol_payload() { + let service = make_server(); + let operation = crate::storage::storage_api::DeleteRecoveryRpcOperation::Discover { + object: None, + cursor: None, + limit: 1, + }; + let message = RecoverDeleteTransactionRequest { + disk: "unknown".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: rmp_serde::to_vec_named(&operation).expect("encode discovery").into(), + }; + let unsigned = service + .recover_delete_transaction(Request::new(message.clone())) + .await + .expect_err("new RPC must reject unsigned requests"); + assert_eq!(unsigned.code(), tonic::Code::PermissionDenied); + for digest in ["", "UNSIGNED-PAYLOAD"] { + let mut request = Request::new(message.clone()); + request + .metadata_mut() + .insert("x-rustfs-content-sha256", digest.parse().expect("metadata")); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("missing or unsigned digest"); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + } + let mut invalid = message.clone(); + invalid.protocol_version = 2; + let mut malformed = message.clone(); + malformed.operation = vec![0xc1].into(); + let mut trailing = message.clone(); + let mut payload = trailing.operation.to_vec(); + payload.push(0xc0); + trailing.operation = payload.into(); + for (message, expected) in [ + (invalid, tonic::Code::Unimplemented), + (malformed, tonic::Code::InvalidArgument), + (trailing, tonic::Code::InvalidArgument), + ] { + let mut request = Request::new(message); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical recovery"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("invalid recovery input"); + assert_eq!(error.code(), expected); + } + let mut request = Request::new(message); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical recovery"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let response = service + .recover_delete_transaction(request) + .await + .expect("valid request reaches disk lookup") + .into_inner(); + assert!(!response.success); + assert!(response.error.is_some()); + } + /// Per-handler wiring check for every mutating disk RPC. A mismatched digest must be rejected /// (catches a handler that omits its `verify_disk_mutation_digest` gate) and a correctly /// body-bound digest must pass the gate (catches a handler wired to the wrong @@ -4659,6 +4839,16 @@ mod tests { }, rustfs_protos::canonical_prepare_part_transaction_request_body ); + assert_gated!( + recover_delete_transaction, + RecoverDeleteTransactionRequest { + disk: disk.clone(), + volume: "dst".into(), + protocol_version: 1, + operation: vec![0x80].into() + }, + rustfs_protos::canonical_recover_delete_transaction_request_body + ); assert_gated!( settle_part_transaction, SettlePartTransactionRequest { @@ -6597,7 +6787,7 @@ mod tests { }; let rollback_dir = Uuid::new_v4(); remote - .delete_version( + .delete_version_recoverable( bucket, object, marker.clone(), diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index c7e73aecc..22c1a88c1 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -19,7 +19,10 @@ use crate::storage::storage_api::rpc_consumer::node_service::{ validate_batch_read_version_item_count, }; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; -use crate::storage::storage_api::{PartTransactionAction, RenameDataResp, SnapshotLeaseToken, verify_tonic_mutation_body_digest}; +use crate::storage::storage_api::{ + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, PartTransactionAction, RenameDataResp, + SnapshotLeaseToken, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, +}; use bytes::Bytes; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::{ @@ -118,6 +121,16 @@ const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512; const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024; const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1; +fn rejects_legacy_delete_transaction(volume: &str, opts: &DeleteOptions) -> bool { + !(volume == ".rustfs.sys" + || volume.starts_with(".rustfs.sys/") + || volume == ".minio.sys" + || volume.starts_with(".minio.sys/")) + && opts.old_data_dir.is_some() + && !opts.recoverable_delete + && !opts.undo_write +} + fn snapshot_lease_response(result: Result) -> Response { match result { Ok(token) => Response::new(SnapshotLeaseResponse { @@ -715,6 +728,14 @@ impl NodeService { } }; + if rejects_legacy_delete_transaction(&request.volume, &opts) { + return Ok(Response::new(DeleteVersionsResponse { + success: false, + errors: Vec::new(), + error: Some(DiskError::UnsupportedDisk.into()), + item_errors: Vec::new(), + })); + } let (errors, item_errors) = encode_delete_versions_errors(disk.delete_versions(&request.volume, versions, opts).await); @@ -782,6 +803,13 @@ impl NodeService { })); } }; + if rejects_legacy_delete_transaction(&request.volume, &opts) { + return Ok(Response::new(DeleteVersionResponse { + success: false, + raw_file_info: String::new(), + error: Some(DiskError::UnsupportedDisk.into()), + })); + } if require_marker_condition && opts.expected_delete_marker.is_none() { return Err(Status::invalid_argument("retired marker deletion requires a marker precondition")); } @@ -1578,6 +1606,131 @@ impl NodeService { } } + pub(super) async fn handle_recover_delete_transaction( + &self, + request: Request, + ) -> Result, Status> { + if request.get_ref().operation.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES { + return Err(Status::resource_exhausted("delete recovery operation exceeds payload limit")); + } + let body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()) + .map_err(|_| Status::invalid_argument("delete recovery request length cannot be represented"))?; + // This additive protocol has no digestless rolling-upgrade exception. + verify_tonic_canonical_body_digest(&request, &body) + .map_err(|_| Status::permission_denied("delete recovery requires body-bound v2 authentication"))?; + let request = request.into_inner(); + if request.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION { + return Err(Status::unimplemented("incompatible delete recovery protocol")); + } + let operation: DeleteRecoveryRpcOperation = rustfs_protos::decode_delete_recovery_payload(&request.operation) + .map_err(|_| Status::invalid_argument("invalid delete recovery operation"))?; + if request.operation.len() > operation.max_wire_payload_bytes() { + return Err(Status::resource_exhausted("delete recovery operation exceeds payload limit")); + } + match &operation { + DeleteRecoveryRpcOperation::ApplyVersions { versions, .. } + if versions.len() > 1000 + || versions + .iter() + .try_fold(0_usize, |count, versions| count.checked_add(versions.versions.len())) + .is_none_or(|count| count > 1000) => + { + return Err(Status::invalid_argument("recoverable batch delete exceeds 1000 object/version requests")); + } + DeleteRecoveryRpcOperation::Discover { limit, .. } if *limit == 0 || *limit > 256 => { + return Err(Status::invalid_argument("delete recovery page limit must be between 1 and 256")); + } + DeleteRecoveryRpcOperation::Inspect { transaction, .. } | DeleteRecoveryRpcOperation::Settle { transaction, .. } + if transaction.is_nil() => + { + return Err(Status::invalid_argument("delete transaction must be a non-nil UUID")); + } + DeleteRecoveryRpcOperation::ApplyVersion { opts, .. } | DeleteRecoveryRpcOperation::ApplyVersions { opts, .. } + if opts.old_data_dir.is_none_or(|transaction| transaction.is_nil()) + || opts.undo_delete + || opts.undo_write + || opts.recursive + || opts.immediate + || opts.expected_delete_marker.is_some() + || opts + .delete_recovery_bucket_incarnation + .is_some_and(|incarnation| incarnation.is_nil()) => + { + return Err(Status::invalid_argument( + "recoverable delete requires a non-nil transaction and valid apply options", + )); + } + DeleteRecoveryRpcOperation::ApplyVersion { object, .. } if object.starts_with('/') => { + return Err(Status::invalid_argument("recoverable delete requires a relative object key")); + } + DeleteRecoveryRpcOperation::ApplyVersions { versions, .. } + if versions.iter().any(|version| version.name.starts_with('/')) => + { + return Err(Status::invalid_argument("recoverable delete requires relative object keys")); + } + _ => {} + } + let result = if let Some(disk) = self.find_disk(&request.disk).await { + match operation { + DeleteRecoveryRpcOperation::ApplyVersion { + object, + file_info, + force_del_marker, + opts, + } => disk + .delete_version_recoverable(&request.volume, &object, *file_info, force_del_marker, opts) + .await + .map(|()| DeleteRecoveryRpcReply::AppliedVersion), + DeleteRecoveryRpcOperation::ApplyVersions { versions, opts } => { + let errors = disk.delete_versions_recoverable(&request.volume, versions, opts).await; + Ok(DeleteRecoveryRpcReply::AppliedVersions( + errors.into_iter().map(|error| error.map(|error| error.to_u32())).collect(), + )) + } + DeleteRecoveryRpcOperation::Discover { object, cursor, limit } => disk + .discover_delete_transactions(&request.volume, object.as_deref(), cursor, limit) + .await + .map(DeleteRecoveryRpcReply::Discovered), + DeleteRecoveryRpcOperation::Inspect { object, transaction } => disk + .inspect_delete_transaction(&request.volume, &object, transaction) + .await + .map(DeleteRecoveryRpcReply::Inspected), + DeleteRecoveryRpcOperation::Settle { + object, + transaction, + expected_fingerprint, + action, + } => disk + .settle_delete_transaction(&request.volume, &object, transaction, expected_fingerprint, action) + .await + .map(DeleteRecoveryRpcReply::Settled), + } + } else { + Err(DiskError::DiskNotFound) + }; + match result { + Ok(reply) => { + let result = encode_msgpack_named(&reply, "delete recovery reply") + .map_err(|_| Status::internal("cannot encode delete recovery reply"))?; + if result.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES { + return Err(Status::resource_exhausted("delete recovery reply exceeds payload limit")); + } + Ok(Response::new(RecoverDeleteTransactionResponse { + success: true, + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + result: result.into(), + error: None, + })) + } + Err(error) => Ok(Response::new(RecoverDeleteTransactionResponse { + success: false, + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + result: Bytes::new(), + error: Some(error.into()), + })), + } + } + pub(super) async fn handle_settle_part_transaction( &self, request: Request, @@ -2007,9 +2160,10 @@ impl NodeService { #[cfg(test)] mod tests { use super::{ - compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, + DeleteOptions, compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, encode_batch_read_version_response_payloads, encode_delete_versions_errors, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, encode_read_multiple_response_payloads, encode_rename_data_response_payloads, + rejects_legacy_delete_transaction, }; use crate::storage::DiskError; use crate::storage::rpc::node_service::make_server; @@ -2022,6 +2176,7 @@ mod tests { use serde::{Deserialize, Serialize}; use serial_test::serial; use tonic::Request; + use uuid::Uuid; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct SamplePayload { @@ -2029,6 +2184,26 @@ mod tests { count: u32, } + #[test] + fn legacy_delete_transaction_policy_preserves_rollback_and_internal_maintenance() { + let mut opts = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + ..Default::default() + }; + assert!(rejects_legacy_delete_transaction("bucket", &opts)); + assert!(!rejects_legacy_delete_transaction(".rustfs.sys", &opts)); + assert!(!rejects_legacy_delete_transaction(".rustfs.sys/tmp", &opts)); + assert!(!rejects_legacy_delete_transaction(".minio.sys", &opts)); + opts.undo_write = true; + assert!(!rejects_legacy_delete_transaction("bucket", &opts)); + opts.undo_write = false; + opts.recoverable_delete = true; + assert!(!rejects_legacy_delete_transaction("bucket", &opts)); + opts.recoverable_delete = false; + opts.old_data_dir = None; + assert!(!rejects_legacy_delete_transaction("bucket", &opts)); + } + #[test] fn response_compat_send_sites_keep_manifest_json_encoders() { // Rolling-upgrade contract (rustfs-protos compat manifest): every diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index adc83492a..3edbe7d4d 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -478,11 +478,12 @@ pub(crate) mod ecstore_data_usage { #[allow(unused_imports)] pub(crate) mod ecstore_disk { pub(crate) use rustfs_ecstore::api::disk::{ - BUCKET_META_PREFIX, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DeleteOptions, - DiskAPI, DiskInfo, DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, OldCurrentSize, - PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, - validate_batch_read_version_item_count, + BUCKET_META_PREFIX, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, + DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, + DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction, + RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, + UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, validate_batch_read_version_item_count, }; #[cfg(test)] pub(crate) use rustfs_ecstore::api::disk::{DiskOption, new_disk}; @@ -734,6 +735,14 @@ pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq; pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp; pub(crate) type ReadOptions = ecstore_disk::ReadOptions; pub(crate) type OldCurrentSize = ecstore_disk::OldCurrentSize; +pub(crate) type DeleteRecoveryCursor = ecstore_disk::DeleteRecoveryCursor; +pub(crate) type DeleteRecoveryPage = ecstore_disk::DeleteRecoveryPage; +pub(crate) type DeleteRecoveryInspection = ecstore_disk::DeleteRecoveryInspection; +pub(crate) type DeleteRecoveryDigest = ecstore_disk::DeleteRecoveryDigest; +pub(crate) type DeleteRecoveryAction = ecstore_disk::DeleteRecoveryAction; +pub(crate) type DeleteRecoveryRpcOperation = ecstore_disk::DeleteRecoveryRpcOperation; +pub(crate) type DeleteRecoveryRpcReply = ecstore_disk::DeleteRecoveryRpcReply; +pub(crate) const DELETE_RECOVERY_PROTOCOL_VERSION: u32 = ecstore_disk::DELETE_RECOVERY_PROTOCOL_VERSION; pub(crate) type PartTransactionAction = ecstore_disk::PartTransactionAction; pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp; pub(crate) type ReplicationStatusType = ecstore_bucket::replication::ReplicationStatusType; @@ -1444,6 +1453,41 @@ pub(crate) trait StorageDiskRpcExt { dst_path: &str, meta: bytes::Bytes, ) -> DiskResult<()>; + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: rustfs_filemeta::FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> DiskResult<()>; + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec>; + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> DiskResult; + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + ) -> DiskResult; + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> DiskResult; async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()>; async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()>; async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; @@ -1616,6 +1660,53 @@ where ecstore_disk::DiskAPI::prepare_part_transaction(self, src_volume, src_path, dst_volume, dst_path, meta).await } + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: rustfs_filemeta::FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete_version_recoverable(self, volume, object, file_info, force_del_marker, opts).await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + ecstore_disk::DiskAPI::delete_versions_recoverable(self, volume, versions, opts).await + } + + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> DiskResult { + ecstore_disk::DiskAPI::discover_delete_transactions(self, volume, object, cursor, limit).await + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + ) -> DiskResult { + ecstore_disk::DiskAPI::inspect_delete_transaction(self, volume, object, transaction).await + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> DiskResult { + ecstore_disk::DiskAPI::settle_delete_transaction(self, volume, object, transaction, expected_fingerprint, action).await + } + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()> { ecstore_disk::DiskAPI::settle_part_transaction(self, volume, path, action).await } diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 2e51ba37e..ecc668e0d 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -180,6 +180,8 @@ pub(crate) mod server { } pub(crate) mod http { + #[cfg(test)] + pub(crate) use crate::storage::storage_api::ecstore_disk::{DiskAPI, DiskOption, new_disk}; pub(crate) use crate::storage::storage_api::{ ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_tonic_rpc_signature_with_bootstrap, diff --git a/scripts/test/delete_recovery_docker.py b/scripts/test/delete_recovery_docker.py new file mode 100644 index 000000000..c9bb96b32 --- /dev/null +++ b/scripts/test/delete_recovery_docker.py @@ -0,0 +1,1257 @@ +#!/usr/bin/env python3 +"""Run delete recovery regressions in disposable local Docker containers. + +Requires host Python 3.9+, Docker, and curl with --aws-sigv4. Supply a native +Linux binary compiled with e2e-test-hooks and a Linux image containing Python 3 +and that binary's runtime libraries. All S3 requests target localhost with +synthetic credentials. Container processes run as UID 10001. The harness labels +its resources, refuses existing volumes, preserves JSON reports/server logs, +and removes only containers, networks, and volumes it created. + +Example: + python3 scripts/test/delete_recovery_docker.py --binary /absolute/path/to/rustfs-candidate --baseline-binary /absolute/path/to/rustfs-old --image local-rustfs-validation:latest --artifact-dir /absolute/path/to/new-results smoke prepared1 applied1 decision1 restore returning untouched mixed + +Available cases include single/multi-node process termination, inline/empty +objects, legacy explicit cleanup, versioned and retained objects, UUID prefix +children, corrupted journals, real permission errors, lost receipt responses, +admin bucket/object repair, and deep keys. Use --help for runtime options. +Logical disks share the Docker host filesystem; this tests process/RPC failure +and namespace safety, not independent hardware failure or power-loss durability. +""" +from pathlib import Path +import base64, hashlib, ipaddress, json, os, subprocess, tempfile, threading, time, uuid, xml.etree.ElementTree as ET +from urllib.parse import quote +ROOT = None +PREFIX = None +IMAGE = None +PORT = 19010 +CANDIDATE = None +BASELINE = None +ACCESS = 'recovery-test' +SECRET = 'recovery-test-secret' + +def run(*args, check=True, **kwargs): + return subprocess.run(args, check=check, capture_output=True, **kwargs) + +def docker(*args, check=True): + return run('docker', *args, check=check) + +class S3: + + def __init__(self, port): + self.url = f'http://127.0.0.1:{port}' + + def request(self, method, path='', body=None, headers=None, expected=None, timeout=60): + call = ['curl', '--noproxy', '*', '--silent', '--show-error', '--max-time', str(timeout), '--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', ACCESS + ':' + SECRET, '--request', method] + for (key, value) in (headers or {}).items(): + call += ['--header', key + ': ' + value] + if body is not None: + call += ['--data-binary', '@-'] + with tempfile.NamedTemporaryFile(dir=ROOT) as header_file: + response = run(*call, self.url + '/' + path, '--dump-header', header_file.name, '--write-out', '\n%{http_code}', input=body, check=False) + self.last_headers = {} + for line in Path(header_file.name).read_text(errors='replace').splitlines(): + if ':' in line: + (key, value) = line.split(':', 1) + self.last_headers[key.lower()] = value.strip() + if response.returncode: + raise RuntimeError(response.stderr.decode(errors='replace')) + (payload, status) = response.stdout.rsplit(b'\n', 1) + code = int(status) + if expected is not None and code not in ((expected,) if isinstance(expected, int) else expected): + raise AssertionError(f'{method} {path}: {code}, expected {expected}: {payload[:1500]!r}') + return (code, payload) + + def put_bucket(self, bucket, lock=False): + headers = {'x-amz-bucket-object-lock-enabled': 'true'} if lock else {} + return self.request('PUT', bucket, headers=headers, expected=200) + + def put(self, bucket, key, body, headers=None): + return self.request('PUT', bucket + '/' + key, body, headers, 200) + + def get(self, bucket, key): + return self.request('GET', bucket + '/' + key, expected=200)[1] + + def delete(self, bucket, key, expected=(204, 200)): + return self.request('DELETE', bucket + '/' + key, expected=expected) + + def delete_bucket(self, bucket, expected=204): + return self.request('DELETE', bucket, expected=expected) + + def list(self, bucket): + (_, body) = self.request('GET', bucket + '?list-type=2', expected=200) + root = ET.fromstring(body) + return [node.text for node in root.iter() if node.tag.rsplit('}', 1)[-1] == 'Key'] + + def versioning(self, bucket, state='Enabled'): + xml = f'{state}'.encode() + return self.request('PUT', bucket + '?versioning', xml, {'Content-Type': 'application/xml'}, 200) + + def batch_delete(self, bucket, keys): + xml = ET.Element('Delete') + for key in keys: + ET.SubElement(ET.SubElement(xml, 'Object'), 'Key').text = key + data = ET.tostring(xml) + (status, body) = self.request('POST', bucket + '?delete', data, {'Content-Type': 'application/xml', 'Content-MD5': base64.b64encode(hashlib.md5(data).digest()).decode()}, 200) + response = ET.fromstring(body) + errors = [node for node in response.iter() if node.tag.rsplit('}', 1)[-1] == 'Error'] + assert not errors, f'DeleteObjects returned per-key errors: {body!r}' + return (status, body) + + def complete_multipart(self, bucket, key, parts): + path = bucket + '/' + key + (_, response) = self.request('POST', path + '?uploads', expected=200) + upload = next(node.text for node in ET.fromstring(response).iter() if node.tag.rsplit('}', 1)[-1] == 'UploadId') + assert upload, 'native multipart initiation must return an upload ID' + query = '?uploadId=' + quote(upload, safe='') + complete = ET.Element('CompleteMultipartUpload') + for number, body in enumerate(parts, 1): + self.request('PUT', path + query + '&partNumber=' + str(number), body, expected=200) + etag = self.last_headers.get('etag') + assert etag, 'native UploadPart must return its ETag' + part = ET.SubElement(complete, 'Part') + ET.SubElement(part, 'PartNumber').text = str(number) + ET.SubElement(part, 'ETag').text = etag + (_, response) = self.request('POST', path + query, ET.tostring(complete), {'Content-Type': 'application/xml'}, 200) + assert ET.fromstring(response).tag.rsplit('}', 1)[-1] == 'CompleteMultipartUploadResult', response + return upload + +class Cluster: + + def __init__(self, case, nodes=1, drives=4, extra_env=None, binary_by_node=None): + self.case = case + self.nodes = nodes + self.drives = drives + self.port = PORT + self.binary = 'candidate' + self.binary_by_node = binary_by_node or {} + self.names = [f'{PREFIX}-{case}-n{i}' for i in range(1, nodes + 1)] + self.network = f'{PREFIX}-{case}-net' + self.env = extra_env or {} + self.created_names = [] + self.created_network = False + self.clients = [S3(self.port + i) for i in range(nodes)] + self.volumes = [] + self.run_args = {} + self.barriers = [ROOT / 'barriers' / case / f'n{i}' for i in range(1, nodes + 1)] + for directory in self.barriers: + directory.mkdir(parents=True, exist_ok=True) + directory.chmod(511) + + def create(self): + docker('network', 'create', '--label', 'rustfs.delete-recovery-test=' + PREFIX, self.network) + self.created_network = True + configuration = json.loads(docker('network', 'inspect', self.network).stdout)[0]['IPAM']['Config'] + subnet = next(ipaddress.ip_network(value['Subnet']) for value in configuration if ':' not in value['Subnet']) + addresses = [str(subnet[index + 2]) for index in range(self.nodes)] + urls = [] + for name in self.names: + for drive in range(1, self.drives + 1): + urls.append(f'http://{name}:9000/data/d{drive}') + for (idx, name) in enumerate(self.names): + args = ['run', '-d', '--user', '10001:10001', '--name', name, '--network', self.network, '--hostname', name, '-p', f'127.0.0.1:{self.port + idx}:9000', '--label', 'rustfs.delete-recovery-test=' + PREFIX, '-v', str(CANDIDATE) + ':/binaries/candidate:ro', '-v', str(self.barriers[idx]) + ':/barrier'] + # Stopped peers must still resolve while the remaining quorum restarts. + args += ['--ip', addresses[idx]] + for peer, address in zip(self.names, addresses): + args += ['--add-host', peer + ':' + address] + if BASELINE is not None: + args += ['-v', str(BASELINE) + ':/binaries/baseline:ro'] + data_mounts = [] + for drive in range(1, self.drives + 1): + volume = f'{name}-d{drive}' + if docker('volume', 'inspect', volume, check=False).returncode == 0: + raise RuntimeError('refusing existing volume: ' + volume) + docker('volume', 'create', '--label', 'rustfs.delete-recovery-test=' + PREFIX, volume) + self.volumes.append(volume) + args += ['-v', f'{volume}:/data/d{drive}'] + data_mounts += ['-v', f'{volume}:/data/d{drive}'] + docker('run', '--rm', *data_mounts, IMAGE, 'python3', '-c', "from pathlib import Path; import os; [os.chown(p,10001,10001) for p in Path('/data').glob('d*')]") + values = {'RUSTFS_VOLUMES': ' '.join(urls) if self.nodes > 1 else ' '.join((f'/data/d{d}' for d in range(1, self.drives + 1))), 'RUSTFS_ADDRESS': '0.0.0.0:9000', 'RUSTFS_CONSOLE_ENABLE': 'false', 'RUSTFS_ACCESS_KEY': ACCESS, 'RUSTFS_SECRET_KEY': SECRET, 'RUSTFS_OBS_LOG_DIRECTORY': '/tmp/logs', 'RUSTFS_OBS_LOGGER_LEVEL': 'info', 'RUSTFS_OBS_LOG_STDOUT_ENABLED': 'true', 'RUSTFS_DELETE_RECOVERY_BARRIER_DIR': '/barrier', 'RUSTFS_UNSAFE_BYPASS_DISK_CHECK': 'true', 'RUSTFS_DURABILITY_MODE': 'strict', **self.env} + for (key, value) in values.items(): + if value is not None: + args += ['-e', key + '=' + value] + self.run_args[idx] = [*args, IMAGE, '/binaries/' + self.binary_by_node.get(idx, self.binary)] + launch = docker(*self.run_args[idx], check=False) + if launch.returncode: + label = docker('inspect', '--format', '{{index .Config.Labels "rustfs.delete-recovery-test"}}', name, check=False) + if label.returncode == 0 and label.stdout.decode().strip() == PREFIX: + self.created_names.append(name) + raise RuntimeError('Docker launch failed: ' + launch.stderr.decode(errors='replace')) + self.created_names.append(name) + self.ready() + + def ready(self, timeout=180, indices=None): + deadline = time.monotonic() + timeout + pending = set(range(self.nodes) if indices is None else indices) + while pending and time.monotonic() < deadline: + for idx in list(pending): + try: + (code, _) = self.clients[idx].request('GET', '', timeout=2) + if code in (200, 403): + pending.remove(idx) + except Exception: + pass + if pending: + for idx in pending: + state = docker('inspect', '--format', '{{.State.Running}}', self.names[idx], check=False) + if state.returncode == 0 and state.stdout.strip() != b'true': + self.logs() + raise RuntimeError('server exited during startup: ' + self.names[idx]) + time.sleep(0.2) + if pending: + self.logs() + raise TimeoutError('cluster not ready: ' + str(pending)) + + def kill(self, index=0): + docker('kill', '--signal', 'KILL', self.names[index]) + + def restart(self, index=None): + for i in range(self.nodes) if index is None else [index]: + docker('start', self.names[i]) + self.ready() + + def replace_node(self, index, binary): + docker('kill', '--signal', 'KILL', self.names[index], check=False) + docker('rm', self.names[index]) + args = self.run_args[index].copy() + args[-1] = '/binaries/' + binary + docker(*args) + self.run_args[index] = args + self.ready(indices=[index]) + + def logs(self): + for name in self.created_names: + response = docker('logs', name, check=False) + (ROOT / (name + '.log')).write_bytes(response.stdout + response.stderr) + + def inspect_data(self, code, indices=None): + """Access only this cluster's named volumes, including stopped peers.""" + result = [] + for index, name in enumerate(self.names): + if indices is not None and index not in indices: + continue + response = docker('run', '--rm', '--volumes-from', name, IMAGE, 'python3', '-c', code) + result.append(json.loads(response.stdout)) + return result + + def remove(self): + """Keep logs and delete only resources successfully created here.""" + self.logs() + for name in self.created_names: + docker('rm', '-f', name, check=False) + for volume in self.volumes: + docker('volume', 'rm', volume, check=False) + if self.created_network: + docker('network', 'rm', self.network, check=False) + +def smoke(): + cluster = Cluster('smoke', drives=1) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'recovery-smoke' + client.put_bucket(bucket) + for (key, size) in [('zero', 0), ('inline', 4096), ('external', 2 * 1024 * 1024)]: + payload = bytes(range(256)) * (size // 256) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + client.batch_delete(bucket, ['zero', 'inline', 'external']) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': 's3-smoke', 'status': 'passed', 'topology': '1 node / 1 named local volume'} + finally: + cluster.remove() + +def seed_legacy(cluster, bucket, key, shape): + transaction = str(uuid.uuid4()) + if shape == 'alias': + directories = {path.split('/')[0] for node in residue_inventory(cluster, bucket, key) for drive in node for path in drive['files'] if path.endswith('/part.1')} + assert len(directories) == 1, 'alias fixture requires one shared data-directory generation' + transaction = directories.pop() + code = "from pathlib import Path\nimport hashlib,json,os,uuid\nbucket=%r; key=%r; shape=%r; transaction=%r\nresult=[]\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/bucket/key\n original=root/'xl.meta'\n if not original.is_file(): continue\n data=original.read_bytes(); backup=root/transaction/'xl.meta.bkp'\n backup.parent.mkdir(exist_ok=shape=='alias'); backup.write_bytes(data); os.chown(backup.parent,10001,10001); os.chown(backup,10001,10001)\n dirs=[p for p in root.iterdir() if p.is_dir() and (p.name!=transaction or shape=='alias') and any(p.glob('part.*'))]\n if shape in ('reserved','alias'):\n assert dirs, 'external fixture must contain erasure parts'\n for d in dirs:\n marker=d/('reserve-delete-data.'+transaction); marker.write_bytes(b''); os.chown(marker,10001,10001)\n elif shape=='backup-only':\n assert not dirs, 'inline fixture must not contain external parts'\n else: raise AssertionError(shape)\n original.unlink()\n owned=[root,*root.rglob('*')]\n assert all(p.stat().st_uid==10001 and p.stat().st_gid==10001 for p in owned), [(str(p),p.stat().st_uid,p.stat().st_gid) for p in owned]\n result.append({'drive':drive.name,'transaction':transaction,'backup_sha256':hashlib.sha256(data).hexdigest(),'data_dirs':[p.name for p in dirs],'fixture_owner':'10001:10001','fixture_modes':{str(p.relative_to(root)):oct(p.stat().st_mode & 0o777) for p in owned}})\nprint(json.dumps(result))\n" % (bucket, key, shape, transaction) + return cluster.inspect_data(code) + +def residue_inventory(cluster, bucket, key): + return cluster.inspect_data("from pathlib import Path\nimport hashlib,json\nresult=[]\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/%r/%r\n result.append({'drive':drive.name,'exists':root.exists(),'files':{str(p.relative_to(root)):hashlib.sha256(p.read_bytes()).hexdigest() for p in root.rglob('*') if p.is_file()}})\nprint(json.dumps(result))\n" % (bucket, key)) + +def legacy_retry(nodes=1): + case = 'legacy' + str(nodes) + cluster = Cluster(case, nodes=nodes, drives=1 if nodes == 1 else 2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'legacy-recovery' + client.put_bucket(bucket) + original = {key: bytes(range(256)) * (size // 256) for (key, size) in [('inline', 4096), ('reserved', 2 * 1024 * 1024)]} + for (key, payload) in original.items(): + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + for i in range(nodes): + cluster.kill(i) + fixtures = {key: seed_legacy(cluster, bucket, key, 'backup-only' if key == 'inline' else 'reserved') for key in original} + before = {key: residue_inventory(cluster, bucket, key) for key in original} + cluster.restart() + time.sleep(2) + after = {key: residue_inventory(cluster, bucket, key) for key in original} + assert before == after, 'automatic recovery must not guess an unjournaled historical outcome' + client.batch_delete(bucket, list(original)) + for key in original: + current = residue_inventory(cluster, bucket, key) + assert all((not drive['exists'] for node in current for drive in node)), f'current explicit DELETE must reclaim {key}: {current}' + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': case, 'status': 'passed', 'fixtures': fixtures, 'topology': f'{nodes} nodes'} + finally: + cluster.remove() + +def arm_barrier(cluster, index, bucket, key, stage, disk=None): + nonce = str(uuid.uuid4()) + arm = {'nonce': nonce, 'bucket': bucket, 'object': key, 'stage': stage} + if disk is not None: + arm['disk'] = disk + (cluster.barriers[index] / 'arm.json').write_text(json.dumps(arm)) + return nonce + +def reached_barrier(cluster, index, nonce, timeout=45): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + for path in cluster.barriers[index].glob('reached.' + nonce + '.*.json'): + try: + return json.loads(path.read_text()) + except (json.JSONDecodeError, OSError): + pass + time.sleep(0.02) + cluster.logs() + raise TimeoutError('crash stage not reached: ' + nonce) + +def clear_barriers(cluster): + for directory in cluster.barriers: + (directory / 'arm.json').unlink(missing_ok=True) + +def async_delete(client, bucket, key): + result = {} + + def target(): + try: + result['response'] = client.request('DELETE', bucket + '/' + key, timeout=90) + except Exception as error: + result['exception'] = str(error) + thread = threading.Thread(target=target, daemon=True) + thread.start() + return (thread, result) + +def wait_inventory(cluster, bucket, key, predicate, timeout=60): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = residue_inventory(cluster, bucket, key) + if predicate(last): + return last + time.sleep(0.1) + raise AssertionError('recovery did not converge: ' + json.dumps(last)) + +def process_crash(stage, nodes=1, size=2 * 1024 * 1024): + case = ('inline-' if size == 4096 else 'zero-' if size == 0 else 'kill-') + stage.lower() + '-' + str(nodes) + cluster = Cluster(case, nodes=nodes, drives=4 if nodes == 1 else 2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'journal-crash' + key = 'payload' + client.put_bucket(bucket) + payload = bytes(range(256)) * (size // 256) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + original = residue_inventory(cluster, bucket, key) + nonce = arm_barrier(cluster, 0, bucket, key, stage, disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + for index in range(nodes): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + cluster.restart() + if stage == 'DECISION': + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + else: + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': case, 'status': 'passed', 'observation': observation, 'request': {key: str(value) for (key, value) in request.items()}} + finally: + cluster.remove() + +def rollback_crash(): + cluster = Cluster('kill-restore', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'rollback-restart' + key = 'payload' + client.put_bucket(bucket) + payload = bytes(range(256)) * 8192 + client.put(bucket, key, payload) + original = residue_inventory(cluster, bucket, key) + first_nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + first = reached_barrier(cluster, 0, first_nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + second_nonce = arm_barrier(cluster, 0, bucket, key, 'RESTORE', disk='/data/d1') + docker('start', cluster.names[0]) + second = reached_barrier(cluster, 0, second_nonce) + cluster.kill() + clear_barriers(cluster) + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'rollback-second-process-kill', 'status': 'passed', 'observations': [first, second]} + finally: + cluster.remove() + +def versioned_crash(stage): + cluster = Cluster('versions-' + stage.lower(), drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'versioned-recovery' + key = 'history' + client.put_bucket(bucket) + client.versioning(bucket) + first = b'old-version-' * 200000 + second = b'new-version-' * 300 + client.put(bucket, key, first) + v1 = client.last_headers.get('x-amz-version-id') + assert v1 and v1 != 'null' + client.put(bucket, key, second) + v2 = client.last_headers.get('x-amz-version-id') + assert v2 and v2 != v1 + assert client.get(bucket, key + '?versionId=' + v1) == first + assert client.get(bucket, key) == second + original = residue_inventory(cluster, bucket, key) + nonce = arm_barrier(cluster, 0, bucket, key, stage, disk='/data/d1') + (thread, request) = async_delete(client, bucket, key + '?versionId=' + v1) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + if stage == 'APPLIED': + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key + '?versionId=' + v1) == first + client.delete(bucket, key + '?versionId=' + v1) + else: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + (code, _) = client.request('GET', bucket + '/' + key + '?versionId=' + v1) + if code == 404: + break + time.sleep(0.1) + else: + raise AssertionError('committed version deletion was resurrected') + assert client.get(bucket, key) == second + assert client.get(bucket, key + '?versionId=' + v2) == second + client.delete(bucket, key + '?versionId=' + v2) + client.delete_bucket(bucket) + return {'case': 'versioned-' + stage.lower(), 'status': 'passed', 'observation': observation, 'deleted_version': v1, 'retained_version': v2} + finally: + cluster.remove() + +def protected_version(): + import datetime + cluster = Cluster('retention', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'protected-recovery' + key = 'retained' + client.put_bucket(bucket, lock=True) + client.versioning(bucket) + until = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ') + payload = b'compliance-and-legal-hold' * 1024 + client.put(bucket, key, payload, {'x-amz-object-lock-mode': 'COMPLIANCE', 'x-amz-object-lock-retain-until-date': until, 'x-amz-object-lock-legal-hold': 'ON'}) + version = client.last_headers.get('x-amz-version-id') + assert version and version != 'null' + original = residue_inventory(cluster, bucket, key) + client.delete(bucket, key + '?versionId=' + version, expected=403) + assert residue_inventory(cluster, bucket, key) == original + cluster.kill() + cluster.restart() + assert client.get(bucket, key + '?versionId=' + version) == payload + assert residue_inventory(cluster, bucket, key) == original + client.delete_bucket(bucket, expected=409) + return {'case': 'compliance-legal-hold-restart', 'status': 'passed', 'retained_version': version} + finally: + cluster.remove() + +def returning_drive_new_generation(): + cluster = Cluster('returning-new-generation', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'returning-drive' + key = 'payload' + client.put_bucket(bucket) + old = b'old-payload-' * 200000 + new = b'new-payload-' * 180000 + client.put(bucket, key, old) + assert client.get(bucket, key) == old + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + for index in range(4): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + for index in range(3): + docker('start', cluster.names[index]) + cluster.ready(indices=range(3)) + client.put(bucket, key, new) + assert client.get(bucket, key) == new + client.delete(bucket, key) + docker('start', cluster.names[3]) + cluster.ready() + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node)), timeout=90) + client.request('GET', bucket + '/' + key, expected=404) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': 'old-prepared-new-put-new-delete-returning-peer', 'status': 'passed', 'observation': observation} + finally: + cluster.remove() + +def mixed_versions(): + cluster = Cluster('mixed', nodes=4, drives=2, binary_by_node={3: 'baseline'}) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'mixed-recovery' + key = 'payload' + payload = b'mixed-version-data' * 100000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + original = residue_inventory(cluster, bucket, key) + results = [] + for initiator in (0, 3): + (code, body) = cluster.clients[initiator].request('DELETE', bucket + '/' + key) + assert code >= 400, f'mixed initiator{initiator} must fail closed: {code} {body!r}' + after = residue_inventory(cluster, bucket, key) + (ROOT / f'mixed-initiator-{initiator}-inventory.json').write_text(json.dumps({'before': original, 'after': after}, indent=2)) + assert after == original, 'mixed rejection must preserve exact metadata/shards after rollback' + assert client.get(bucket, key) == payload + results.append({'initiator': initiator, 'delete_status': code}) + cluster.replace_node(3, 'candidate') + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'three-current-one-pre-extension-peer-both-initiators', 'status': 'passed', 'results': results} + finally: + cluster.remove() + +def permission_fault(): + cluster = Cluster('permission-error', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'permission-fault' + key = 'payload' + payload = b'permission-test' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + transaction = str(uuid.UUID(observation['transaction'])) + code = "from pathlib import Path; import json,os; p=Path('/data/d1')/%r/%r/%r; os.chmod(p,0o555); print(json.dumps({'path':str(p),'mode':oct(p.stat().st_mode&0o777)}))" % (bucket, key, transaction) + fault = cluster.inspect_data(code) + (cluster.barriers[0] / ('release.' + nonce)).write_bytes(b'') + thread.join(30) + assert not thread.is_alive(), 'permission failure must return rather than hang' + clear_barriers(cluster) + current = residue_inventory(cluster, bucket, key) + assert any((drive['exists'] for node in current for drive in node)) + client.delete_bucket(bucket, expected=409) + repair = "from pathlib import Path; import json,os; p=Path('/data/d1')/%r/%r/%r; os.chmod(p,0o755); print(json.dumps({'repaired':True}))" % (bucket, key, transaction) + cluster.inspect_data(repair) + cluster.kill() + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'actual-eacces-decision-publication', 'status': 'passed', 'fault': fault, 'request': {key: str(value) for (key, value) in request.items()}, 'observation': observation} + finally: + cluster.remove() + +def injected_error(stage): + cluster = Cluster('injected-' + stage.lower(), drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'injected-fault' + key = 'payload' + payload = b'injected-error' * 170000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + nonce = arm_barrier(cluster, 0, bucket, key, stage, disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + (cluster.barriers[0] / ('release.' + nonce)).write_bytes(b'io_error') + thread.join(30) + assert not thread.is_alive(), 'injected error must finish the caller' + clear_barriers(cluster) + cluster.kill() + cluster.restart() + if stage == 'PREPARED': + (code, body) = client.request('GET', bucket + '/' + key) + if code == 200: + assert body == payload + client.delete(bucket, key) + else: + assert code == 404 + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'post-persist-' + stage.lower() + '-io-error', 'status': 'passed', 'observation': observation, 'request': {key: str(value) for (key, value) in request.items()}} + finally: + cluster.remove() + +def missing_version_returning_peer(): + cluster = Cluster('returning-specific-version', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'missing-version-recovery' + key = 'history' + client.put_bucket(bucket) + client.versioning(bucket) + old = b'first-version' * 160000 + newer = b'latest-version' * 1000 + client.put(bucket, key, old) + v1 = client.last_headers['x-amz-version-id'] + client.put(bucket, key, newer) + v2 = client.last_headers['x-amz-version-id'] + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key + '?versionId=' + v1) + observation = reached_barrier(cluster, 0, nonce) + for index in range(4): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + for index in range(3): + docker('start', cluster.names[index]) + cluster.ready(indices=range(3)) + (code, body) = client.request('DELETE', bucket + '/' + key + '?versionId=' + v1) + initial_delete_status = code + assert code >= 400, f'missing version with unresolved offline journal must not acknowledge204: {code} {body!r}' + docker('start', cluster.names[3]) + cluster.ready() + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + (code, body) = client.request('GET', bucket + '/' + key + '?versionId=' + v1) + if code == 200: + assert body == old + break + time.sleep(0.2) + else: + raise AssertionError('undecided historical version did not recover after peer returned') + client.delete(bucket, key + '?versionId=' + v1) + client.request('GET', bucket + '/' + key + '?versionId=' + v1, expected=404) + assert client.get(bucket, key) == newer + client.delete(bucket, key + '?versionId=' + v2) + client.delete_bucket(bucket) + return {'case': 'missing-specific-version-offline-fence', 'status': 'passed', 'observation': observation, 'initial_delete_status': initial_delete_status} + finally: + cluster.remove() + +def untouched_returning_peer(): + cluster = Cluster('untouched-returning', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'untouched-peer' + key = 'payload' + payload = b'untouched-original' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + original = residue_inventory(cluster, bucket, key) + cluster.kill(3) + client.delete(bucket, key) + after = residue_inventory(cluster, bucket, key) + assert after[3] == original[3], 'offline unprepared peer must remain byte-for-byte untouched' + client.delete_bucket(bucket, expected=409) + cluster.restart(3) + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node)), timeout=90) + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'acknowledged-delete-untouched-offline-peer-return', 'status': 'passed'} + finally: + cluster.remove() + +def uuid_prefix_crash(): + cluster = Cluster('uuid-prefix', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'uuid-prefix' + key = 'parent' + payload = b'parent-payload' * 160000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + inventory = residue_inventory(cluster, bucket, key) + part = next((name for name in inventory[0][0]['files'] if name.endswith('/part.1'))) + data_dir = str(uuid.UUID(part.split('/')[0])) + child = key + '/' + data_dir + '/child' + child_payload = b'preserve-live-prefix-child' * 800 + client.put(bucket, child, child_payload) + assert client.get(bucket, child) == child_payload + nonce = arm_barrier(cluster, 0, bucket, key, 'DECISION', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + assert client.get(bucket, child) == child_payload, 'recovery must preserve actual data-dir UUID-prefix descendants' + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket, expected=409) + client.delete(bucket, child) + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'committed-delete-live-uuid-data-prefix', 'status': 'passed', 'observation': observation, 'protected_child': child} + finally: + cluster.remove() + +def corruption_repair(): + cluster = Cluster('corrupt-journal', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'corrupt-journal' + key = 'payload' + payload = b'original-protected-data' * 100000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + transaction = str(uuid.UUID(observation['transaction'])) + code = "from pathlib import Path; import base64,json; p=Path('/data/d1')/%r/%r/%r/'delete-prepared'; old=p.read_bytes(); p.write_bytes(b'corrupt-record'); print(json.dumps({'original':base64.b64encode(old).decode()}))" % (bucket, key, transaction) + saved = cluster.inspect_data(code)[0]['original'] + before = residue_inventory(cluster, bucket, key) + cluster.restart() + time.sleep(2) + assert residue_inventory(cluster, bucket, key) == before, 'corrupt authority must preserve all backup/payload files' + (code, body) = client.request('DELETE', bucket + '/' + key) + assert code >= 400, f'corrupt hidden transaction must not acknowledge cleanup: {code} {body!r}' + assert residue_inventory(cluster, bucket, key) == before + repair = "from pathlib import Path; import base64,json; p=Path('/data/d1')/%r/%r/%r/'delete-prepared'; p.write_bytes(base64.b64decode(%r)); print(json.dumps({'repaired':True}))" % (bucket, key, transaction, saved) + cluster.inspect_data(repair) + cluster.kill() + cluster.restart() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + (code, body) = client.request('GET', bucket + '/' + key) + if code == 200: + assert body == payload + break + time.sleep(0.1) + else: + raise AssertionError('valid original journal must recover original bytes after repair') + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'corrupt-journal-fails-closed-then-repair', 'status': 'passed', 'observation': observation} + finally: + cluster.remove() + +def admin_heal(client, bucket, dry_run=False, remove=False, key=None, recursive=None): + path = 'rustfs/admin/v3/heal/' + bucket + ('/' + key if key else '') + options = {'recursive': key is None if recursive is None else recursive, 'dryRun': dry_run, 'remove': remove, 'recreate': False, 'scanMode': 1, 'updateParity': False, 'nolock': False, 'readRepair': False} + (_, body) = client.request('POST', path, json.dumps(options).encode(), {'Content-Type': 'application/json'}, 200) + token = json.loads(body)['clientToken'] + deadline = time.monotonic() + 90 + + def states(value): + if isinstance(value, dict): + for (k, v) in value.items(): + if k == 'state' and isinstance(v, str): + yield v.lower() + yield from states(v) + elif isinstance(value, list): + for v in value: + yield from states(v) + while time.monotonic() < deadline: + (_, body) = client.request('POST', path + '?clientToken=' + token, headers={'Content-Type': 'application/json'}, expected=200) + result = json.loads(body) + with (ROOT / 'admin-responses.jsonl').open('a') as report: + report.write(json.dumps({'bucket': bucket, 'key': key, 'dry_run': dry_run, 'remove': remove, 'recursive': options['recursive'], 'response': result}) + '\n') + status = set(states(result)) + if status & {'failed', 'completed_with_errors', 'blocked'}: + raise AssertionError('admin heal failed: ' + json.dumps(result)) + if 'completed' in status or result.get('summary') == 'finished': + return result + time.sleep(0.1) + raise TimeoutError('admin heal did not complete: ' + token) + +def admin_legacy_dry_run(direct=False): + cluster = Cluster('admin-legacy', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'admin-legacy' + client.put_bucket(bucket) + for (key, payload) in [('inline', b'i' * 4096), ('external', b'e' * (2 * 1024 * 1024))]: + client.put(bucket, key, payload) + cluster.kill() + seed_legacy(cluster, bucket, 'inline', 'backup-only') + seed_legacy(cluster, bucket, 'external', 'reserved') + before = {key: residue_inventory(cluster, bucket, key) for key in ('inline', 'external')} + cluster.restart() + time.sleep(2) + dry = admin_heal(client, bucket, dry_run=True, remove=True) + assert {key: residue_inventory(cluster, bucket, key) for key in before} == before, 'dry run must never mutate ambiguous residue' + actual = [admin_heal(client, bucket, dry_run=False, remove=True, key=key) for key in before] if direct else admin_heal(client, bucket, dry_run=False, remove=True) + for key in before: + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + client.delete_bucket(bucket) + return {'case': 'admin-legacy-dry-run-and-explicit-cleanup', 'status': 'passed', 'dry_run_report': dry, 'cleanup_report': actual} + finally: + cluster.remove() + +def admin_prefix_scoped(): + cluster = Cluster('admin-prefix', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket, prefix = 'admin-prefix', 'selected/' + selected = {'selected/inline': b'i' * 4096, 'selected/external': b'e' * (2 * 1024 * 1024)} + outside = {'selected-neighbor/hidden': b'n' * 4096, 'outside/hidden': b'o' * 4096} + live_key, live_body = 'selected/live', b'live-prefix-object-must-survive' + client.put_bucket(bucket) + for key, body in {**selected, **outside, live_key: live_body}.items(): + client.put(bucket, key, body) + assert client.get(bucket, key) == body + cluster.kill() + for key in {**selected, **outside}: + seed_legacy(cluster, bucket, key, 'reserved' if key == 'selected/external' else 'backup-only') + before = {key: residue_inventory(cluster, bucket, key) for key in {**selected, **outside}} + cluster.restart() + dry = admin_heal(client, bucket, dry_run=True, remove=True, key=prefix, recursive=True) + assert any('delete recovery (dry run)' in item.get('detail', '') and 'discovered=2' in item.get('detail', '') for item in dry.get('items', [])), 'prefix dry run must inspect the two matching hidden transactions' + assert {key: residue_inventory(cluster, bucket, key) for key in before} == before, 'prefix dry run must preserve every hidden transaction' + actual = admin_heal(client, bucket, remove=True, key=prefix, recursive=True) + after = {key: residue_inventory(cluster, bucket, key) for key in before} + (ROOT / 'prefix-inventory.json').write_text(json.dumps({'before': before, 'after': after}, indent=2)) + for key in selected: + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + for key in outside: + assert residue_inventory(cluster, bucket, key) == before[key], f'prefix recovery must preserve outside transaction {key}' + assert client.get(bucket, live_key) == live_body, 'prefix remove must preserve the live object' + assert client.list(bucket) == [live_key] + client.delete(bucket, live_key) + client.delete_bucket(bucket, expected=409) + for key in outside: + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'admin-prefix-hidden-transactions-only', 'status': 'passed', 'prefix': prefix, 'selected_keys': list(selected), 'protected_keys': list(outside), 'dry_run_report': dry, 'cleanup_report': actual} + finally: + cluster.remove() + +def deep_key_crash(large_stack=False): + extra_env = {'RUSTFS_RUNTIME_THREAD_STACK_SIZE': str(32 * 1024 * 1024)} if large_stack else None + cluster = Cluster('deep-key-large-stack' if large_stack else 'deep-key', drives=4, extra_env=extra_env) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'deep-recovery' + key = '/'.join(['a'] * 300) + '/payload' + payload = b'deep-inline-content' * 200 + client.put_bucket(bucket) + client.put(bucket, key, payload) + original = residue_inventory(cluster, bucket, key) + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'deep-key-recovery-large-stack' if large_stack else 'deep-key-recovery', 'status': 'passed', 'segments': 301, 'runtime_thread_stack_size': 32 * 1024 * 1024 if large_stack else 'default', 'observation': observation} + finally: + cluster.remove() + +def receipt_response_loss(): + cluster = Cluster('receipt-response-loss', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'receipt-response-loss' + key = 'payload' + payload = b'lost-response-data' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + nonces = {index: arm_barrier(cluster, index, bucket, key, 'DECISION') for index in (1, 2)} + (thread, request) = async_delete(client, bucket, key) + observations = [] + for (index, nonce) in nonces.items(): + observations.append(reached_barrier(cluster, index, nonce)) + deadline = time.monotonic() + 40 + while len(list(cluster.barriers[index].glob('reached.' + nonce + '.*.json'))) < 2: + if time.monotonic() > deadline: + raise TimeoutError('both drive receipts must persist before peer kill') + time.sleep(0.02) + for index in (1, 2): + cluster.kill(index) + thread.join(75) + assert not thread.is_alive(), 'DELETE must return after peer response loss' + if 'response' in request: + assert request['response'][0] >= 400, 'four online responses cannot satisfy five-drive receipt quorum' + before = residue_inventory(cluster, bucket, key) + assert any(('delete-quorum-committed' in path for node in before for drive in node for path in drive['files'])), 'stored decisions must remain while peers are offline' + clear_barriers(cluster) + for index in (1, 2): + docker('start', cluster.names[index]) + cluster.ready() + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node)), timeout=90) + client.request('GET', bucket + '/' + key, expected=404) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': 'receipt-publication-response-loss', 'status': 'passed', 'observations': observations, 'request': {key: str(value) for (key, value) in request.items()}} + finally: + cluster.remove() + +def retained_decision_newer_undecided(): + cluster = Cluster('inverse-null-history', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + (bucket, key) = ('inverse-null-history', 'payload') + older = b'older-generation' * 150000 + newer = b'newer-generation' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, older) + cluster.kill(3) + client.delete(bucket, key) + witness = residue_inventory(cluster, bucket, key) + assert any(('delete-quorum-committed' in path for node in witness[:3] for drive in node for path in drive['files'])), 'offline old peer requires retained committed authority' + client.put(bucket, key, newer) + assert client.get(bucket, key) == newer + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + for index in range(3): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + cluster.restart() + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + (status, body) = client.request('GET', bucket + '/' + key) + if status == 200: + assert body == newer, 'older committed nil receipt must never promote a newer undecided generation' + break + time.sleep(0.1) + else: + raise AssertionError('newer undecided generation was not restored after complete peer return') + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'retained-older-decision-newer-undecided-null', 'status': 'passed', 'observation': observation} + finally: + cluster.remove() + + +def completed_multipart_sidecars(protected=False, crash=False): + case = 'multipart-integrity' if protected else 'multipart-sidecars' + if crash: + case += '-applied' + mode = 'true' if protected else 'false' + cluster = Cluster(case, drives=4, extra_env={'RUSTFS_SHARD_INTEGRITY_WRITE': mode, 'RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED': mode}) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'completed-multipart' + key = 'payload' + parts = [bytes(range(256)) * 20480, bytes(reversed(range(256))) * 4096] + payload = b''.join(parts) + client.put_bucket(bucket) + upload = client.complete_multipart(bucket, key, parts) + assert client.get(bucket, key) == payload, 'complete MPU must expose all uploaded bytes' + original = residue_inventory(cluster, bucket, key) + (ROOT / 'multipart-original-inventory.json').write_text(json.dumps(original, indent=2)) + for node in original: + for drive in node: + assert drive['exists'] and 'xl.meta' in drive['files'], 'complete MPU must publish metadata on every drive' + for number in (1, 2): + sidecars = [name for name in drive['files'] if Path(name).name == f'part.{number}.meta'] + assert len(sidecars) == 1, 'native completed MPU must retain each real part metadata sidecar' + prefix = f'part.{number}.integrity.' + proofs = [name for name in drive['files'] if Path(name).name.startswith(prefix)] + assert len(proofs) == (1 if protected else 0), 'native completion must retain the selected integrity format' + for proof in proofs: + assert uuid.UUID(Path(proof).name[len(prefix):]).int != 0, 'integrity proof generation must be non-nil' + observation = None + if crash: + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + restored = residue_inventory(cluster, bucket, key) + assert restored == original, 'rollback must preserve exact metadata, all shards, part sidecars and integrity proofs' + (ROOT / 'multipart-restored-inventory.json').write_text(json.dumps(restored, indent=2)) + assert client.get(bucket, key) == payload, 'rolled-back native MPU must retain the complete readable body' + client.batch_delete(bucket, [key]) + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + client.request('GET', bucket + '/' + key, expected=404) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': case, 'status': 'passed', 'native_upload_id': upload, 'integrity': protected, 'part_sizes': [len(part) for part in parts], 'bytes': len(payload), 'observation': observation, 'bucket_deletion': 'ordinary non-force204'} + finally: + cluster.remove() + +def ordinary_missing_with_offline_peer(): + cluster = Cluster('missing-offline', nodes=4, drives=1) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'missing-offline' + client.put_bucket(bucket) + cluster.kill(3) + client.delete(bucket, 'never-existed-single', expected=204) + client.batch_delete(bucket, ['never-existed-batch-a', 'never-existed-batch-b']) + assert client.list(bucket) == [], 'ordinary missing DELETE must leave the existing bucket valid' + payload = b'bucket-remains-usable' * 200 + client.put(bucket, 'live', payload) + assert client.get(bucket, 'live') == payload + docker('start', cluster.names[3]) + cluster.ready() + assert client.get(bucket, 'live') == payload + client.delete(bucket, 'live') + wait_inventory(cluster, bucket, 'live', lambda inventory: all(not drive['exists'] for node in inventory for drive in node), timeout=90) + client.delete_bucket(bucket) + return {'case': 'ordinary-missing-delete-offline-single-and-batch', 'status': 'passed', 'cleanup_claim': 'no residue recovery was requested'} + finally: + cluster.remove() + +def legacy_transaction_data_dir_alias(): + cluster = Cluster('legacy-data-dir-alias', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'legacy-data-dir-alias', 'payload' + payload = b'legacy-data-dir-content' * 100000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + cluster.kill() + fixture = seed_legacy(cluster, bucket, key, 'alias') + before = residue_inventory(cluster, bucket, key) + cluster.restart() + time.sleep(2) + assert residue_inventory(cluster, bucket, key) == before, 'automatic recovery must preserve ambiguous alias residue' + client.delete(bucket, key) + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + client.delete_bucket(bucket) + return {'case': 'legacy-transaction-equals-data-directory', 'status': 'passed', 'fixture': fixture} + finally: + cluster.remove() + +def admin_smoke(): + cluster = Cluster('admin-smoke', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'admin-smoke', 'payload' + payload = b'healthy-admin-object' * 1000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + bucket_report = admin_heal(client, bucket, dry_run=True) + object_report = admin_heal(client, bucket, key=key) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'admin-healthy-bucket-and-object-client', 'status': 'passed', 'bucket_report': bucket_report, 'object_report': object_report} + finally: + cluster.remove() + +def large_batch_delete(): + cluster = Cluster('batch-1000', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'batch-1000' + client.put_bucket(bucket) + payloads = {f'object-{index:04d}': (bytes(range(256)) * 8192 if index % 250 == 0 else f'content-{index:04d}'.encode()) for index in range(1000)} + for key, payload in payloads.items(): + client.put(bucket, key, payload) + for key, payload in payloads.items(): + assert client.get(bucket, key) == payload + client.batch_delete(bucket, list(payloads)) + assert client.list(bucket) == [] + for key in ('object-0000', 'object-0500', 'object-0999'): + assert all(not drive['exists'] for node in residue_inventory(cluster, bucket, key) for drive in node) + client.delete_bucket(bucket) + return {'case': 'maximum-1000-key-remote-delete-batch', 'status': 'passed', 'exact_gets': 1000, 'keys': 1000} + finally: + cluster.remove() + +def large_version_history(): + cluster = Cluster('versions-3000', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'versions-3000', 'history' + client.put_bucket(bucket) + client.versioning(bucket) + versions = [] + for index in range(3000): + payload = f'version-content-{index:04d}'.encode() + client.put(bucket, key, payload) + versions.append((client.last_headers['x-amz-version-id'], payload)) + for index in (0, 1500, 2999): + version, payload = versions[index] + assert client.request('GET', bucket + '/' + key + '?versionId=' + version, expected=200)[1] == payload + oldest = versions.pop(0)[0] + client.delete(bucket, key + '?versionId=' + oldest) + client.request('GET', bucket + '/' + key + '?versionId=' + oldest, expected=404) + assert client.get(bucket, key) == versions[-1][1] + for offset in range(0, len(versions), 1000): + group = versions[offset:offset + 1000] + xml = ET.Element('Delete') + for version, payload in group: + item = ET.SubElement(xml, 'Object') + ET.SubElement(item, 'Key').text = key + ET.SubElement(item, 'VersionId').text = version + data = ET.tostring(xml) + status, body = client.request('POST', bucket + '?delete', data, {'Content-Type': 'application/xml', 'Content-MD5': base64.b64encode(hashlib.md5(data).digest()).decode()}, 200, timeout=180) + response = ET.fromstring(body) + assert not [node for node in response.iter() if node.tag.rsplit('}', 1)[-1] == 'Error'], body + deleted = [child.text for node in response.iter() if node.tag.rsplit('}', 1)[-1] == 'Deleted' for child in node if child.tag.rsplit('}', 1)[-1] == 'VersionId'] + assert sorted(deleted) == sorted(version for version, payload in group), body + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'large-version-history-remote-single-and-batch-delete', 'status': 'passed', 'versions': 3000, 'survivor_bytes_verified': True} + finally: + cluster.remove() + +def unknown_predecessor_fresh_delete(): + cluster = Cluster('unknown-predecessor', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'unknown-predecessor', 'payload' + client.put_bucket(bucket) + client.put(bucket, key, b'older-inline' * 300) + nonce = arm_barrier(cluster, 3, bucket, key, 'APPLIED', disk='/data/d1') + thread, request = async_delete(cluster.clients[3], bucket, key) + observation = reached_barrier(cluster, 3, nonce) + transaction = str(uuid.UUID(observation['transaction'])) + verify = "from pathlib import Path; import json; result=[];\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/%r/%r; tx=root/%r; result.append(not (root/'xl.meta').exists() and (tx/'xl.meta.bkp').is_file())\nprint(json.dumps(result))" % (bucket, key, transaction) + deadline = time.monotonic() + 30 + while not all(cluster.inspect_data(verify, indices=[3])[0]): + if time.monotonic() >= deadline: + raise TimeoutError('both returning-peer preparations must reach metadata absence') + time.sleep(.05) + cluster.kill(3) + for index in range(3): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + restore = "from pathlib import Path; import json,os; result=[];\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/%r/%r; tx=root/%r; backup=tx/'xl.meta.bkp';\n if backup.exists(): os.replace(backup,root/'xl.meta')\n if tx.exists():\n for path in tx.iterdir():\n assert path.is_file() and path.name in ('delete-prepared','delete-target'), path\n path.unlink()\n tx.rmdir()\n assert (root/'xl.meta').is_file(); result.append(True)\nprint(json.dumps(result))" % (bucket, key, transaction) + cluster.inspect_data(restore, indices=range(3)) + for index in range(3): + docker('start', cluster.names[index]) + cluster.ready(indices=range(3)) + newer = b'newer-inline' * 300 + client.put(bucket, key, newer) + assert client.get(bucket, key) == newer + client.delete(bucket, key) + docker('start', cluster.names[3]) + cluster.ready() + time.sleep(2) + pending = residue_inventory(cluster, bucket, key) + assert all(transaction + '/xl.meta.bkp' in drive['files'] for drive in pending[3]), 'automatic recovery must preserve unknown predecessor evidence' + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket, expected=409) + client.delete(bucket, key, expected=204) + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + client.delete_bucket(bucket) + return {'case': 'unknown-predecessor-preserved-until-fresh-current-delete', 'status': 'passed', 'observation': observation, 'fixture': 'authentic prepared records retained only on stopped returning peer'} + finally: + cluster.remove() + +def main(): + global ROOT, PREFIX, IMAGE, PORT, CANDIDATE, BASELINE + import argparse, re, sys + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--binary', required=True, type=Path, help='Linux candidate binary built with e2e-test-hooks') + parser.add_argument('--baseline-binary', type=Path, help='Older Linux binary, required by mixed case') + parser.add_argument('--image', required=True, help='Local Linux image with Python 3 and the binary runtime libraries') + parser.add_argument('--artifact-dir', required=True, type=Path, help='New directory for logs and stage handshakes') + parser.add_argument('--namespace', default='rustfs-delete-' + uuid.uuid4().hex[:12]) + parser.add_argument('--port', type=int, default=19010, help='First of four sequential localhost ports') + parser.add_argument('cases', nargs='*', default=['smoke']) + options = parser.parse_args() + if not re.fullmatch('[a-z][a-z0-9-]{0,45}', options.namespace): + parser.error('namespace must be a short lowercase Docker name') + if not 1024 <= options.port <= 65532: + parser.error('port must leave room for four localhost ports') + CANDIDATE = options.binary.resolve(strict=True) + if not CANDIDATE.is_file(): + parser.error('binary must be a regular file') + BASELINE = options.baseline_binary.resolve(strict=True) if options.baseline_binary else None + if BASELINE is not None and (not BASELINE.is_file()): + parser.error('baseline binary must be a regular file') + ROOT = options.artifact_dir.resolve() + ROOT.mkdir(parents=True, exist_ok=False) + PREFIX = options.namespace + IMAGE = options.image + PORT = options.port + for resource in ('container', 'volume', 'network'): + existing = docker(resource, 'ls', '-q', '--filter', 'label=rustfs.delete-recovery-test=' + PREFIX).stdout.strip() + if existing: + parser.error('namespace already owns Docker resources; choose a new namespace') + cases = {'unknown-fresh': unknown_predecessor_fresh_delete, 'batch1000': large_batch_delete, 'versions3000': large_version_history, 'admin-smoke': admin_smoke, 'legacy-alias': legacy_transaction_data_dir_alias, 'missing-offline': ordinary_missing_with_offline_peer, 'inverse': retained_decision_newer_undecided, 'deep': deep_key_crash, 'deep-large-stack': lambda : deep_key_crash(True), 'admin-prefix': admin_prefix_scoped, 'response-loss': receipt_response_loss, 'admin-object': lambda : admin_legacy_dry_run(True), 'smoke': smoke, 'legacy1': lambda : legacy_retry(1), 'legacy4': lambda : legacy_retry(4), 'restore': rollback_crash, 'retention': protected_version, 'version-applied': lambda : versioned_crash('APPLIED'), 'version-decision': lambda : versioned_crash('DECISION'), 'returning': returning_drive_new_generation, 'untouched': untouched_returning_peer, 'missing-version': missing_version_returning_peer, 'mixed': mixed_versions, 'admin': admin_legacy_dry_run, 'prefix': uuid_prefix_crash, 'corruption': corruption_repair, 'permission': permission_fault, 'error-prepared': lambda : injected_error('PREPARED'), 'error-applied': lambda : injected_error('APPLIED'), 'error-decision': lambda : injected_error('DECISION')} + cases.update({'multipart-sidecars': completed_multipart_sidecars, 'multipart-integrity': lambda: completed_multipart_sidecars(True), 'multipart-sidecars-applied': lambda: completed_multipart_sidecars(False, True), 'multipart-integrity-applied': lambda: completed_multipart_sidecars(True, True)}) + for stage in ('PREACTIVATION', 'PREPARED', 'APPLIED', 'DECISION'): + for nodes in (1, 4): + cases[f'{stage.lower()}{nodes}'] = lambda stage=stage, nodes=nodes: process_crash(stage, nodes) + cases[f'inline-{stage.lower()}'] = lambda stage=stage: process_crash(stage, 1, 4096) + cases[f'zero-{stage.lower()}'] = lambda stage=stage: process_crash(stage, 1, 0) + if 'mixed' in options.cases and BASELINE is None: + parser.error('mixed requires --baseline-binary') + unknown = set(options.cases) - set(cases) + if unknown: + parser.error('unknown cases: ' + ','.join(sorted(unknown))) + results = [] + for case in options.cases: + started = time.monotonic() + try: + result = cases[case]() + result['elapsed_seconds'] = round(time.monotonic() - started, 3) + except BaseException as error: + result = {'case': case, 'status': 'failed', 'error': str(error), 'elapsed_seconds': round(time.monotonic() - started, 3)} + (ROOT / (case + '.json')).write_text(json.dumps(result, indent=2, sort_keys=True)) + raise + results.append(result) + (ROOT / (case + '.json')).write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, sort_keys=True), flush=True) + (ROOT / 'results.json').write_text(json.dumps(results, indent=2, sort_keys=True)) +if __name__ == '__main__': + main()