diff --git a/crates/e2e_test/src/reliant/grpc_lock_server.rs b/crates/e2e_test/src/reliant/grpc_lock_server.rs index ccc466d23..b97e96555 100644 --- a/crates/e2e_test/src/reliant/grpc_lock_server.rs +++ b/crates/e2e_test/src/reliant/grpc_lock_server.rs @@ -465,6 +465,13 @@ impl NodeService for MinimalLockNodeService { Err(Status::unimplemented("lock-only test server")) } + async fn recover_delete_transaction( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("delete recovery is not supported by the lock fixture")) + } + async fn settle_part_transaction( &self, _request: Request, diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e5e22995f..a43116a77 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -423,11 +423,12 @@ pub mod disk { pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage}; pub use crate::disk::{ BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, - CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, - DiskStore, FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, - PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, - validate_batch_read_version_item_count, + CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, + DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, + DeleteRecoveryRpcReply, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, + FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction, + RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, + SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count, }; pub use bytes::Bytes; pub use endpoint::Endpoint; diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 27bd0542f..75706254c 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -124,6 +124,22 @@ pub async fn node_service_time_out_client_for_class( .max_encoding_message_size(max_message_size)) } +pub(crate) async fn node_service_delete_recovery_client( + addr: &str, + interceptor: TonicInterceptor, + timeout: std::time::Duration, +) -> Result>, DiskError> { + let interceptor = interceptor.with_rpc_audience(addr).map_err(DiskError::from)?; + let channel = rustfs_protos::create_channel_with_timeout(addr, timeout) + .await + .map_err(|error| DiskError::RemoteClientUnavailable(error.to_string()))?; + let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); + let max_message_size = rustfs_protos::internode_rpc_max_message_size(); + Ok(NodeServiceClient::with_interceptor(channel, interceptor) + .max_decoding_message_size(max_message_size) + .max_encoding_message_size(max_message_size)) +} + pub async fn node_service_time_out_client_no_auth( addr: &String, ) -> Result>, Box> { @@ -528,6 +544,92 @@ mod tests { use tracing_opentelemetry::OpenTelemetrySpanExt; use tracing_subscriber::{Registry, layer::SubscriberExt}; + #[tokio::test] + async fn delete_recovery_channel_uses_its_scoped_deadline() { + use bytes::Bytes; + use http_body_util::StreamBody; + use hyper::{body::Frame, service::service_fn}; + use hyper_util::rt::{TokioExecutor, TokioIo}; + use rustfs_protos::proto_gen::node_service::{RecoverDeleteTransactionRequest, RecoverDeleteTransactionResponse}; + use tonic_prost::prost::Message; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind recovery peer"); + let addr = format!("http://{}", listener.local_addr().expect("peer address")); + let server = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.expect("accept recovery connection"); + tokio::spawn(async move { + let service = service_fn(|_request: HttpRequest| async { + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + let payload = RecoverDeleteTransactionResponse { + success: true, + protocol_version: 1, + result: Bytes::new(), + error: None, + } + .encode_to_vec(); + let mut data = vec![0]; + data.extend_from_slice(&u32::try_from(payload.len()).expect("small response").to_be_bytes()); + data.extend_from_slice(&payload); + let mut trailers = http::HeaderMap::new(); + trailers.insert("grpc-status", "0".parse().expect("status")); + let frames: Vec, std::convert::Infallible>> = + vec![Ok(Frame::data(data.into())), Ok(Frame::trailers(trailers))]; + let mut response = HttpResponse::new(StreamBody::new(futures::stream::iter(frames))); + response + .headers_mut() + .insert("content-type", "application/grpc".parse().expect("content type")); + Ok::<_, std::convert::Infallible>(response) + }); + let _ = hyper::server::conn::http2::Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + let request = || RecoverDeleteTransactionRequest { + disk: "disk".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: Bytes::new(), + }; + // Seed an existing short-lived control channel to reproduce the inherited deadline. + let control = tonic::transport::Endpoint::from_shared(addr.clone()) + .expect("test endpoint") + .timeout(std::time::Duration::from_millis(50)) + .connect() + .await + .expect("control connect"); + rustfs_common::cache_connection(addr.clone(), control).await; + let mut foreground = node_service_time_out_client(&addr, TonicInterceptor::NoOp(NoOpInterceptor)) + .await + .expect("control client"); + let error = foreground + .recover_delete_transaction(request()) + .await + .expect_err("short control deadline"); + assert!(matches!(error.code(), tonic::Code::Cancelled | tonic::Code::DeadlineExceeded)); + // The 2-second test override bypasses the production range parser, which rejects 2. + let mut recovery = node_service_delete_recovery_client( + &addr, + TonicInterceptor::NoOp(NoOpInterceptor), + std::time::Duration::from_secs(2), + ) + .await + .expect("scoped recovery client"); + assert!( + recovery + .recover_delete_transaction(request()) + .await + .expect("scoped deadline must survive control deadline") + .into_inner() + .success + ); + server.abort(); + rustfs_protos::evict_failed_connection(&addr).await; + } + #[tokio::test] async fn rpc_client_factories_do_not_reuse_https_channels_without_tls_generation() { let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve local address"); diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 0ba276567..e067609a0 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -14,7 +14,8 @@ use crate::cluster::rpc::client::{ AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, - node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth, + node_service_delete_recovery_client, node_service_time_out_client, node_service_time_out_client_for_class, + node_service_time_out_client_no_auth, }; use crate::cluster::rpc::internode_data_transport::{ InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest, @@ -22,10 +23,11 @@ use crate::cluster::rpc::internode_data_transport::{ }; use crate::disk::error::{Error, Result}; use crate::disk::{ - BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DeleteOptions, DiskAPI, DiskInfo, - DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, - ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, - batch_read_version_one_by_one, + BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION, + DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, + DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, + DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions, + RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one, disk_store::{ DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE, get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout, @@ -59,10 +61,10 @@ use rustfs_protos::proto_gen::node_service::{ CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, - ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, - SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, - SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, - node_service_client::NodeServiceClient, + ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest, + RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, + SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, node_service_client::NodeServiceClient, }; use serde::{Serialize, de::DeserializeOwned}; use std::{ @@ -142,6 +144,45 @@ fn decode_delete_versions_errors(response: DeleteVersionsResponse, expected_len: .collect() } +fn decode_delete_recovery_batch_errors(codes: Vec>, expected_len: usize) -> Result>> { + if codes.len() != expected_len { + return Err(Error::other("malformed recoverable batch delete response length")); + } + codes + .into_iter() + .map(|code| match code { + None => Ok(None), + Some(code) => DiskError::from_u32(code) + .map(Some) + .ok_or_else(|| Error::other("invalid recoverable batch delete error code")), + }) + .collect() +} + +fn validate_delete_recovery_inspection( + volume: &str, + object: &str, + transaction: Uuid, + inspection: &DeleteRecoveryInspection, +) -> Result<()> { + if inspection.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || inspection.id.object != object + || inspection.id.transaction != transaction + || transaction.is_nil() + { + return Err(Error::other("delete recovery inspection does not match request")); + } + if let Some(journal) = &inspection.journal + && (journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || journal.volume != volume + || journal.object != object + || journal.transaction != transaction) + { + return Err(Error::other("delete recovery journal does not match request")); + } + Ok(()) +} + fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result { if !response.success { return Err(response.error.unwrap_or_default().into()); @@ -858,6 +899,75 @@ fn spawn_control_channel_prewarm(addr: String) { } impl RemoteDisk { + async fn recover_delete_transaction( + &self, + volume: &str, + operation: DeleteRecoveryRpcOperation, + ) -> Result { + let payload = rmp_serde::to_vec_named(&operation)?; + if payload.len() > operation.max_wire_payload_bytes() { + return Err(Error::other("delete recovery operation exceeds payload limit")); + } + let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) { + Some(crate::disk::get_delete_recovery_timeout()?) + } else { + None + }; + self.execute_with_timeout( + || async { + let mut client = if let Some(timeout) = settle_timeout { + if let Some(error) = self.offline_bypass_error() { + return Err(error); + } + node_service_delete_recovery_client( + &self.addr, + TonicInterceptor::Signature(gen_tonic_signature_interceptor()), + timeout, + ) + .await? + } else { + self.get_client().await? + }; + let mut request = Request::new(RecoverDeleteTransactionRequest { + disk: self.endpoint.to_string(), + volume: volume.to_owned(), + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + operation: payload.into(), + }); + let canonical_body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()) + .map_err(|_| Error::other("recover_delete_transaction request length cannot be represented"))?; + crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?; + // UNIMPLEMENTED is deliberately propagated: older peers cannot settle transactions. + let response = client + .recover_delete_transaction(request) + .await + .map_err(|status| { + if status.code() == Code::Unimplemented { + DiskError::UnsupportedDisk + } else { + status.into() + } + })? + .into_inner(); + if response.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION { + return Err(DiskError::UnsupportedDisk); + } + if !response.success { + return Err(response + .error + .ok_or_else(|| Error::other("missing delete recovery error"))? + .into()); + } + if response.error.is_some() { + return Err(Error::other("malformed delete recovery success")); + } + rustfs_protos::decode_delete_recovery_payload(&response.result).map_err(Into::into) + }, + settle_timeout.unwrap_or_else(get_max_timeout_duration), + ) + .await + } + async fn rename_file_with_durability( &self, src_volume: &str, @@ -1920,6 +2030,12 @@ fn encode_msgpack(value: &T) -> Result> { encode_msgpack_with_capacity(value, MSGPACK_ENCODE_CAPACITY_HINT) } +// Optional fields are omitted by serde; named maps keep rollback flags and +// transaction UUIDs independent of field order across peer versions. +fn encode_delete_options_msgpack(value: &DeleteOptions) -> Result> { + Ok(rmp_serde::to_vec_named(value)?) +} + fn encode_file_info_msgpack(value: &FileInfo) -> Result> { encode_msgpack_with_capacity(value, FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT) } @@ -2566,7 +2682,7 @@ impl DiskAPI for RemoteDisk { // `_bin` support for DeleteVersion is new (grpc-optimization P2); always dual-write // JSON + msgpack until its fallback counter has read zero across a release window. let file_info_bin = encode_file_info_msgpack(&fi)?; - let opts_bin = encode_msgpack(&opts)?; + let opts_bin = encode_delete_options_msgpack(&opts)?; let conditional_marker = opts.expected_delete_marker.is_some(); let file_info = serde_json::to_string(&fi)?; let opts = serde_json::to_string(&opts)?; @@ -2638,7 +2754,7 @@ impl DiskAPI for RemoteDisk { // `_bin` support for DeleteVersions is new (grpc-optimization P2); always dual-write JSON + // msgpack until its fallback counter has read zero across a release window. - let opts_bin = match encode_msgpack(&opts) { + let opts_bin = match encode_delete_options_msgpack(&opts) { Ok(opts_bin) => opts_bin, Err(err) => { let mut errors = Vec::with_capacity(versions.len()); @@ -3617,6 +3733,138 @@ impl DiskAPI for RemoteDisk { } #[tracing::instrument(level = "trace", skip_all)] + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> Result<()> { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::ApplyVersion { + object: object.to_owned(), + file_info: Box::new(file_info), + force_del_marker, + opts, + }, + ) + .await? + { + DeleteRecoveryRpcReply::AppliedVersion => Ok(()), + _ => Err(Error::other("unexpected recoverable delete reply")), + } + } + + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + let expected_len = versions.len(); + let result = match self + .recover_delete_transaction(volume, DeleteRecoveryRpcOperation::ApplyVersions { versions, opts }) + .await + { + Ok(DeleteRecoveryRpcReply::AppliedVersions(codes)) => decode_delete_recovery_batch_errors(codes, expected_len), + Ok(_) => Err(Error::other("unexpected recoverable batch delete reply")), + Err(error) => Err(error), + }; + match result { + Ok(errors) => errors, + Err(error) => vec![Some(error); expected_len], + } + } + + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::Discover { + object: object.map(str::to_owned), + cursor, + limit, + }, + ) + .await? + { + DeleteRecoveryRpcReply::Discovered(page) => { + if object.is_some_and(|object| page.transactions.iter().any(|id| id.object != object)) { + return Err(Error::other("delete recovery discovery escaped requested object")); + } + if page.transactions.len() > usize::try_from(limit).map_err(|_| Error::other("invalid discovery limit"))? + || page + .transactions + .iter() + .any(|id| id.transaction.is_nil() || id.object.is_empty()) + { + return Err(Error::other("malformed delete discovery page")); + } + Ok(page) + } + _ => Err(Error::other("unexpected delete discovery reply")), + } + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::Inspect { + object: object.to_owned(), + transaction, + }, + ) + .await? + { + DeleteRecoveryRpcReply::Inspected(inspection) => { + validate_delete_recovery_inspection(volume, object, transaction, &inspection)?; + Ok(inspection) + } + _ => Err(Error::other("unexpected delete inspection reply")), + } + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> Result { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::Settle { + object: object.to_owned(), + transaction, + expected_fingerprint, + action, + }, + ) + .await? + { + DeleteRecoveryRpcReply::Settled(inspection) => { + validate_delete_recovery_inspection(volume, object, transaction, &inspection)?; + Ok(inspection) + } + _ => Err(Error::other("unexpected delete settlement reply")), + } + } + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> { self.execute_with_timeout( || async { @@ -4030,6 +4278,45 @@ mod tests { static INIT: Once = Once::new(); + #[test] + fn recoverable_batch_delete_rejects_invalid_error_codes_and_length() { + let errors = decode_delete_recovery_batch_errors(vec![None, Some(DiskError::FileNotFound.to_u32())], 2) + .expect("valid per-item errors"); + assert!(errors[0].is_none()); + assert_eq!(errors[1], Some(DiskError::FileNotFound)); + assert!(decode_delete_recovery_batch_errors(vec![None], 2).is_err()); + assert!(decode_delete_recovery_batch_errors(vec![Some(0)], 1).is_err()); + assert!(decode_delete_recovery_batch_errors(vec![Some(u32::MAX)], 1).is_err()); + } + + #[test] + fn delete_recovery_inspection_rejects_wrong_transaction_object_and_protocol() { + let transaction = Uuid::new_v4(); + let inspection = DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: crate::disk::DeleteTransactionId { + object: "object".into(), + transaction, + }, + state: crate::disk::DeleteRecoveryState::Absent, + journal: None, + current_digest: None, + current_versions: Vec::new(), + backup_digest: None, + rollback_data_complete: false, + commit_data_safe: false, + explicit_delete_safe: false, + fingerprint: [0; 32], + }; + validate_delete_recovery_inspection("bucket", "object", transaction, &inspection).expect("matching inspection"); + assert!(validate_delete_recovery_inspection("bucket", "other", transaction, &inspection).is_err()); + assert!(validate_delete_recovery_inspection("bucket", "object", Uuid::new_v4(), &inspection).is_err()); + assert!(validate_delete_recovery_inspection("bucket", "object", Uuid::nil(), &inspection).is_err()); + let mut incompatible = inspection; + incompatible.protocol_version += 1; + assert!(validate_delete_recovery_inspection("bucket", "object", transaction, &incompatible).is_err()); + } + #[test] fn compare_and_update_wire_outcome_fails_closed() { assert_eq!( @@ -4063,32 +4350,19 @@ mod tests { } #[test] - fn retired_marker_options_preserve_legacy_positional_wire_shape() { + fn delete_options_named_wire_preserves_rollback_and_legacy_marker_fields() { #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)] struct LegacyDeleteOptions { recursive: bool, immediate: bool, undo_write: bool, + #[serde(default)] undo_delete: bool, old_data_dir: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + expected_delete_marker: Option, } - let legacy = LegacyDeleteOptions { - recursive: false, - immediate: false, - undo_write: false, - undo_delete: false, - old_data_dir: None, - }; - let original = encode_msgpack(&legacy).unwrap(); - let current = encode_msgpack(&DeleteOptions::default()).unwrap(); - assert_eq!(current, original, "ordinary deletes must retain the older peer's positional payload"); - assert_eq!(rmp_serde::from_slice::(¤t).unwrap(), legacy); - assert!( - rmp_serde::from_slice::(&original) - .unwrap() - .expected_delete_marker - .is_none() - ); + let transaction = Uuid::new_v4(); let mut marker = FileInfo { deleted: true, version_id: Some(Uuid::new_v4()), @@ -4097,12 +4371,69 @@ mod tests { }; marker.set_delete_marker_incarnation(Uuid::new_v4()); let marker = rustfs_filemeta::MetaDeleteMarker::from(marker); + for expected_delete_marker in [None, Some(marker)] { + for (undo_write, undo_delete) in [(true, false), (false, true)] { + let legacy = LegacyDeleteOptions { + recursive: false, + immediate: false, + undo_write, + undo_delete, + old_data_dir: Some(transaction), + expected_delete_marker: expected_delete_marker.clone(), + }; + // Older senders use positional arrays. Their unchanged field prefix and + // missing optional fields must also decode on the named-map sender's peer. + for (encoding, payload) in [ + ("positional", rmp_serde::to_vec(&legacy).expect("encode legacy positional options")), + ("named", rmp_serde::to_vec_named(&legacy).expect("encode legacy named options")), + ] { + let decoded: DeleteOptions = rmp_serde::from_slice(&payload).expect("decode legacy options on current peer"); + assert_eq!(decoded.undo_write, undo_write, "{encoding} undo-write flag"); + assert_eq!(decoded.undo_delete, undo_delete, "{encoding} undo-delete flag"); + assert_eq!(decoded.old_data_dir, Some(transaction), "{encoding} transaction UUID"); + assert_eq!(decoded.expected_delete_marker, expected_delete_marker, "{encoding} marker precondition"); + assert!(decoded.delete_recovery_bucket_incarnation.is_none(), "{encoding} omitted incarnation"); + assert!(!decoded.recoverable_delete, "{encoding} omitted recovery flag"); + } + for incarnation in [None, Some(Uuid::new_v4())] { + let current = DeleteOptions { + undo_write, + undo_delete, + old_data_dir: Some(transaction), + expected_delete_marker: expected_delete_marker.clone(), + delete_recovery_bucket_incarnation: incarnation, + recoverable_delete: true, + ..Default::default() + }; + let payload = encode_delete_options_msgpack(¤t).expect("encode current named options"); + let decoded: DeleteOptions = rmp_serde::from_slice(&payload).expect("decode current options on current peer"); + assert_eq!(decoded.undo_write, undo_write); + assert_eq!(decoded.undo_delete, undo_delete); + assert_eq!(decoded.old_data_dir, Some(transaction)); + assert_eq!(decoded.expected_delete_marker, expected_delete_marker); + assert_eq!(decoded.delete_recovery_bucket_incarnation, incarnation); + assert!(decoded.recoverable_delete); + let decoded: LegacyDeleteOptions = + rmp_serde::from_slice(&payload).expect("decode current options on legacy peer"); + assert_eq!(decoded, legacy, "unknown recovery fields cannot shift rollback or marker values"); + } + } + } + } + + #[test] + fn malformed_nonempty_delete_options_do_not_fall_back_to_json() { let options = DeleteOptions { - expected_delete_marker: Some(marker.clone()), + undo_write: true, + old_data_dir: Some(Uuid::new_v4()), ..Default::default() }; - let decoded: DeleteOptions = rmp_serde::from_slice(&encode_msgpack(&options).unwrap()).unwrap(); - assert_eq!(decoded.expected_delete_marker, Some(marker)); + let json = serde_json::to_string(&options).expect("valid compatibility JSON"); + assert!(decode_msgpack_or_json::(&[0xc1], &json, "DeleteOptions").is_err()); + let decoded: DeleteOptions = + decode_msgpack_or_json(&[], &json, "DeleteOptions").expect("absent MessagePack permits JSON compatibility"); + assert!(decoded.undo_write); + assert_eq!(decoded.old_data_dir, options.old_data_dir); } #[test] diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 98de238f5..01091325b 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -466,12 +466,11 @@ impl LocalDiskWrapper { self.track_disk_health_mutation( "delete_version", DiskMetricMutation::Delete, - || async { + || { Box::pin( self.disk .delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner), ) - .await }, get_max_timeout_duration(), ) @@ -1816,6 +1815,8 @@ impl LocalDiskWrapper { undo_delete: false, old_data_dir: None, expected_delete_marker: None, + recoverable_delete: false, + delete_recovery_bucket_incarnation: None, }, ) .await?; @@ -2001,10 +2002,86 @@ impl LocalDiskWrapper { } /// Get the current disk ID + pub(crate) async fn delete_versions_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + let version_count = versions.len(); + self.metrics.record_operation_call("delete_versions"); + // Check if disk is faulty before proceeding + if self.health.is_faulty() { + self.metrics.record_availability_error(); + return vec![Some(DiskError::FaultyDisk); versions.len()]; + } + + // Check if disk is stale + if let Err(e) = self.check_disk_stale().await { + self.metrics.record_availability_error(); + return vec![Some(e); versions.len()]; + } + + let _io_permit = match self.acquire_disk_io_permit().await { + Ok(permit) => permit, + Err(error) => return vec![Some(error); version_count], + }; + + // Record operation start + self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed); + let waiting_guard = self.health.waiting_guard(); + let metric_waiting_guard = self.metrics.waiting_guard(); + let started = Instant::now(); + + // Execute the operation + let result = self + .disk + .delete_versions_with_namespace_owner(volume, versions, opts, namespace_owner) + .await; + self.metrics.record_operation_latency("delete_versions", started.elapsed()); + self.record_batch_delete_error_metrics(&result); + + drop(waiting_guard); + drop(metric_waiting_guard); + let has_err = result.iter().any(|e| e.is_some()); + if !has_err { + // Log success and decrement waiting counter + self.health.record_operation_success(&self.endpoint(), "operation_success"); + self.metrics.record_mutation_success(DiskMetricMutation::Delete); + } + + result + } + pub async fn get_current_disk_id(&self) -> Option { *self.disk_id.read().await } + pub(crate) async fn settle_delete_transaction_with_namespace_owner( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: crate::disk::DeleteRecoveryDigest, + action: crate::disk::DeleteRecoveryAction, + namespace_owner: Option>, + ) -> Result { + self.track_disk_health_mutation( + "settle_delete_transaction", + DiskMetricMutation::Delete, + || { + Box::pin(async { + self.disk + .delete_recovery_settle(volume, object, transaction, expected_fingerprint, action, namespace_owner) + .await + }) + }, + crate::disk::get_delete_recovery_timeout()?, + ) + .await + } + /// Track disk health for an operation. /// This method should wrap disk operations to ensure health checking. pub async fn track_disk_health(&self, operation: F, timeout_duration: Duration) -> Result @@ -2195,6 +2272,87 @@ impl LocalDiskWrapper { #[async_trait::async_trait] impl DiskAPI for LocalDiskWrapper { + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + mut opts: DeleteOptions, + ) -> Result<()> { + crate::disk::validate_delete_recovery_options(&opts)?; + if object.starts_with('/') { + return Err(DiskError::InvalidPath); + } + opts.recoverable_delete = true; + self.delete_version(volume, object, file_info, force_del_marker, opts).await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + mut opts: DeleteOptions, + ) -> Vec> { + if let Err(err) = crate::disk::validate_delete_recovery_options(&opts) { + return vec![Some(err); versions.len()]; + } + if versions.iter().any(|version| version.name.starts_with('/')) { + return vec![Some(DiskError::InvalidPath); versions.len()]; + } + opts.recoverable_delete = true; + self.delete_versions(volume, versions, opts).await + } + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + self.track_disk_health_with_op( + "discover_delete_transactions", + || async { + self.disk + .discover_delete_transactions(volume, object, cursor.clone(), limit) + .await + }, + get_max_timeout_duration(), + ) + .await + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + ) -> Result { + self.track_disk_health_with_op( + "inspect_delete_transaction", + || async { self.disk.inspect_delete_transaction(volume, object, transaction).await }, + get_max_timeout_duration(), + ) + .await + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: crate::disk::DeleteRecoveryDigest, + action: crate::disk::DeleteRecoveryAction, + ) -> Result { + self.track_disk_health_mutation( + "settle_delete_transaction", + DiskMetricMutation::Delete, + || async { + self.disk + .settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + }, + crate::disk::get_delete_recovery_timeout()?, + ) + .await + } fn has_replacement_mount_lease(&self) -> bool { self.disk.has_replacement_mount_lease() } @@ -2408,46 +2566,7 @@ impl DiskAPI for LocalDiskWrapper { } async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) -> Vec> { - let version_count = versions.len(); - self.metrics.record_operation_call("delete_versions"); - // Check if disk is faulty before proceeding - if self.health.is_faulty() { - self.metrics.record_availability_error(); - return vec![Some(DiskError::FaultyDisk); versions.len()]; - } - - // Check if disk is stale - if let Err(e) = self.check_disk_stale().await { - self.metrics.record_availability_error(); - return vec![Some(e); versions.len()]; - } - - let _io_permit = match self.acquire_disk_io_permit().await { - Ok(permit) => permit, - Err(error) => return vec![Some(error); version_count], - }; - - // Record operation start - self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed); - let waiting_guard = self.health.waiting_guard(); - let metric_waiting_guard = self.metrics.waiting_guard(); - let started = Instant::now(); - - // Execute the operation - let result = self.disk.delete_versions(volume, versions, opts).await; - self.metrics.record_operation_latency("delete_versions", started.elapsed()); - self.record_batch_delete_error_metrics(&result); - - drop(waiting_guard); - drop(metric_waiting_guard); - let has_err = result.iter().any(|e| e.is_some()); - if !has_err { - // Log success and decrement waiting counter - self.health.record_operation_success(&self.endpoint(), "operation_success"); - self.metrics.record_mutation_success(DiskMetricMutation::Delete); - } - - result + self.delete_versions_with_namespace_owner(volume, versions, opts, None).await } async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index be9497308..11134035f 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -17,6 +17,7 @@ pub(in crate::disk) use self::commit::LocalRenamePreflightRejection; use self::commit::lock_rename_commit_directories; mod commit; +pub(super) mod delete_recovery; #[cfg(all(unix, test))] #[path = "fd_cache_tests.rs"] mod fd_cache_tests; @@ -130,8 +131,10 @@ tokio::task_local! { const RUSTFS_META_TMP_OLD_BUCKET: &str = ".rustfs.sys/tmp-old"; const INLINE_METADATA_ROLLBACK_DIR_XOR: u128 = 0x7275737466735f696e6c696e655f7262; const DELETE_MARKER_ROLLBACK_FILE: &str = "xl.meta.delete-marker.rollback"; +pub(crate) const DELETE_QUORUM_COMMITTED_FILE: &str = "delete-quorum-committed"; pub(crate) const DELETE_DATA_DIR_MARKER_PREFIX: &str = "delete-data."; pub(crate) const RESERVED_DELETE_DATA_DIR_MARKER_PREFIX: &str = "reserve-delete-data."; +pub(crate) const RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX: &str = "recovery-delete-data."; /// Largest directory read the delete-residue probe issues before it must /// fall back to a complete read. Residue holds one or two data dirs, so an /// under-filled batch settles the common case without materializing large @@ -161,6 +164,16 @@ pub(crate) fn metadata_less_part_file(entry: &str) -> bool { .is_some_and(|part_number| part_number.parse::().is_ok_and(|part_number| part_number > 0)) } +/// Native multipart sidecars travel with their part during delete recovery. +/// Callers must additionally require a regular file inside a bound data dir. +fn delete_recovery_part_file(entry: &str) -> bool { + metadata_less_part_file(entry) + || entry.strip_suffix(".meta").is_some_and(metadata_less_part_file) + || entry.split_once(".integrity.").is_some_and(|(part, generation)| { + metadata_less_part_file(part) && Uuid::parse_str(generation).is_ok_and(|id| !id.is_nil()) + }) +} + fn delete_transaction_marker_id(entry: &str, prefix: &str) -> Option { entry .strip_prefix(prefix) @@ -177,10 +190,15 @@ fn is_delete_transaction_marker(entry: &str, prefix: &str) -> bool { /// an `xl.meta`, an unknown file) means the directory is not plain delete residue. fn is_metadata_less_data_dir_entry(entry: &str) -> bool { !entry.ends_with(SLASH_SEPARATOR) - && (metadata_less_part_file(entry) + && (delete_recovery_part_file(entry) || entry == STORAGE_FORMAT_FILE_BACKUP + || entry == DELETE_QUORUM_COMMITTED_FILE + || entry == delete_recovery::PREPARED + || entry == delete_recovery::TARGET + || entry == delete_recovery::FINISHED || is_delete_transaction_marker(entry, DELETE_DATA_DIR_MARKER_PREFIX) - || is_delete_transaction_marker(entry, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX)) + || is_delete_transaction_marker(entry, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX) + || is_delete_transaction_marker(entry, RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX)) } const STARTUP_CLEANUP_WAIT_TIMEOUT: Duration = Duration::from_secs(2); const ENV_BITROT_SIZE_MISMATCH_RETRY_COUNT: &str = "RUSTFS_BITROT_SIZE_MISMATCH_RETRY_COUNT"; @@ -276,10 +294,21 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r #[cfg(test)] async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> { - write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, data, None).await + write_delete_rollback_file( + object_dir.parent().unwrap_or(object_dir), + "test-backup", + object_dir, + rollback_dir, + STORAGE_FORMAT_FILE_BACKUP, + data, + None, + ) + .await } async fn write_delete_rollback_file( + io_root: &Path, + volume: &str, object_dir: &Path, rollback_dir: Uuid, name: &str, @@ -290,16 +319,33 @@ async fn write_delete_rollback_file( let path = backup_dir.join(name); let lease = os::acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await; let data = data.to_vec(); - let object_dir = object_dir.to_path_buf(); + let sync = effective_durability(volume).syncs_commit_metadata(); + let staging_root = io_root.join(RUSTFS_META_TMP_BUCKET); + let io_root = io_root.to_path_buf(); os::run_blocking_namespace_operation(lease, move || { + reject_local_disk_symlink_components(&io_root, &path).map_err(std::io::Error::from)?; + reject_local_disk_symlink_components(&io_root, &staging_root).map_err(std::io::Error::from)?; std::fs::create_dir_all(&backup_dir)?; + std::fs::create_dir_all(&staging_root)?; #[cfg(test)] run_owned_file_write_before_open(&path); - // Rollback must never consume a partially written backup. Stage outside - // the rollback directory so interrupted writes cannot become staged data. - let mut staged = tempfile::Builder::new().prefix(".delete-rollback-").tempfile_in(object_dir)?; + // Private directory staging is reclaimed by existing tmp cleanup after + // a crash and cannot leave a regular artifact in the user's bucket. + let staging_dir = tempfile::Builder::new() + .prefix(".delete-rollback-") + .tempdir_in(&staging_root)?; + let mut staged = tempfile::NamedTempFile::new_in(staging_dir.path())?; std::io::Write::write_all(&mut staged, &data)?; + if sync { + staged.as_file().sync_all()?; + } staged.persist(path).map_err(|err| err.error)?; + if sync { + os::fsync_dir_std(&backup_dir)?; + if let Some(parent) = backup_dir.parent() { + os::fsync_dir_std(parent)?; + } + } Ok(()) }) .await @@ -336,14 +382,35 @@ async fn restore_metadata_backup_with_namespace_owner( } async fn restore_delete_rollback_with_namespace_owner( + volume: &str, object_dir: &Path, xl_path: &Path, rollback_dir: Uuid, publication_root: &os::PublicationRoot, namespace_owner: Option>, ) -> Result<()> { - remove_version_delete_markers(object_dir, rollback_dir, namespace_owner.clone()).await?; let rollback_path = object_dir.join(rollback_dir.to_string()); + match fs::symlink_metadata(rollback_path.join(DELETE_QUORUM_COMMITTED_FILE)).await { + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + let recovery_journal = match fs::symlink_metadata(rollback_path.join(delete_recovery::PREPARED)).await { + Ok(metadata) if metadata.is_file() => { + let journal: super::DeleteRecoveryJournal = serde_json::from_slice( + &fs::read(rollback_path.join(delete_recovery::PREPARED)) + .await + .map_err(to_file_error)?, + )?; + if journal.volume != volume || journal.transaction != rollback_dir { + return Err(DiskError::FileCorrupt); + } + Some(journal) + } + Ok(_) => return Err(DiskError::InvalidPath), + Err(err) if err.kind() == ErrorKind::NotFound => None, + Err(err) => return Err(to_file_error(err).into()), + }; let mut staged_paths = Vec::new(); let mut remove_new_metadata = false; match fs::read_dir(&rollback_path).await { @@ -352,65 +419,166 @@ async fn restore_delete_rollback_with_namespace_owner( let name = entry.file_name(); if name == DELETE_MARKER_ROLLBACK_FILE { remove_new_metadata = true; - } else if name != STORAGE_FORMAT_FILE_BACKUP { + } else if name != STORAGE_FORMAT_FILE_BACKUP + && name != DELETE_QUORUM_COMMITTED_FILE + && name != delete_recovery::PREPARED + && name != delete_recovery::TARGET + && name != delete_recovery::FINISHED + { + if let Some(journal) = &recovery_journal { + let id = name + .to_str() + .and_then(|name| Uuid::parse_str(name).ok()) + .ok_or(DiskError::FileCorrupt)?; + if !journal.data_dirs.contains(&id) || !entry.file_type().await.map_err(to_file_error)?.is_dir() { + return Err(DiskError::FileCorrupt); + } + } staged_paths.push((entry.path(), object_dir.join(name))); } } } - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) if err.kind() == ErrorKind::NotFound => { + return remove_version_delete_markers(volume, object_dir, rollback_dir, namespace_owner).await; + } Err(err) => return Err(to_file_error(err).into()), } let had_staged_paths = !staged_paths.is_empty(); for (src, dst) in staged_paths { - os::rename_all_with_owner(&src, &dst, object_dir, publication_root, namespace_owner.clone()).await?; + if recovery_journal.is_some() { + let mut entries = fs::read_dir(&src).await.map_err(to_file_error)?; + let mut files = Vec::new(); + while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { + let name = entry.file_name(); + let name = name.to_str().ok_or(DiskError::FileCorrupt)?; + if !entry.file_type().await.map_err(to_file_error)?.is_file() + || (!delete_recovery_part_file(name) + && ![ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .any(|prefix| name == format!("{prefix}{rollback_dir}"))) + { + return Err(DiskError::FileCorrupt); + } + files.push((entry.path(), dst.join(name))); + } + os::create_dir_all_with_namespace_owner(&dst, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + for (source, destination) in files { + match fs::symlink_metadata(&destination).await { + Ok(metadata) if metadata.is_file() => { + let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, namespace_owner.clone()).await; + let identical = os::run_blocking_namespace_operation(lease, move || { + let source_meta = std::fs::symlink_metadata(&source)?; + let destination_meta = std::fs::symlink_metadata(&destination)?; + if !source_meta.is_file() + || !destination_meta.is_file() + || source_meta.len() != destination_meta.len() + { + return Ok(false); + } + let mut source_file = std::fs::File::open(&source)?; + let mut destination_file = std::fs::File::open(&destination)?; + let mut left = [0u8; 65536]; + let mut right = [0u8; 65536]; + let mut remaining = source_meta.len(); + while remaining > 0 { + let length = usize::try_from(remaining.min(left.len() as u64)).map_err(std::io::Error::other)?; + std::io::Read::read_exact(&mut source_file, &mut left[..length])?; + std::io::Read::read_exact(&mut destination_file, &mut right[..length])?; + if left[..length] != right[..length] { + return Ok(false); + } + remaining -= length as u64; + } + std::fs::remove_file(source)?; + Ok(true) + }) + .await + .map_err(to_file_error)?; + if !identical { + return Err(DiskError::FileCorrupt); + } + } + Ok(_) => return Err(DiskError::InvalidPath), + Err(err) if err.kind() == ErrorKind::NotFound => { + os::rename_all_with_owner(&source, &destination, object_dir, publication_root, namespace_owner.clone()) + .await?; + } + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&dst, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + os::fsync_dir_with_owner(&src, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } + let _ = os::remove_dir_with_owner(&src, namespace_owner.clone()).await; + } else { + os::rename_all_with_owner(&src, &dst, object_dir, publication_root, namespace_owner.clone()).await?; + } } let backup_path = rollback_path.join(STORAGE_FORMAT_FILE_BACKUP); match os::rename_all_with_owner(&backup_path, xl_path, object_dir, publication_root, namespace_owner.clone()).await { - Ok(()) => { - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - // A missing backup only means "remove the newly-created delete marker" - // when the marker proves there was no old metadata to restore. + Ok(()) => {} Err(DiskError::FileNotFound) if remove_new_metadata => { match os::remove_file_with_owner(xl_path, namespace_owner.clone()).await { - Ok(()) => { - let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()) - .await; - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - Err(err) if err.kind() == ErrorKind::NotFound => { - let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()) - .await; - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - Err(err) => Err(to_file_error(err).into()), + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), } } - Err(DiskError::FileNotFound) if had_staged_paths => Err(DiskError::FileNotFound), + Err(DiskError::FileNotFound) if had_staged_paths => return Err(DiskError::FileNotFound), Err(DiskError::FileNotFound) => match fs::metadata(xl_path).await { - Ok(_) => { - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - Err(err) if err.kind() == ErrorKind::NotFound => Err(DiskError::FileNotFound), - Err(err) => Err(to_file_error(err).into()), + Ok(_) => {} + Err(err) if err.kind() == ErrorKind::NotFound => return Err(DiskError::FileNotFound), + Err(err) => return Err(to_file_error(err).into()), }, - Err(err) => Err(err), + Err(err) => return Err(err), } + delete_recovery::restore_barrier(volume, object_dir, rollback_dir).await?; + // Persist both sides of the restore rename before discarding its evidence. + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(object_dir, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + os::fsync_dir_with_owner(&rollback_path, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } + remove_version_delete_markers(volume, object_dir, rollback_dir, namespace_owner.clone()).await?; + if remove_new_metadata { + let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()).await; + } + for name in [delete_recovery::TARGET, delete_recovery::PREPARED, delete_recovery::FINISHED] { + match os::remove_file_with_owner(rollback_path.join(name), namespace_owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner).await; + Ok(()) } async fn remove_version_delete_markers( + volume: &str, object_dir: &Path, rollback_dir: Uuid, namespace_owner: Option>, ) -> Result<()> { let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); let committed_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); + let recovery_name = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); let mut entries = match fs::read_dir(object_dir).await { Ok(entries) => entries, Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), @@ -422,13 +590,20 @@ async fn remove_version_delete_markers( { continue; } - for marker_name in [&reserved_name, &committed_name] { + for marker_name in [&reserved_name, &committed_name, &recovery_name] { match os::remove_file_with_owner(entry.path().join(marker_name), namespace_owner.clone()).await { Ok(()) => {} Err(err) if err.kind() == ErrorKind::NotFound => {} Err(err) => return Err(to_file_error(err).into()), } } + // Also sync on retry: a previous unlink may have succeeded before its + // directory sync failed. Keep the journal until those removals persist. + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&entry.path(), namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } } Ok(()) } @@ -464,7 +639,8 @@ async fn restore_delete_rollback_after_error( }; if let Err(restore_err) = - restore_delete_rollback_with_namespace_owner(object_dir, xl_path, rollback_dir, publication_root, namespace_owner).await + restore_delete_rollback_with_namespace_owner(volume, object_dir, xl_path, rollback_dir, publication_root, namespace_owner) + .await { warn!( volume, @@ -6300,6 +6476,32 @@ impl LocalDisk { result.map(|()| DataDirDeleteStatus::Deleted) } + pub(crate) async fn delete_versions_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + let mut errs = Vec::with_capacity(versions.len()); + for _ in 0..versions.len() { + errs.push(None); + } + + for (i, ver) in versions.iter().enumerate() { + if let Err(e) = self + .delete_versions_internal(volume, ver.name.as_str(), &ver.versions, &opts, namespace_owner.clone()) + .await + { + errs[i] = Some(e); + } else { + errs[i] = None; + } + } + + errs + } + async fn delete_version_inner(&self, volume: &str, path: &str, fi: FileInfo, mutation: DeleteVersionMutation) -> Result<()> { let DeleteVersionMutation { force_del_marker, @@ -6347,10 +6549,24 @@ impl LocalDisk { check_path_length(file_path.to_string_lossy().as_ref())?; let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]); - let namespace_owner: Option> = + let mut namespace_owner: Option> = Some(os::acquire_metadata_mutation_lease(&self.get_object_path(volume, path)?, namespace_owner).await); + if opts.recoverable_delete || (opts.undo_write && opts.undo_delete) { + let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await; + namespace_owner = Some(Arc::new((namespace_owner, volume_guard))); + } if opts.old_data_dir.is_some() && opts.undo_write { - return self.undo_write(file_path.as_path(), &fi, &opts, namespace_owner).await; + if opts.undo_delete + && let Some(transaction) = opts.old_data_dir + && self + .rollback_prepared_delete(volume, path, transaction, namespace_owner.clone()) + .await? + { + return Ok(()); + } + return self + .undo_write(volume, file_path.as_path(), &fi, &opts, namespace_owner) + .await; } let rollback_dir = opts.old_data_dir; @@ -6360,12 +6576,12 @@ impl LocalDisk { if err != DiskError::FileNotFound { return Err(err); } - self.finish_committed_delete_residue(volume, path, namespace_owner.clone()) + self.finish_internal_delete_residue(volume, path, namespace_owner.clone()) .await; if fi.deleted && force_del_marker { return self - .write_missing_delete_marker(volume, path, fi, file_path.as_path(), rollback_dir, namespace_owner.clone()) + .write_missing_delete_marker(volume, path, fi, file_path.as_path(), &opts, namespace_owner.clone()) .await; } @@ -6397,15 +6613,35 @@ impl LocalDisk { } let old_dir = meta.delete_version(&fi)?; let mut reserved_version_delete = false; + let mut recovery_target = None; if let Some(rollback_dir) = rollback_dir { - write_delete_rollback_file( - file_path.as_path(), - rollback_dir, - STORAGE_FORMAT_FILE_BACKUP, - &buf, - namespace_owner.clone(), - ) - .await?; + if opts.recoverable_delete { + recovery_target = self + .prepare_delete_recovery( + volume, + path, + Some(&buf), + std::slice::from_ref(&fi), + None, + delete_recovery::DeletePreparationContext { + transaction: rollback_dir, + bucket_incarnation: opts.delete_recovery_bucket_incarnation, + namespace_owner: namespace_owner.clone(), + }, + ) + .await?; + } else { + write_delete_rollback_file( + self.io_root(), + volume, + file_path.as_path(), + rollback_dir, + STORAGE_FORMAT_FILE_BACKUP, + &buf, + namespace_owner.clone(), + ) + .await?; + } } if let Some(uuid) = old_dir { @@ -6562,7 +6798,7 @@ impl LocalDisk { } let commit_result = if !meta.versions.is_empty() { - let buf = match meta.marshal_msg() { + let buf = match recovery_target.map(Ok).unwrap_or_else(|| meta.marshal_msg()) { Ok(buf) => buf, Err(err) => { let err: DiskError = err.into(); @@ -6642,6 +6878,10 @@ impl LocalDisk { .await); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } + if reserved_version_delete && let Some(rollback_dir) = rollback_dir && let Err(err) = self @@ -6698,6 +6938,7 @@ impl LocalDisk { async fn undo_write( &self, + volume: &str, file_path: &Path, fi: &FileInfo, opts: &DeleteOptions, @@ -6707,6 +6948,7 @@ impl LocalDisk { let xl_path = path_join(&[file_path, Path::new(STORAGE_FORMAT_FILE)]); if opts.undo_delete { restore_delete_rollback_with_namespace_owner( + volume, file_path, &xl_path, old_data_dir, @@ -6836,13 +7078,29 @@ impl LocalDisk { namespace_owner: Option>, ) -> Result<()> { crate::hp_guard!("LocalDisk::delete"); + let mut opt = opt; + let mut namespace_owner = namespace_owner; let handled_version_delete = if opt.recursive && opt.immediate && let Some((object, transaction_id)) = path.rsplit_once('/') && let Ok(transaction_id) = Uuid::parse_str(transaction_id) + && !transaction_id.is_nil() { - self.finish_version_delete(volume, object, transaction_id, namespace_owner.clone()) - .await? + if opt.old_data_dir == Some(transaction_id) { + let object_path = self.get_object_path(volume, object)?; + namespace_owner = Some(os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await); + let handled = self + .finish_delete_transaction_with_namespace_owner(volume, object, transaction_id, namespace_owner.clone()) + .await?; + // A key below the UUID prefix may be created concurrently with + // cleanup. Never recursively remove newly published descendants. + opt.recursive = false; + handled + } else { + self.finish_version_delete(volume, object, transaction_id, false, namespace_owner.clone()) + .await? + .0 + } } else { false }; @@ -6855,6 +7113,104 @@ impl LocalDisk { } } + // The caller owns the object metadata lease; retry recovery must not + // reacquire it while holding the same mutation domain. + async fn finish_delete_transaction_with_namespace_owner( + &self, + volume: &str, + object: &str, + transaction_id: Uuid, + namespace_owner: Option>, + ) -> Result { + let path = format!("{object}/{transaction_id}"); + let transaction_path = self.io_get_object_path(volume, &path)?; + let mut transaction_cleanup = false; + match fs::read_dir(&transaction_path).await { + Ok(mut entries) => { + while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { + let name = entry.file_name(); + if !entry.file_type().await.map_err(to_file_error)?.is_file() + || (name != STORAGE_FORMAT_FILE_BACKUP + && name != DELETE_MARKER_ROLLBACK_FILE + && name != DELETE_QUORUM_COMMITTED_FILE) + { + return Err(DiskError::FileCorrupt); + } + transaction_cleanup = true; + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + if transaction_cleanup { + // The set invokes transaction cleanup only after observing write + // quorum. Unlike data-dir markers, this receipt proves that + // decision and also covers inline objects with no data directory. + let receipt_path = transaction_path.join(DELETE_QUORUM_COMMITTED_FILE); + let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, namespace_owner.clone()).await; + let sync = effective_durability(volume).syncs_commit_metadata(); + os::run_blocking_namespace_operation(lease, move || { + let receipt = match std::fs::OpenOptions::new().write(true).create_new(true).open(&receipt_path) { + Ok(receipt) => receipt, + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + if !std::fs::symlink_metadata(&receipt_path)?.is_file() { + return Err(std::io::Error::new(ErrorKind::InvalidData, "delete receipt is not a regular file")); + } + std::fs::File::open(receipt_path)? + } + Err(err) => return Err(err), + }; + if sync { + receipt.sync_all()?; + os::fsync_dir_std(&transaction_path)?; + if let Some(parent) = transaction_path.parent() { + os::fsync_dir_std(parent)?; + } + } + Ok(()) + }) + .await + .map_err(to_file_error)?; + // Retain the receipt until both the backup and marked data are + // gone. Every interrupted cleanup remains recognizable. + for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] { + match os::remove_file_with_owner(self.io_get_object_path(volume, &path)?.join(name), namespace_owner.clone()) + .await + { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&self.io_get_object_path(volume, &path)?, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } + } + let (found, physically_deleted) = self + .finish_version_delete(volume, object, transaction_id, true, namespace_owner.clone()) + .await?; + if !physically_deleted { + // Snapshot readers still own these parts. Keep the quorum receipt + // until their release finishes the physical data removal. + return Err(DiskError::DiskOngoingReq); + } + if transaction_cleanup { + match os::remove_file_with_owner( + self.io_get_object_path(volume, &path)?.join(DELETE_QUORUM_COMMITTED_FILE), + namespace_owner, + ) + .await + { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + Ok(found || transaction_cleanup) + } + async fn delete_unleased(&self, volume: &str, path: &str, opt: &DeleteOptions) -> Result<()> { self.delete_unleased_with_namespace_owner(volume, path, opt, None).await } @@ -6875,7 +7231,94 @@ impl LocalDisk { let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; - self.delete_file_with_namespace_owner(&volume_dir, &file_path, opt.recursive, opt.immediate, namespace_owner) + let mut recursive = opt.recursive; + if recursive + && let Some(transaction) = opt.old_data_dir + && path + .rsplit_once('/') + .is_some_and(|(_, name)| Uuid::parse_str(name).is_ok_and(|id| id != transaction || opt.recoverable_delete)) + { + // Snapshot release replays these same options. Remove only this + // transaction's regular files; UUID-prefix child keys can appear + // after classification and must never be recursively removed. + let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner.clone()).await; + let data_path = file_path.clone(); + let sync = effective_durability(volume).syncs_commit_metadata(); + let keep_rollback_witnesses = opt.recoverable_delete; + os::run_blocking_namespace_operation(lease, move || { + let entries = match std::fs::read_dir(&data_path) { + Ok(entries) => entries, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(err), + }; + let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"); + let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"); + let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"); + let corrupt = + || std::io::Error::new(ErrorKind::InvalidData, "delete data directory contains unrecognized entries"); + let mut parts = Vec::new(); + let mut markers = Vec::new(); + let mut has_committed_marker = false; + for entry in entries { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_str().ok_or_else(corrupt)?; + if !entry.file_type()?.is_file() { + return Err(corrupt()); + } + if delete_recovery_part_file(name) { + parts.push(entry.path()); + } else if name == committed || name == recovery_committed || name == reserved { + has_committed_marker |= name == committed || name == recovery_committed; + markers.push(entry.path()); + } else if keep_rollback_witnesses + && [ + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + ] + .contains(&name) + { + // Explicit legacy deletion keeps rollback witnesses until + // every snapshot-protected payload has disappeared. + } else { + return Err(corrupt()); + } + } + if !parts.is_empty() && !has_committed_marker { + return Err(corrupt()); + } + for files in [parts, markers] { + for entry in files { + #[cfg(all(any(test, feature = "test-util"), not(windows)))] + os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &entry); + match std::fs::remove_file(entry) { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + } + if sync { + os::fsync_dir_std(&data_path)?; + } + } + match std::fs::remove_dir(data_path) { + Ok(()) => Ok(()), + Err(err) if is_benign_object_rmdir_error(&err) => Ok(()), + Err(err) => Err(err), + } + }) + .await + .map_err(|err| { + if err.kind() == ErrorKind::InvalidData { + DiskError::FileCorrupt + } else { + to_file_error(err).into() + } + })?; + recursive = false; + } + self.delete_file_with_namespace_owner(&volume_dir, &file_path, recursive, opt.immediate, namespace_owner) .await?; // A deleted shard must not remain readable through the io_uring fd cache. self.io_backend.invalidate_cached_fds_under(volume, path); @@ -7245,20 +7688,63 @@ impl LocalDisk { path: &str, fi: FileInfo, object_dir: &Path, - rollback_dir: Option, + opts: &DeleteOptions, namespace_owner: Option>, ) -> Result<()> { + let rollback_dir = opts.old_data_dir; + let incarnation = opts.delete_recovery_bucket_incarnation; let xl_path = object_dir.join(STORAGE_FORMAT_FILE); + let mut recovery_target = None; if let Some(rollback_dir) = rollback_dir { - write_delete_rollback_file(object_dir, rollback_dir, DELETE_MARKER_ROLLBACK_FILE, &[], namespace_owner.clone()) + if opts.recoverable_delete { + recovery_target = self + .prepare_delete_recovery( + volume, + path, + None, + std::slice::from_ref(&fi), + None, + delete_recovery::DeletePreparationContext { + transaction: rollback_dir, + bucket_incarnation: incarnation, + namespace_owner: namespace_owner.clone(), + }, + ) + .await?; + } else { + write_delete_rollback_file( + self.io_root(), + volume, + object_dir, + rollback_dir, + DELETE_MARKER_ROLLBACK_FILE, + &[], + namespace_owner.clone(), + ) .await?; + } } - if let Err(err) = self - .write_metadata_with_namespace_owner(volume, path, fi, namespace_owner.clone()) + let committed = if let Some(target) = recovery_target { + async { + fi.validate_for_metadata_read()?; + self.write_all_meta_with_namespace_owner( + volume, + &format!("{path}/{STORAGE_FORMAT_FILE}"), + &target, + true, + namespace_owner.clone(), + ) + .await + } .await - { + } else { + self.write_metadata_with_namespace_owner(volume, path, fi, namespace_owner.clone()) + .await + }; + if let Err(err) = committed { if let Some(rollback_dir) = rollback_dir && let Err(restore_err) = restore_delete_rollback_with_namespace_owner( + volume, object_dir, &xl_path, rollback_dir, @@ -7281,13 +7767,28 @@ impl LocalDisk { } return Err(err); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } Ok(()) } - async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> { + async fn delete_versions_internal( + &self, + volume: &str, + path: &str, + fis: &[FileInfo], + opts: &DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { let volume_dir = self.io_get_bucket_path(volume)?; let object_path = self.get_object_path(volume, path)?; - let namespace_owner: Option> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await); + let mut namespace_owner: Option> = + Some(os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await); + if opts.recoverable_delete || (opts.undo_write && opts.undo_delete) { + let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await; + namespace_owner = Some(Arc::new((namespace_owner, volume_guard))); + } let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?; let object_dir = xlpath .parent() @@ -7297,7 +7798,14 @@ impl LocalDisk { && opts.undo_write { if opts.undo_delete { + if self + .rollback_prepared_delete(volume, path, rollback_dir, namespace_owner.clone()) + .await? + { + return Ok(()); + } return restore_delete_rollback_with_namespace_owner( + volume, object_dir, &xlpath, rollback_dir, @@ -7323,7 +7831,7 @@ impl LocalDisk { { Ok(data) => data, Err(DiskError::FileNotFound) => { - self.finish_committed_delete_residue(volume, path, namespace_owner.clone()) + self.finish_internal_delete_residue(volume, path, namespace_owner.clone()) .await; // `deleted` alone can be an explicit marker purge; only // `mark_deleted` may create metadata that was not present. @@ -7331,14 +7839,7 @@ impl LocalDisk { return Err(DiskError::FileNotFound); }; return self - .write_missing_delete_marker( - volume, - path, - delete_marker, - object_dir, - opts.old_data_dir, - namespace_owner.clone(), - ) + .write_missing_delete_marker(volume, path, delete_marker, object_dir, opts, namespace_owner.clone()) .await; } Err(err) => return Err(err), @@ -7353,9 +7854,35 @@ impl LocalDisk { fm.unmarshal_msg(&data)?; let rollback_dir = opts.old_data_dir; let mut reserved_version_delete = false; + let mut recovery_target = None; if let Some(rollback_dir) = rollback_dir { - write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, &data, namespace_owner.clone()) + if opts.recoverable_delete { + recovery_target = self + .prepare_delete_recovery( + volume, + path, + Some(&data), + fis, + None, + delete_recovery::DeletePreparationContext { + transaction: rollback_dir, + bucket_incarnation: opts.delete_recovery_bucket_incarnation, + namespace_owner: namespace_owner.clone(), + }, + ) + .await?; + } else { + write_delete_rollback_file( + self.io_root(), + volume, + object_dir, + rollback_dir, + STORAGE_FORMAT_FILE_BACKUP, + &data, + namespace_owner.clone(), + ) .await?; + } } for fi in fis.iter() { @@ -7601,6 +8128,9 @@ impl LocalDisk { ) .await); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } if reserved_version_delete && let Some(rollback_dir) = rollback_dir && let Err(err) = self @@ -7629,7 +8159,7 @@ impl LocalDisk { // Update xl.meta atomically: a concurrent reader or crash mid-write must // never observe a truncated xl.meta for versions that were not deleted. - let buf = match fm.marshal_msg() { + let buf = match recovery_target.map(Ok).unwrap_or_else(|| fm.marshal_msg()) { Ok(buf) => buf, Err(err) => { let err: DiskError = err.into(); @@ -7706,6 +8236,10 @@ impl LocalDisk { .await); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } + if reserved_version_delete && let Some(rollback_dir) = rollback_dir && let Err(err) = self @@ -9237,7 +9771,16 @@ impl LocalDisk { Err(err) => return Err(to_file_error(err).into()), }; let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); - let committed_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); + let recovery_owned = + fs::try_exists(self.io_get_object_path(volume, &format!("{object}/{rollback_dir}/{}", delete_recovery::PREPARED))?) + .await + .map_err(to_file_error)?; + let prefix = if recovery_owned { + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX + } else { + DELETE_DATA_DIR_MARKER_PREFIX + }; + let committed_name = format!("{prefix}{rollback_dir}"); while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { if !entry.file_type().await.map_err(to_file_error)?.is_dir() || !entry.file_name().to_str().is_some_and(|name| Uuid::parse_str(name).is_ok()) @@ -9272,14 +9815,21 @@ impl LocalDisk { /// removed the object's metadata on this disk and committed, so the data /// it marked can no longer be read, healed or rolled back here. Finishing /// removes what `T`'s own cleanup pass would have removed. Reserved - /// markers, rollback backups without a committed marker, and object + /// transaction receipts also prove quorum commitment for inline objects. + /// Reserved markers, rollback backups without either proof, and object /// directories that hold `xl.meta` are left alone. - async fn finish_committed_delete_residue( + async fn finish_internal_delete_residue( &self, volume: &str, path: &str, namespace_owner: Option>, ) -> usize { + // RUSTFS_COMPAT_TODO(backlog#2807): retain legacy cleanup for internal + // maintenance until old system-bucket records drain. User-bucket + // residue requires the explicit set-wide recovery decision. + if !crate::bucket::utils::is_meta_bucketname(volume) { + return 0; + } let transactions = match self.committed_delete_transactions(volume, path).await { Ok(transactions) => transactions, Err(err) => { @@ -9289,7 +9839,7 @@ impl LocalDisk { subsystem = LOG_SUBSYSTEM_DISK_LOCAL, volume, path, - operation = "scan_committed_delete_residue", + operation = "scan_internal_delete_residue", error = ?err, "Disk local delete failed" ); @@ -9303,17 +9853,49 @@ impl LocalDisk { }; let mut finished = 0; for transaction in transactions { - // Unlike the delete's own pass, drop the rollback backup before the - // marked data: if this is cut short, the marker that proves the - // transaction is still in place for the next attempt. - let result = match self - .delete_unleased_with_namespace_owner(volume, &format!("{path}/{transaction}"), &opts, namespace_owner.clone()) - .await - { - Ok(()) | Err(DiskError::FileNotFound) => self - .finish_version_delete(volume, path, transaction, namespace_owner.clone()) - .await - .map(|_| ()), + let transaction_path = format!("{path}/{transaction}"); + let receipt = self + .io_get_object_path(volume, &transaction_path) + .map(|transaction_path| transaction_path.join(DELETE_QUORUM_COMMITTED_FILE)); + let result = match receipt { + Ok(receipt) => match fs::symlink_metadata(receipt).await { + Ok(metadata) if metadata.is_file() => { + match self + .finish_delete_transaction_with_namespace_owner(volume, path, transaction, namespace_owner.clone()) + .await + { + Ok(_) => { + self.delete_unleased_with_namespace_owner( + volume, + &transaction_path, + &DeleteOptions { + recursive: false, + ..opts.clone() + }, + namespace_owner.clone(), + ) + .await + } + Err(err) => Err(err), + } + } + Ok(_) => Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => { + // Preserve the legacy marker-only cleanup policy without + // upgrading disk-local evidence into a quorum receipt. + match self + .delete_unleased_with_namespace_owner(volume, &transaction_path, &opts, namespace_owner.clone()) + .await + { + Ok(()) | Err(DiskError::FileNotFound) => self + .finish_version_delete(volume, path, transaction, false, namespace_owner.clone()) + .await + .map(|_| ()), + Err(err) => Err(err), + } + } + Err(err) => Err(to_file_error(err).into()), + }, Err(err) => Err(err), }; match result { @@ -9325,7 +9907,7 @@ impl LocalDisk { volume, path, transaction = %transaction, - operation = "finish_committed_delete_residue", + operation = "finish_internal_delete_residue", error = ?err, "Disk local delete failed" ), @@ -9385,7 +9967,22 @@ impl LocalDisk { Err(err) if err.kind() == ErrorKind::NotFound => continue, Err(err) => return Err(to_file_error(err).into()), }; + let mut has_quorum_receipt = false; + let mut safe_receipt_dir = true; while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { + let name = entry.file_name(); + let file_type = entry.file_type().await.map_err(to_file_error)?; + if !file_type.is_file() + || (name != STORAGE_FORMAT_FILE_BACKUP + && name != DELETE_MARKER_ROLLBACK_FILE + && name != DELETE_QUORUM_COMMITTED_FILE) + { + safe_receipt_dir = false; + } + if entry.file_name() == DELETE_QUORUM_COMMITTED_FILE { + has_quorum_receipt = file_type.is_file(); + continue; + } let Some(transaction) = entry .file_name() .to_str() @@ -9402,8 +9999,31 @@ impl LocalDisk { Err(err) => return Err(to_file_error(err).into()), } } + if has_quorum_receipt + && safe_receipt_dir + && let Some(transaction) = data_dir + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| Uuid::parse_str(name).ok()) + { + transactions.insert(transaction); + } } - Ok(transactions) + let mut legacy = BTreeSet::new(); + for transaction in transactions { + let rollback = object_path.join(transaction.to_string()); + let mut recovery_owned = false; + for name in [delete_recovery::PREPARED, delete_recovery::TARGET, delete_recovery::FINISHED] { + recovery_owned |= fs::try_exists(rollback.join(name)).await.map_err(to_file_error)?; + } + if let Ok(metadata) = fs::symlink_metadata(rollback.join(DELETE_QUORUM_COMMITTED_FILE)).await { + recovery_owned |= metadata.len() > 0; + } + if !recovery_owned { + legacy.insert(transaction); + } + } + Ok(legacy) } async fn finish_version_delete( @@ -9411,17 +10031,30 @@ impl LocalDisk { volume: &str, object: &str, rollback_dir: Uuid, + preserve_prefix_children: bool, namespace_owner: Option>, - ) -> Result { + ) -> Result<(bool, bool)> { let object_path = self.io_get_object_path(volume, object)?; let mut entries = match fs::read_dir(object_path).await { Ok(entries) => entries, - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(false), + Err(err) if err.kind() == ErrorKind::NotFound => return Ok((false, true)), Err(err) => return Err(to_file_error(err).into()), }; - let marker_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); + let recovery_owned = preserve_prefix_children + && fs::try_exists( + self.io_get_object_path(volume, &format!("{object}/{rollback_dir}/{}", delete_recovery::PREPARED))?, + ) + .await + .map_err(to_file_error)?; + let prefix = if recovery_owned { + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX + } else { + DELETE_DATA_DIR_MARKER_PREFIX + }; + let marker_name = format!("{prefix}{rollback_dir}"); let mut first_err = None; let mut found = false; + let mut deferred = false; while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { let Some(data_dir) = entry.file_name().to_str().and_then(|data_dir| Uuid::parse_str(data_dir).ok()) else { continue; @@ -9432,25 +10065,27 @@ impl LocalDisk { Err(err) if err.kind() == ErrorKind::NotFound => continue, Err(err) => return Err(to_file_error(err).into()), } - if let Err(err) = self + match self .delete_data_dir_with_namespace_owner( volume, &format!("{object}/{data_dir}"), DeleteOptions { recursive: true, + old_data_dir: preserve_prefix_children.then_some(rollback_dir), ..Default::default() }, namespace_owner.clone(), ) .await - && first_err.is_none() - && err != DiskError::FileNotFound - && err != DiskError::VolumeNotFound { - first_err = Some(err); + Ok(DataDirDeleteStatus::Deferred) => deferred = true, + Ok(DataDirDeleteStatus::Deleted) => {} + Err(err) if err == DiskError::FileNotFound || err == DiskError::VolumeNotFound => {} + Err(err) if first_err.is_none() => first_err = Some(err), + Err(_) => {} } } - first_err.map_or(Ok(found), Err) + first_err.map_or(Ok((found, !deferred)), Err) } async fn abort_reserved_version_delete_with_failure( @@ -9560,6 +10195,64 @@ pub(crate) async fn batch_shard_pread(requests: Vec<(std::path::PathBuf, usize, #[async_trait::async_trait] impl DiskAPI for LocalDisk { + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + mut opts: DeleteOptions, + ) -> Result<()> { + super::validate_delete_recovery_options(&opts)?; + if object.starts_with('/') { + return Err(DiskError::InvalidPath); + } + opts.recoverable_delete = true; + self.delete_version(volume, object, file_info, force_del_marker, opts).await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + mut opts: DeleteOptions, + ) -> Vec> { + if let Err(err) = super::validate_delete_recovery_options(&opts) { + return vec![Some(err); versions.len()]; + } + if versions.iter().any(|version| version.name.starts_with('/')) { + return vec![Some(DiskError::InvalidPath); versions.len()]; + } + opts.recoverable_delete = true; + self.delete_versions(volume, versions, opts).await + } + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + self.delete_recovery_discover(volume, object, cursor, limit).await + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + self.delete_recovery_inspect(volume, object, transaction).await + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: super::DeleteRecoveryDigest, + action: super::DeleteRecoveryAction, + ) -> Result { + self.delete_recovery_settle(volume, object, transaction, expected_fingerprint, action, None) + .await + } fn to_string(&self) -> String { self.root.to_string_lossy().to_string() } @@ -11261,23 +11954,7 @@ impl DiskAPI for LocalDisk { } #[tracing::instrument(level = "trace", skip_all)] async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) -> Vec> { - let mut errs = Vec::with_capacity(versions.len()); - for _ in 0..versions.len() { - errs.push(None); - } - - for (i, ver) in versions.iter().enumerate() { - if let Err(e) = self - .delete_versions_internal(volume, ver.name.as_str(), &ver.versions, &opts) - .await - { - errs[i] = Some(e); - } else { - errs[i] = None; - } - } - - errs + self.delete_versions_with_namespace_owner(volume, versions, opts, None).await } #[tracing::instrument(level = "trace", skip_all)] @@ -11632,7 +12309,7 @@ mod test { } } - fn test_file_info(name: &str, version_id: Uuid, data_dir: Option, data: Option) -> FileInfo { + pub(super) fn test_file_info(name: &str, version_id: Uuid, data_dir: Option, data: Option) -> FileInfo { let size = data .as_ref() .map(|data| i64::try_from(data.len()).expect("test data length should fit i64")) @@ -11694,7 +12371,7 @@ mod test { assert!(!should_read_legacy_inline_part(&ec12, &config)); } - fn test_meta(fi: FileInfo) -> Vec { + pub(super) fn test_meta(fi: FileInfo) -> Vec { let mut meta = FileMeta::default(); meta.add_version(fi).expect("test metadata should accept file info"); meta.marshal_msg().expect("test metadata should encode") @@ -11958,7 +12635,7 @@ mod test { assert!(!child_path.exists(), "the exact locked directory should be removed"); } - async fn ensure_test_volume(disk: &LocalDisk, volume: &str) { + pub(super) async fn ensure_test_volume(disk: &LocalDisk, volume: &str) { match disk.make_volume(volume).await { Ok(()) | Err(DiskError::VolumeExists) => {} Err(err) => panic!("test volume should be available: {err:?}"), @@ -17890,7 +18567,7 @@ mod test { .await .expect("existing metadata should be written"); - disk.delete_versions_internal(bucket, object, &[missing_fi, existing_fi], &DeleteOptions::default()) + disk.delete_versions_internal(bucket, object, &[missing_fi, existing_fi], &DeleteOptions::default(), None) .await .expect("missing non-deleted version should not abort deletion"); @@ -18047,6 +18724,7 @@ mod test { old_data_dir: Some(rollback_dir), ..Default::default() }, + None, ) .await .expect("missing inline data dir should not fail batch deletion"); @@ -18260,8 +18938,7 @@ mod test { } } - /// Leave the state a delete reaches once it committed on this disk but - /// before its cleanup pass: what a cancelled request used to strand. + /// A legacy disk-local mutation without a structured set decision. async fn write_committed_delete_residue(disk: &LocalDisk, bucket: &str, object: &str) -> CommittedDeleteResidue { let version_id = Uuid::new_v4(); let data_dir = Uuid::new_v4(); @@ -18301,6 +18978,56 @@ mod test { residue } + #[tokio::test] + async fn delete_recovery_user_bucket_apply_preserves_ambiguous_legacy_residue() { + for batch in [false, true] { + for recoverable in [false, true] { + let dir = tempfile::tempdir().expect("residue fixture directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 endpoint")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "user-legacy-residue"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let original = fs::read(residue.backup()).await.expect("legacy metadata backup"); + let request = FileInfo { + name: object.to_owned(), + version_id: Some(Uuid::new_v4()), + ..Default::default() + }; + let opts = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + recoverable_delete: recoverable, + delete_recovery_bucket_incarnation: Some(Uuid::new_v4()), + ..Default::default() + }; + if batch { + let errors = disk + .delete_versions( + bucket, + vec![FileInfoVersions { + name: object.to_owned(), + versions: vec![request], + ..Default::default() + }], + opts, + ) + .await; + assert!(matches!(errors.as_slice(), [Some(DiskError::FileNotFound)]), "{errors:?}"); + } else { + assert!(matches!( + disk.delete_version(bucket, object, request, false, opts).await, + Err(DiskError::FileVersionNotFound) + )); + } + assert_eq!(fs::read(residue.backup()).await.expect("preserved backup"), original); + assert_eq!(fs::read(residue.part()).await.expect("preserved shard"), b"stale"); + assert!(residue.marker().is_file(), "a missing local apply cannot certify historical deletion"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); + } + } + } + fn residue_delete_request(object: &str) -> Vec { vec![FileInfoVersions { name: object.to_string(), @@ -18312,14 +19039,14 @@ mod test { }] } - // #6898: repeating the delete of a key whose earlier delete committed but - // never cleaned up reclaims the residue and still reports the key missing. + // Internal maintenance retains its legacy cleanup behavior while user + // buckets require the separate set-wide authorization boundary. #[tokio::test] async fn delete_versions_finishes_committed_delete_residue() { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "prefix/object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18339,7 +19066,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18359,14 +19086,13 @@ mod test { assert!(!residue.object_dir.exists(), "the committed residue must be reclaimed"); } - // The rollback backup may already be gone; the committed marker alone - // proves the delete. + // Internal maintenance can finish its legacy marker after backup cleanup. #[tokio::test] async fn delete_versions_finishes_committed_marker_without_rollback_backup() { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18382,15 +19108,14 @@ mod test { assert!(!residue.object_dir.exists(), "the committed marker alone must be enough"); } - // Only a committed marker on this disk proves the delete. A reserved marker, - // a rollback backup alone, or live metadata beside the marker is left for - // the owning transaction, rollback, or heal. + // Internal maintenance preserves reserved markers, backup-only states and + // live metadata for the owning transaction, rollback or heal. #[tokio::test] async fn committed_delete_residue_finish_leaves_unproven_state_alone() { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; ensure_test_volume(&disk, bucket).await; // A rollback backup without a committed marker, as an inline object leaves it. @@ -18419,7 +19144,7 @@ mod test { for object in ["backup-only", "reserved", "live"] { assert_eq!( - disk.finish_committed_delete_residue(bucket, object, None).await, + disk.finish_internal_delete_residue(bucket, object, None).await, 0, "{object} carries no committed proof" ); @@ -18446,7 +19171,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "logs"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18458,7 +19183,7 @@ mod test { .await .expect("nested metadata should be written"); - assert_eq!(disk.finish_committed_delete_residue(bucket, object, None).await, 0); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); assert!(residue.backup().exists() && residue.marker().exists() && residue.part().exists()); assert!(nested.exists()); } @@ -18474,7 +19199,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18492,7 +19217,7 @@ mod test { let _ = release_rx.recv(); }); - let mut finish = Box::pin(disk.finish_committed_delete_residue(bucket, object, None)); + let mut finish = Box::pin(disk.finish_internal_delete_residue(bucket, object, None)); tokio::time::timeout(Duration::from_secs(10), async { tokio::select! { entered = entered_rx => entered.expect("finish should reach the marked data dir"), @@ -18508,7 +19233,7 @@ mod test { // The abandoned rename is still parked; the next attempt recognizes the // residue by its marker and finishes it. - assert_eq!(disk.finish_committed_delete_residue(bucket, object, None).await, 1); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); assert!(!residue.object_dir.exists(), "the retried finish must reclaim the residue"); // Let the abandoned rename run into its missing source and wait until @@ -18530,7 +19255,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let committed = write_committed_delete_residue(&disk, bucket, object).await; @@ -18549,12 +19274,620 @@ mod test { .await .expect("unproven backup should be written"); - assert_eq!(disk.finish_committed_delete_residue(bucket, object, None).await, 1); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); assert!(!committed.object_dir.join(committed.data_dir.to_string()).exists()); assert!(unproven_backup.exists(), "a backup without its own committed marker must remain"); assert_eq!(recorder.counter_value(METRIC_DISK_DELETE_RESIDUE_FINISHED_TOTAL, &[]), 1); } + #[tokio::test] + async fn delete_quorum_receipt_finishes_inline_backup_after_interrupted_cleanup() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let object_dir = disk.io_get_object_path(bucket, object).expect("object path"); + let transaction = Uuid::new_v4(); + let rollback_path = object_dir.join(transaction.to_string()); + fs::create_dir_all(&rollback_path).await.expect("transaction directory"); + fs::write(rollback_path.join(STORAGE_FORMAT_FILE_BACKUP), b"inline metadata") + .await + .expect("inline rollback backup"); + // Backup alone remains ambiguous. Only the set-quorum receipt permits cleanup. + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); + fs::write(rollback_path.join(DELETE_QUORUM_COMMITTED_FILE), b"") + .await + .expect("quorum receipt"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!object_dir.exists(), "inline backup and receipt must be reclaimed"); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_cleanup_removes_backup_before_marked_data() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "quorum-cleanup-order"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let data_path = residue.part(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install_at(hooks::Stage::Remove, &data_path, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let transaction_path = format!("{object}/{}", residue.transaction); + let mut cleanup = Box::pin(disk.delete( + bucket, + &transaction_path, + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + )); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("data cleanup starts"), + result = cleanup.as_mut() => panic!("cleanup returned before data removal: {result:?}"), + } + }) + .await + .expect("cleanup reaches data removal"); + assert!(!residue.backup().exists(), "backup must disappear before data"); + assert!(residue.marker().exists(), "data proof must remain during removal"); + assert!( + residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE) + .is_file() + ); + drop(release); + tokio::time::timeout(Duration::from_secs(10), cleanup) + .await + .expect("cleanup completes") + .expect("cleanup succeeds"); + assert!(!residue.object_dir.exists(), "completed cleanup reclaims every transaction file"); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_rollback_keeps_markers_until_metadata_restore() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "rollback-marker-order"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let backup = residue.backup(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install_at(hooks::Stage::Rename, &backup, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let request = residue_delete_request(object); + let mut rollback = Box::pin(disk.delete_versions( + bucket, + request, + DeleteOptions { + undo_write: true, + undo_delete: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + )); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("metadata restore starts"), + result = rollback.as_mut() => panic!("rollback returned before restore: {result:?}"), + } + }) + .await + .expect("rollback reaches metadata restore"); + assert!(residue.marker().exists(), "rollback must retain its marker until metadata is restored"); + assert!(residue.backup().exists(), "backup must remain before restore rename"); + drop(release); + let errors = tokio::time::timeout(Duration::from_secs(10), rollback) + .await + .expect("rollback completes"); + assert!(errors.iter().all(Option::is_none), "{errors:?}"); + assert!(residue.object_dir.join(STORAGE_FORMAT_FILE).is_file()); + assert!(!residue.marker().exists()); + assert!(residue.part().exists()); + } + + #[tokio::test] + async fn delete_rollback_rejects_quorum_committed_transaction() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "quorum-rollback-rejection"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + fs::write( + residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE), + b"", + ) + .await + .expect("quorum receipt"); + let errors = disk + .delete_versions( + bucket, + residue_delete_request(object), + DeleteOptions { + undo_write: true, + undo_delete: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(matches!(errors.as_slice(), [Some(DiskError::FileCorrupt)]), "{errors:?}"); + assert!( + !residue.object_dir.join(STORAGE_FORMAT_FILE).exists(), + "a committed delete must not resurrect metadata" + ); + assert!(residue.backup().exists() && residue.marker().exists()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_receipt_sync_failure_preserves_rollback_backup() { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let rollback_path = residue.object_dir.join(residue.transaction.to_string()); + os::fsync_dir_recorder::set_failure(&rollback_path, ErrorKind::Other); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(result.is_err(), "failed receipt durability must stop cleanup"); + assert!(residue.backup().exists() && residue.part().exists() && residue.marker().exists()); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists(), "retry must complete the already-decided deletion"); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_quorum_backup_removal_sync_failure_retains_receipt() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let transaction_path = residue.object_dir.join(residue.transaction.to_string()); + let receipt = transaction_path.join(DELETE_QUORUM_COMMITTED_FILE); + let _hook = hooks::install_at(hooks::Stage::Remove, &residue.backup(), move || { + // Arm after the receipt's initial fsync to fail the ordering fence + // between backup unlink and eventual receipt removal. + os::fsync_dir_recorder::set_failure(&transaction_path, ErrorKind::Other); + }); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(result.is_err(), "backup unlink must become durable before removing commitment proof"); + assert!(receipt.is_file() && residue.part().is_file() && residue.marker().is_file()); + assert!(!residue.backup().exists()); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_quorum_payload_removal_sync_failure_retains_markers_and_receipt() { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + os::fsync_dir_recorder::set_failure(&residue.object_dir.join(residue.data_dir.to_string()), ErrorKind::Other); + let receipt = residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(result.is_err(), "payload unlink sync failure must stop marker cleanup"); + assert!(receipt.is_file() && residue.marker().is_file()); + assert!(!residue.part().exists()); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_generic_uuid_backup_directory_does_not_publish_quorum_receipt() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "generic-uuid-delete"; + ensure_test_volume(&disk, bucket).await; + let path = format!("object/{}", Uuid::new_v4()); + let directory = disk.io_get_object_path(bucket, &path).expect("UUID directory"); + fs::create_dir_all(&directory).await.expect("UUID directory"); + fs::write(directory.join("part.1"), b"data").await.expect("part"); + fs::write(directory.join(STORAGE_FORMAT_FILE_BACKUP), b"write transaction backup") + .await + .expect("non-delete rollback backup"); + let receipt = directory.join(DELETE_QUORUM_COMMITTED_FILE); + let (receipt_tx, receipt_rx) = tokio::sync::oneshot::channel(); + let _hook = hooks::install_at(hooks::Stage::Rename, &directory, move || { + let _ = receipt_tx.send(receipt.exists()); + }); + disk.delete( + bucket, + &path, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + .expect("generic directory deletion"); + assert!( + !receipt_rx.await.expect("inspect directory before deletion"), + "ordinary UUID data is not a transaction" + ); + } + + #[tokio::test] + async fn delete_quorum_receipt_preserves_uuid_prefix_child() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let transaction = Uuid::new_v4(); + let transaction_path = format!("{object}/{transaction}"); + let rollback = disk.io_get_object_path(bucket, &transaction_path).expect("transaction path"); + let child = rollback.join("child").join(STORAGE_FORMAT_FILE); + fs::create_dir_all(child.parent().expect("child directory")) + .await + .expect("child directory"); + fs::write(&child, b"live child metadata").await.expect("live child"); + fs::write(rollback.join(DELETE_QUORUM_COMMITTED_FILE), b"") + .await + .expect("receipt"); + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), b"old backup") + .await + .expect("backup"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); + let result = disk + .delete( + bucket, + &transaction_path, + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await; + assert!(matches!(result, Err(DiskError::FileCorrupt)), "{result:?}"); + assert_eq!(fs::read(&child).await.expect("preserved child"), b"live child metadata"); + assert!(rollback.join(STORAGE_FORMAT_FILE_BACKUP).is_file()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_cleanup_preserves_child_created_after_classification() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "receipt-prefix-race"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let child = residue + .object_dir + .join(residue.transaction.to_string()) + .join("child") + .join(STORAGE_FORMAT_FILE); + let published_child = child.clone(); + let _hook = hooks::install_at(hooks::Stage::Remove, &residue.part(), move || { + std::fs::create_dir_all(published_child.parent().expect("child directory")).expect("concurrent child directory"); + std::fs::write(&published_child, b"concurrent child").expect("concurrent child metadata"); + }); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + result.expect("cleanup preserves a concurrent child and removes only deleted payload"); + assert_eq!(fs::read(child).await.expect("preserved concurrent child"), b"concurrent child"); + assert!(!residue.part().exists(), "the original deleted payload must still be reclaimed"); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_rollback_backup_sync_failure_precedes_metadata_mutation() { + for mode in [DurabilityMode::Strict, DurabilityMode::None] { + let _mode = durability_mode_override::set(mode); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "backup-durability-gate"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let root = disk.io_get_object_path(bucket, object).expect("object path"); + let data_dir = Uuid::new_v4(); + let data_path = root.join(data_dir.to_string()); + fs::create_dir_all(&data_path).await.expect("data directory"); + fs::write(data_path.join("part.1"), b"original payload") + .await + .expect("payload"); + let original = test_meta(test_file_info(object, Uuid::nil(), Some(data_dir), None)); + fs::write(root.join(STORAGE_FORMAT_FILE), &original) + .await + .expect("original metadata"); + let transaction = Uuid::new_v4(); + let rollback = root.join(transaction.to_string()); + os::fsync_dir_recorder::set_failure(&rollback, ErrorKind::Other); + let errors = disk + .delete_versions( + bucket, + residue_delete_request(object), + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await; + if mode == DurabilityMode::Strict { + assert!(errors.iter().any(Option::is_some), "backup sync failure must stop deletion"); + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("intact metadata"), original); + assert_eq!(fs::read(data_path.join("part.1")).await.expect("intact payload"), b"original payload"); + } else { + assert!(errors.iter().all(Option::is_none), "{errors:?}"); + assert!( + !os::fsync_dir_recorder::was_fsynced(&rollback), + "none mode must skip backup directory sync" + ); + assert!( + os::fsync_dir_recorder::take_failure(&rollback).is_some(), + "none mode must not consume a sync failure" + ); + assert!(!root.join(STORAGE_FORMAT_FILE).exists(), "none mode still applies the requested delete"); + } + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_rollback_restore_sync_failure_retains_markers() { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "restore-durability-gate"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + os::fsync_dir_recorder::set_failure(&residue.object_dir, ErrorKind::Other); + let opts = DeleteOptions { + undo_write: true, + undo_delete: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }; + let errors = disk + .delete_versions(bucket, residue_delete_request(object), opts.clone()) + .await; + assert!(errors.iter().any(Option::is_some), "restore sync failure must remain visible"); + let bytes = fs::read(residue.object_dir.join(STORAGE_FORMAT_FILE)) + .await + .expect("restored metadata is readable"); + assert_eq!(FileMeta::load(&bytes).expect("valid restored metadata").versions.len(), 1); + assert!(residue.marker().exists(), "failed durability must retain transaction evidence"); + assert!(residue.part().exists(), "rollback must retain payload"); + let errors = disk.delete_versions(bucket, residue_delete_request(object), opts).await; + assert!(errors.iter().all(Option::is_none), "retry must finish rollback: {errors:?}"); + assert!(!residue.marker().exists()); + assert!(residue.object_dir.join(STORAGE_FORMAT_FILE).is_file()); + } + + #[tokio::test] + async fn delete_quorum_receipt_survives_deferred_snapshot_cleanup() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let data_path = format!("{object}/{}", residue.data_dir); + let snapshot = disk + .acquire_snapshot_lease(bucket, &data_path) + .await + .expect("snapshot protects deleted payload"); + let receipt = residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(matches!(result, Err(DiskError::DiskOngoingReq)), "{result:?}"); + assert!(receipt.is_file() && residue.marker().is_file() && residue.part().is_file()); + assert!(!residue.backup().exists(), "committed backup is no longer needed"); + disk.release_snapshot_lease(bucket, &data_path, snapshot) + .await + .expect("snapshot release finishes physical cleanup"); + assert!(!residue.part().exists()); + assert!(receipt.is_file(), "receipt remains until transaction cleanup observes physical removal"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_data_cleanup_preserves_concurrent_uuid_prefix_child() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "quorum-data-prefix-race"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let child = residue + .object_dir + .join(residue.data_dir.to_string()) + .join("child") + .join(STORAGE_FORMAT_FILE); + let published = child.clone(); + let _hook = hooks::install_at(hooks::Stage::Remove, &residue.part(), move || { + std::fs::create_dir_all(published.parent().expect("child directory")).expect("concurrent child directory"); + std::fs::write(published, b"new child").expect("concurrent child metadata"); + }); + disk.delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await + .expect("safe cleanup preserves new descendants"); + assert_eq!(fs::read(child).await.expect("preserved child"), b"new child"); + assert!(!residue.part().exists()); + assert!(!residue.marker().exists()); + } + + #[tokio::test] + async fn delete_quorum_snapshot_replay_preserves_live_uuid_prefix_child() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let data_path = format!("{object}/{}", residue.data_dir); + let snapshot = disk.acquire_snapshot_lease(bucket, &data_path).await.expect("snapshot lease"); + let receipt = residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE); + assert!(matches!( + disk.delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + } + ) + .await, + Err(DiskError::DiskOngoingReq) + )); + let child = residue + .object_dir + .join(residue.data_dir.to_string()) + .join("child") + .join(STORAGE_FORMAT_FILE); + fs::create_dir_all(child.parent().expect("child directory")) + .await + .expect("live child directory"); + fs::write(&child, b"live child").await.expect("live child metadata"); + assert!( + disk.release_snapshot_lease(bucket, &data_path, snapshot).await.is_err(), + "replay must reject mixed live contents" + ); + assert_eq!(fs::read(&child).await.expect("preserved child"), b"live child"); + assert!( + receipt.is_file() && residue.part().is_file(), + "blocked replay retains payload and commitment evidence" + ); + fs::remove_file(&child).await.expect("remove test child metadata"); + fs::remove_dir(child.parent().expect("child directory")) + .await + .expect("remove test child directory"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + #[tokio::test] async fn test_delete_version_error_after_staging_restores_data_dir() { use tempfile::tempdir; @@ -21825,6 +23158,8 @@ mod test { undo_delete: false, old_data_dir: None, expected_delete_marker: None, + delete_recovery_bucket_incarnation: None, + recoverable_delete: false, }; disk.delete("test-volume", "test-file.txt", delete_opts) .await diff --git a/crates/ecstore/src/disk/local/delete_recovery.rs b/crates/ecstore/src/disk/local/delete_recovery.rs new file mode 100644 index 000000000..c1b90219e --- /dev/null +++ b/crates/ecstore/src/disk/local/delete_recovery.rs @@ -0,0 +1,3646 @@ +use super::*; +use crate::disk::{ + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, + DeleteRecoveryJournal, DeleteRecoveryPage, DeleteRecoveryState, DeleteTransactionId, +}; +use sha2::{Digest, Sha256}; + +pub(super) const PREPARED: &str = "delete-prepared"; +pub(super) const TARGET: &str = "delete-target"; +pub(super) const FINISHED: &str = "delete-commit-finished"; +const MAX_METADATA: u64 = 64 * 1024 * 1024; +struct DiscoverySession { + disk_root: PathBuf, + volume: String, + object: Option, + root: PathBuf, + _mount: std::fs::File, + frames: Vec<(String, std::fs::ReadDir)>, + touched: Instant, +} + +static DISCOVERY_SESSIONS: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| ParkingLotMutex::new(HashMap::new())); + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct CommitReceipt { + journal: DeleteRecoveryJournal, + finished: bool, +} + +pub(super) struct DeletePreparationContext { + pub transaction: Uuid, + pub bucket_incarnation: Option, + pub namespace_owner: Option>, +} + +pub(super) fn digest(bytes: &[u8]) -> DeleteRecoveryDigest { + Sha256::digest(bytes).into() +} + +pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result { + canonical.data = None; + canonical.erasure.index = 0; + canonical.erasure.checksums.clear(); + canonical.is_latest = false; + canonical.num_versions = 0; + canonical.successor_mod_time = None; + rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut canonical.metadata); + // Only schema-declared string fields and the inline presence marker are + // aliases here. Unknown suffixes and opaque binary fields stay distinct. + use rustfs_utils::http::metadata_compat as compat; + for suffix in [ + compat::SUFFIX_INLINE_DATA, + compat::SUFFIX_COMPRESSION, + compat::SUFFIX_COMPRESSION_SIZE, + compat::SUFFIX_ACTUAL_SIZE, + compat::SUFFIX_ACTUAL_OBJECT_SIZE, + compat::SUFFIX_ACTUAL_OBJECT_SIZE_CAP, + compat::SUFFIX_TRANSITION_STATUS, + compat::SUFFIX_TRANSITIONED_OBJECTNAME, + compat::SUFFIX_TRANSITION_TIER, + compat::SUFFIX_TRANSITIONED_VERSION_STATE, + compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + compat::SUFFIX_PURGESTATUS, + compat::SUFFIX_REPLICA_STATUS, + compat::SUFFIX_REPLICATION_STATUS, + compat::SUFFIX_REPLICA_TIMESTAMP, + compat::SUFFIX_REPLICATION_TIMESTAMP, + compat::SUFFIX_TAGGING_TIMESTAMP, + compat::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, + compat::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, + ] { + if !compat::contains_key_str(&canonical.metadata, suffix) { + continue; + } + // The inline marker is a presence flag whose persisted value is empty. + // Compare both declared aliases explicitly before normalizing it; the + // general compatibility reader rejects empty strings by design. + let value = if suffix == compat::SUFFIX_INLINE_DATA { + let rustfs_key = compat::internal_key_rustfs(suffix); + let minio_key = format!("{}{suffix}", compat::MINIO_INTERNAL_PREFIX); + let mut value = None; + for (key, candidate) in &canonical.metadata { + if !key.eq_ignore_ascii_case(&rustfs_key) && !key.eq_ignore_ascii_case(&minio_key) { + continue; + } + if value.is_some_and(|current: &String| current != candidate) { + return Err(DiskError::FileCorrupt); + } + value = Some(candidate); + } + value.ok_or(DiskError::FileCorrupt)?.clone() + } else { + compat::get_consistent_str(&canonical.metadata, suffix) + .ok_or(DiskError::FileCorrupt)? + .to_owned() + }; + if value.is_empty() && suffix != compat::SUFFIX_INLINE_DATA { + continue; + } + compat::insert_str(&mut canonical.metadata, suffix, value); + } + fn ordered(value: serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let sorted: std::collections::BTreeMap<_, _> = map.into_iter().collect(); + serde_json::Value::Object(sorted.into_iter().map(|(key, value)| (key, ordered(value))).collect()) + } + serde_json::Value::Array(values) => serde_json::Value::Array(values.into_iter().map(ordered).collect()), + other => other, + } + } + Ok(crate::disk::DeleteRecoveryVersionIdentity { + version_id: canonical.version_id, + mod_time: canonical.mod_time, + data_dir: canonical.data_dir, + deleted: canonical.deleted, + metadata_digest: digest(&serde_json::to_vec(&ordered(serde_json::to_value(canonical)?))?), + }) +} + +fn safe_path(path: &str, empty: bool) -> bool { + (empty && path.is_empty()) + || (!path.is_empty() && !path.starts_with('/') && crate::bucket::utils::is_valid_object_prefix(path)) +} + +fn regular_metadata_bytes(path: &Path) -> Result>> { + match std::fs::symlink_metadata(path) { + Ok(meta) if meta.is_file() => Ok(Some(std::fs::read(path).map_err(to_file_error)?)), + Ok(_) => Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(err) => Err(to_file_error(err).into()), + } +} + +fn regular_bytes(path: &Path) -> Result>> { + if std::fs::symlink_metadata(path).is_ok_and(|meta| meta.len() > MAX_METADATA) { + return Err(DiskError::FileCorrupt); + } + regular_metadata_bytes(path) +} + +fn journal_bytes(journal: &DeleteRecoveryJournal) -> Result> { + let bytes = serde_json::to_vec(journal)?; + // The same journal is carried by adoption requests and decision receipts. + if bytes.len() > usize::try_from(MAX_METADATA).map_err(|_| DiskError::FileCorrupt)? - 128 { + return Err(DiskError::FileCorrupt); + } + let projected = crate::disk::DeleteRecoveryRpcReply::Inspected(DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: DeleteTransactionId { + object: journal.object.clone(), + transaction: journal.transaction, + }, + state: DeleteRecoveryState::NewPrepared, + journal: Some(journal.clone()), + current_digest: journal.original_digest, + current_versions: journal.original_versions.clone(), + backup_digest: journal.original_digest, + rollback_data_complete: true, + commit_data_safe: true, + explicit_delete_safe: false, + fingerprint: [0; 32], + }); + if rmp_serde::to_vec_named(&projected)?.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES - 4096 { + return Err(DiskError::FileCorrupt); + } + Ok(bytes) +} + +fn regular_marker(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(meta) if meta.is_file() && meta.len() == 0 => Ok(true), + Ok(_) => Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(false), + Err(err) => Err(to_file_error(err).into()), + } +} + +fn complete_part_sources(live: &Path, staged: &Path, info: &FileInfo) -> Result { + let layout = info.validate(ValidationMode::RequireErasure)?.ok_or(DiskError::FileCorrupt)?; + for part in &info.parts { + let expected = u64::try_from(layout.shard_file_size(part.size).ok_or(DiskError::FileCorrupt)?) + .map_err(|_| DiskError::FileCorrupt)?; + let name = format!("part.{}", part.number); + if ![live, staged] + .iter() + .any(|root| std::fs::symlink_metadata(root.join(&name)).is_ok_and(|meta| meta.is_file() && meta.len() >= expected)) + { + return Ok(false); + } + if let Some(integrity) = &part.integrity { + let name = integrity.file_name(); + let length = u64::try_from(integrity.index_size()?).map_err(|_| DiskError::FileCorrupt)?; + if ![live, staged] + .iter() + .any(|root| std::fs::symlink_metadata(root.join(&name)).is_ok_and(|meta| meta.is_file() && meta.len() == length)) + { + return Ok(false); + } + } + } + Ok(true) +} + +async fn verify_recovery_integrity_sources( + live: &Path, + staged: &Path, + integrity: &rustfs_filemeta::shard_integrity::PartIntegrity, + coding_index: usize, + algorithm: HashAlgorithm, + owner: Option>, +) -> Result<()> { + let lease = os::acquire_namespace_mutation_lease_with_owner(live, owner).await; + let roots = [live.to_path_buf(), staged.to_path_buf()]; + let integrity = integrity.clone(); + os::run_blocking_namespace_operation(lease, move || { + Ok((|| { + integrity.validate()?; + if coding_index >= integrity.layout.shards() + || !matches!(algorithm, HashAlgorithm::HighwayHash256S | HashAlgorithm::HighwayHash256SLegacy) + { + return Err(DiskError::FileCorrupt); + } + let codec = coding::Erasure::try_new_with_options( + usize::from(integrity.layout.data), + usize::from(integrity.layout.parity), + usize::try_from(integrity.layout.block_size).map_err(|_| DiskError::FileCorrupt)?, + integrity.layout.legacy, + ) + .map_err(DiskError::from)?; + let shard_size = codec.shard_size(); + let part_size = + usize::try_from(codec.shard_file_size(i64::try_from(integrity.size).map_err(|_| DiskError::FileCorrupt)?)) + .map_err(|_| DiskError::FileCorrupt)?; + let encoded_size = u64::try_from(coding::bitrot_shard_file_size(part_size, shard_size, algorithm.clone())) + .map_err(|_| DiskError::FileCorrupt)?; + let expected = u64::try_from(integrity.index_size()?).map_err(|_| DiskError::FileCorrupt)?; + let header = integrity.index_header(); + let mut checked = false; + for root in &roots { + let path = root.join(integrity.file_name()); + match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_file() && metadata.len() == expected => {} + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(to_file_error(err).into()), + } + let mut file = std::fs::File::open(path).map_err(to_file_error)?; + let mut actual = [0u8; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE]; + std::io::Read::read_exact(&mut file, &mut actual).map_err(to_file_error)?; + if actual != header { + return Err(DiskError::FileCorrupt); + } + // An interrupted restore can split payload and proof between + // live and staged roots. Verify every present payload copy + // against the declared proof, independently of its location. + let mut shards = Vec::new(); + for root in &roots { + let path = root.join(format!("part.{}", integrity.number)); + match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_file() && metadata.len() == encoded_size => { + shards.push(std::fs::File::open(path).map_err(to_file_error)?); + } + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if shards.is_empty() { + return Err(DiskError::FileCorrupt); + } + let mut record = vec![0; integrity.record_size()]; + let mut payload = vec![0; shard_size]; + let mut bitrot_hash = vec![0; algorithm.size()]; + let mut remaining = part_size; + for stripe in 0..integrity.stripes { + std::io::Read::read_exact(&mut file, &mut record).map_err(to_file_error)?; + integrity.verify_record(stripe, &record)?; + let length = remaining.min(shard_size); + if length == 0 { + return Err(DiskError::FileCorrupt); + } + for shard in &mut shards { + std::io::Read::read_exact(shard, &mut bitrot_hash).map_err(to_file_error)?; + std::io::Read::read_exact(shard, &mut payload[..length]).map_err(to_file_error)?; + integrity.verify_shard(stripe, coding_index, &payload[..length], &record)?; + } + remaining -= length; + } + if remaining != 0 { + return Err(DiskError::FileCorrupt); + } + if integrity.stripes == 0 && integrity.root != integrity.root_digest(&integrity.padding_digest(0)) { + return Err(DiskError::FileCorrupt); + } + checked = true; + } + if !checked { + return Err(DiskError::FileCorrupt); + } + Ok(()) + })()) + }) + .await + .map_err(to_file_error)? +} + +struct ExplicitDeletePlan { + data_dirs: Vec, + proof_files: Vec, +} + +fn explicit_delete_plan( + root: &Path, + transaction: Uuid, + original: Option<&FileMeta>, + journal: Option<&DeleteRecoveryJournal>, + current_missing: bool, +) -> Result> { + if !current_missing + || journal + .is_some_and(|journal| journal.target_digest.is_some() || (journal.original_digest.is_some() && original.is_none())) + { + return Ok(None); + } + let mut referenced = HashSet::new(); + if let Some(metadata) = original { + if metadata.versions.len() != 1 || metadata.versions[0].header.version_type != rustfs_filemeta::VersionType::Object { + return Ok(None); + } + let info = metadata.versions[0].into_fileinfo("", "", false)?; + if info.version_id.is_some_and(|id| !id.is_nil()) + || info.deleted + || info.is_remote() + || info.metadata.keys().any(|key| { + let key = key.to_ascii_lowercase(); + key.starts_with("x-amz-object-lock-") + || rustfs_utils::http::metadata_compat::internal_key_strip_suffix_prefix(&key, "objectlock-").is_some() + }) + { + return Ok(None); + } + referenced.extend(metadata.get_data_dirs()?.into_iter().flatten().filter(|id| !id.is_nil())); + } + let rollback = root.join(transaction.to_string()); + let marker_names = [ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .map(|prefix| format!("{prefix}{transaction}")); + let mut data_dirs = Vec::new(); + let mut proof_files = Vec::new(); + let mut candidates = Vec::new(); + match std::fs::read_dir(&rollback) { + Ok(entries) => { + for entry in entries { + let entry = entry.map_err(to_file_error)?; + let kind = entry.file_type().map_err(to_file_error)?; + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + if kind.is_dir() && Uuid::parse_str(&name).is_ok_and(|id| referenced.contains(&id)) { + candidates.push(entry.path()); + } else if kind.is_file() + && [ + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + PREPARED, + TARGET, + FINISHED, + ] + .contains(&name.as_str()) + { + proof_files.push(entry.path()); + } else if kind.is_file() + && (marker_names.contains(&name) || (delete_recovery_part_file(&name) && referenced.contains(&transaction))) + { + candidates.push(rollback.clone()); + } else { + return Ok(None); + } + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + match std::fs::read_dir(root) { + Ok(entries) => { + for entry in entries { + let entry = entry.map_err(to_file_error)?; + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + let Some(id) = Uuid::parse_str(&name).ok().filter(|id| !id.is_nil()) else { + continue; + }; + if !entry.file_type().map_err(to_file_error)?.is_dir() { + continue; + } + let own_marker = marker_names + .iter() + .any(|name| std::fs::symlink_metadata(entry.path().join(name)).is_ok_and(|metadata| metadata.is_file())); + if referenced.contains(&id) || own_marker { + candidates.push(entry.path()); + } + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + candidates.sort(); + candidates.dedup(); + for dir in candidates { + let mut has_parts = false; + for entry in std::fs::read_dir(&dir).map_err(to_file_error)? { + let entry = entry.map_err(to_file_error)?; + if !entry.file_type().map_err(to_file_error)?.is_file() { + return Ok(None); + } + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + if delete_recovery_part_file(&name) { + has_parts = true; + } else if !(marker_names.contains(&name) + || (dir == rollback + && [ + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + PREPARED, + TARGET, + FINISHED, + ] + .contains(&name.as_str()))) + { + return Ok(None); + } + } + if has_parts && original.is_none() { + return Ok(None); + } + data_dirs.push(dir); + } + data_dirs.sort_by_key(|path| std::cmp::Reverse(path.components().count())); + Ok(Some(ExplicitDeletePlan { data_dirs, proof_files })) +} + +fn inspection(root: &Path, volume: &str, object: &str, transaction: Uuid) -> Result { + let rollback = root.join(transaction.to_string()); + let before = regular_metadata_bytes(&rollback.join(STORAGE_FORMAT_FILE_BACKUP))?; + let current = regular_metadata_bytes(&root.join(STORAGE_FORMAT_FILE))?; + let prepared = regular_bytes(&rollback.join(PREPARED))?; + let target = regular_metadata_bytes(&rollback.join(TARGET))?; + let receipt_bytes = regular_bytes(&rollback.join(DELETE_QUORUM_COMMITTED_FILE))?; + let receipt: Option = receipt_bytes + .as_deref() + .filter(|bytes| !bytes.is_empty()) + .map(serde_json::from_slice) + .transpose()?; + let committed = receipt.is_some(); + let finished = regular_marker(&rollback.join(FINISHED))? || receipt.as_ref().is_some_and(|receipt| receipt.finished); + let journal: Option = prepared + .as_deref() + .map(serde_json::from_slice) + .transpose()? + .or_else(|| receipt.as_ref().map(|receipt| receipt.journal.clone())); + if receipt + .as_ref() + .is_some_and(|receipt| journal.as_ref() != Some(&receipt.journal)) + { + return Err(DiskError::FileCorrupt); + } + if let Some(journal) = &journal + && (journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || journal.volume != volume + || journal.object != object + || journal.transaction != transaction + || journal.transaction.is_nil() + || journal.data_dirs.iter().any(|id| id.is_nil() || *id == transaction)) + { + return Err(DiskError::FileCorrupt); + } + let backup_digest = before.as_deref().map(digest); + let current_digest = current.as_deref().map(digest); + let prepared_digest = prepared.as_deref().map(digest); + let target_digest = target.as_deref().map(digest); + let receipt_digest = receipt_bytes.as_deref().map(digest); + if let Some(journal) = &journal { + if (!finished + && journal.target_digest != target_digest + && !(target.is_none() + && !committed + && before.is_none() + && journal.original_digest.is_some() + && current_digest == journal.original_digest)) + || target_digest.is_some_and(|hash| journal.target_digest != Some(hash)) + || backup_digest.is_some_and(|hash| journal.original_digest != Some(hash)) + { + return Err(DiskError::FileCorrupt); + } + if let Some(bytes) = &target { + FileMeta::load(bytes)?; + } + } + let current_meta = current.as_deref().map(FileMeta::load).transpose()?; + let before_meta = before.as_deref().map(FileMeta::load).transpose()?; + let mut fingerprint = Sha256::new(); + for hash in [prepared_digest, target_digest, backup_digest, current_digest, receipt_digest] { + fingerprint.update([u8::from(hash.is_some())]); + if let Some(hash) = hash { + fingerprint.update(hash); + } + } + fingerprint.update([u8::from(committed), u8::from(finished)]); + let mut any_marker = false; + let mut descriptions = Vec::new(); + let entries = match std::fs::read_dir(root) { + Ok(entries) => Some(entries), + Err(err) if err.kind() == ErrorKind::NotFound => None, + Err(err) => return Err(to_file_error(err).into()), + }; + if let Some(entries) = entries { + for entry in entries { + let entry = entry.map_err(to_file_error)?; + let kind = entry.file_type().map_err(to_file_error)?; + if !kind.is_file() && !kind.is_dir() { + return Err(DiskError::InvalidPath); + } + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + descriptions.push((name.clone(), kind.is_dir(), entry.metadata().map_err(to_file_error)?.len())); + if kind.is_dir() && Uuid::parse_str(&name).is_ok_and(|id| !id.is_nil()) { + for child in std::fs::read_dir(entry.path()).map_err(to_file_error)? { + let child = child.map_err(to_file_error)?; + let kind = child.file_type().map_err(to_file_error)?; + if !kind.is_dir() && !kind.is_file() { + return Err(DiskError::InvalidPath); + } + let child_name = child.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + let metadata = child.metadata().map_err(to_file_error)?; + descriptions.push((format!("{name}/{child_name}"), kind.is_dir(), metadata.len())); + if kind.is_file() + && [ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .any(|prefix| child_name == format!("{prefix}{transaction}")) + { + any_marker = true; + } + let staged_id = Uuid::parse_str(&child_name).ok(); + let known_staged = kind.is_dir() + && staged_id.is_some_and(|dir| { + journal.as_ref().is_some_and(|journal| journal.data_dirs.contains(&dir)) + || before_meta + .as_ref() + .is_some_and(|metadata| metadata.get_data_dirs().is_ok_and(|dirs| dirs.contains(&Some(dir)))) + }); + if name == transaction.to_string() + && !(kind.is_dir() && known_staged) + && !(journal.is_none() + && kind.is_file() + && (delete_recovery_part_file(&child_name) + || [DELETE_DATA_DIR_MARKER_PREFIX, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX] + .iter() + .any(|prefix| child_name == format!("{prefix}{transaction}")))) + && (kind.is_dir() + || ![ + PREPARED, + TARGET, + FINISHED, + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + ] + .contains(&child_name.as_str())) + { + return Err(DiskError::FileCorrupt); + } + if name == transaction.to_string() && kind.is_dir() { + for file in std::fs::read_dir(child.path()).map_err(to_file_error)? { + let file = file.map_err(to_file_error)?; + let kind = file.file_type().map_err(to_file_error)?; + let file_name = file.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + if !kind.is_file() + || (!delete_recovery_part_file(&file_name) + && ![ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .any(|prefix| file_name == format!("{prefix}{transaction}"))) + { + return Err(DiskError::FileCorrupt); + } + descriptions.push(( + format!("{name}/{child_name}/{file_name}"), + false, + file.metadata().map_err(to_file_error)?.len(), + )); + } + } + } + } + } + } + descriptions.sort(); + fingerprint.update(serde_json::to_vec(&descriptions)?); + let mut rollback_data_complete = false; + let mut commit_data_safe = false; + if let Some(journal) = &journal { + rollback_data_complete = (journal.original_digest == backup_digest || current_digest == journal.original_digest) + && (current_digest == journal.original_digest || current_digest == journal.target_digest); + if let Some(meta) = before_meta + .as_ref() + .or_else(|| current_meta.as_ref().filter(|_| current_digest == journal.original_digest)) + { + let inline: HashMap<&str, &[u8]> = if meta.data.as_slice().is_empty() { + HashMap::new() + } else { + if !meta.data.version_ok() { + return Err(DiskError::FileCorrupt); + } + rmp_serde::from_slice(meta.data.after_version())? + }; + for version in &meta.versions { + if version.header.version_type != rustfs_filemeta::VersionType::Object { + continue; + } + let mut info = version.into_fileinfo(volume, object, true)?; + info.hydrate_shard_integrity()?; + if info.is_remote() { + continue; + } + if info.inline_data() { + let id = info.version_id.unwrap_or_default(); + rollback_data_complete &= + inline.contains_key(id.to_string().as_str()) || (id.is_nil() && inline.contains_key("null")); + continue; + } + let Some(data_dir) = info.data_dir.filter(|id| !id.is_nil()) else { + rollback_data_complete = false; + continue; + }; + rollback_data_complete &= + complete_part_sources(&root.join(data_dir.to_string()), &rollback.join(data_dir.to_string()), &info)?; + } + } + commit_data_safe = current_digest == journal.original_digest || current_digest == journal.target_digest; + if current_digest != journal.original_digest && current_digest != journal.target_digest { + // A later certified delete can remove the remaining metadata of + // this committed version delete. Absence has no live references, + // but cannot itself authorize an undecided transaction. + commit_data_safe = (committed && current.is_none()) + || current_meta.as_ref().is_some_and(|meta| { + meta.get_data_dirs() + .is_ok_and(|dirs| !dirs.into_iter().flatten().any(|dir| journal.data_dirs.contains(&dir))) + && !meta.versions.iter().any(|version| { + version + .header + .version_id + .is_some_and(|id| !id.is_nil() && journal.deleted_versions.contains(&id)) + }) + }); + } + } + // In-place shard repair leaves a backup in the live data directory. + // Preserve it without interpreting it as an unfinished delete. + let live_write_backup = journal.is_none() + && !any_marker + && !regular_marker(&rollback.join(DELETE_MARKER_ROLLBACK_FILE))? + && prepared.is_none() + && target.is_none() + && receipt_bytes.is_none() + && !finished + && current_meta.as_ref().is_some_and(|meta| { + meta.get_data_dirs() + .is_ok_and(|dirs| dirs.into_iter().flatten().any(|dir| dir == transaction)) + }) + && before_meta.as_ref().is_some_and(|meta| { + meta.get_data_dirs() + .is_ok_and(|dirs| dirs.into_iter().flatten().any(|dir| dir == transaction)) + }); + let state = if live_write_backup { + DeleteRecoveryState::Absent + } else if journal.is_some() { + if committed { + if finished { + DeleteRecoveryState::CommitFinished + } else { + DeleteRecoveryState::QuorumCommitted + } + } else { + DeleteRecoveryState::NewPrepared + } + } else if before.is_some() || any_marker || receipt_bytes.is_some() || prepared.is_some() || target.is_some() { + DeleteRecoveryState::LegacyAmbiguous + } else { + DeleteRecoveryState::Absent + }; + let explicit_delete_safe = + explicit_delete_plan(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none())?.is_some(); + Ok(DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: DeleteTransactionId { + object: object.to_owned(), + transaction, + }, + state, + journal, + current_digest, + backup_digest, + rollback_data_complete, + commit_data_safe, + explicit_delete_safe, + current_versions: current_meta + .as_ref() + .map(|meta| { + meta.versions + .iter() + .map(|version| generation(version.into_fileinfo(volume, object, true)?)) + .collect::>>() + }) + .transpose()? + .unwrap_or_default(), + fingerprint: fingerprint.finalize().into(), + }) +} + +impl LocalDisk { + async fn delete_recovery_predecessors( + &self, + volume: &str, + object: &str, + transaction: Uuid, + incarnation: Option, + owner: Option>, + ) -> Result> { + let root = self.io_get_object_path(volume, object)?; + let volume = volume.to_owned(); + let object = object.to_owned(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await; + os::run_blocking_namespace_operation(lease, move || { + Ok((|| { + let entries = match std::fs::read_dir(&root) { + Ok(entries) => entries, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Vec::new()), + Err(err) => return Err(to_file_error(err).into()), + }; + let mut predecessors = BTreeSet::new(); + let mut visited = 0usize; + for entry in entries { + visited += 1; + if visited > 65536 { + return Err(DiskError::DiskOngoingReq); + } + let entry = entry.map_err(to_file_error)?; + let Some(id) = entry + .file_name() + .to_str() + .and_then(|name| Uuid::parse_str(name).ok()) + .filter(|id| !id.is_nil() && *id != transaction) + else { + continue; + }; + if !entry.file_type().map_err(to_file_error)?.is_dir() { + return Err(DiskError::InvalidPath); + } + let prepared = regular_bytes(&entry.path().join(PREPARED))?; + let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?; + let mut records = Vec::new(); + if let Some(bytes) = prepared { + records.push(serde_json::from_slice::(&bytes)?); + } + if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) { + records.push(serde_json::from_slice::(&bytes)?.journal); + } + for record in records { + if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || record.volume != volume + || record.object != object + || record.transaction != id + || (incarnation.is_some() + && record.bucket_incarnation.is_some() + && record.bucket_incarnation != incarnation) + || record.predecessors.iter().any(|before| before.is_nil() || *before == id) + { + return Err(DiskError::FileCorrupt); + } + predecessors.insert(id); + } + } + Ok(predecessors.into_iter().collect()) + })()) + }) + .await + .map_err(to_file_error)? + } + pub(super) async fn delete_recovery_discover( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + if !(1..=256).contains(&limit) + || !safe_path(volume, false) + || volume.contains('/') + || object.is_some_and(|object| !safe_path(object, false)) + { + return Err(DiskError::InvalidPath); + } + let bucket_path = match object { + Some(object) => self.io_get_object_path(volume, object)?, + None => self.io_get_bucket_path(volume)?, + }; + let disk_root = self.io_root().to_path_buf(); + let (token, mut session) = if let Some(cursor) = cursor { + let session = DISCOVERY_SESSIONS + .lock() + .remove(&cursor.token) + .ok_or(DiskError::FileNotFound)?; + if session.disk_root != disk_root + || session.volume != volume + || session.object.as_deref() != object + || session.touched.elapsed() > Duration::from_secs(300) + { + return Err(DiskError::InvalidPath); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let before = session._mount.metadata().map_err(to_file_error)?; + #[cfg(target_os = "linux")] + let current = self.mount_lease.metadata().map_err(to_file_error)?; + #[cfg(not(target_os = "linux"))] + let current = std::fs::metadata(self.io_root()).map_err(to_file_error)?; + if before.dev() != current.dev() || before.ino() != current.ino() { + return Err(DiskError::InvalidPath); + } + } + (cursor.token, session) + } else { + #[cfg(target_os = "linux")] + let (mount, root) = { + use std::os::fd::AsRawFd; + let mount = self.mount_lease.try_clone().map_err(to_file_error)?; + let root = PathBuf::from(format!("/proc/self/fd/{}/.", mount.as_raw_fd())).join(volume); + let root = match object { + Some(object) => root.join(object), + None => root, + }; + (mount, root) + }; + #[cfg(windows)] + let (mount, root) = { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_FLAG_BACKUP_SEMANTICS, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_TRAVERSE, + }; + // Match the existing publication-root directory open contract. + let mount = std::fs::OpenOptions::new() + .access_mode(FILE_TRAVERSE | FILE_READ_ATTRIBUTES) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(self.io_root()) + .map_err(to_file_error)?; + (mount, bucket_path.clone()) + }; + #[cfg(all(not(target_os = "linux"), not(windows)))] + let (mount, root) = (std::fs::File::open(self.io_root()).map_err(to_file_error)?, bucket_path.clone()); + ( + Uuid::new_v4(), + DiscoverySession { + disk_root, + volume: volume.to_owned(), + object: object.map(str::to_owned), + root, + _mount: mount, + frames: Vec::new(), + touched: Instant::now(), + }, + ) + }; + let lease = os::acquire_namespace_mutation_lease_with_owner(&bucket_path, None).await; + os::run_blocking_namespace_operation(lease, move || { + if session.frames.is_empty() { + match std::fs::read_dir(&session.root) { + Ok(entries) => session.frames.push((String::new(), entries)), + Err(err) if err.kind() == ErrorKind::NotFound => { + return Ok(DeleteRecoveryPage { + transactions: Vec::new(), + next: None, + }); + } + Err(err) => return Err(err), + } + } + let mut transactions = BTreeSet::new(); + let mut visited = 0usize; + while !session.frames.is_empty() + && transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))? + && visited < 4096 + { + let (parent, entries) = session + .frames + .last_mut() + .ok_or_else(|| std::io::Error::from(ErrorKind::InvalidData))?; + let entry = match entries.next() { + Some(entry) => entry?, + None => { + session.frames.pop(); + continue; + } + }; + let name = entry + .file_name() + .into_string() + .map_err(|_| std::io::Error::from(ErrorKind::InvalidData))?; + if !safe_path(&name, false) || name.contains('/') { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + let relative = if parent.is_empty() { + name.clone() + } else { + format!("{parent}/{name}") + }; + let path = session.root.join(&relative); + reject_local_disk_symlink_components(&session.root, &path).map_err(std::io::Error::from)?; + let kind = match std::fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; + if !kind.is_dir() && !kind.is_file() { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + visited += 1; + if kind.is_file() { + let bound = match session.object.as_deref() { + Some(object) if !parent.is_empty() && !parent.contains('/') => Some((object, parent.as_str())), + Some(_) => None, + None => parent.rsplit_once('/'), + }; + if let Some((object, id)) = bound { + let transaction = + if [PREPARED, STORAGE_FORMAT_FILE_BACKUP, DELETE_QUORUM_COMMITTED_FILE].contains(&name.as_str()) { + Uuid::parse_str(id).ok().filter(|id| !id.is_nil()) + } else { + [ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .find_map(|prefix| delete_transaction_marker_id(&name, prefix)) + }; + if let Some(transaction) = transaction { + let physical = match session.object.as_ref() { + Some(_) => session.root.clone(), + None => session.root.join(object), + }; + let proof_root = physical.join(transaction.to_string()); + let prepared = regular_bytes(&proof_root.join(PREPARED)).map_err(std::io::Error::from)?; + let receipt = + regular_bytes(&proof_root.join(DELETE_QUORUM_COMMITTED_FILE)).map_err(std::io::Error::from)?; + let record = match prepared { + Some(bytes) => { + Some(serde_json::from_slice::(&bytes).map_err(std::io::Error::other)?) + } + None => receipt + .filter(|bytes| !bytes.is_empty()) + .map(|bytes| serde_json::from_slice::(&bytes).map(|receipt| receipt.journal)) + .transpose() + .map_err(std::io::Error::other)?, + }; + let logical = if let Some(journal) = record { + let bound_path = match session.object.as_ref() { + Some(scope) if scope == &journal.object => physical.clone(), + Some(_) => return Err(std::io::Error::from(ErrorKind::InvalidData)), + None => build_local_disk_object_path(&session.root, "", &journal.object).1, + }; + if journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || journal.volume != session.volume + || journal.transaction != transaction + || !safe_path(&journal.object, false) + || bound_path != physical + { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + journal.object + } else { + object.to_owned() + }; + transactions.insert((logical, transaction)); + } + } + } else { + if session.object.is_some() && (!parent.is_empty() || !Uuid::parse_str(&name).is_ok_and(|id| !id.is_nil())) { + continue; + } + // A valid 1024-byte key can exceed 256 one-byte path + // components; allow its transaction/data directories too. + if session.frames.len() >= 1024 { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + session.frames.push((relative, std::fs::read_dir(path)?)); + } + } + let next = if session.frames.is_empty() { + None + } else { + session.touched = Instant::now(); + let mut sessions = DISCOVERY_SESSIONS.lock(); + sessions.retain(|_, session| session.touched.elapsed() <= Duration::from_secs(300)); + if sessions.len() >= 128 { + return Err(std::io::Error::from(ErrorKind::WouldBlock)); + } + sessions.insert(token, session); + Some(DeleteRecoveryCursor { token }) + }; + Ok(DeleteRecoveryPage { + transactions: transactions + .into_iter() + .map(|(object, transaction)| DeleteTransactionId { object, transaction }) + .collect(), + next, + }) + }) + .await + .map_err(|err| to_file_error(err).into()) + } + + pub(super) async fn delete_recovery_inspect( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + if !safe_path(object, false) || transaction.is_nil() { + return Err(DiskError::InvalidPath); + } + let object_path = self.get_object_path(volume, object)?; + let owner: Option> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await); + self.delete_recovery_inspect_owned(volume, object, transaction, owner).await + } + + pub(super) async fn delete_recovery_inspect_owned( + &self, + volume: &str, + object: &str, + transaction: Uuid, + owner: Option>, + ) -> Result { + let root = self.io_get_object_path(volume, object)?; + self.io_get_object_path(volume, &format!("{object}/{transaction}"))?; + let volume = volume.to_owned(); + let object = object.to_owned(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await; + os::run_blocking_namespace_operation(lease, move || Ok(inspection(&root, &volume, &object, transaction))) + .await + .map_err(to_file_error)? + } + + async fn activate_delete_preparation( + &self, + journal: &DeleteRecoveryJournal, + original: Option<&[u8]>, + target: Option>, + owner: Option>, + ) -> Result>> { + let volume = journal.volume.as_str(); + let object = journal.object.as_str(); + let transaction = journal.transaction; + let journal = journal_bytes(journal)?; + self.wait_for_startup_cleanup().await; + let temporary = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?; + let root = self.io_get_object_path(volume, object)?; + let destination = root.join(transaction.to_string()); + let original = original.map(<[u8]>::to_vec); + let sync = effective_durability(volume).syncs_commit_metadata(); + #[cfg(test)] + let record_destination = destination.clone(); + let staging_root = temporary.clone(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, owner.clone()).await; + let (staged, target) = os::run_blocking_namespace_operation(lease, move || { + std::fs::create_dir_all(&staging_root)?; + let staged = Arc::new( + tempfile::Builder::new() + .prefix("delete-preparation-") + .tempdir_in(&staging_root)?, + ); + let missing_marker = original.is_none() && target.is_some(); + for (name, bytes) in [ + (STORAGE_FORMAT_FILE_BACKUP, original.as_deref()), + (TARGET, target.as_deref()), + (PREPARED, Some(journal.as_slice())), + (DELETE_MARKER_ROLLBACK_FILE, missing_marker.then_some(&[][..])), + ] { + let Some(bytes) = bytes else { + continue; + }; + #[cfg(test)] + run_owned_file_write_before_open(&record_destination.join(name)); + let mut file = std::fs::File::create(staged.path().join(name))?; + std::io::Write::write_all(&mut file, bytes)?; + if sync { + file.sync_all()?; + } + } + if sync { + os::fsync_dir_std(staged.path())?; + } + Ok((staged, target)) + }) + .await + .map_err(to_file_error)?; + self.delete_recovery_barrier(volume, object, transaction, "PREACTIVATION") + .await?; + match fs::symlink_metadata(&destination).await { + Ok(_) => return Err(DiskError::OutdatedXLMeta), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + // The directory owner accompanies the blocking rename, so cancelling + // its waiter cannot delete staged files while activation is running. + let publication_owner: Option> = Some(Arc::new((owner.clone(), staged.clone()))); + os::rename_all_with_owner( + staged.path(), + &destination, + &self.io_get_bucket_path(volume)?, + &self.publication_root, + publication_owner, + ) + .await?; + if sync { + let bucket = self.io_get_bucket_path(volume)?; + let mut path = root.as_path(); + loop { + os::fsync_dir_with_owner(path, owner.clone()).await.map_err(to_file_error)?; + if path == bucket { + break; + } + path = path.parent().ok_or(DiskError::InvalidPath)?; + } + os::fsync_dir_with_owner(&temporary, owner.clone()) + .await + .map_err(to_file_error)?; + } + self.delete_recovery_barrier(volume, object, transaction, "PREPARED").await?; + Ok(target) + } + + pub(super) async fn prepare_delete_recovery( + &self, + volume: &str, + object: &str, + original: Option<&[u8]>, + requests: &[FileInfo], + authority: Option<&DeleteRecoveryJournal>, + context: DeletePreparationContext, + ) -> Result>> { + let DeletePreparationContext { + transaction, + bucket_incarnation: incarnation, + namespace_owner: owner, + } = context; + let mut plan = original.map(FileMeta::load).transpose()?.unwrap_or_else(FileMeta::new); + let mut deleted_headers: Vec<_> = plan + .versions + .iter() + .map(|version| (version.header.version_id, std::mem::discriminant(&version.header.version_type))) + .collect(); + let mut original_versions = Vec::new(); + for version in &plan.versions { + original_versions.push(generation(version.into_fileinfo(volume, object, true)?)?); + } + let mut data_dirs = Vec::with_capacity(requests.len()); + for request in requests { + if original.is_none() { + plan.add_version(request.clone())?; + continue; + } + match plan.delete_version(request) { + Ok(Some(dir)) => { + let _ = plan.data.remove(vec![request.version_id.unwrap_or_default(), dir]); + data_dirs.push(dir); + } + Ok(None) => {} + Err(err) => { + let err: DiskError = err.into(); + if request.deleted || (err != DiskError::FileVersionNotFound && err != DiskError::FileNotFound) { + return Err(err); + } + } + } + } + if data_dirs.iter().any(|dir| *dir == transaction || dir.is_nil()) { + return Err(DiskError::FileCorrupt); + } + let target = if plan.versions.is_empty() { + None + } else { + Some(plan.marshal_msg()?) + }; + let predecessors = match authority { + Some(authority) => authority.predecessors.clone(), + None => { + self.delete_recovery_predecessors(volume, object, transaction, incarnation, owner.clone()) + .await? + } + }; + let surviving: HashSet<_> = plan + .versions + .iter() + .map(|version| (version.header.version_id, std::mem::discriminant(&version.header.version_type))) + .collect(); + deleted_headers.retain(|header| !surviving.contains(header)); + let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect(); + let requests = requests + .iter() + .map(|request| { + let mut sparse = request.clone(); + sparse.data = None; + sparse + }) + .collect(); + let mut journal = DeleteRecoveryJournal { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + volume: volume.to_owned(), + object: object.to_owned(), + transaction, + bucket_incarnation: incarnation, + original_digest: original.map(digest), + target_digest: target.as_deref().map(digest), + data_dirs, + deleted_versions, + requests, + original_versions, + predecessors, + }; + if let Some(authority) = authority { + if authority.volume != volume + || authority.object != object + || authority.transaction != transaction + || authority.bucket_incarnation != incarnation + { + return Err(DiskError::OutdatedXLMeta); + } + journal.original_versions.clone_from(&authority.original_versions); + journal.deleted_versions.clone_from(&authority.deleted_versions); + } + drop(plan); + self.activate_delete_preparation(&journal, original, target, owner).await + } + + pub(crate) async fn delete_recovery_settle( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + namespace_owner: Option>, + ) -> Result { + if !safe_path(object, false) || transaction.is_nil() { + return Err(DiskError::InvalidPath); + } + let object_path = self.get_object_path(volume, object)?; + let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await; + let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await; + let owner: Option> = Some(Arc::new((metadata_owner, volume_guard))); + self.delete_recovery_settle_owned(volume, object, transaction, expected_fingerprint, action, owner) + .await + } + + pub(super) async fn rollback_prepared_delete( + &self, + volume: &str, + object: &str, + transaction: Uuid, + owner: Option>, + ) -> Result { + let prepared = self.io_get_object_path(volume, &format!("{object}/{transaction}/{PREPARED}"))?; + if regular_bytes(&prepared)?.is_none() { + return Ok(false); + } + let state = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + self.delete_recovery_settle_owned(volume, object, transaction, state.fingerprint, DeleteRecoveryAction::Rollback, owner) + .await?; + Ok(true) + } + + async fn delete_recovery_settle_owned( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + owner: Option>, + ) -> Result { + let observed = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + if observed.fingerprint != expected_fingerprint { + return Err(DiskError::OutdatedXLMeta); + } + if let DeleteRecoveryAction::AdoptCommit { authority } = &action { + if authority.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || authority.volume != volume + || authority.object != object + || authority.transaction != transaction + { + return Err(DiskError::OutdatedXLMeta); + } + if observed.state == DeleteRecoveryState::NewPrepared { + let local = observed.journal.as_ref().ok_or(DiskError::FileCorrupt)?; + if observed.current_digest.is_some() || !local.matches_prepared_authority(authority) { + return Err(DiskError::OutdatedXLMeta); + } + // The coordinator supplies an inspected same-transaction + // receipt. Retain this drive's immutable hashes, data dirs and + // causal history when completing its partial publication. + let root = self.io_get_object_path(volume, object)?; + let receipt = CommitReceipt { + journal: local.clone(), + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + if observed.state != DeleteRecoveryState::Absent { + return Err(DiskError::OutdatedXLMeta); + } + if observed.current_digest.is_none() { + // A replay witness inherits the original decision's causal + // history; observing later records cannot backdate new edges. + let mut witness = authority.as_ref().clone(); + witness.original_digest = None; + witness.target_digest = None; + witness.data_dirs.clear(); + let root = self.io_get_object_path(volume, object)?; + let _ = self.activate_delete_preparation(&witness, None, None, owner.clone()).await?; + let receipt = CommitReceipt { + journal: witness, + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + if !authority.matches_adoption_source(&observed.current_versions) { + return Err(DiskError::OutdatedXLMeta); + } + let root = self.io_get_object_path(volume, object)?; + let bytes = self.read_all(volume, &format!("{object}/{STORAGE_FORMAT_FILE}")).await?; + let _ = self + .prepare_delete_recovery( + volume, + object, + Some(&bytes), + &authority.requests, + Some(authority.as_ref()), + DeletePreparationContext { + transaction, + bucket_incarnation: authority.bucket_incarnation, + namespace_owner: owner.clone(), + }, + ) + .await?; + let adopted = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + let adopted_journal = adopted.journal.ok_or(DiskError::FileCorrupt)?; + let receipt = CommitReceipt { + journal: adopted_journal, + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + if observed.state == DeleteRecoveryState::Absent { + if action == DeleteRecoveryAction::PublishCommit { + if observed.current_digest.is_some() { + return Err(DiskError::OutdatedXLMeta); + } + self.prepare_delete_recovery( + volume, + object, + None, + &[], + None, + DeletePreparationContext { + transaction, + bucket_incarnation: None, + namespace_owner: owner.clone(), + }, + ) + .await?; + } else { + return Ok(observed); + } + } + let observed = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + if action == DeleteRecoveryAction::DiscardLegacy { + if observed.state != DeleteRecoveryState::LegacyAmbiguous || !observed.explicit_delete_safe { + return Err(DiskError::FileCorrupt); + } + let root = self.io_get_object_path(volume, object)?; + let rollback = root.join(transaction.to_string()); + let plan_root = root.clone(); + let missing = observed.current_digest.is_none(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await; + let plan = os::run_blocking_namespace_operation(lease, move || { + Ok((|| { + let before = + regular_metadata_bytes(&plan_root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))? + .as_deref() + .map(FileMeta::load) + .transpose()?; + explicit_delete_plan(&plan_root, transaction, before.as_ref(), None, missing)?.ok_or(DiskError::FileCorrupt) + })()) + }) + .await + .map_err(to_file_error)??; + for dir in plan.data_dirs { + let relative = dir.strip_prefix(&root).map_err(|_| DiskError::InvalidPath)?; + let data_dir = relative + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| Uuid::parse_str(name).ok()) + .ok_or(DiskError::FileCorrupt)?; + let parent = relative.parent().filter(|parent| !parent.as_os_str().is_empty()); + let staged_object = + parent.map_or_else(|| object.to_owned(), |parent| format!("{object}/{}", parent.to_string_lossy())); + self.reserve_version_delete_with_namespace_owner(volume, &staged_object, data_dir, transaction, owner.clone()) + .await?; + let reserved = dir.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + let private = dir.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + match os::rename_with_namespace_owner(&reserved, &private, owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&dir, owner.clone()).await.map_err(to_file_error)?; + } + if self + .delete_data_dir_with_namespace_owner( + volume, + &format!("{object}/{}", relative.to_string_lossy()), + DeleteOptions { + recursive: true, + old_data_dir: Some(transaction), + recoverable_delete: true, + ..Default::default() + }, + owner.clone(), + ) + .await? + == DataDirDeleteStatus::Deferred + { + return Err(DiskError::DiskOngoingReq); + } + } + for file in plan.proof_files { + match os::remove_file_with_owner(file, owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() && fs::try_exists(&rollback).await.map_err(to_file_error)? { + os::fsync_dir_with_owner(&rollback, owner.clone()) + .await + .map_err(to_file_error)?; + } + self.delete_unleased_with_namespace_owner( + volume, + &format!("{object}/{transaction}"), + &DeleteOptions { + immediate: true, + ..Default::default() + }, + owner.clone(), + ) + .await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + let journal = observed.journal.as_ref().ok_or(DiskError::FileCorrupt)?; + let root = self.io_get_object_path(volume, object)?; + let rollback = root.join(transaction.to_string()); + match action { + DeleteRecoveryAction::PublishCommit => { + if !observed.commit_data_safe { + return Err(DiskError::OutdatedXLMeta); + } + if matches!(observed.state, DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished) { + return Ok(observed); + } + let receipt = CommitReceipt { + journal: journal.clone(), + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + } + DeleteRecoveryAction::FinishCommit => { + if !matches!(observed.state, DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished) + || !observed.commit_data_safe + { + return Err(DiskError::OutdatedXLMeta); + } + if observed.state == DeleteRecoveryState::CommitFinished { + return Ok(observed); + } + for dir in &journal.data_dirs { + self.reserve_version_delete_with_namespace_owner(volume, object, *dir, transaction, owner.clone()) + .await?; + } + if observed.current_digest == journal.original_digest { + if journal.target_digest.is_some() { + let target = self.read_all(volume, &format!("{object}/{transaction}/{TARGET}")).await?; + self.write_all_meta_with_namespace_owner( + volume, + &format!("{object}/{STORAGE_FORMAT_FILE}"), + &target, + true, + owner.clone(), + ) + .await?; + } else { + self.delete_file_with_namespace_owner( + &self.io_get_bucket_path(volume)?, + &root.join(STORAGE_FORMAT_FILE), + false, + false, + owner.clone(), + ) + .await?; + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&root, owner.clone()).await.map_err(to_file_error)?; + } + } + } + self.commit_reserved_version_delete_with_namespace_owner(volume, object, transaction, owner.clone()) + .await?; + for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] { + match os::remove_file_with_owner(rollback.join(name), owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&rollback, owner.clone()) + .await + .map_err(to_file_error)?; + } + let (_, complete) = self + .finish_version_delete(volume, object, transaction, true, owner.clone()) + .await?; + if !complete { + return Err(DiskError::DiskOngoingReq); + } + for dir in &journal.data_dirs { + let staged = rollback.join(dir.to_string()); + match fs::symlink_metadata(&staged).await { + Ok(metadata) if metadata.is_dir() => { + let staged_object = format!("{object}/{transaction}"); + self.reserve_version_delete_with_namespace_owner( + volume, + &staged_object, + *dir, + transaction, + owner.clone(), + ) + .await?; + let source = staged.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + let destination = staged.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + match os::rename_with_namespace_owner(&source, &destination, owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&staged, owner.clone()) + .await + .map_err(to_file_error)?; + } + if self + .delete_data_dir_with_namespace_owner( + volume, + &format!("{staged_object}/{dir}"), + DeleteOptions { + recursive: true, + old_data_dir: Some(transaction), + ..Default::default() + }, + owner.clone(), + ) + .await? + == DataDirDeleteStatus::Deferred + { + return Err(DiskError::DiskOngoingReq); + } + } + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + write_delete_rollback_file(self.io_root(), volume, &root, transaction, FINISHED, &[], owner.clone()).await?; + let receipt = CommitReceipt { + journal: journal.clone(), + finished: true, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + } + DeleteRecoveryAction::FinalizeCommit => { + if observed.state != DeleteRecoveryState::CommitFinished { + return Err(DiskError::DiskOngoingReq); + } + for name in [TARGET, PREPARED, FINISHED] { + match os::remove_file_with_owner(rollback.join(name), owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&rollback, owner.clone()) + .await + .map_err(to_file_error)?; + } + os::remove_file_with_owner(rollback.join(DELETE_QUORUM_COMMITTED_FILE), owner.clone()) + .await + .map_err(to_file_error)?; + self.delete_unleased_with_namespace_owner( + volume, + &format!("{object}/{transaction}"), + &DeleteOptions { + immediate: true, + ..Default::default() + }, + owner.clone(), + ) + .await?; + } + DeleteRecoveryAction::Rollback => { + if observed.state != DeleteRecoveryState::NewPrepared || !observed.rollback_data_complete { + return Err(DiskError::FileCorrupt); + } + let source = if observed.backup_digest.is_some() { + format!("{object}/{transaction}/{STORAGE_FORMAT_FILE_BACKUP}") + } else { + format!("{object}/{STORAGE_FORMAT_FILE}") + }; + if journal.original_digest.is_some() { + let bytes = self.read_all(volume, &source).await?; + let metadata = FileMeta::load(&bytes)?; + for version in &metadata.versions { + if version.header.version_type != rustfs_filemeta::VersionType::Object { + continue; + } + let mut info = version.into_fileinfo(volume, object, true)?; + info.hydrate_shard_integrity()?; + if info.inline_data() || info.is_remote() { + continue; + } + let dir = info.data_dir.filter(|id| !id.is_nil()).ok_or(DiskError::FileCorrupt)?; + let codec = coding::Erasure::try_new_with_options( + info.erasure.data_blocks, + info.erasure.parity_blocks, + info.erasure.block_size, + info.uses_legacy_checksum, + ) + .map_err(DiskError::from)?; + let live = self.io_get_object_path(volume, &format!("{object}/{dir}"))?; + let staged = self.io_get_object_path(volume, &format!("{object}/{transaction}/{dir}"))?; + for part in &info.parts { + let name = format!("part.{}", part.number); + let checksum = info.erasure.get_checksum_info(part.number); + let algo = if info.uses_legacy_checksum && checksum.algorithm == HashAlgorithm::HighwayHash256S { + HashAlgorithm::HighwayHash256SLegacy + } else { + checksum.algorithm + }; + if let Some(integrity) = &part.integrity { + verify_recovery_integrity_sources( + &live, + &staged, + integrity, + info.erasure.index.checked_sub(1).ok_or(DiskError::FileCorrupt)?, + algo.clone(), + owner.clone(), + ) + .await?; + } + let mut checked = false; + for part_root in [&live, &staged] { + let path = part_root.join(&name); + match fs::symlink_metadata(&path).await { + Ok(metadata) if metadata.is_file() => {} + Ok(_) => return Err(DiskError::InvalidPath), + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(to_file_error(err).into()), + } + self.bitrot_verify( + &path, + usize::try_from( + codec.shard_file_size(i64::try_from(part.size).map_err(|_| DiskError::FileCorrupt)?), + ) + .map_err(|_| DiskError::FileCorrupt)?, + algo.clone(), + codec.shard_size(), + ) + .await?; + checked = true; + } + if !checked { + return Err(DiskError::FileCorrupt); + } + } + } + } + if journal.original_digest.is_none() && journal.target_digest.is_none() { + for name in [TARGET, PREPARED, FINISHED] { + match os::remove_file_with_owner(rollback.join(name), owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + self.delete_unleased_with_namespace_owner( + volume, + &format!("{object}/{transaction}"), + &DeleteOptions { + immediate: true, + ..Default::default() + }, + owner.clone(), + ) + .await?; + } else { + restore_delete_rollback_with_namespace_owner( + volume, + &root, + &root.join(STORAGE_FORMAT_FILE), + transaction, + &self.publication_root, + owner.clone(), + ) + .await?; + } + } + DeleteRecoveryAction::AdoptCommit { .. } => return Err(DiskError::MethodNotAllowed), + DeleteRecoveryAction::DiscardLegacy => return Err(DiskError::MethodNotAllowed), + } + self.delete_recovery_inspect_owned(volume, object, transaction, owner).await + } + + pub(super) async fn delete_recovery_barrier(&self, volume: &str, object: &str, transaction: Uuid, stage: &str) -> Result<()> { + barrier(volume, object, transaction, stage, &self.root).await + } +} + +#[cfg(not(feature = "e2e-test-hooks"))] +async fn barrier(_volume: &str, _object: &str, _transaction: Uuid, _stage: &str, _drive: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(feature = "e2e-test-hooks")] +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct BarrierArm { + nonce: String, + bucket: String, + object: String, + stage: String, + #[serde(default)] + disk: Option, +} + +#[cfg(feature = "e2e-test-hooks")] +async fn barrier(volume: &str, object: &str, transaction: Uuid, stage: &str, drive: &Path) -> Result<()> { + let Some(directory) = std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").map(PathBuf::from) else { + return Ok(()); + }; + let bytes = match fs::read(directory.join("arm.json")).await { + Ok(bytes) => bytes, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(to_file_error(err).into()), + }; + let arm: BarrierArm = serde_json::from_slice(&bytes)?; + if arm.bucket != volume + || arm.object != object + || arm.stage != stage + || arm.disk.as_deref().is_some_and(|disk| Path::new(disk) != drive) + { + return Ok(()); + } + if arm.nonce.is_empty() + || arm.nonce.len() > 64 + || !arm + .nonce + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') + { + return Err(DiskError::InvalidPath); + } + let drive_hash = rustfs_utils::crypto::hex(digest(drive.to_string_lossy().as_bytes())); + let observation = serde_json::json!({"transaction": transaction, "bucket": volume, "object": object, "stage": stage, "pid": std::process::id(), "drive": drive}); + fs::write( + directory.join(format!("reached.{}.{}.{}.json", arm.nonce, std::process::id(), &drive_hash[..16])), + serde_json::to_vec(&observation)?, + ) + .await + .map_err(to_file_error)?; + let deadline = Instant::now() + Duration::from_secs(60); + while !fs::try_exists(directory.join(format!("release.{}", arm.nonce))) + .await + .map_err(to_file_error)? + { + if Instant::now() >= deadline { + return Err(DiskError::Timeout); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + if fs::read(directory.join(format!("release.{}", arm.nonce))) + .await + .map_err(to_file_error)? + == b"io_error" + { + return Err(DiskError::Io(std::io::Error::new( + ErrorKind::Other, + "injected delete recovery barrier failure", + ))); + } + Ok(()) +} + +pub(super) async fn restore_barrier(volume: &str, object_dir: &Path, transaction: Uuid) -> Result<()> { + #[cfg(feature = "e2e-test-hooks")] + if let Some(directory) = std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").map(PathBuf::from) { + let bytes = match fs::read(directory.join("arm.json")).await { + Ok(bytes) => bytes, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(to_file_error(err).into()), + }; + let arm: BarrierArm = serde_json::from_slice(&bytes)?; + if arm.bucket == volume && arm.stage == "RESTORE" && object_dir.ends_with(Path::new(volume).join(&arm.object)) { + let suffix = Path::new(volume).join(&arm.object); + let canonical = fs::canonicalize(object_dir).await.map_err(to_file_error)?; + let mut drive = canonical; + for _ in suffix.components() { + drive.pop(); + } + return barrier(volume, &arm.object, transaction, "RESTORE", &drive).await; + } + } + let _ = (volume, object_dir, transaction); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::disk::local::test::{ensure_test_volume, test_file_info, test_meta}; + use tokio::io::AsyncReadExt; + + #[test] + fn delete_recovery_native_sidecar_classifier_is_bounded() { + assert!(delete_recovery_part_file("part.1")); + assert!(delete_recovery_part_file("part.1.meta")); + assert!(delete_recovery_part_file(&format!("part.1.integrity.{}", Uuid::new_v4()))); + for name in [ + "part.0.meta", + "part.1.meta.extra", + "part.1.integrity.invalid", + "xl.meta", + "child/", + ] { + assert!(!delete_recovery_part_file(name), "unknown sidecar {name} must stay protected"); + } + assert!(!delete_recovery_part_file(&format!("part.1.integrity.{}", Uuid::nil()))); + assert!(!metadata_less_part_file("part.1.meta"), "the heal shard classifier remains unchanged"); + } + + #[tokio::test] + #[serial_test::serial(shard_integrity_rollout)] + async fn delete_recovery_multipart_rollback_moves_native_sidecars_and_preserves_proofs() { + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use crate::storage_api_contracts::multipart::{CompletePart, MultipartOperations}; + use crate::storage_api_contracts::object::ObjectIO; + use http::HeaderMap; + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some("true")), + (rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, Some("true")), + ], + async { + for damage in [None, Some("missing"), Some("corrupt"), Some("payload")] { + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "recovery-multipart"; + let object = "object"; + store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap(); + let upload = store + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .unwrap(); + let payload = vec![63; 6 * 1024 * 1024]; + let mut reader = PutObjReader::from_vec(payload.clone()); + let part = store + .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + store + .clone() + .complete_multipart_upload( + bucket, + object, + &upload.upload_id, + vec![CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + .unwrap(); + let disks = store.pools[0].disk_set[0].disks.read().await.clone(); + let transaction = Uuid::new_v4(); + let mut originals = Vec::new(); + for (dir, disk) in dirs.iter().zip(&disks) { + let root = dir.path().join(bucket).join(object); + let original = fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let metadata = FileMeta::load(&original).unwrap(); + let mut info = metadata.versions[0].into_fileinfo(bucket, object, true).unwrap(); + info.hydrate_shard_integrity().unwrap(); + let data = info.data_dir.unwrap(); + let proof_name = info.parts[0] + .integrity + .as_ref() + .expect("real protected MPU proof") + .file_name(); + assert!(root.join(data.to_string()).join("part.1.meta").is_file()); + disk.as_ref() + .unwrap() + .delete_version_recoverable( + bucket, + object, + FileInfo { + name: object.to_owned(), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .unwrap(); + let staged = root.join(transaction.to_string()).join(data.to_string()); + fs::rename(root.join(data.to_string()), &staged).await.unwrap(); + originals.push((root, data, proof_name, original)); + } + let (root, data, proof, _) = &originals[0]; + let proof_path = root.join(transaction.to_string()).join(data.to_string()).join(proof); + if let Some(damage) = damage { + if damage == "missing" { + fs::remove_file(&proof_path).await.unwrap(); + } else if damage == "corrupt" { + let mut bytes = fs::read(&proof_path).await.unwrap(); + bytes[rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE] ^= 1; + fs::write(&proof_path, bytes).await.unwrap(); + } else { + let metadata = FileMeta::load(&originals[0].3).unwrap(); + let info = metadata.versions[0].into_fileinfo(bucket, object, true).unwrap(); + let codec = coding::Erasure::try_new_with_options( + info.erasure.data_blocks, + info.erasure.parity_blocks, + info.erasure.block_size, + info.uses_legacy_checksum, + ) + .unwrap(); + let algorithm = info.erasure.get_checksum_info(1).algorithm; + let path = proof_path.parent().unwrap().join("part.1"); + let mut bytes = fs::read(&path).await.unwrap(); + let hash_size = algorithm.size(); + let part_size = usize::try_from(codec.shard_file_size(info.parts[0].size as i64)).unwrap(); + let length = part_size.min(codec.shard_size()); + bytes[hash_size] ^= 1; + let rewritten = algorithm.hash_encode(&bytes[hash_size..hash_size + length]).as_ref().to_vec(); + bytes[..hash_size].copy_from_slice(&rewritten); + coding::bitrot_verify( + std::io::Cursor::new(bytes.clone()), + bytes.len(), + part_size, + algorithm, + codec.shard_size(), + ) + .await + .expect("recomputed self-described bitrot checksum passes the former rollback gate"); + fs::write(path, bytes).await.unwrap(); + } + let disk = disks[0].as_ref().unwrap(); + let observed = disk.inspect_delete_transaction(bucket, object, transaction).await.unwrap(); + let backup = fs::read(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP)) + .await + .unwrap(); + assert!(matches!( + disk.settle_delete_transaction( + bucket, + object, + transaction, + observed.fingerprint, + DeleteRecoveryAction::Rollback + ) + .await, + Err(DiskError::FileCorrupt) + )); + assert!(!root.join(STORAGE_FORMAT_FILE).exists()); + assert!(root.join(transaction.to_string()).join(PREPARED).is_file()); + assert_eq!( + fs::read(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP)) + .await + .unwrap(), + backup + ); + continue; + } + // Both directions of a partially restored native layout: + // proof still staged with live payload, and the inverse. + for (index, (root, data, proof, _)) in originals.iter().take(2).enumerate() { + let live = root.join(data.to_string()); + fs::create_dir(&live).await.unwrap(); + let name = if index == 0 { "part.1" } else { proof.as_str() }; + fs::rename(root.join(transaction.to_string()).join(data.to_string()).join(name), live.join(name)) + .await + .unwrap(); + } + for (disk, (root, data, proof, original)) in disks.iter().zip(originals) { + let disk = disk.as_ref().unwrap(); + let observed = disk.inspect_delete_transaction(bucket, object, transaction).await.unwrap(); + assert!(observed.rollback_data_complete); + disk.settle_delete_transaction( + bucket, + object, + transaction, + observed.fingerprint, + DeleteRecoveryAction::Rollback, + ) + .await + .expect("rollback restores native multipart sidecars"); + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), original); + assert!(root.join(data.to_string()).join("part.1.meta").is_file()); + assert!(root.join(data.to_string()).join(proof).is_file()); + assert!(!root.join(transaction.to_string()).exists()); + } + let mut reader = store.pools[0].disk_set[0] + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .unwrap(); + let mut restored = Vec::new(); + reader.stream.read_to_end(&mut restored).await.unwrap(); + assert_eq!(restored, payload, "restored protected MPU remains readable through integrity checks"); + } + }, + ) + .await; + } + + async fn fixture() -> (tempfile::TempDir, LocalDisk, FileInfo, Vec) { + let temp = tempfile::tempdir().expect("temporary disk"); + let endpoint = Endpoint::try_from(temp.path().to_str().expect("UTF-8 disk path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + ensure_test_volume(&disk, "recovery").await; + let mut info = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"complete original inline payload"))); + info.set_inline_data(); + let bytes = test_meta(info.clone()); + let path = disk.io_get_object_path("recovery", "object/xl.meta").expect("metadata path"); + fs::create_dir_all(path.parent().expect("object directory")) + .await + .expect("object directory"); + fs::write(path, &bytes).await.expect("original metadata"); + (temp, disk, info, bytes) + } + + fn request() -> FileInfo { + FileInfo { + name: "object".to_owned(), + version_id: Some(Uuid::nil()), + ..Default::default() + } + } + + #[test] + fn delete_recovery_generation_known_aliases_are_equal_unknowns_stay_distinct() { + use rustfs_utils::http::metadata_compat as compat; + let mut legacy = request(); + legacy + .metadata + .insert(format!("{}{}", compat::MINIO_INTERNAL_PREFIX, compat::SUFFIX_INLINE_DATA), String::new()); + legacy + .metadata + .insert(format!("{}{}", compat::MINIO_INTERNAL_PREFIX, compat::SUFFIX_COMPRESSION), "gzip".into()); + let mut modern = legacy.clone(); + compat::insert_str(&mut modern.metadata, compat::SUFFIX_INLINE_DATA, String::new()); + compat::insert_str(&mut modern.metadata, compat::SUFFIX_COMPRESSION, "gzip".into()); + assert_eq!( + generation(legacy.clone()).expect("legacy identity"), + generation(modern.clone()).expect("modern identity") + ); + modern.metadata.insert( + format!("{}{}", compat::RUSTFS_INTERNAL_PREFIX, compat::SUFFIX_COMPRESSION), + "different".into(), + ); + assert!(matches!(generation(modern), Err(DiskError::FileCorrupt))); + let mut unknown = legacy.clone(); + legacy + .metadata + .insert("x-minio-internal-unknown-extension".into(), "same".into()); + unknown + .metadata + .insert("x-rustfs-internal-unknown-extension".into(), "same".into()); + assert_ne!( + generation(legacy).expect("legacy unknown field"), + generation(unknown).expect("modern unknown field") + ); + } + + #[tokio::test] + async fn delete_recovery_inactive_preparation_artifacts_use_existing_tmp_cleanup() { + let (_temp, disk, _, original) = fixture().await; + let temporary = disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).unwrap(); + fs::create_dir_all(temporary.join("delete-preparation-interrupted")) + .await + .unwrap(); + fs::write(temporary.join("delete-preparation-interrupted").join(TARGET), b"unfinished target") + .await + .unwrap(); + fs::create_dir_all(temporary.join(".delete-rollback-interrupted")) + .await + .unwrap(); + fs::write( + temporary.join(".delete-rollback-interrupted").join("metadata"), + b"unpublished replacement", + ) + .await + .unwrap(); + assert!( + disk.discover_delete_transactions("recovery", Some("object"), None, 16) + .await + .unwrap() + .transactions + .is_empty() + ); + let ready = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let notify = Arc::new(Notify::new()); + LocalDisk::cleanup_tmp_on_startup(disk.io_root(), &disk.publication_root, ready.clone(), notify) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + while ready.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!temporary.join("delete-preparation-interrupted").exists()); + assert!(!temporary.join(".delete-rollback-interrupted").exists()); + assert!(!disk.io_root().join(RUSTFS_META_TMP_OLD_BUCKET).exists()); + assert!(temporary.join(".trash").is_dir(), "startup recreates the active trash directory"); + assert_eq!(disk.read_all("recovery", "object/xl.meta").await.unwrap().as_ref(), original); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_activation_is_complete_before_metadata_mutation_even_after_cancel() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + use futures::FutureExt; + let (_temp, disk, _, original) = fixture().await; + let disk = Arc::new(disk); + let transaction = Uuid::new_v4(); + let destination = disk.io_get_object_path("recovery", &format!("object/{transaction}")).unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Rename, &destination, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let deleting = disk.clone(); + let mut request = request(); + request.expire_restored = true; + let task = tokio::spawn(async move { + deleting + .delete_version_recoverable( + "recovery", + "object", + request, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + }); + let observations = std::panic::AssertUnwindSafe(async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .unwrap() + .unwrap(); + assert!(!destination.exists()); + assert_eq!(disk.read_all("recovery", "object/xl.meta").await.unwrap().as_ref(), original); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + }) + .catch_unwind() + .await; + let _ = release_tx.send(()); + drop(hook); + tokio::time::timeout(Duration::from_secs(5), async { + while !destination.exists() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + for name in [PREPARED, TARGET, STORAGE_FORMAT_FILE_BACKUP] { + assert!(destination.join(name).is_file()); + } + let state = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(state.state, DeleteRecoveryState::NewPrepared); + assert_eq!(state.current_digest, Some(digest(&original))); + if let Err(panic) = observations { + std::panic::resume_unwind(panic); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_receiver_volume_guard_outlives_cancelled_restore() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + use futures::FutureExt; + let (_temp, disk, _, _) = fixture().await; + let disk = Arc::new(disk); + let transaction = Uuid::new_v4(); + let state = apply(&disk, transaction).await; + let metadata = disk.io_get_object_path("recovery", "object/xl.meta").unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Rename, &metadata, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let restoring = disk.clone(); + let task = tokio::spawn(async move { + restoring + .settle_delete_transaction("recovery", "object", transaction, state.fingerprint, DeleteRecoveryAction::Rollback) + .await + }); + let mut deletion = None; + let observations = std::panic::AssertUnwindSafe(async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .unwrap() + .unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let removing = disk.clone(); + deletion = Some(tokio::spawn(async move { removing.delete_volume("recovery", true).await })); + assert!( + tokio::time::timeout(Duration::from_millis(100), deletion.as_mut().unwrap()) + .await + .is_err() + ); + }) + .catch_unwind() + .await; + let _ = release_tx.send(()); + drop(hook); + if let Some(deletion) = deletion { + tokio::time::timeout(Duration::from_secs(5), deletion) + .await + .unwrap() + .unwrap() + .unwrap(); + } + disk.make_volume("recovery").await.unwrap(); + assert!(!metadata.exists(), "late restore must finish before volume recreation"); + if let Err(panic) = observations { + std::panic::resume_unwind(panic); + } + } + + async fn apply(disk: &LocalDisk, transaction: Uuid) -> DeleteRecoveryInspection { + disk.delete_version_recoverable( + "recovery", + "object", + request(), + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .expect("prepared metadata application"); + let inspected = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("inspect applied transaction"); + assert_eq!( + inspected.current_digest, + inspected.journal.as_ref().unwrap().target_digest, + "planned target must match the unchanged mutation loop" + ); + inspected + } + + #[tokio::test] + async fn delete_recovery_uncommitted_inline_rolls_back_exact_bytes() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let observed = apply(&disk, transaction).await; + assert_eq!(observed.state, DeleteRecoveryState::NewPrepared); + assert!(observed.rollback_data_complete); + let restored = disk + .settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("rollback intact prepared metadata"); + assert_eq!(restored.state, DeleteRecoveryState::Absent); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("readable original metadata") + .as_ref(), + original + ); + } + + #[tokio::test] + async fn delete_recovery_preparation_records_existing_decision_before_new_generation() { + let (_temp, disk, info, original) = fixture().await; + let first = Uuid::new_v4(); + let applied = apply(&disk, first).await; + disk.settle_delete_transaction("recovery", "object", first, applied.fingerprint, DeleteRecoveryAction::PublishCommit) + .await + .expect("persist first decision"); + let mut newer = info; + newer.mod_time = newer.mod_time.map(|time| time + time::Duration::seconds(1)); + let newer_bytes = test_meta(newer); + assert_ne!(newer_bytes, original); + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").unwrap(), &newer_bytes) + .await + .unwrap(); + let second = Uuid::new_v4(); + let second_state = apply(&disk, second).await; + assert_eq!(second_state.journal.unwrap().predecessors, vec![first]); + } + + #[tokio::test] + async fn delete_recovery_large_intent_fails_before_metadata_application() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let mut large = request(); + large + .metadata + .insert("oversized-intent".to_owned(), "x".repeat(usize::try_from(MAX_METADATA).unwrap())); + assert!( + disk.delete_version_recoverable( + "recovery", + "object", + large, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + } + ) + .await + .is_err() + ); + assert_eq!(disk.read_all("recovery", "object/xl.meta").await.unwrap().as_ref(), original); + assert!( + !disk + .io_get_object_path("recovery", &format!("object/{transaction}/{PREPARED}")) + .unwrap() + .exists() + ); + } + + #[tokio::test] + async fn delete_recovery_metadata_updates_publish_exact_prepared_target_bytes() { + for batch in [false, true] { + for transition in [false, true] { + let (_temp, disk, mut info, _) = fixture().await; + info.metadata.insert("x-amz-meta-first".into(), "retained-a".into()); + info.metadata.insert("x-amz-meta-second".into(), "retained-b".into()); + info.metadata + .insert(rustfs_filemeta::metadata_keys::RESTORE.into(), "ongoing-request=\"false\"".into()); + let original = test_meta(info); + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").unwrap(), &original) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + let mut change = request(); + if transition { + change.transition_status = "complete".into(); + change.transition_tier = "test-tier".into(); + change.transitioned_objname = "remote/object".into(); + } else { + change.expire_restored = true; + } + let opts = DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }; + if batch { + let errors = disk + .delete_versions_recoverable( + "recovery", + vec![FileInfoVersions { + name: "object".into(), + volume: "recovery".into(), + versions: vec![change], + ..Default::default() + }], + opts, + ) + .await; + assert_eq!(errors, vec![None]); + } else { + disk.delete_version_recoverable("recovery", "object", change, false, opts) + .await + .unwrap(); + } + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(observed.current_digest, observed.journal.unwrap().target_digest); + let applied = disk.read_all("recovery", "object/xl.meta").await.unwrap(); + let prepared = disk + .read_all("recovery", &format!("object/{transaction}/{TARGET}")) + .await + .unwrap(); + assert_eq!(applied, prepared, "batch={batch}, transition={transition}"); + assert_ne!(applied.as_ref(), original); + } + } + } + + async fn legacy_fixture(disk: &LocalDisk, original: &[u8], transaction: Uuid) -> PathBuf { + let root = disk.io_get_object_path("recovery", "object").unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + fs::create_dir(root.join(transaction.to_string())).await.unwrap(); + fs::write(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP), original) + .await + .unwrap(); + root + } + + #[tokio::test] + async fn delete_recovery_live_repair_backup_preserves_legacy_delete_guards() { + for shape in ["live", "rollback-marker", "missing-current", "foreign-backup"] { + let (_temp, disk, mut info, _) = fixture().await; + let transaction = Uuid::new_v4(); + info.data_dir = Some(transaction); + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let current = test_meta(info.clone()); + fs::write(root.join(STORAGE_FORMAT_FILE), ¤t).await.unwrap(); + let rollback = root.join(transaction.to_string()); + fs::create_dir(&rollback).await.unwrap(); + if shape == "foreign-backup" { + info.data_dir = Some(Uuid::new_v4()); + } + let backup = test_meta(info); + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), &backup).await.unwrap(); + if shape == "rollback-marker" { + fs::write(rollback.join(DELETE_MARKER_ROLLBACK_FILE), []).await.unwrap(); + } else if shape == "missing-current" { + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + } + let inspection = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!( + inspection.state, + if shape == "live" { + DeleteRecoveryState::Absent + } else { + DeleteRecoveryState::LegacyAmbiguous + }, + "{shape}" + ); + assert_eq!(fs::read(rollback.join(STORAGE_FORMAT_FILE_BACKUP)).await.unwrap(), backup); + if shape != "missing-current" { + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), current); + } + } + } + + #[tokio::test] + async fn delete_recovery_explicit_legacy_discard_accepts_old_empty_receipt() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let root = legacy_fixture(&disk, &original, transaction).await; + fs::write(root.join(transaction.to_string()).join(DELETE_QUORUM_COMMITTED_FILE), []) + .await + .unwrap(); + let state = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(state.state, DeleteRecoveryState::LegacyAmbiguous); + assert!(state.explicit_delete_safe); + let cleared = disk + .settle_delete_transaction("recovery", "object", transaction, state.fingerprint, DeleteRecoveryAction::DiscardLegacy) + .await + .unwrap(); + assert_eq!(cleared.state, DeleteRecoveryState::Absent); + assert!(!root.join(transaction.to_string()).exists()); + } + + #[tokio::test] + async fn delete_recovery_prepared_explicit_delete_checks_original_header() { + for protected in ["none", "version", "lock", "internal-lock"] { + let (_temp, disk, mut info, _) = fixture().await; + match protected { + "version" => info.version_id = Some(Uuid::new_v4()), + "lock" => { + info.metadata.insert("x-amz-object-lock-mode".into(), "COMPLIANCE".into()); + } + "internal-lock" => rustfs_utils::http::metadata_compat::insert_str( + &mut info.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, + "known-lock".to_owned(), + ), + _ => {} + } + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").unwrap(), test_meta(info.clone())) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + disk.delete_version_recoverable( + "recovery", + "object", + FileInfo { + version_id: info.version_id, + ..request() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .unwrap(); + let state = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(state.state, DeleteRecoveryState::NewPrepared); + assert_eq!(state.explicit_delete_safe, protected == "none", "original header {protected}"); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_cancelled_restore_retains_actual_key_and_bucket_guards() { + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::disk::os::prepared_publication_test_hooks as hooks; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use crate::storage_api_contracts::object::ObjectIO; + use futures::FutureExt; + use rustfs_heal_contracts::heal_channel::HealOpts; + + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "recover-owned-timeout"; + store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap(); + store + .put_object(bucket, "object", &mut PutObjReader::from_vec(vec![61; 4096]), &ObjectOptions::default()) + .await + .unwrap(); + let disks = store.pools[0].disk_set[0].disk_inventory().await; + let transaction = Uuid::new_v4(); + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); + for disk in disks.iter().flatten() { + disk.delete_version_recoverable( + bucket, + "object", + request(), + false, + DeleteOptions { + old_data_dir: Some(transaction), + delete_recovery_bucket_incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .unwrap(); + } + let metadata = disks[0] + .as_ref() + .unwrap() + .get_object_path_for_io_if_local(bucket, "object/xl.meta") + .unwrap() + .unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Rename, &metadata, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let before = store.ctx.namespace_commit_generation(); + let recovering = store.clone(); + let mut recovery = + tokio::spawn(async move { recovering.recover_delete_transactions(bucket, &HealOpts::default()).await }); + let mut writer = None; + let mut bucket_writer = None; + let observations = std::panic::AssertUnwindSafe(async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .unwrap() + .unwrap(); + assert!(tokio::time::timeout(Duration::from_millis(100), &mut recovery).await.is_err()); + assert!(store.ctx.namespace_commits_pending()); + assert_eq!(store.ctx.namespace_commit_generation(), before + 1); + recovery.abort(); + assert!(recovery.await.unwrap_err().is_cancelled()); + assert!(store.ctx.namespace_commits_pending(), "late restore must retain namespace accounting"); + let writing = store.clone(); + writer = Some(tokio::spawn(async move { + writing + .put_object(bucket, "object", &mut PutObjReader::from_vec(vec![62; 4096]), &ObjectOptions::default()) + .await + })); + assert!( + tokio::time::timeout(Duration::from_millis(100), writer.as_mut().unwrap()) + .await + .is_err(), + "same-key PUT must wait for the late restore" + ); + let removing = store.clone(); + bucket_writer = Some(tokio::spawn(async move { removing.acquire_bucket_lifecycle_write_lock(bucket).await })); + assert!( + tokio::time::timeout(Duration::from_millis(100), bucket_writer.as_mut().unwrap()) + .await + .is_err(), + "bucket incarnation mutation must wait for the late restore" + ); + }) + .catch_unwind() + .await; + let _ = release_tx.send(()); + drop(hook); + if let Some(writer) = writer { + tokio::time::timeout(Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap() + .unwrap(); + } + if let Some(writer) = bucket_writer { + drop( + tokio::time::timeout(Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap() + .unwrap(), + ); + } + if let Err(panic) = observations { + std::panic::resume_unwind(panic); + } + } + + #[tokio::test] + async fn delete_recovery_explicit_legacy_discard_preserves_versions_locks_and_live_children() { + for protected in ["version", "lock", "internal-lock", "child", "unknown"] { + let (_temp, disk, mut info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let root = legacy_fixture(&disk, &original, transaction).await; + let rollback = root.join(transaction.to_string()); + match protected { + "version" => { + info.version_id = Some(Uuid::new_v4()); + } + "lock" => { + info.metadata.insert("X-Amz-Object-Lock-Legal-Hold".into(), "ON".into()); + } + "internal-lock" => { + rustfs_utils::http::metadata_compat::insert_str( + &mut info.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, + "old-lock-state".to_owned(), + ); + } + "child" => { + fs::create_dir(rollback.join("live-child")).await.unwrap(); + } + _ => { + fs::write(rollback.join("unknown"), b"preserve").await.unwrap(); + } + } + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), test_meta(info)) + .await + .unwrap(); + match disk.inspect_delete_transaction("recovery", "object", transaction).await { + Ok(state) => { + assert!(!state.explicit_delete_safe, "protected shape {protected}"); + assert!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + state.fingerprint, + DeleteRecoveryAction::DiscardLegacy + ) + .await + .is_err() + ); + } + Err(error) => assert!(matches!(error, DiskError::FileCorrupt | DiskError::InvalidPath)), + } + assert!(rollback.join(STORAGE_FORMAT_FILE_BACKUP).exists()); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_explicit_legacy_discard_rejects_selected_symlink() { + let (temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let root = legacy_fixture(&disk, &original, transaction).await; + std::os::unix::fs::symlink(temp.path(), root.join(transaction.to_string()).join("linked")).unwrap(); + assert_eq!( + disk.inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap_err(), + DiskError::InvalidPath + ); + assert!(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP).exists()); + } + + #[tokio::test] + async fn delete_recovery_commit_retains_proof_until_explicit_finalization() { + let (_temp, disk, _info, _original) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + let committed = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .expect("durable set decision"); + let finished = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + committed.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("physical cleanup"); + assert_eq!(finished.state, DeleteRecoveryState::CommitFinished); + assert_eq!( + disk.finish_internal_delete_residue("recovery", "object", None).await, + 0, + "legacy retry must not GC new decision proof" + ); + let receipt = disk + .io_get_object_path("recovery", &format!("object/{transaction}/{DELETE_QUORUM_COMMITTED_FILE}")) + .expect("receipt path"); + assert!(receipt.is_file()); + let settled = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + finished.fingerprint, + DeleteRecoveryAction::FinalizeCommit, + ) + .await + .expect("finalize after coordinator convergence proof"); + assert_eq!(settled.state, DeleteRecoveryState::Absent); + assert!(!disk.io_get_object_path("recovery", "object").expect("object path").exists()); + } + + #[tokio::test] + async fn delete_recovery_fingerprint_rejects_new_generation() { + let (_temp, disk, mut info, _original) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + info.mod_time = Some(OffsetDateTime::now_utc()); + info.data = Some(Bytes::from_static(b"new generation")); + let newer = test_meta(info); + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").expect("metadata path"), &newer) + .await + .expect("concurrent new generation"); + assert!(matches!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("new generation remains") + .as_ref(), + newer + ); + } + + #[tokio::test] + async fn delete_recovery_untouched_peer_adopts_only_matching_generation() { + let (_temp, source, info, original) = fixture().await; + let (_peer_temp, peer, _peer_info, _) = fixture().await; + fs::write(peer.io_get_object_path("recovery", "object/xl.meta").expect("peer metadata"), &original) + .await + .expect("same original generation"); + let transaction = Uuid::new_v4(); + let applied = apply(&source, transaction).await; + let committed = source + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .expect("source durable decision"); + let authority = committed.journal.expect("source intent"); + let untouched = peer + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("untouched peer"); + let adopted = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + untouched.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority.clone()), + }, + ) + .await + .expect("generation-bound adoption"); + assert_eq!(adopted.state, DeleteRecoveryState::QuorumCommitted); + let finished = peer + .settle_delete_transaction("recovery", "object", transaction, adopted.fingerprint, DeleteRecoveryAction::FinishCommit) + .await + .expect("converge untouched peer"); + assert_eq!(finished.state, DeleteRecoveryState::CommitFinished); + assert!(finished.current_digest.is_none()); + let (_foreign_temp, foreign, _, _) = fixture().await; + let mut replacement = info; + replacement.mod_time = Some(OffsetDateTime::now_utc()); + let newer = test_meta(replacement); + fs::write( + foreign + .io_get_object_path("recovery", "object/xl.meta") + .expect("foreign metadata"), + &newer, + ) + .await + .expect("foreign generation"); + let observed = foreign + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("inspect foreign generation"); + assert!(matches!( + foreign + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority) + } + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert_eq!( + foreign + .read_all("recovery", "object/xl.meta") + .await + .expect("protected foreign generation") + .as_ref(), + newer + ); + } + + #[tokio::test] + async fn delete_recovery_adoption_preserves_unrelated_new_version() { + let (_source_temp, source, original_info, original) = fixture().await; + let (_peer_temp, peer, _, _) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&source, transaction).await; + let committed = source + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + let mut unrelated = original_info; + unrelated.version_id = Some(Uuid::new_v4()); + unrelated.mod_time = unrelated.mod_time.map(|time| time + time::Duration::seconds(1)); + let mut peer_meta = FileMeta::load(&original).unwrap(); + peer_meta.add_version(unrelated.clone()).unwrap(); + fs::write( + peer.io_get_object_path("recovery", "object/xl.meta").unwrap(), + peer_meta.marshal_msg().unwrap(), + ) + .await + .unwrap(); + let observed = peer + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + let adopted = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(committed.journal.unwrap()), + }, + ) + .await + .unwrap(); + peer.settle_delete_transaction( + "recovery", + "object", + transaction, + adopted.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .unwrap(); + let remaining = FileMeta::load(&peer.read_all("recovery", "object/xl.meta").await.unwrap()).unwrap(); + assert_eq!(remaining.versions.len(), 1); + assert_eq!(remaining.versions[0].header.version_id, unrelated.version_id); + } + + #[tokio::test] + async fn delete_recovery_absent_replay_witness_cannot_acquire_future_predecessors() { + let (_source_temp, source, _, _) = fixture().await; + let (_peer_temp, peer, _, _) = fixture().await; + let transaction = Uuid::new_v4(); + let source_state = apply(&source, transaction).await; + let committed = source + .settle_delete_transaction( + "recovery", + "object", + transaction, + source_state.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + let authority = committed.journal.unwrap(); + assert!(authority.predecessors.is_empty()); + let future = Uuid::new_v4(); + apply(&peer, future).await; + let absent = peer + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(absent.state, DeleteRecoveryState::Absent); + assert!(absent.current_digest.is_none()); + let adopted = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + absent.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority.clone()), + }, + ) + .await + .unwrap(); + let witness = adopted.journal.unwrap(); + assert_eq!(witness.predecessors, authority.predecessors); + assert!(!witness.predecessors.contains(&future)); + assert_eq!(witness.original_versions, authority.original_versions); + assert_eq!(witness.deleted_versions, authority.deleted_versions); + assert_eq!(witness.bucket_incarnation, authority.bucket_incarnation); + assert!(witness.original_digest.is_none() && witness.target_digest.is_none() && witness.data_dirs.is_empty()); + } + + #[tokio::test] + async fn delete_recovery_raw_discovery_paginates_and_binds_exact_key_scope() { + let (temp, disk, _info, _original) = fixture().await; + for index in 0..512 { + let path = temp + .path() + .join("recovery") + .join(format!("key-{index:04}")) + .join(Uuid::new_v4().to_string()); + fs::create_dir_all(&path).await.expect("hidden transaction directory"); + fs::write(path.join(STORAGE_FORMAT_FILE_BACKUP), b"legacy backup") + .await + .expect("hidden backup"); + } + let mut cursor = None; + let mut found = BTreeSet::new(); + loop { + let page = disk + .discover_delete_transactions("recovery", None, cursor, 13) + .await + .expect("bounded raw discovery"); + found.extend(page.transactions.into_iter().map(|id| id.object)); + cursor = page.next; + if cursor.is_none() { + break; + } + } + assert_eq!(found.len(), 512, "every hidden legacy transaction must be discovered across pages"); + let scoped = disk + .discover_delete_transactions("recovery", Some("key-0000"), None, 1) + .await + .expect("shallow exact-key discovery"); + assert_eq!(scoped.transactions.len(), 1); + assert_eq!(scoped.transactions[0].object, "key-0000"); + if let Some(cursor) = scoped.next { + assert!( + disk.discover_delete_transactions("recovery", Some("key-0001"), Some(cursor), 1) + .await + .is_err(), + "a cursor cannot cross object scopes" + ); + } + } + + #[tokio::test] + async fn delete_recovery_gc_interruption_replays_finished_receipt_without_journal() { + let (_temp, disk, _info, _) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + let committed = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .expect("record decision"); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + committed.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("finish physical cleanup"); + let root = disk + .io_get_object_path("recovery", &format!("object/{transaction}")) + .expect("transaction root"); + fs::remove_file(root.join(PREPARED)) + .await + .expect("simulate partially completed proof GC"); + fs::remove_file(root.join(FINISHED)) + .await + .expect("simulate removed completion marker"); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("receipt preserves completed intent"); + assert_eq!(observed.state, DeleteRecoveryState::CommitFinished); + let repeated = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("finish is idempotent after interrupted GC"); + let finalized = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + repeated.fingerprint, + DeleteRecoveryAction::FinalizeCommit, + ) + .await + .expect("finish proof GC"); + assert_eq!(finalized.state, DeleteRecoveryState::Absent); + } + + #[tokio::test] + async fn delete_recovery_completed_restore_without_backup_can_finish_rollback() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + apply(&disk, transaction).await; + let root = disk.io_get_object_path("recovery", "object").expect("object root"); + fs::rename( + root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP), + root.join(STORAGE_FORMAT_FILE), + ) + .await + .expect("simulate crash after rollback metadata rename"); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("already restored generation"); + assert_eq!(observed.state, DeleteRecoveryState::NewPrepared); + assert!( + observed.rollback_data_complete, + "restored original replaces consumed backup as rollback evidence" + ); + disk.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("finish interrupted rollback"); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("original remains readable") + .as_ref(), + original + ); + } + + #[tokio::test] + async fn delete_recovery_completed_restore_without_target_can_finish_versioned_rollback() { + let (_temp, disk, info, original) = fixture().await; + let mut remaining = info; + remaining.version_id = Some(Uuid::new_v4()); + remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1)); + let mut metadata = FileMeta::load(&original).unwrap(); + metadata.add_version(remaining).unwrap(); + let original = metadata.marshal_msg().unwrap(); + let root = disk.io_get_object_path("recovery", "object").unwrap(); + fs::write(root.join(STORAGE_FORMAT_FILE), &original).await.unwrap(); + let transaction = Uuid::new_v4(); + let observed = apply(&disk, transaction).await; + assert!(observed.journal.as_ref().unwrap().target_digest.is_some()); + let rollback = root.join(transaction.to_string()); + let target = fs::read(rollback.join(TARGET)).await.unwrap(); + fs::remove_file(rollback.join(TARGET)).await.unwrap(); + assert!(matches!( + disk.inspect_delete_transaction("recovery", "object", transaction).await, + Err(DiskError::FileCorrupt) + )); + fs::write(rollback.join(TARGET), target).await.unwrap(); + fs::rename(rollback.join(STORAGE_FORMAT_FILE_BACKUP), root.join(STORAGE_FORMAT_FILE)) + .await + .unwrap(); + fs::remove_file(rollback.join(TARGET)).await.unwrap(); + let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE); + fs::write( + &receipt_path, + serde_json::to_vec(&CommitReceipt { + journal: observed.journal.unwrap(), + finished: false, + }) + .unwrap(), + ) + .await + .unwrap(); + assert!(matches!( + disk.inspect_delete_transaction("recovery", "object", transaction).await, + Err(DiskError::FileCorrupt) + )); + fs::remove_file(receipt_path).await.unwrap(); + let retry = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(retry.state, DeleteRecoveryState::NewPrepared); + assert!(retry.rollback_data_complete); + disk.settle_delete_transaction("recovery", "object", transaction, retry.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("finish cleanup of an exactly restored versioned generation"); + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), original); + assert!(!rollback.exists()); + } + + #[tokio::test] + async fn delete_recovery_absent_remaining_metadata_requires_existing_commit_receipt() { + let (_temp, disk, info, original) = fixture().await; + let mut remaining = info; + remaining.version_id = Some(Uuid::new_v4()); + remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1)); + let mut metadata = FileMeta::load(&original).unwrap(); + metadata.add_version(remaining).unwrap(); + let root = disk.io_get_object_path("recovery", "object").unwrap(); + fs::write(root.join(STORAGE_FORMAT_FILE), metadata.marshal_msg().unwrap()) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + assert!(applied.journal.as_ref().unwrap().target_digest.is_some()); + let rollback = root.join(transaction.to_string()); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let undecided = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(undecided.state, DeleteRecoveryState::NewPrepared); + assert!( + !undecided.commit_data_safe, + "absence cannot authorize a target-Some undecided transaction" + ); + assert!(matches!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + undecided.fingerprint, + DeleteRecoveryAction::PublishCommit + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert!(rollback.join(PREPARED).is_file()); + assert!(rollback.join(STORAGE_FORMAT_FILE_BACKUP).is_file()); + assert!(!rollback.join(DELETE_QUORUM_COMMITTED_FILE).exists()); + fs::copy(rollback.join(TARGET), root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let applied = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let certified = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(certified.state, DeleteRecoveryState::QuorumCommitted); + assert!(certified.commit_data_safe); + let finished = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + certified.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("certified obsolete version cleanup has no live metadata references"); + assert_eq!(finished.state, DeleteRecoveryState::CommitFinished); + assert!( + !root.join(STORAGE_FORMAT_FILE).exists(), + "cleanup must never republish the old remaining version" + ); + assert!( + rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file(), + "set-wide GC still owns the final witness" + ); + } + + #[tokio::test] + async fn delete_recovery_partial_publication_adopts_same_intent_without_replacing_local_journal() { + let (_temp, disk, mut original, _) = fixture().await; + let (_peer_temp, peer, _, _) = fixture().await; + let mut remaining = original.clone(); + remaining.version_id = Some(Uuid::new_v4()); + remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1)); + for (index, target) in [(1, &disk), (2, &peer)] { + original.erasure.index = index; + remaining.erasure.index = index; + let mut metadata = FileMeta::load(&test_meta(original.clone())).unwrap(); + metadata.add_version(remaining.clone()).unwrap(); + fs::write( + target.io_get_object_path("recovery", "object/xl.meta").unwrap(), + metadata.marshal_msg().unwrap(), + ) + .await + .unwrap(); + } + let transaction = Uuid::new_v4(); + let local = apply(&disk, transaction).await; + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let rollback = root.join(transaction.to_string()); + let prepared = fs::read(rollback.join(PREPARED)).await.unwrap(); + let target = fs::read(rollback.join(TARGET)).await.unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let applied = apply(&peer, transaction).await; + let committed = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + let authority = committed.journal.unwrap(); + assert_ne!( + local.journal.as_ref().unwrap().original_digest, + authority.original_digest, + "each drive owns distinct raw metadata" + ); + let mut invalid = Vec::new(); + let mut changed = authority.clone(); + changed.bucket_incarnation = Some(Uuid::new_v4()); + invalid.push(changed); + let mut changed = authority.clone(); + changed.deleted_versions.push(Uuid::new_v4()); + invalid.push(changed); + let mut changed = authority.clone(); + changed.requests[0].version_id = Some(Uuid::new_v4()); + invalid.push(changed); + let mut changed = authority.clone(); + changed + .original_versions + .iter_mut() + .find(|version| version.version_id.unwrap_or_default().is_nil()) + .unwrap() + .metadata_digest[0] ^= 1; + invalid.push(changed); + for authority in invalid { + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert!(matches!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority) + } + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert!(!rollback.join(DELETE_QUORUM_COMMITTED_FILE).exists()); + } + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert!(!observed.commit_data_safe); + let adopted = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority), + }, + ) + .await + .expect("same-T certified intent completes partial publication"); + assert_eq!( + adopted.journal, local.journal, + "receipt preserves local hashes and data-directory ownership" + ); + assert_eq!(fs::read(rollback.join(PREPARED)).await.unwrap(), prepared); + assert_eq!(fs::read(rollback.join(TARGET)).await.unwrap(), target); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + adopted.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .unwrap(); + assert!(!root.join(STORAGE_FORMAT_FILE).exists()); + assert!(rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file()); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_rollback_marker_sync_failure_retains_journal() { + for mode in [DurabilityMode::Strict, DurabilityMode::None] { + let _mode = durability_mode_override::set(mode); + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + apply(&disk, transaction).await; + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let data = root.join(Uuid::new_v4().to_string()); + fs::create_dir(&data).await.unwrap(); + let marker = data.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + fs::write(&marker, []).await.unwrap(); + let prepared = root.join(transaction.to_string()).join(PREPARED); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + os::fsync_dir_recorder::set_failure(&data, ErrorKind::Other); + let result = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::Rollback, + ) + .await; + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), original); + assert!(!marker.exists(), "the unlink precedes the durability barrier"); + if mode == DurabilityMode::Strict { + assert!(result.is_err(), "a failed marker-directory sync must stop proof removal"); + assert!(prepared.exists(), "rollback evidence survives the sync failure"); + let retry = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + retry.fingerprint, + DeleteRecoveryAction::Rollback, + ) + .await + .expect("retry syncs the directory even though its marker is already absent"); + assert!(os::fsync_dir_recorder::was_fsynced(&data)); + } else { + result.expect("none mode does not add a marker durability barrier"); + assert!(!os::fsync_dir_recorder::was_fsynced(&data)); + assert!(os::fsync_dir_recorder::take_failure(&data).is_some()); + } + assert!(!prepared.exists()); + } + } + + #[tokio::test] + async fn delete_recovery_validates_staged_parts_before_restoring_metadata() { + use crate::erasure::coding::BitrotWriter; + let (_temp, disk, _info, _) = fixture().await; + let data_dir = Uuid::new_v4(); + let payload = Bytes::from_static(b"complete staged external payload"); + let mut info = test_file_info("object", Uuid::nil(), Some(data_dir), None); + info.size = i64::try_from(payload.len()).expect("payload length"); + info.erasure.block_size = payload.len(); + info.parts[0].size = payload.len(); + info.parts[0].actual_size = info.size; + let mut second = info.parts[0].clone(); + second.number = 2; + info.parts.push(second); + info.size *= 2; + let original = test_meta(info.clone()); + let root = disk.io_get_object_path("recovery", "object").expect("object root"); + fs::write(root.join(STORAGE_FORMAT_FILE), &original) + .await + .expect("external original metadata"); + let mut writer = BitrotWriter::new( + std::io::Cursor::new(Vec::new()), + info.erasure.shard_size(), + HashAlgorithm::HighwayHash256S, + ); + assert_eq!(writer.write(&payload).await.expect("encode intact external shard"), payload.len()); + writer.shutdown().await.expect("flush shard encoding"); + let encoded = writer.into_inner().into_inner(); + fs::create_dir_all(root.join(data_dir.to_string())) + .await + .expect("external data directory"); + fs::write(root.join(data_dir.to_string()).join("part.1"), &encoded) + .await + .expect("intact external part"); + fs::write(root.join(data_dir.to_string()).join("part.2"), &encoded) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + let _ = disk + .prepare_delete_recovery( + "recovery", + "object", + Some(&original), + &[request()], + None, + DeletePreparationContext { + transaction, + bucket_incarnation: None, + namespace_owner: None, + }, + ) + .await + .expect("prepared intent"); + fs::rename( + root.join(data_dir.to_string()), + root.join(transaction.to_string()).join(data_dir.to_string()), + ) + .await + .expect("simulate staged legacy-compatible data"); + fs::create_dir(root.join(data_dir.to_string())).await.unwrap(); + fs::rename( + root.join(transaction.to_string()).join(data_dir.to_string()).join("part.1"), + root.join(data_dir.to_string()).join("part.1"), + ) + .await + .unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)) + .await + .expect("simulate applied metadata removal"); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("inspect complete staged data"); + assert!(observed.rollback_data_complete); + disk.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("verified staged rollback"); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("restored metadata") + .as_ref(), + original + ); + assert_eq!( + disk.read_all("recovery", &format!("object/{data_dir}/part.1")) + .await + .expect("restored shard") + .as_ref(), + encoded + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_raw_discovery_rejects_symlink_and_forged_cursor() { + use std::os::unix::fs::symlink; + let (temp, disk, _info, _) = fixture().await; + let outside = tempfile::tempdir().expect("outside sentinel"); + fs::write(outside.path().join("sentinel"), b"outside bytes") + .await + .expect("outside sentinel"); + symlink(outside.path(), temp.path().join("recovery").join("untrusted-prefix")).expect("untrusted symlink"); + assert!(disk.discover_delete_transactions("recovery", None, None, 16).await.is_err()); + assert!( + disk.discover_delete_transactions("recovery", None, Some(DeleteRecoveryCursor { token: Uuid::new_v4() }), 16) + .await + .is_err() + ); + assert_eq!( + fs::read(outside.path().join("sentinel")) + .await + .expect("outside remains intact"), + b"outside bytes" + ); + } +} diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 89fa4c76e..c1166f405 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -230,6 +230,105 @@ pub enum Disk { } impl Disk { + pub(crate) async fn delete_versions_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + match self { + Self::Local(disk) => { + disk.delete_versions_with_namespace_owner(volume, versions, opts, namespace_owner) + .await + } + Self::Remote(disk) => { + let _owner = namespace_owner; + disk.delete_versions(volume, versions, opts).await + } + } + } + + pub(crate) async fn delete_versions_recoverable_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + mut opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + if let Err(error) = validate_delete_recovery_options(&opts) { + return vec![Some(error); versions.len()]; + } + if versions.iter().any(|version| version.name.starts_with('/')) { + return vec![Some(DiskError::InvalidPath); versions.len()]; + } + opts.recoverable_delete = true; + match self { + Self::Local(disk) => { + disk.delete_versions_with_namespace_owner(volume, versions, opts, namespace_owner) + .await + } + Self::Remote(disk) => { + let _owner = namespace_owner; + disk.delete_versions_recoverable(volume, versions, opts).await + } + } + } + + pub(crate) async fn settle_delete_transaction_with_namespace_owner( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + namespace_owner: Option>, + ) -> Result { + match self { + Self::Local(disk) => { + disk.settle_delete_transaction_with_namespace_owner( + volume, + object, + transaction, + expected_fingerprint, + action, + namespace_owner, + ) + .await + } + Self::Remote(disk) => { + let _owner = namespace_owner; + disk.settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + } + } + } + + pub(crate) async fn delete_version_recoverable_with_namespace_owner( + &self, + volume: &str, + path: &str, + fi: FileInfo, + force_del_marker: bool, + mut opts: DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { + validate_delete_recovery_options(&opts)?; + if path.starts_with('/') { + return Err(DiskError::InvalidPath); + } + opts.recoverable_delete = true; + match self { + Self::Local(disk) => { + disk.delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner) + .await + } + Self::Remote(disk) => { + disk.delete_version_recoverable(volume, path, fi, force_del_marker, opts) + .await + } + } + } pub(crate) async fn set_disk_id_state(&self, id: Option) -> Result<()> { match self { Disk::Local(local_disk) => { @@ -257,6 +356,70 @@ impl Disk { #[async_trait::async_trait] impl DiskAPI for Disk { + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> Result<()> { + self.delete_version_recoverable_with_namespace_owner(volume, object, file_info, force_del_marker, opts, None) + .await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + match self { + Self::Local(disk) => disk.delete_versions_recoverable(volume, versions, opts).await, + Self::Remote(disk) => disk.delete_versions_recoverable(volume, versions, opts).await, + } + } + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + match self { + Self::Local(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await, + Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await, + } + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + match self { + Self::Local(disk) => disk.inspect_delete_transaction(volume, object, transaction).await, + Self::Remote(disk) => disk.inspect_delete_transaction(volume, object, transaction).await, + } + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> Result { + match self { + Self::Local(disk) => { + disk.settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + } + Self::Remote(disk) => { + disk.settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + } + } + } fn to_string(&self) -> String { match self { Disk::Local(local_disk) => local_disk.to_string(), @@ -1246,6 +1409,51 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { Err(DiskError::MethodNotAllowed) } async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; + async fn discover_delete_transactions( + &self, + _volume: &str, + _object: Option<&str>, + _cursor: Option, + _limit: u32, + ) -> Result { + Err(DiskError::MethodNotAllowed) + } + async fn delete_version_recoverable( + &self, + _volume: &str, + _object: &str, + _file_info: FileInfo, + _force_del_marker: bool, + _opts: DeleteOptions, + ) -> Result<()> { + Err(DiskError::MethodNotAllowed) + } + async fn delete_versions_recoverable( + &self, + _volume: &str, + versions: Vec, + _opts: DeleteOptions, + ) -> Vec> { + vec![Some(DiskError::MethodNotAllowed); versions.len()] + } + async fn inspect_delete_transaction( + &self, + _volume: &str, + _object: &str, + _transaction: Uuid, + ) -> Result { + Err(DiskError::MethodNotAllowed) + } + async fn settle_delete_transaction( + &self, + _volume: &str, + _object: &str, + _transaction: Uuid, + _expected_fingerprint: DeleteRecoveryDigest, + _action: DeleteRecoveryAction, + ) -> Result { + Err(DiskError::MethodNotAllowed) + } // VerifyFile async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result; // CheckParts @@ -1582,6 +1790,269 @@ pub struct DeleteOptions { /// Remote calls carrying it must use DeleteRetiredMarker, never DeleteVersion. #[serde(default, skip_serializing_if = "Option::is_none")] pub expected_delete_marker: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete_recovery_bucket_incarnation: Option, + #[serde(default)] + pub recoverable_delete: bool, +} + +pub const DELETE_RECOVERY_PROTOCOL_VERSION: u32 = 1; +pub(crate) fn get_delete_recovery_timeout() -> Result { + let value = std::env::var("RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS"); + match value { + Ok(value) => parse_delete_recovery_timeout(Some(&value)), + Err(std::env::VarError::NotPresent) => parse_delete_recovery_timeout(None), + Err(std::env::VarError::NotUnicode(_)) => Err(DiskError::other("invalid delete recovery timeout configuration")), + } +} + +fn parse_delete_recovery_timeout(value: Option<&str>) -> Result { + let seconds = match value { + None => 900, + Some(value) => value + .parse::() + .ok() + .filter(|seconds| (30..=86400).contains(seconds)) + .ok_or_else(|| DiskError::other("delete recovery timeout must be an integer between 30 and 86400 seconds"))?, + }; + Ok(std::time::Duration::from_secs(seconds)) +} + +pub(crate) fn validate_delete_recovery_options(opts: &DeleteOptions) -> Result<()> { + if opts.old_data_dir.is_none_or(|id| id.is_nil()) + || opts.undo_write + || opts.undo_delete + || opts.recursive + || opts.immediate + || opts.expected_delete_marker.is_some() + || opts.delete_recovery_bucket_incarnation.is_some_and(|id| id.is_nil()) + { + return Err(DiskError::FileCorrupt); + } + Ok(()) +} +pub type DeleteRecoveryDigest = [u8; 32]; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteTransactionId { + pub object: String, + pub transaction: Uuid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryCursor { + pub token: Uuid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryPage { + pub transactions: Vec, + pub next: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryJournal { + pub protocol_version: u32, + pub volume: String, + pub object: String, + pub transaction: Uuid, + pub bucket_incarnation: Option, + pub original_digest: Option, + pub target_digest: Option, + pub data_dirs: Vec, + pub deleted_versions: Vec, + pub requests: Vec, + pub original_versions: Vec, + #[serde(default)] + pub predecessors: Vec, +} + +impl DeleteRecoveryJournal { + /// Checks source generations only; the coordinator must supply a certified + /// decision and the receiver must retain its namespace/incarnation fences. + pub(crate) fn matches_adoption_source(&self, current: &[DeleteRecoveryVersionIdentity]) -> bool { + if self.requests.is_empty() { + return false; + } + let mut originals = std::collections::HashMap::with_capacity(self.original_versions.len()); + for version in &self.original_versions { + originals.entry(version.version_id.unwrap_or_default()).or_insert(version); + } + let mut locals = std::collections::HashMap::with_capacity(current.len()); + for version in current { + locals.entry(version.version_id.unwrap_or_default()).or_insert(version); + } + self.requests.iter().all(|request| { + if request.deleted && request.mark_deleted { + return current.first() == self.original_versions.first(); + } + let id = request.version_id.unwrap_or_default(); + let Some(original) = originals.get(&id) else { + return false; + }; + match locals.get(&id) { + Some(local) => local == original, + None => !id.is_nil(), + } + }) + } + + /// Matches immutable local preparation intent against an independently + /// inspected same-transaction commit receipt. This match is not authority + /// by itself; the coordinator must supply the certified receipt. + pub(crate) fn matches_prepared_authority(&self, authority: &Self) -> bool { + if self.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || authority.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || self.transaction.is_nil() + || self.transaction != authority.transaction + || self.volume != authority.volume + || self.object != authority.object + || self.bucket_incarnation != authority.bucket_incarnation + || self.bucket_incarnation.is_some_and(|id| id.is_nil()) + || self.requests.is_empty() + || self.requests != authority.requests + || self.deleted_versions.iter().collect::>() + != authority.deleted_versions.iter().collect::>() + { + return false; + } + let sources: std::collections::HashMap<_, _> = self + .original_versions + .iter() + .map(|version| (version.version_id.unwrap_or_default(), version)) + .collect(); + let certified: std::collections::HashMap<_, _> = authority + .original_versions + .iter() + .map(|version| (version.version_id.unwrap_or_default(), version)) + .collect(); + if sources.len() != self.original_versions.len() || certified.len() != authority.original_versions.len() { + return false; + } + self.requests.iter().all(|request| { + let (local, remote) = if request.deleted && request.mark_deleted { + (self.original_versions.first(), authority.original_versions.first()) + } else { + let id = request.version_id.unwrap_or_default(); + (sources.get(&id).copied(), certified.get(&id).copied()) + }; + match (local, remote) { + (Some(local), Some(remote)) => local == remote, + (None, None) => self.original_digest.is_none() && authority.original_digest.is_none(), + _ => false, + } + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryVersionIdentity { + pub version_id: Option, + pub mod_time: Option, + pub data_dir: Option, + pub deleted: bool, + pub metadata_digest: DeleteRecoveryDigest, +} + +impl DeleteRecoveryVersionIdentity { + pub(crate) fn from_file_info(info: FileInfo) -> error::Result { + local::delete_recovery::generation(info) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum DeleteRecoveryState { + Absent, + NewPrepared, + QuorumCommitted, + CommitFinished, + LegacyAmbiguous, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryInspection { + pub protocol_version: u32, + pub id: DeleteTransactionId, + pub state: DeleteRecoveryState, + pub journal: Option, + pub current_digest: Option, + pub current_versions: Vec, + pub backup_digest: Option, + pub rollback_data_complete: bool, + pub commit_data_safe: bool, + pub explicit_delete_safe: bool, + pub fingerprint: DeleteRecoveryDigest, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum DeleteRecoveryAction { + PublishCommit, + FinishCommit, + FinalizeCommit, + Rollback, + DiscardLegacy, + AdoptCommit { authority: Box }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum DeleteRecoveryRpcOperation { + Discover { + object: Option, + cursor: Option, + limit: u32, + }, + Inspect { + object: String, + transaction: Uuid, + }, + Settle { + object: String, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + }, + ApplyVersion { + object: String, + file_info: Box, + force_del_marker: bool, + opts: DeleteOptions, + }, + ApplyVersions { + versions: Vec, + opts: DeleteOptions, + }, +} + +impl DeleteRecoveryRpcOperation { + /// Full metadata application and adoption share the existing mutation envelope. + pub fn max_wire_payload_bytes(&self) -> usize { + match self { + Self::ApplyVersion { .. } + | Self::ApplyVersions { .. } + | Self::Settle { + action: DeleteRecoveryAction::AdoptCommit { .. }, + .. + } => rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES, + _ => rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum DeleteRecoveryRpcReply { + Discovered(DeleteRecoveryPage), + Inspected(DeleteRecoveryInspection), + Settled(DeleteRecoveryInspection), + AppliedVersion, + AppliedVersions(Vec>), } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -1872,6 +2343,8 @@ mod tests { undo_delete: false, old_data_dir: Some(Uuid::new_v4()), expected_delete_marker: None, + delete_recovery_bucket_incarnation: None, + recoverable_delete: false, }; assert!(opts.recursive); @@ -2296,3 +2769,183 @@ mod tests { } } } + +#[cfg(test)] +mod delete_recovery_timeout_tests { + use super::*; + #[test] + fn recovery_timeout_defaults_and_validates_range() { + assert_eq!( + parse_delete_recovery_timeout(None).expect("default timeout"), + std::time::Duration::from_secs(900) + ); + for seconds in ["30", "900", "86400"] { + assert!(parse_delete_recovery_timeout(Some(seconds)).is_ok()); + } + for value in ["", "invalid", "0", "2", "29", "86401", "-1", "18446744073709551616"] { + assert!(parse_delete_recovery_timeout(Some(value)).is_err()); + } + } +} + +#[cfg(test)] +mod delete_recovery_wire_limit_tests { + use super::*; + + #[test] + fn boxed_apply_version_preserves_existing_wire_payload() { + #[derive(Serialize)] + enum PriorOperation<'a> { + ApplyVersion { + object: &'a str, + file_info: &'a FileInfo, + force_del_marker: bool, + opts: &'a DeleteOptions, + }, + } + let file_info = FileInfo { + name: "object".into(), + version_id: Some(Uuid::new_v4()), + metadata: std::collections::HashMap::from([("metadata".into(), "value".into())]), + ..Default::default() + }; + let opts = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + recoverable_delete: true, + delete_recovery_bucket_incarnation: Some(Uuid::new_v4()), + ..Default::default() + }; + let prior = PriorOperation::ApplyVersion { + object: "object", + file_info: &file_info, + force_del_marker: true, + opts: &opts, + }; + let encoded = rmp_serde::to_vec_named(&prior).expect("encode the previous unboxed request shape"); + let operation = DeleteRecoveryRpcOperation::ApplyVersion { + object: "object".into(), + file_info: Box::new(file_info.clone()), + force_del_marker: true, + opts: opts.clone(), + }; + assert_eq!(rmp_serde::to_vec_named(&operation).expect("encode boxed request"), encoded); + assert_eq!( + serde_json::to_value(&operation).expect("boxed request JSON"), + serde_json::to_value(prior).expect("previous request JSON") + ); + let decoded: DeleteRecoveryRpcOperation = + rustfs_protos::decode_delete_recovery_payload(&encoded).expect("decode the previous payload"); + assert_eq!(rmp_serde::to_vec_named(&decoded).expect("re-encode decoded prior request"), encoded); + } + #[test] + fn adoption_and_inspection_reply_support_large_version_journals() { + let transaction = Uuid::new_v4(); + let request = FileInfo { + metadata: std::collections::HashMap::from([("metadata".into(), "x".repeat(2048))]), + ..Default::default() + }; + let journal = DeleteRecoveryJournal { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + volume: "bucket".into(), + object: "object".into(), + transaction, + bucket_incarnation: None, + original_digest: None, + target_digest: None, + data_dirs: Vec::new(), + deleted_versions: Vec::new(), + requests: vec![request; 1000], + original_versions: Vec::new(), + predecessors: Vec::new(), + }; + let operation = DeleteRecoveryRpcOperation::Settle { + object: "object".into(), + transaction, + expected_fingerprint: [0; 32], + action: DeleteRecoveryAction::AdoptCommit { + authority: Box::new(journal.clone()), + }, + }; + let encoded = rmp_serde::to_vec_named(&operation).expect("encode large adoption"); + assert!(encoded.len() > rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES); + assert!(encoded.len() < operation.max_wire_payload_bytes()); + let decoded: DeleteRecoveryRpcOperation = + rustfs_protos::decode_delete_recovery_payload(&encoded).expect("large adoption can decode"); + assert_eq!(decoded.max_wire_payload_bytes(), rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES); + let reply = DeleteRecoveryRpcReply::Inspected(DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: DeleteTransactionId { + object: "object".into(), + transaction, + }, + state: DeleteRecoveryState::QuorumCommitted, + journal: Some(journal), + current_digest: None, + current_versions: Vec::new(), + backup_digest: None, + rollback_data_complete: false, + commit_data_safe: true, + explicit_delete_safe: false, + fingerprint: [0; 32], + }); + let encoded = rmp_serde::to_vec_named(&reply).expect("encode large inspection"); + assert!(encoded.len() > rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES); + assert!(rustfs_protos::decode_delete_recovery_payload::(&encoded).is_ok()); + let inspect = DeleteRecoveryRpcOperation::Inspect { + object: "object".into(), + transaction, + }; + assert_eq!(inspect.max_wire_payload_bytes(), rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES); + } +} + +#[cfg(test)] +mod delete_recovery_adoption_tests { + use super::*; + + #[test] + fn source_matching_preserves_non_null_absence_and_rejects_foreign_or_missing_null() { + let id = Uuid::new_v4(); + let source = DeleteRecoveryVersionIdentity { + version_id: Some(id), + mod_time: None, + data_dir: None, + deleted: false, + metadata_digest: [1; 32], + }; + let mut journal = DeleteRecoveryJournal { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + volume: "bucket".into(), + object: "object".into(), + transaction: Uuid::new_v4(), + bucket_incarnation: None, + original_digest: None, + target_digest: None, + data_dirs: Vec::new(), + deleted_versions: vec![id], + requests: vec![FileInfo { + version_id: Some(id), + ..Default::default() + }], + original_versions: vec![source.clone()], + predecessors: Vec::new(), + }; + assert!(journal.matches_adoption_source(&[]), "certified non-null removal can already be absent"); + assert!(journal.matches_adoption_source(std::slice::from_ref(&source))); + let mut foreign = source.clone(); + foreign.metadata_digest = [2; 32]; + assert!(!journal.matches_adoption_source(std::slice::from_ref(&foreign))); + journal.original_versions.push(foreign.clone()); + assert!( + journal.matches_adoption_source(&[source.clone(), foreign.clone()]), + "retain original first-match semantics" + ); + assert!(!journal.matches_adoption_source(&[foreign, source])); + journal.requests[0].version_id = None; + journal.original_versions[0].version_id = None; + assert!(!journal.matches_adoption_source(&[]), "null absence does not establish its generation"); + assert!(journal.matches_adoption_source(std::slice::from_ref(&journal.original_versions[0]))); + journal.requests.clear(); + assert!(!journal.matches_adoption_source(&journal.original_versions)); + } +} diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 78f8df7a7..1b9048bba 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -938,6 +938,10 @@ pub struct ObjectOptions { /// A genuine target write guard; loss fences alone do not authorize a borrowed write. #[doc(hidden)] pub write_lock_context: Option, + /// In-process delete ownership for late filesystem completion. Never + /// serialized or copied into object metadata. + #[doc(hidden)] + pub delete_mutation_owner: Option>, /// Internal read-only inspection must not enqueue metadata or payload repairs. pub suppress_read_repair: bool, /// Control-plane writers that immediately read or CAS the same namespace @@ -1158,6 +1162,7 @@ impl std::fmt::Debug for ObjectOptions { .field("http_preconditions", &self.http_preconditions.is_some()) .field("delete_replication", &self.delete_replication.is_some()) .field("delete_replication_config_snapshot", &self.delete_replication_config_snapshot) + .field("delete_mutation_owner", &self.delete_mutation_owner.is_some()) .field("namespace_lock_fence", &self.namespace_lock_fence.is_some()) .field("bucket_lifecycle_lock_fence", &self.bucket_lifecycle_lock_fence.is_some()) .field("replication_request", &self.replication_request) diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index bf4bf7a3f..c63244ea8 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -874,6 +874,7 @@ mod ctx; mod metadata; mod ops; pub(crate) use ops::bucket::BucketInfoQuorum; +pub(crate) use ops::delete_recovery::{DeleteRecoveryContext, DeleteRecoverySummary}; pub(crate) use ops::heal::AbsenceProofRequest; #[cfg(test)] pub(crate) use ops::heal::DanglingDeleteFailure; diff --git a/crates/ecstore/src/set_disk/ops/delete_recovery.rs b/crates/ecstore/src/set_disk/ops/delete_recovery.rs new file mode 100644 index 000000000..71d76f8b8 --- /dev/null +++ b/crates/ecstore/src/set_disk/ops/delete_recovery.rs @@ -0,0 +1,1992 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Set-wide delete decisions and replay. Decisions remain until every +//! participant and related transaction for the exact key converges. + +use crate::disk::error::{DiskError, Result, RpcStatusError}; +use crate::disk::{ + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteRecoveryAction, DeleteRecoveryInspection, DeleteRecoveryState, DeleteTransactionId, + DiskAPI, DiskStore, +}; +use crate::set_disk::SetDisks; +use futures::future::join_all; +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::sync::Arc; +use uuid::Uuid; + +const DISCOVERY_PAGE_SIZE: u32 = 128; +const MAX_DISCOVERED_TRANSACTIONS: usize = 1_000_000; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DeleteRecoverySummary { + pub discovered: usize, + pub committed: usize, + pub rolled_back: usize, + pub blocked: usize, + pub pending: usize, + pub reclaimed: usize, + /// Read-only evidence for ordinary missing-key S3 semantics. This never + /// authorizes settlement while a configured participant is unavailable. + pub no_transactions_at_quorum: bool, +} + +#[derive(Clone)] +pub(crate) struct DeleteRecoveryContext<'a> { + pub incarnation: Option, + pub dry_run: bool, + pub explicit_delete: bool, + /// Only the locked heal path may repair a live marker-creation source. + pub repair_marker_source: bool, + pub namespace_owner: Option>, + pub can_commit: &'a (dyn Fn() -> bool + Sync), +} + +impl<'a> DeleteRecoveryContext<'a> { + pub(crate) fn new(incarnation: Option, can_commit: &'a (dyn Fn() -> bool + Sync)) -> Self { + Self { + incarnation, + dry_run: false, + explicit_delete: false, + repair_marker_source: false, + namespace_owner: None, + can_commit, + } + } +} + +impl DeleteRecoverySummary { + pub(crate) fn add(&mut self, other: Self) { + self.discovered += other.discovered; + self.committed += other.committed; + self.rolled_back += other.rolled_back; + self.blocked += other.blocked; + self.pending += other.pending; + self.reclaimed += other.reclaimed; + } +} + +pub(in crate::set_disk) fn unsupported_recovery(error: &DiskError) -> bool { + matches!(error, DiskError::UnsupportedDisk) + || matches!(error, DiskError::Io(source) if source.get_ref() + .and_then(|source| source.downcast_ref::()) + .is_some_and(|status| status.status().code() == tonic::Code::Unimplemented)) +} + +/// Set authority needs generation identities, not the potentially large +/// per-version apply requests carried by returning-drive adoption. +struct DecisionEvidence { + transaction: Uuid, + deleted_versions: HashSet, + original_versions: HashSet, + predecessors: HashSet, + target_absent: bool, +} + +fn receipt(state: DeleteRecoveryState) -> bool { + matches!(state, DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished) +} + +fn validate(inspection: &DeleteRecoveryInspection, object: &str, transaction: Uuid) -> Result<()> { + if inspection.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || inspection.id.object != object + || inspection.id.transaction != transaction + { + return Err(DiskError::FileCorrupt); + } + Ok(()) +} + +async fn inspect( + disks: &[Option], + bucket: &str, + object: &str, + transaction: Uuid, +) -> Vec> { + join_all(disks.iter().map(|disk| async move { + let state = disk + .as_ref() + .ok_or(DiskError::DiskNotFound)? + .inspect_delete_transaction(bucket, object, transaction) + .await?; + validate(&state, object, transaction)?; + Ok(state) + })) + .await +} + +async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Result> { + let mut cursor = None; + let mut ids = BTreeSet::new(); + loop { + let page = disk + .discover_delete_transactions(bucket, object, cursor, DISCOVERY_PAGE_SIZE) + .await?; + if page.transactions.len() > usize::try_from(DISCOVERY_PAGE_SIZE).map_err(|_| DiskError::FileCorrupt)? { + return Err(DiskError::FileCorrupt); + } + for id in page.transactions { + if id.transaction.is_nil() || object.is_some_and(|object| object != id.object) { + return Err(DiskError::FileCorrupt); + } + ids.insert((id.object, id.transaction)); + if ids.len() > MAX_DISCOVERED_TRANSACTIONS { + return Err(DiskError::other("delete recovery transaction limit exceeded")); + } + } + cursor = page.next; + if cursor.is_none() { + break; + } + } + Ok(ids + .into_iter() + .map(|(object, transaction)| DeleteTransactionId { object, transaction }) + .collect()) +} + +async fn settle( + disk: &DiskStore, + bucket: &str, + state: &DeleteRecoveryInspection, + action: DeleteRecoveryAction, + can_commit: &(impl Fn() -> bool + Sync + ?Sized), + namespace_owner: Option<&Arc>, +) -> Result { + if !can_commit() { + return Err(DiskError::other("delete recovery namespace fence was lost")); + } + let result = disk + .settle_delete_transaction_with_namespace_owner( + bucket, + &state.id.object, + state.id.transaction, + state.fingerprint, + action, + namespace_owner.cloned(), + ) + .await; + let next = result?; + validate(&next, &state.id.object, state.id.transaction)?; + Ok(next) +} + +/// Each phase fans out across independent drives, and awaits all started work +/// before evaluating the next decision or cleanup barrier. +async fn settle_group( + disks: &[Option], + bucket: &str, + copies: &mut [DeleteRecoveryInspection], + action: impl Fn(&DeleteRecoveryInspection) -> Option + Sync, + can_commit: &(impl Fn() -> bool + Sync + ?Sized), + namespace_owner: Option<&Arc>, +) -> Result { + let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| { + let action = action(copy); + async move { + match action { + Some(action) => settle( + disk.as_ref().ok_or(DiskError::DiskNotFound)?, + bucket, + copy, + action, + can_commit, + namespace_owner, + ) + .await + .map(Some), + None => Ok(None), + } + } + })) + .await; + let mut pending = false; + for (copy, result) in copies.iter_mut().zip(results) { + match result { + Ok(Some(next)) => *copy = next, + Ok(None) => {} + Err(DiskError::DiskOngoingReq) => pending = true, + Err(error) => return Err(error), + } + } + Ok(pending) +} + +/// A later decision may require settling its prepared predecessor before an +/// untouched participant can adopt that decision. Ordering provides no commit +/// authority; reject contradictory causal records before any settlement. +fn causal_transaction_order(states: &BTreeMap>) -> Result> { + let mut indegrees: BTreeMap<_, usize> = states.keys().map(|id| (*id, 0)).collect(); + let mut successors: BTreeMap> = BTreeMap::new(); + for (transaction, copies) in states { + for predecessor in copies + .iter() + .filter_map(|copy| copy.journal.as_ref()) + .flat_map(|journal| &journal.predecessors) + { + if states.contains_key(predecessor) && successors.entry(*predecessor).or_default().insert(*transaction) { + let degree = indegrees.get_mut(transaction).ok_or(DiskError::FileCorrupt)?; + *degree = degree.checked_add(1).ok_or(DiskError::FileCorrupt)?; + } + } + } + let mut ready: BTreeSet<_> = indegrees + .iter() + .filter_map(|(id, degree)| (*degree == 0).then_some(*id)) + .collect(); + let mut order = Vec::with_capacity(states.len()); + while let Some(transaction) = ready.pop_first() { + order.push(transaction); + if let Some(children) = successors.get(&transaction) { + for child in children { + let degree = indegrees.get_mut(child).ok_or(DiskError::FileCorrupt)?; + *degree = degree.checked_sub(1).ok_or(DiskError::FileCorrupt)?; + if *degree == 0 { + ready.insert(*child); + } + } + } + } + if order.len() != states.len() { + return Err(DiskError::FileCorrupt); + } + Ok(order) +} + +impl SetDisks { + /// Publication follows the ordinary metadata quorum. Once a publication is + /// attempted, response loss must never trigger blind rollback. + pub(in crate::set_disk) async fn publish_delete_commit( + &self, + bucket: &str, + object: &str, + transaction: Uuid, + disks: &[Option], + write_quorum: usize, + mut context: DeleteRecoveryContext<'_>, + ) -> Result<()> { + if context.namespace_owner.is_none() { + context.namespace_owner = Some(self.ctx.begin_namespace_commit()); + } + let can_commit = context.can_commit; + let states = inspect(disks, bucket, object, transaction).await; + let untouched_absence = |state: &Result| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none()); + if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum + && states.iter().all(|state| { + untouched_absence(state) || matches!(state, Err(error) if unavailable(error) && !unsupported_recovery(error)) + }) + { + // An ordinary missing-key no-op requires no recovery decision. + // Do not manufacture witness transactions while a drive is offline. + return Ok(()); + } + let published = join_all(disks.iter().zip(&states).map(|(disk, state)| async { + let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?; + let state = state.as_ref().map_err(Clone::clone)?; + if state.state == DeleteRecoveryState::LegacyAmbiguous { + return Err(DiskError::FileCorrupt); + } + settle( + disk, + bucket, + state, + DeleteRecoveryAction::PublishCommit, + can_commit, + context.namespace_owner.as_ref(), + ) + .await + })) + .await; + if published + .iter() + .filter(|state| matches!(state, Ok(state) if receipt(state.state))) + .count() + < write_quorum + { + return Err(DiskError::ErasureWriteQuorum); + } + // Physical cleanup is retryable after the certified decision. An + // unavailable participant retains the decision for its eventual return. + let _ = self.recover_delete_key(bucket, object, context).await; + Ok(()) + } + + pub(crate) async fn recover_delete_bucket( + &self, + bucket: &str, + context: DeleteRecoveryContext<'_>, + ) -> Result { + self.recover_delete_prefix(bucket, "", context).await + } + + pub(crate) async fn recover_delete_prefix( + &self, + bucket: &str, + prefix: &str, + context: DeleteRecoveryContext<'_>, + ) -> Result { + let disks = self.disk_inventory().await; + if disks.len() != self.set_drive_count || disks.iter().any(Option::is_none) { + return Ok(DeleteRecoverySummary { + pending: 1, + ..Default::default() + }); + } + let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, None))).await; + let mut keys = BTreeSet::new(); + for page in pages { + match page { + Ok(ids) => { + keys.extend( + ids.into_iter() + .map(|id| id.object) + .filter(|object| object.starts_with(prefix)), + ); + } + Err(error) if unavailable(&error) => { + return Ok(DeleteRecoverySummary { + pending: 1, + ..Default::default() + }); + } + Err(error) => return Err(error), + } + } + let mut summary = DeleteRecoverySummary::default(); + // Lock order: caller's bucket lifecycle fence, then one exact key. + for object in keys { + summary.add(self.recover_delete_object(bucket, &object, context.clone()).await?); + } + Ok(summary) + } + + pub(crate) async fn recover_delete_object( + &self, + bucket: &str, + object: &str, + mut context: DeleteRecoveryContext<'_>, + ) -> Result { + // Lock order: caller's bucket lifecycle fence, then one exact key. + let guard = Arc::new( + self.acquire_write_lock_diag("recover_delete", bucket, object) + .await + .map_err(DiskError::other)?, + ); + if !context.dry_run { + let namespace_owner = context + .namespace_owner + .clone() + .unwrap_or_else(|| self.ctx.begin_namespace_commit()); + context.namespace_owner = Some(Arc::new((namespace_owner, guard.clone()))); + } + let can_commit = context.can_commit; + let fence = || can_commit() && !guard.is_lock_lost(); + context.can_commit = &fence; + self.recover_delete_key(bucket, object, context).await + } + + /// Caller retains the exact key's write lock and its lifecycle owner. + pub(in crate::set_disk) async fn recover_delete_key( + &self, + bucket: &str, + object: &str, + context: DeleteRecoveryContext<'_>, + ) -> Result { + let DeleteRecoveryContext { + incarnation, + dry_run, + explicit_delete, + repair_marker_source, + namespace_owner, + can_commit, + } = context; + let disks = self.disk_inventory().await; + let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some); + let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, Some(object)))).await; + let mut transactions = BTreeSet::new(); + let mut inspected = 0; + let mut offline = !complete_inventory; + let mut unsupported = false; + for page in pages { + match page { + Ok(ids) => { + inspected += 1; + transactions.extend(ids.into_iter().map(|id| id.transaction)); + } + Err(error) if unavailable(&error) => { + offline = true; + unsupported |= unsupported_recovery(&error); + } + Err(error) => return Err(error), + } + } + if offline { + return Ok(DeleteRecoverySummary { + discovered: transactions.len(), + pending: 1, + no_transactions_at_quorum: transactions.is_empty() && !unsupported && inspected > self.set_drive_count / 2, + ..Default::default() + }); + } + let mut summary = DeleteRecoverySummary { + discovered: transactions.len(), + ..Default::default() + }; + let mut all_states = BTreeMap::new(); + for transaction in transactions { + let mut complete = Vec::with_capacity(disks.len()); + for state in inspect(&disks, bucket, object, transaction).await { + match state { + Ok(state) => { + if let Some(journal) = &state.journal { + let witness = journal.original_digest.is_none() + && journal.target_digest.is_none() + && journal.requests.is_empty(); + if incarnation.is_some_and(|expected| journal.bucket_incarnation != Some(expected) && !witness) { + summary.blocked += 1; + return Ok(summary); + } + } + complete.push(state); + } + Err(error) if unavailable(&error) => { + summary.pending += 1; + return Ok(summary); + } + Err(error) => return Err(error), + } + } + all_states.insert(transaction, complete); + } + let prerequisite = if repair_marker_source && !dry_run { + let mut active = all_states + .iter() + .filter(|(_, copies)| copies.iter().any(|copy| copy.state != DeleteRecoveryState::Absent)); + let first = active.next(); + if active.next().is_none() { + first.and_then(|(&transaction, copies)| { + marker_source_to_repair(copies, incarnation).map(|source| (transaction, source)) + }) + } else { + None + } + } else { + None + }; + if let Some((transaction, source)) = prerequisite { + Box::pin(self.heal_delete_recovery_source(bucket, object, &source, can_commit)).await?; + let refreshed = inspect(&disks, bucket, object, transaction) + .await + .into_iter() + .collect::>>()?; + if !can_commit() { + return Err(DiskError::DiskOngoingReq); + } + if refreshed.iter().any(|copy| { + copy.journal + .as_ref() + .is_some_and(|journal| journal.bucket_incarnation != incarnation) + }) { + return Err(DiskError::OutdatedXLMeta); + } + all_states.insert(transaction, refreshed); + } + let transaction_order = causal_transaction_order(&all_states)?; + // Legacy snapshots contain no set decision. A fresh authorized removal + // applies only after the entire key is absent and every raw shape is safe. + let legacy_only = explicit_delete + && all_states.values().flatten().all(|copy| { + copy.current_digest.is_none() + && matches!(copy.state, DeleteRecoveryState::Absent | DeleteRecoveryState::LegacyAmbiguous) + && (copy.state != DeleteRecoveryState::LegacyAmbiguous || copy.explicit_delete_safe) + }); + // This is a fresh authorized delete of an absent unversioned key, + // rather than evidence about the outcome of a historical transaction. + let fresh_absence_decision = explicit_delete + && all_states.values().flatten().all(|copy| { + copy.current_digest.is_none() + && match copy.state { + DeleteRecoveryState::Absent => true, + DeleteRecoveryState::NewPrepared => copy.explicit_delete_safe, + DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished => { + copy.journal.as_ref().is_some_and(|journal| journal.target_digest.is_none()) + } + DeleteRecoveryState::LegacyAmbiguous => false, + } + }); + // Receipt cleanup alone does not mutate xl.meta. Returning peers and + // undecided preparations still need fences before and after recovery. + let metadata_may_change = all_states.values().any(|copies| { + !copies.iter().all(|copy| copy.state == DeleteRecoveryState::Absent) + && copies.iter().any(|copy| { + copy.state == DeleteRecoveryState::NewPrepared + || (copy.state == DeleteRecoveryState::Absent && copy.current_digest.is_some()) + || copy + .journal + .as_ref() + .is_some_and(|journal| copy.current_digest != journal.target_digest) + }) + }); + let namespace_owner = if dry_run { + None + } else { + Some(namespace_owner.unwrap_or_else(|| self.ctx.begin_namespace_commit())) + }; + if !dry_run && metadata_may_change { + self.invalidate_get_object_metadata_cache(bucket, object).await; + } + let authorities = all_states + .values() + .flatten() + .filter(|state| receipt(state.state)) + .filter_map(|state| state.journal.as_ref()) + .map(|journal| DecisionEvidence { + transaction: journal.transaction, + deleted_versions: journal.deleted_versions.iter().copied().collect(), + original_versions: journal.original_versions.iter().cloned().collect(), + predecessors: journal.predecessors.iter().copied().collect(), + target_absent: journal.target_digest.is_none(), + }) + .collect::>(); + let mut unresolved = false; + let mut mutated = false; + for transaction in transaction_order { + let copies = all_states.get_mut(&transaction).ok_or(DiskError::FileCorrupt)?; + if mutated { + // A sibling may have restored metadata or reclaimed shared + // evidence. Refresh fingerprints before acting on this group. + *copies = inspect(&disks, bucket, object, transaction) + .await + .into_iter() + .collect::>>()?; + } + if copies.iter().all(|state| state.state == DeleteRecoveryState::Absent) { + continue; + } + if copies.iter().any(|state| state.state == DeleteRecoveryState::LegacyAmbiguous) { + if legacy_only { + if dry_run { + summary.reclaimed += 1; + continue; + } + mutated = true; + if settle_group( + &disks, + bucket, + copies, + |copy| { + (copy.state == DeleteRecoveryState::LegacyAmbiguous).then_some(DeleteRecoveryAction::DiscardLegacy) + }, + can_commit, + namespace_owner.as_ref(), + ) + .await? + { + summary.pending += 1; + unresolved = true; + } else { + summary.reclaimed += 1; + } + } else { + summary.blocked += 1; + unresolved = true; + } + continue; + } + let authority = copies + .iter() + .filter(|state| receipt(state.state)) + .filter_map(|state| state.journal.as_ref()) + .max_by_key(|journal| !journal.requests.is_empty()) + .cloned(); + let prepared = copies.iter().find_map(|state| state.journal.as_ref()); + let covered = prepared + .filter(|journal| !journal.deleted_versions.is_empty()) + .is_some_and(|journal| { + authorities.iter().any(|authority| { + journal.deleted_versions.iter().all(|version| { + authority.deleted_versions.contains(version) + && (!version.is_nil() + || journal + .original_versions + .iter() + .filter(|source| source.version_id.is_none_or(|id| id.is_nil())) + .all(|source| authority.original_versions.contains(source)) + || (authority.predecessors.contains(&journal.transaction) + && journal.target_digest.is_none() + && authority.target_absent + && copies.iter().all(|copy| { + copy.current_digest.is_none() || copy.current_versions == journal.original_versions + }))) + }) + }) + }); + let original_quorum = prepared.is_some_and(|journal| { + copies + .iter() + .filter(|copy| copy.current_versions == journal.original_versions) + .count() + > self.set_drive_count / 2 + }); + let all_decisions_precede = !authorities.is_empty() + && authorities.iter().all(|authority| { + copies + .iter() + .filter_map(|copy| copy.journal.as_ref()) + .any(|journal| journal.predecessors.contains(&authority.transaction)) + }); + if authority.is_some() || covered || fresh_absence_decision { + let certified_same_transaction = authority.is_some(); + let reference = authority + .or_else(|| copies.iter().find_map(|state| state.journal.clone())) + .ok_or(DiskError::FileCorrupt)?; + if dry_run { + if copies.iter().any(|copy| { + if certified_same_transaction + && copy.state == DeleteRecoveryState::NewPrepared + && copy.current_digest.is_none() + && !copy.commit_data_safe + { + copy.journal + .as_ref() + .is_none_or(|local| !local.matches_prepared_authority(&reference)) + } else if copy.state == DeleteRecoveryState::Absent { + copy.current_digest.is_some() && !reference.matches_adoption_source(©.current_versions) + } else { + !copy.commit_data_safe + } + }) { + summary.blocked += 1; + } else { + summary.committed += 1; + } + continue; + } + mutated = true; + let mut pending = settle_group( + &disks, + bucket, + copies, + |copy| { + Some( + if copy.state == DeleteRecoveryState::Absent + || (certified_same_transaction + && copy.state == DeleteRecoveryState::NewPrepared + && copy.current_digest.is_none() + && !copy.commit_data_safe) + { + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(reference.clone()), + } + } else { + DeleteRecoveryAction::PublishCommit + }, + ) + }, + can_commit, + namespace_owner.as_ref(), + ) + .await?; + if !pending { + pending = settle_group( + &disks, + bucket, + copies, + |_| Some(DeleteRecoveryAction::FinishCommit), + can_commit, + namespace_owner.as_ref(), + ) + .await?; + } + if pending { + summary.pending += 1; + unresolved = true; + } else { + summary.committed += 1; + } + } else { + // A sibling decision or foreign generation can supersede this + // snapshot. Never guess by restoring the whole old backup. + if (!authorities.is_empty() && !original_quorum && !all_decisions_precede) + || copies + .iter() + .any(|state| state.state != DeleteRecoveryState::Absent && !state.rollback_data_complete) + { + summary.blocked += 1; + unresolved = true; + continue; + } + let originals = copies + .iter() + .filter_map(|state| state.journal.as_ref()) + .flat_map(|journal| journal.original_versions.iter()) + .collect::>(); + if copies.iter().any(|state| { + state.state == DeleteRecoveryState::Absent + && state.current_versions.iter().any(|version| !originals.contains(version)) + }) { + summary.blocked += 1; + unresolved = true; + continue; + } + if dry_run { + summary.rolled_back += 1; + continue; + } + mutated = true; + if settle_group( + &disks, + bucket, + copies, + |copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback), + can_commit, + namespace_owner.as_ref(), + ) + .await? + { + summary.pending += 1; + unresolved = true; + } else { + summary.rolled_back += 1; + } + } + } + if !unresolved && !dry_run { + // GC is a key-wide barrier: every predecessor must have settled + // before the last surviving commit witness can disappear. + let mut settled = Vec::with_capacity(all_states.len()); + for transaction in all_states.keys() { + let fresh = inspect(&disks, bucket, object, *transaction).await; + if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) { + summary.pending += 1; + return Ok(summary); + } + settled.push(fresh.into_iter().collect::>>()?); + } + for (index, mut copies) in settled.into_iter().enumerate() { + if index > 0 { + copies = inspect(&disks, bucket, object, copies[0].id.transaction) + .await + .into_iter() + .collect::>>()?; + if copies + .iter() + .any(|copy| !matches!(copy.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished)) + { + summary.pending += 1; + return Ok(summary); + } + } + if settle_group( + &disks, + bucket, + &mut copies, + |copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit), + can_commit, + namespace_owner.as_ref(), + ) + .await? + { + summary.pending += 1; + } + } + } + + if !dry_run && metadata_may_change { + self.invalidate_get_object_metadata_cache(bucket, object).await; + } + Ok(summary) + } +} + +// A lagging member may have the complete older suffix of a marker creation's +// live source. Repair that source before applying the original strict matcher. +fn marker_source_to_repair( + copies: &[DeleteRecoveryInspection], + incarnation: Option, +) -> Option { + let expected = incarnation.filter(|id| !id.is_nil())?; + let authority = copies + .iter() + .filter(|copy| receipt(copy.state)) + .find_map(|copy| copy.journal.as_ref())?; + let source = authority.original_versions.first()?; + let request = authority.requests.first()?; + if authority.bucket_incarnation != Some(expected) + || authority.requests.len() != 1 + || !request.deleted + || !request.mark_deleted + || request.version_id.is_none_or(|id| id.is_nil()) + || authority + .original_versions + .iter() + .any(|version| version.version_id == request.version_id) + || !authority.deleted_versions.is_empty() + || !authority.data_dirs.is_empty() + || authority.target_digest.is_none() + || source.version_id.is_none_or(|id| id.is_nil()) + || source.deleted + { + return None; + } + let mut missing_source = false; + for copy in copies { + match copy.state { + DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished => { + let journal = copy.journal.as_ref()?; + if journal.bucket_incarnation != Some(expected) + || journal.original_versions != authority.original_versions + || !journal.matches_prepared_authority(authority) + || copy.current_digest != journal.target_digest + { + return None; + } + } + DeleteRecoveryState::Absent => { + if copy.journal.is_some() + || copy.current_versions.is_empty() + || !authority.original_versions.ends_with(©.current_versions) + { + return None; + } + missing_source |= copy.current_versions.first() != Some(source); + } + _ => return None, + } + } + missing_source.then(|| source.clone()) +} + +fn unavailable(error: &DiskError) -> bool { + if unsupported_recovery(error) { + return true; + } + match error { + DiskError::DiskNotFound + | DiskError::FaultyDisk + | DiskError::DiskOngoingReq + | DiskError::Timeout + | DiskError::RemoteClientUnavailable(_) => true, + DiskError::Io(source) => { + if let Some(status) = crate::cluster::rpc::client::embedded_tonic_status(source) { + return crate::cluster::rpc::client::is_network_like_status(status); + } + matches!( + source.kind(), + std::io::ErrorKind::TimedOut + | std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::NotConnected + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::UnexpectedEof + ) + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::metadata_sys::{self, init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::disk::{DeleteOptions, ReadOptions, STORAGE_FORMAT_FILE}; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions}; + use crate::storage_api_contracts::object::{ObjectIO, ObjectOperations}; + use crate::store::ECStore; + use rustfs_filemeta::FileInfo; + use rustfs_heal_contracts::heal_channel::HealOpts; + use std::sync::Arc; + use tokio::io::AsyncReadExt; + + async fn fixture(bucket: &str) -> (Vec, Arc, Vec) { + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative recovery bucket"); + let disks = store.pools[0].disk_set[0] + .disk_inventory() + .await + .into_iter() + .map(|disk| disk.expect("fixture drive is online")) + .collect(); + (dirs, store, disks) + } + + async fn put(store: &Arc, bucket: &str, body: &[u8], versioned: bool) -> Option { + let mut reader = PutObjReader::from_vec(body.to_vec()); + store + .put_object( + bucket, + "object", + &mut reader, + &ObjectOptions { + versioned, + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("put recovery source") + .version_id + } + + async fn applied(store: &Arc, disks: &[DiskStore], bucket: &str, version: Option, count: usize) -> Uuid { + applied_at(store, disks, bucket, version, count, Uuid::new_v4()).await + } + + async fn applied_at( + store: &Arc, + disks: &[DiskStore], + bucket: &str, + version: Option, + count: usize, + transaction: Uuid, + ) -> Uuid { + let incarnation = store + .bucket_incarnation_id_from_disk(bucket) + .await + .expect("source bucket identity"); + for disk in disks.iter().take(count) { + disk.delete_version_recoverable( + bucket, + "object", + FileInfo { + name: "object".to_string(), + version_id: version, + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + delete_recovery_bucket_incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("apply prepared delete without publishing a set decision"); + } + transaction + } + + async fn publish(disk: &DiskStore, bucket: &str, transaction: Uuid) { + let state = disk + .inspect_delete_transaction(bucket, "object", transaction) + .await + .expect("inspect prepared source"); + disk.settle_delete_transaction(bucket, "object", transaction, state.fingerprint, DeleteRecoveryAction::PublishCommit) + .await + .expect("persist a decision after the simulated metadata quorum"); + } + + async fn body(store: &Arc, bucket: &str, version: Option) -> Vec { + let opts = ObjectOptions { + version_id: version.map(|version| version.to_string()), + ..Default::default() + }; + let mut reader = store + .get_object_reader(bucket, "object", None, http::HeaderMap::new(), &opts) + .await + .expect("open recovered object"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("read exact recovered bytes"); + body + } + + #[tokio::test] + async fn delete_recovery_set_rolls_back_undecided_inline_bytes() { + let bucket = "recover-undecided"; + let (_dirs, store, disks) = fixture(bucket).await; + let original = vec![21; 4096]; + put(&store, bucket, &original, false).await; + applied(&store, &disks, bucket, None, disks.len()).await; + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("recover undisputed prepared delete"); + assert!( + report.detail.contains("rolled_back=1"), + "actual rollback must be reported: {}", + report.detail + ); + assert_eq!(body(&store, bucket, None).await, original); + } + + #[tokio::test] + async fn delete_recovery_marker_source_heal_preserves_foreign_history_and_damaged_data() { + fn snapshot(root: &std::path::Path) -> BTreeMap> { + let mut files = BTreeMap::new(); + let mut pending = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + pending.push(path); + } else { + files.insert(path.strip_prefix(root).unwrap().to_path_buf(), std::fs::read(path).unwrap()); + } + } + } + files + } + + for shape in ["live-backup", "foreign", "partial-history", "damaged-data"] { + let bucket = "recover-marker-source-guards"; + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket( + bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); + let set = &store.pools[0].disk_set[0]; + let disks: Vec<_> = set.disk_inventory().await.into_iter().map(Option::unwrap).collect(); + let original = vec![71; 1024 * 1024]; + let oldest = put(&store, bucket, &original, true).await.unwrap(); + let first = put(&store, bucket, &original, true).await.unwrap(); + set.disks.write().await[3] = None; + let payload = vec![72; 1024 * 1024]; + let source = put(&store, bucket, &payload, true).await.unwrap(); + store + .delete_object( + bucket, + "object", + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("publish marker while one member lacks the source version"); + set.disks.write().await[3] = Some(disks[3].clone()); + let mut first_info = disks[3] + .read_version("", bucket, "object", &first.to_string(), &ReadOptions::default()) + .await + .unwrap(); + match shape { + "live-backup" => { + let root = dirs[3].path().join(bucket).join("object"); + let backup = root.join(first_info.data_dir.unwrap().to_string()).join("xl.meta.bkp"); + tokio::fs::write(backup, tokio::fs::read(root.join("xl.meta")).await.unwrap()) + .await + .unwrap(); + } + "foreign" => { + first_info.version_id = Some(Uuid::new_v4()); + first_info.mod_time = Some(time::OffsetDateTime::now_utc()); + disks[3].write_metadata("", bucket, "object", first_info).await.unwrap(); + } + "partial-history" => { + disks[3] + .delete_version( + bucket, + "object", + FileInfo { + name: "object".into(), + version_id: Some(oldest), + ..Default::default() + }, + false, + DeleteOptions::default(), + ) + .await + .unwrap(); + } + "damaged-data" => { + // Three metadata sources meet write quorum, but corrupting + // two shards leaves fewer than EC2+2's two required readers. + for (index, disk) in disks.iter().take(2).enumerate() { + let info = disk + .read_version("", bucket, "object", &source.to_string(), &ReadOptions::default()) + .await + .unwrap(); + let part = dirs[index] + .path() + .join(bucket) + .join("object") + .join(info.data_dir.unwrap().to_string()) + .join("part.1"); + let mut bytes = tokio::fs::read(&part).await.unwrap(); + *bytes.last_mut().unwrap() ^= 1; + tokio::fs::write(part, bytes).await.unwrap(); + } + } + _ => unreachable!(), + } + let before: Vec<_> = dirs + .iter() + .map(|dir| snapshot(&dir.path().join(bucket).join("object"))) + .collect(); + let result = store + .heal_object_at_incarnation( + bucket, + "object", + &source.to_string(), + incarnation, + &HealOpts { + scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep, + ..Default::default() + }, + ) + .await; + if shape == "live-backup" { + assert!(result.unwrap().error.is_none(), "plain live backups cannot block source repair"); + assert_eq!(body(&store, bucket, Some(source)).await, payload); + assert_eq!(body(&store, bucket, Some(first)).await, original); + } else { + assert!(result.is_err() || result.unwrap().error.is_some(), "{shape} must fail safely"); + for (dir, expected) in dirs.iter().zip(before) { + assert_eq!(snapshot(&dir.path().join(bucket).join("object")), expected, "{shape}"); + } + } + } + } + + #[tokio::test] + async fn delete_recovery_admin_prefix_excludes_other_keys_and_restores_hidden_match() { + let bucket = "recover-prefix-scope"; + let (dirs, store, disks) = fixture(bucket).await; + let original = vec![27; 4096]; + put(&store, bucket, &original, false).await; + let transaction = applied(&store, &disks, bucket, None, disks.len()).await; + let outside = store + .recover_delete_prefix_at_incarnation(bucket, "other/", None, &HealOpts::default()) + .await + .expect("inspect unrelated prefix"); + assert!(outside.is_none()); + for dir in &dirs { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(transaction.to_string()) + .join("xl.meta.bkp") + .exists() + ); + } + let matching = store + .recover_delete_prefix_at_incarnation(bucket, "obj", None, &HealOpts::default()) + .await + .expect("recover matching hidden key") + .expect("report matching transaction"); + assert!(matching.detail.contains("rolled_back=1")); + assert_eq!(body(&store, bucket, None).await, original); + } + + #[tokio::test] + async fn delete_recovery_set_commits_receipt_and_unblocks_bucket() { + let bucket = "recover-committed"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![22; 1024 * 1024], false).await; + let transaction = applied(&store, &disks, bucket, None, disks.len()).await; + publish(&disks[0], bucket, transaction).await; + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("complete certified delete"); + assert!( + report.detail.contains("committed=1"), + "certified replay must be reported: {}", + report.detail + ); + assert!( + store + .get_object_info(bucket, "object", &ObjectOptions::default()) + .await + .is_err() + ); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("non-force bucket deletion must succeed after proof GC"); + } + + #[tokio::test] + async fn delete_recovery_set_handles_prepared_subset() { + let bucket = "recover-subset"; + let (_dirs, store, disks) = fixture(bucket).await; + let original = vec![23; 4096]; + put(&store, bucket, &original, false).await; + applied(&store, &disks, bucket, None, 1).await; + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("untouched peers remain valid evidence"); + assert!(report.detail.contains("rolled_back=1")); + assert_eq!(body(&store, bucket, None).await, original); + } + + #[tokio::test] + async fn delete_recovery_set_adopts_untouched_returning_peer() { + let bucket = "recover-untouched"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![24; 1024 * 1024], false).await; + let transaction = applied(&store, &disks, bucket, None, 3).await; + for disk in disks.iter().take(3) { + publish(disk, bucket, transaction).await; + } + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("matching untouched generation must converge"); + assert!(report.detail.contains("committed=1")); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("returning old replica must not leave bucket residue"); + } + + #[tokio::test] + async fn delete_recovery_set_retains_decisions_until_offline_peer_returns() { + let bucket = "recover-offline"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![25; 4096], false).await; + let transaction = applied(&store, &disks, bucket, None, 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let available = set.disk_inventory().await; + set.publish_delete_commit(bucket, "object", transaction, &available, 3, DeleteRecoveryContext::new(None, &|| true)) + .await + .expect("commit quorum remains available"); + for dir in dirs.iter().take(3) { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(transaction.to_string()) + .join(crate::disk::local::DELETE_QUORUM_COMMITTED_FILE) + .exists(), + "decision must survive an unavailable participant" + ); + } + set.disks.write().await[3] = Some(disks[3].clone()); + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("returning prepared peer must roll forward"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("proof collection follows complete convergence"); + } + + #[tokio::test] + async fn delete_recovery_set_never_resurrects_prepared_old_null_after_new_delete() { + let bucket = "recover-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![26; 4096], false).await; + applied(&store, &disks, bucket, None, 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + put(&store, bucket, &vec![27; 4096], false).await; + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("later visible generation can commit with quorum"); + set.disks.write().await[3] = Some(disks[3].clone()); + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("later deletion must supersede hidden old null residue"); + assert!( + store + .get_object_info(bucket, "object", &ObjectOptions::default()) + .await + .is_err(), + "old bytes must never become visible again" + ); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("both generations' transactions converge"); + } + + #[tokio::test] + async fn delete_recovery_set_missing_version_does_not_ack_hidden_rollback() { + let bucket = "recover-version-history"; + let (_dirs, store, disks) = fixture(bucket).await; + metadata_sys::update_in(&store.ctx, bucket, crate::bucket::metadata::BUCKET_VERSIONING_CONFIG, + br#"Enabled"#.to_vec()) + .await.expect("enable versioning"); + let first = put(&store, bucket, &vec![28; 4096], true).await.expect("first version"); + let survivor = vec![29; 4096]; + let second = put(&store, bucket, &survivor, true).await.expect("second version"); + applied(&store, &disks, bucket, Some(first), 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let opts = ObjectOptions { + version_id: Some(first.to_string()), + versioned: true, + ..Default::default() + }; + assert!( + store.delete_object(bucket, "object", opts.clone()).await.is_err(), + "hidden prepared version cannot receive an idempotent success while a peer is unavailable" + ); + set.disks.write().await[3] = Some(disks[3].clone()); + store + .delete_object(bucket, "object", opts) + .await + .expect("recover then execute the currently authorized exact-version delete"); + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(first.to_string()), + ..Default::default() + } + ) + .await + .is_err() + ); + assert_eq!(body(&store, bucket, Some(second)).await, survivor); + } + + #[tokio::test] + async fn delete_recovery_set_dry_run_preserves_every_file() { + let bucket = "recover-dry-run"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![30; 4096], false).await; + let transaction = applied(&store, &disks, bucket, None, 4).await; + let before = disks[0] + .inspect_delete_transaction(bucket, "object", transaction) + .await + .expect("before inspection"); + let report = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect("dry-run report"); + assert!(report.detail.contains("dry run")); + let after = disks[0] + .inspect_delete_transaction(bucket, "object", transaction) + .await + .expect("after inspection"); + assert_eq!(before.fingerprint, after.fingerprint, "dry run may not modify recovery evidence"); + for dir in dirs { + assert!(!dir.path().join(bucket).join("object").join(STORAGE_FORMAT_FILE).exists()); + } + } + #[tokio::test] + async fn delete_recovery_set_admin_object_restores_hidden_metadata() { + let bucket = "recover-admin-object"; + let (_dirs, store, disks) = fixture(bucket).await; + let payload = vec![41; 4096]; + put(&store, bucket, &payload, false).await; + applied(&store, &disks, bucket, None, 4).await; + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("bucket identity"); + let result = store + .heal_object_at_incarnation(bucket, "object", "", incarnation, &HealOpts::default()) + .await + .expect("admin object recovery"); + assert!(result.error.is_none(), "recovered object should heal cleanly: {:?}", result.error); + assert_eq!(body(&store, bucket, None).await, payload); + } + + #[tokio::test] + async fn delete_recovery_current_version_delete_preserves_legacy_on_missing_participant() { + use crate::storage_api_contracts::object::ObjectToDelete; + for batch in [false, true] { + let bucket = "recover-current-with-legacy"; + let (dirs, store, disks) = fixture(bucket).await; + let original = vec![44; 1024 * 1024]; + let old_version = put(&store, bucket, &original, true).await.expect("historical version"); + let legacy = Uuid::new_v4(); + disks[3] + .delete_version( + bucket, + "object", + FileInfo { + name: "object".into(), + version_id: Some(old_version), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(legacy), + ..Default::default() + }, + ) + .await + .expect("historical disk-local mutation without a set decision"); + let backup = dirs[3] + .path() + .join(bucket) + .join("object") + .join(legacy.to_string()) + .join("xl.meta.bkp"); + let before = tokio::fs::read(&backup).await.expect("historical backup"); + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let current = put(&store, bucket, &vec![45; 4096], true) + .await + .expect("current version at quorum"); + set.disks.write().await[3] = Some(disks[3].clone()); + if batch { + let (_, errors) = store + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: "object".into(), + version_id: Some(current), + ..Default::default() + }], + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await; + assert!(errors.iter().all(Option::is_none), "current deletion at quorum: {errors:?}"); + } else { + store + .delete_object( + bucket, + "object", + ObjectOptions { + versioned: true, + version_id: Some(current.to_string()), + ..Default::default() + }, + ) + .await + .expect("current deletion at quorum"); + } + assert_eq!(tokio::fs::read(&backup).await.expect("legacy backup retained"), before); + let state = disks[3] + .inspect_delete_transaction(bucket, "object", legacy) + .await + .expect("legacy inspection"); + assert_eq!(state.state, DeleteRecoveryState::LegacyAmbiguous); + assert_eq!(body(&store, bucket, Some(old_version)).await, original); + } + } + + #[tokio::test] + async fn delete_recovery_dry_run_matches_adoption_of_already_absent_non_null_version() { + let bucket = "recover-preview-non-null"; + let (_dirs, store, disks) = fixture(bucket).await; + let first = put(&store, bucket, &vec![46; 4096], true).await.expect("first version"); + let remaining = vec![47; 4096]; + let second = put(&store, bucket, &remaining, true).await.expect("remaining version"); + disks[3] + .delete_version( + bucket, + "object", + FileInfo { + name: "object".into(), + version_id: Some(first), + ..Default::default() + }, + false, + DeleteOptions::default(), + ) + .await + .expect("returning peer already lacks the explicit version"); + let transaction = applied(&store, &disks, bucket, Some(first), 3).await; + publish(&disks[0], bucket, transaction).await; + let preview = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect("preview certified recovery"); + assert!( + preview.detail.contains("committed=1") && preview.detail.contains("blocked=0"), + "{}", + preview.detail + ); + let actual = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("recover certified removal"); + assert!( + actual.detail.contains("committed=1") && actual.detail.contains("blocked=0"), + "{}", + actual.detail + ); + assert_eq!(body(&store, bucket, Some(second)).await, remaining); + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(first.to_string()), + ..Default::default() + } + ) + .await + .is_err() + ); + } + + #[tokio::test] + async fn delete_recovery_set_legacy_requires_explicit_removal() { + let bucket = "recover-legacy-remove"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![42; 4096], false).await; + let transaction = Uuid::new_v4(); + for disk in &disks { + disk.delete_version( + bucket, + "object", + FileInfo { + name: "object".to_owned(), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .expect("create undecided historical delete shape"); + } + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("automatic report"); + assert!(report.detail.contains("blocked=1")); + for dir in &dirs { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(transaction.to_string()) + .join("xl.meta.bkp") + .exists() + ); + } + let dry = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + remove: true, + ..Default::default() + }, + ) + .await + .expect("authorized-removal dry run"); + assert!(dry.detail.contains("reclaimed=1")); + for dir in &dirs { + assert!(dir.path().join(bucket).join("object").exists()); + } + let report = store + .recover_delete_transactions( + bucket, + &HealOpts { + remove: true, + ..Default::default() + }, + ) + .await + .expect("explicit admin removal"); + assert!(report.detail.contains("reclaimed=1")); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("non-force bucket deletion"); + } + + #[tokio::test] + async fn delete_recovery_set_old_null_decision_cannot_commit_new_undecided_generation() { + let bucket = "recover-inverse-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![51; 4096], false).await; + let old_transaction = applied(&store, &disks, bucket, None, 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + set.publish_delete_commit( + bucket, + "object", + old_transaction, + &set.disk_inventory().await, + 3, + DeleteRecoveryContext::new(None, &|| true), + ) + .await + .expect("old generation decision while peer offline"); + let survivor = vec![52; 4096]; + put(&store, bucket, &survivor, false).await; + let new_transaction = applied(&store, &disks, bucket, None, 3).await; + set.disks.write().await[3] = Some(disks[3].clone()); + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("causal recovery across old receipt"); + assert!(report.detail.contains("rolled_back=1")); + assert_eq!( + body(&store, bucket, None).await, + survivor, + "an old null-version receipt cannot authorize deleting a later generation" + ); + for disk in &disks { + assert_eq!( + disk.inspect_delete_transaction(bucket, "object", new_transaction) + .await + .expect("new transaction settled") + .state, + DeleteRecoveryState::Absent + ); + } + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("delete recovered generation explicitly"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("all proof is eventually collected"); + } + + #[tokio::test] + async fn delete_recovery_set_ordinary_missing_single_and_batch_keep_quorum_semantics() { + let bucket = "recover-missing-quorum"; + let (_dirs, store, disks) = fixture(bucket).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let error = store + .delete_object(bucket, "single-missing", ObjectOptions::default()) + .await + .expect_err("ordinary missing storage result"); + assert!( + crate::error::is_err_object_not_found(&error), + "S3 may translate missing key into idempotent204; not a bucket/quorum error: {error}" + ); + let objects = ["batch-one", "batch-two"] + .into_iter() + .map(|name| crate::storage_api_contracts::object::ObjectToDelete { + object_name: name.to_owned(), + ..Default::default() + }) + .collect(); + let (deleted, errors) = store.delete_objects(bucket, objects, ObjectOptions::default()).await; + assert!( + errors.iter().all(Option::is_none), + "ordinary batch no-op must succeed at available quorum: {errors:?}" + ); + assert_eq!(deleted.len(), 2); + for disk in disks.iter().take(3) { + for key in ["single-missing", "batch-one", "batch-two"] { + assert!( + disk.discover_delete_transactions(bucket, Some(key), None, 128) + .await + .expect("no-op leaves no recovery journals") + .transactions + .is_empty() + ); + } + } + set.disks.write().await[3] = Some(disks[3].clone()); + let payload = vec![53; 4096]; + put(&store, bucket, &payload, false).await; + assert_eq!(body(&store, bucket, None).await, payload); + } + #[tokio::test] + async fn delete_recovery_set_fresh_delete_resolves_unknown_null_predecessor() { + let bucket = "recover-unknown-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![54; 4096], false).await; + let transaction = Uuid::new_v4(); + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("bucket identity"); + disks[3] + .delete_version_recoverable( + bucket, + "object", + FileInfo { + name: "object".to_owned(), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + delete_recovery_bucket_incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("only future-offline peer prepared old delete"); + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + put(&store, bucket, &vec![55; 4096], false).await; + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("later generation deleted at quorum"); + set.disks.write().await[3] = Some(disks[3].clone()); + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("unknown causal history is reported"); + assert!( + report.detail.contains("blocked=1"), + "automatic recovery cannot invent causality: {}", + report.detail + ); + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("fresh authorized delete certifies current absence"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("fresh intent drains all proven unprotected null history"); + } + #[test] + fn delete_recovery_network_errors_do_not_mask_application_corruption() { + assert!(unavailable(&DiskError::RemoteClientUnavailable( + "transport error: connection refused".to_owned() + ))); + assert!(unavailable(&DiskError::from(tonic::Status::unavailable("peer disconnected")))); + assert!( + unavailable(&DiskError::FaultyDisk), + "a drive disabled by health cannot vote in recovery inspection" + ); + assert!(!unavailable(&DiskError::FileCorrupt)); + assert!(!unavailable(&DiskError::other("invalid journal containing connection refused"))); + assert!(!unavailable(&DiskError::from(tonic::Status::internal("connection refused")))); + assert!(!unavailable(&DiskError::other("bucket unavailable-logs has corrupt metadata"))); + } + + #[tokio::test] + async fn delete_recovery_chained_committed_versions_finish_after_current_metadata_disappears() { + let bucket = "recover-chained-versions"; + let (_dirs, store, disks) = fixture(bucket).await; + let first = put(&store, bucket, &vec![61; 1024 * 1024], true) + .await + .expect("first version"); + let second = put(&store, bucket, &vec![62; 4096], true).await.expect("second version"); + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + for version in [first, second] { + store + .delete_object( + bucket, + "object", + ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), + ..Default::default() + }, + ) + .await + .expect("commit each explicit version at quorum while peer is absent"); + } + assert!( + store + .get_object_info(bucket, "object", &ObjectOptions::default()) + .await + .is_err() + ); + set.disks.write().await[3] = Some(disks[3].clone()); + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("older certified target may be superseded by a later certified absence"); + assert!( + report.detail.contains("committed=2"), + "both retained decisions must converge: {}", + report.detail + ); + for version in [first, second] { + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(version.to_string()), + ..Default::default() + } + ) + .await + .is_err(), + "certified cleanup must not resurrect either deleted version" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("no retained version decision may keep the empty bucket nonempty"); + } + + #[tokio::test] + async fn delete_recovery_partial_receipt_converges_after_later_version_absence() { + let bucket = "recover-partial-chained-versions"; + let (dirs, store, disks) = fixture(bucket).await; + let first = put(&store, bucket, &vec![63; 1024 * 1024], true) + .await + .expect("first version"); + let second = put(&store, bucket, &vec![64; 4096], true).await.expect("second version"); + let transaction = applied(&store, &disks, bucket, Some(first), 4).await; + // A crash after the first durable decision leaves other copies prepared. + publish(&disks[0], bucket, transaction).await; + let later = applied(&store, &disks, bucket, Some(second), 4).await; + publish(&disks[0], bucket, later).await; + let dry = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect("dry-run uses the same certified prepared-copy authority"); + assert!( + dry.detail.contains("committed=2") && dry.detail.contains("blocked=0"), + "accurate dry-run: {}", + dry.detail + ); + for dir in &dirs { + for id in [transaction, later] { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(id.to_string()) + .join("delete-prepared") + .exists(), + "dry-run must retain every preparation record" + ); + } + } + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("same-transaction receipt certifies each prepared absent copy"); + for dir in &dirs { + assert!(!dir.path().join(bucket).join("object").exists(), "all related proof must converge"); + } + for version in [first, second] { + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(version.to_string()), + ..Default::default() + } + ) + .await + .is_err(), + "no erased version may be recreated by receipt adoption" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("non-force bucket removal"); + } + + #[tokio::test] + async fn delete_recovery_causal_cycle_rejects_before_mutation_and_retains_proof() { + let bucket = "recover-causal-cycle"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![58; 4096], false).await; + let first = applied(&store, &disks, bucket, None, 4).await; + put(&store, bucket, &vec![59; 4096], false).await; + let second = applied(&store, &disks, bucket, None, 4).await; + let mut evidence = Vec::new(); + for (dir, disk) in dirs.iter().zip(&disks) { + let root = dir.path().join(bucket).join("object"); + let mut journal = disk + .inspect_delete_transaction(bucket, "object", first) + .await + .expect("inspect first preparation") + .journal + .expect("original journal"); + journal.predecessors.push(second); + let path = root.join(first.to_string()).join("delete-prepared"); + std::fs::write(&path, serde_json::to_vec(&journal).expect("encode cyclic fixture")).expect("write cyclic edge"); + for transaction in [first, second] { + for file in ["delete-prepared", "xl.meta.bkp"] { + let path = root.join(transaction.to_string()).join(file); + evidence.push((path.clone(), std::fs::read(path).expect("capture untouched proof"))); + } + } + } + let error = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect_err("cyclic history cannot settle"); + assert!( + matches!(error, crate::error::StorageError::FileCorrupt), + "typed corruption must reject cyclic evidence: {error:?}" + ); + for (path, bytes) in evidence { + assert_eq!(std::fs::read(path).expect("all proof retained"), bytes); + } + for dir in &dirs { + assert!( + !dir.path().join(bucket).join("object").join(STORAGE_FORMAT_FILE).exists(), + "cycle rejection must not restore or publish metadata" + ); + } + } + + #[tokio::test] + async fn delete_recovery_set_later_null_decision_covers_unapplied_original_peer() { + let bucket = "recover-unapplied-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![56; 1024 * 1024], false).await; + // Every generated v4 UUID sorts before MAX, so the former UUID order + // deterministically tried the newer decision before its predecessor. + applied_at(&store, &disks, bucket, None, 3, Uuid::from_u128(u128::MAX)).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + put(&store, bucket, &vec![57; 1024 * 1024], false).await; + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("later null decision"); + set.disks.write().await[3] = Some(disks[3].clone()); + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("causal older original can be removed"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("no stale old body or receipts remain"); + } +} diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 07df7f69c..200c13690 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -25,11 +25,14 @@ use super::super::{ use crate::bucket::retirement::MarkerRetirementContext; use crate::diagnostics::object_lock::ObjectLockAttempt; use crate::set_disk::{ObjectLockDiagGuard, is_object_lock_diag_enabled}; +use std::sync::Arc; #[derive(Clone, Copy)] struct ExplicitVersionHeal<'a> { opts: &'a HealOpts, allow_regeneration: bool, + preserve_unrepairable: bool, + commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>, retirement: Option<&'a MarkerRetirementContext<'a>>, } @@ -770,6 +773,8 @@ impl SetDisks { ExplicitVersionHeal { opts, allow_regeneration: true, + preserve_unrepairable: false, + commit_fence: None, retirement: None, }, &mut None, @@ -907,6 +912,8 @@ impl SetDisks { let ExplicitVersionHeal { opts, allow_regeneration: allow_explicit_version_regen, + preserve_unrepairable, + commit_fence, retirement, } = run; trace!( @@ -925,6 +932,9 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + return Err(DiskError::DiskOngoingReq); + } let mut result = HealResultItem { heal_item_type: HealItemType::Object.to_string(), @@ -1248,7 +1258,7 @@ impl SetDisks { // return here and never reach the post-heal reclaim tail // below. Sweep the strays on this path too (issues #3231, // #3191). Skipped on dry-run, like every mutating step. - if !opts.dry_run { + if !opts.dry_run && !preserve_unrepairable { self.reclaim_orphan_data_dirs_best_effort(bucket, object).await; } return Ok((result, None)); @@ -1328,7 +1338,7 @@ impl SetDisks { ); } - if read_repair_uses_shared_lock { + if read_repair_uses_shared_lock || preserve_unrepairable { return Ok((result, Some(cannot_heal_err))); } @@ -1527,6 +1537,9 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + return Err(DiskError::DiskOngoingReq); + } let disk = outdated_disk.as_ref().expect("checked above").clone(); let endpoint = disk.endpoint().to_string(); match disk.make_volume(bucket).await { @@ -1562,6 +1575,9 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + return Err(DiskError::DiskOngoingReq); + } if disks_to_heal_count == 0 { return Ok((result, volume_creation_error.or(Some(DiskError::ErasureWriteQuorum)))); } @@ -1873,6 +1889,10 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; + return Err(DiskError::DiskOngoingReq); + } rename_attempts += 1; // record the index of the updated disks parts_metadata[index].erasure.index = index + 1; @@ -2004,7 +2024,9 @@ impl SetDisks { // by pre-#3510 unversioned overwrites, which the dangling paths // above never touch (issues #3231, #3191). Best effort — a // failure must not fail the heal. - self.reclaim_orphan_data_dirs_best_effort(bucket, object).await; + if !preserve_unrepairable { + self.reclaim_orphan_data_dirs_best_effort(bucket, object).await; + } Ok((result, None)) } @@ -2070,7 +2092,7 @@ impl SetDisks { .await; } - if opts.dry_run { + if opts.dry_run || preserve_unrepairable { return Ok(( self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id) .await, @@ -3347,6 +3369,74 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { } impl SetDisks { + /// The recovery coordinator retains the key write lock and lifecycle fence. + /// Only current live metadata can supply this exact source; backups and + /// dangling cleanup cannot authorize prerequisite repair. + pub(in crate::set_disk) async fn heal_delete_recovery_source( + &self, + bucket: &str, + object: &str, + source: &crate::disk::DeleteRecoveryVersionIdentity, + can_commit: &(dyn Fn() -> bool + Sync), + ) -> disk::error::Result<()> { + if !can_commit() { + return Err(DiskError::DiskOngoingReq); + } + let version = source.version_id.ok_or(DiskError::OutdatedXLMeta)?.to_string(); + let disks = self.get_disks_internal().await; + let (metadata, errors, _) = Self::read_metadata(&disks, "", bucket, object, &version, true, false, false) + .await? + .into_legacy(); + let (read_quorum, write_quorum) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?; + let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?; + let write_quorum = usize::try_from(write_quorum).map_err(|_| DiskError::ErasureWriteQuorum)?; + let live_sources = metadata + .iter() + .zip(&errors) + .filter(|(info, error)| { + error.is_none() + && !info.deleted + && !info.is_remote() + && crate::disk::DeleteRecoveryVersionIdentity::from_file_info((*info).clone()) + .is_ok_and(|identity| identity == *source) + }) + .count(); + if live_sources < write_quorum || !can_commit() { + return Err(DiskError::OutdatedXLMeta); + } + let (_, mod_time, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum); + let selected = Self::pick_valid_fileinfo(&metadata, mod_time, etag, read_quorum)?; + if crate::disk::DeleteRecoveryVersionIdentity::from_file_info(selected)? != *source { + return Err(DiskError::OutdatedXLMeta); + } + let opts = HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }; + let (_, error) = Box::pin(self.heal_object_with_explicit_version_regen( + bucket, + object, + &version, + ExplicitVersionHeal { + opts: &opts, + allow_regeneration: false, + preserve_unrepairable: true, + commit_fence: Some(can_commit), + retirement: None, + }, + &mut None, + )) + .await?; + if let Some(error) = error { + return Err(error); + } + if !can_commit() { + return Err(DiskError::DiskOngoingReq); + } + Ok(()) + } + pub(crate) async fn heal_object_with_absence( &self, bucket: &str, @@ -3375,11 +3465,11 @@ impl SetDisks { .new_ns_lock(bucket, object) .await .map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?; - Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| { + Some(Arc::new(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| { self.map_namespace_lock_error(bucket, object, "write", e) .narrow_to_disk() .unwrap_or_else(DiskError::other) - })?) + })?)) } else { None }; @@ -3389,6 +3479,75 @@ impl SetDisks { return Ok((result, err.map(|e| e.into()))); } + let mut recovery_removed_version = false; + if !opts.read_repair + && _write_lock_guard.is_some() + && let Some(scope) = crate::store::bucket_heal_scope_for_object(bucket, object) + && !crate::bucket::utils::is_meta_bucketname(bucket) + { + let requested = Uuid::parse_str(version_id).ok(); + if !opts.dry_run && (version_id.is_empty() || requested.is_some()) { + let disks = self.get_disks_internal().await; + let before = Self::read_metadata(&disks, "", bucket, object, version_id, false, false, false) + .await + .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; + recovery_removed_version = before + .slots + .iter() + .filter_map(|slot| slot.file_info()) + .any(|info| version_id.is_empty() || Some(info.version_id.unwrap_or_default()) == requested); + } + let explicit_delete = opts.remove + && version_id.is_empty() + && crate::bucket::metadata_sys::has_authoritative_never_versioned_state_in(&self.ctx, bucket).await? + && matches!( + crate::bucket::metadata_sys::get_object_lock_config_state_in(&self.ctx, bucket).await?, + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent + ); + let recovery = self + .recover_delete_key( + bucket, + object, + crate::set_disk::DeleteRecoveryContext { + dry_run: opts.dry_run, + explicit_delete, + repair_marker_source: true, + namespace_owner: (!opts.dry_run).then(|| { + Arc::new((scope.clone(), self.ctx.begin_namespace_commit(), _write_lock_guard.clone())) + as Arc + }), + ..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &|| { + scope.check().is_ok() && !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()) + }) + }, + ) + .await?; + if opts.dry_run && recovery.discovered > 0 { + return Ok(( + HealResultItem { + heal_item_type: HealItemType::Object.to_string(), + bucket: bucket.to_owned(), + object: object.to_owned(), + version_id: version_id.to_owned(), + detail: format!( + "delete recovery (dry run): discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}, pending={}", + recovery.discovered, + recovery.committed, + recovery.rolled_back, + recovery.reclaimed, + recovery.blocked, + recovery.pending + ), + ..Default::default() + }, + None, + )); + } + if recovery.pending > 0 || recovery.blocked > 0 { + return Err(StorageError::SlowDown); + } + } + // The inner heal and missing-object report read the registry again; // release this snapshot guard before a topology writer can queue between reads. let disks = self.get_disks_internal().await; @@ -3440,12 +3599,12 @@ impl SetDisks { && errs .iter() .all(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound))) - && !_write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + && !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()) { *absence = Some(HealedObjectAbsence { pool_index: self.pool_index, set_index: self.set_index, - removed: false, + removed: recovery_removed_version, }); } return Ok((result, Some(err))); @@ -3463,6 +3622,8 @@ impl SetDisks { ExplicitVersionHeal { opts: &inner_opts, allow_regeneration: true, + preserve_unrepairable: false, + commit_fence: None, retirement: proof.retirement, }, absence, @@ -3483,6 +3644,8 @@ impl SetDisks { ExplicitVersionHeal { opts: &inner_opts, allow_regeneration: true, + preserve_unrepairable: false, + commit_fence: None, retirement: proof.retirement, }, absence, @@ -3493,7 +3656,7 @@ impl SetDisks { result, err.map(Into::into), absence, - _write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()), + _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()), bucket, object, )); @@ -3505,7 +3668,7 @@ impl SetDisks { result, err.map(Into::into), absence, - _write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()), + _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()), bucket, object, )) diff --git a/crates/ecstore/src/set_disk/ops/mod.rs b/crates/ecstore/src/set_disk/ops/mod.rs index 4cea63793..5dfaa66b5 100644 --- a/crates/ecstore/src/set_disk/ops/mod.rs +++ b/crates/ecstore/src/set_disk/ops/mod.rs @@ -19,6 +19,7 @@ pub(crate) mod bitrot_self_verify; pub(crate) mod bucket; +pub(crate) mod delete_recovery; pub(crate) mod heal; pub(crate) mod heal_walk; pub(crate) mod list; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 791873248..b35eb4cb7 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -64,6 +64,34 @@ use super::super::{ take_prepared_get_object_metadata, to_object_err, try_read_inline_data_shards_direct, warn, }; use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards}; +use super::delete_recovery::DeleteRecoveryContext; + +#[derive(Default)] +struct DeleteCommitContext<'a> { + incarnation: Option, + options: Option<&'a ObjectOptions>, + guard: Option>, +} + +impl<'a> DeleteCommitContext<'a> { + fn request(options: &'a ObjectOptions, guard: Option<&Arc>) -> Self { + Self { + incarnation: options.expected_bucket_incarnation_id, + options: Some(options), + guard: guard.cloned(), + } + } + + fn can_commit(&self, bucket: &str, object: &str) -> bool { + self.options.is_none_or(|options| { + ensure_delete_commit_locks_held(self.guard.as_deref(), bucket, object, options).is_ok() + && options + .scanner_publication_commit_scope + .as_ref() + .is_none_or(|scope| scope.can_commit()) + }) + } +} fn delete_marker_purge_candidate( file_info: &FileInfo, @@ -338,7 +366,7 @@ use crate::storage_api_contracts::object::HTTPPreconditions; use crate::store::ECStore; use crate::store::utils::clean_metadata; use crate::store::{RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence}; -use futures::FutureExt as _; +use futures::{FutureExt as _, StreamExt as _}; use http::HeaderValue; #[cfg(test)] use rustfs_filemeta::FileMeta; @@ -3308,6 +3336,76 @@ pub(in crate::set_disk) fn merge_replication_metadata_lww( } impl SetDisks { + async fn reclaim_missing_delete_residue( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + lock_guard: Option<&Arc>, + ) -> Result { + if is_meta_bucketname(bucket) + || opts.version_id.is_some() + || opts.versioned + || opts.version_suspended + || opts.http_preconditions.is_some() + || opts.data_movement + || opts.transition.expire_restored + || opts.replication_request + || opts.lifecycle_delete_all.is_some() + || opts.delete_prefix + || opts.expected_current_version_id.is_some() + || !metadata_sys::has_authoritative_never_versioned_state_in(&self.ctx, bucket) + .await + .unwrap_or(false) + { + return Ok(false); + } + let lock_state = match opts.object_lock_config_snapshot.as_ref() { + Some(snapshot) => Some(snapshot.state().clone()), + None => metadata_sys::get_object_lock_config_state_in(&self.ctx, bucket).await.ok(), + }; + if !matches!(lock_state, Some(ObjectLockConfigState::ConfirmedAbsent)) { + return Ok(false); + } + if let Some(expected) = opts.expected_bucket_incarnation_id { + self.validate_bucket_incarnation(bucket, expected, opts).await?; + } + ensure_delete_commit_locks_held(lock_guard.map(Arc::as_ref), bucket, object, opts)?; + begin_scanner_publication_delete_mutation(opts.scanner_publication_commit_scope.as_ref())?; + let result = self + .recover_delete_key( + bucket, + object, + DeleteRecoveryContext { + explicit_delete: true, + namespace_owner: Some(Arc::new(( + self.ctx.begin_namespace_commit(), + lock_guard.cloned(), + opts.object_lock_config_snapshot.clone(), + opts.scanner_publication_commit_scope.clone(), + opts.write_lock_context.clone(), + opts.delete_mutation_owner.clone(), + ))), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(lock_guard.map(Arc::as_ref), bucket, object, opts).is_ok() + && opts + .scanner_publication_commit_scope + .as_ref() + .is_none_or(|scope| scope.can_commit()) + }) + }, + ) + .await; + ensure_delete_commit_locks_held(lock_guard.map(Arc::as_ref), bucket, object, opts)?; + let recovered = result.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?; + if recovered.pending > 0 && !recovered.no_transactions_at_quorum { + return Err(StorageError::SlowDown); + } + if recovered.blocked > 0 { + return Err(StorageError::SlowDown); + } + Ok(recovered.reclaimed > 0 || recovered.committed > 0) + } pub(in crate::set_disk) async fn cleanup_rename_tail( &self, targets: Vec, @@ -7612,7 +7710,7 @@ impl SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(opts, None), ) .await?; if let Some((_, deleted_object)) = replication_delete { @@ -7686,7 +7784,7 @@ impl SetDisks { Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended)) } - #[tracing::instrument(skip(self, delete_marker_purge))] + #[tracing::instrument(skip(self, delete_marker_purge, commit_context))] async fn delete_object_version_with_purge( &self, bucket: &str, @@ -7694,12 +7792,28 @@ impl SetDisks { fi: &FileInfo, force_del_marker: bool, delete_marker_purge: Option, - source_bucket_incarnation_id: Option, + commit_context: DeleteCommitContext<'_>, ) -> Result<()> { + let source_bucket_incarnation_id = commit_context.incarnation; let transported = delete_file_info_with_replication_transport_metadata(fi); let fi = &transported; let disks = self.disk_inventory().await; - let namespace_owner = (!is_meta_bucketname(bucket)).then(|| self.ctx.begin_namespace_commit()); + let namespace_owner = (!is_meta_bucketname(bucket)).then(|| { + Arc::new(( + self.ctx.begin_namespace_commit(), + commit_context.guard.clone(), + commit_context + .options + .and_then(|options| options.object_lock_config_snapshot.clone()), + commit_context + .options + .and_then(|options| options.scanner_publication_commit_scope.clone()), + commit_context.options.and_then(|options| options.write_lock_context.clone()), + commit_context + .options + .and_then(|options| options.delete_mutation_owner.clone()), + )) as Arc + }); // Quorum is a property of the configured set, not of the current // online snapshot. A decommission/offline refresh may temporarily // shorten the snapshot; deriving quorum from it would turn a @@ -7714,20 +7828,33 @@ impl SetDisks { let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc); futures.push(async move { if let Some(disk) = disk { - match disk - .delete_version_with_namespace_owner( + let options = DeleteOptions { + old_data_dir: Some(rollback_dir), + delete_recovery_bucket_incarnation: source_bucket_incarnation_id, + ..Default::default() + }; + let result = if is_meta_bucketname(bucket) { + disk.delete_version_with_namespace_owner( bucket, object, fi.clone(), force_del_marker, - DeleteOptions { - old_data_dir: Some(rollback_dir), - ..Default::default() - }, + options, disk_namespace_owner, ) .await - { + } else { + disk.delete_version_recoverable_with_namespace_owner( + bucket, + object, + fi.clone(), + force_del_marker, + options, + disk_namespace_owner, + ) + .await + }; + match result { Ok(r) => Ok(r), Err(e) => Err(e), } @@ -7749,12 +7876,34 @@ impl SetDisks { } } - let quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object); + let mut quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object); + if !is_meta_bucketname(bucket) && errs.iter().flatten().any(super::delete_recovery::unsupported_recovery) { + quorum_result = Err(DiskError::UnsupportedDisk.into()); + } let should_rollback = quorum_result.is_err(); + let structured_cleanup = !is_meta_bucketname(bucket) && !should_rollback; + if structured_cleanup { + self.publish_delete_commit( + bucket, + object, + rollback_dir, + &disks, + write_quorum, + DeleteRecoveryContext { + namespace_owner: namespace_owner.clone().map(|owner| owner as Arc), + ..DeleteRecoveryContext::new(source_bucket_incarnation_id, &|| commit_context.can_commit(bucket, object)) + }, + ) + .await + .map_err(StorageError::from)?; + } #[cfg(test)] pause_delete_before_cleanup(bucket).await; let mut rollback_futures = Vec::new(); for (index, err) in errs.iter().enumerate() { + if structured_cleanup { + break; + } // backlog#1158: when rolling back, fan the idempotent undo out to every // online disk (each self-decides from its staged backup: restore if the // rollback dir is present, no-op otherwise). This covers a disk that @@ -7810,6 +7959,7 @@ impl SetDisks { DeleteOptions { recursive: true, immediate: true, + old_data_dir: Some(rollback_dir), ..Default::default() }, disk_namespace_owner, @@ -8192,8 +8342,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } else { self.bucket_incarnation_id_from_disk(bucket).await.ok() }; - self.delete_object_version_with_purge(bucket, object, fi, force_del_marker, None, source_bucket_incarnation_id) - .await + self.delete_object_version_with_purge( + bucket, + object, + fi, + force_del_marker, + None, + DeleteCommitContext { + incarnation: source_bucket_incarnation_id, + ..Default::default() + }, + ) + .await } #[tracing::instrument(skip(self, objects, opts))] @@ -8313,6 +8473,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let mut tier_reference_leases: Vec<(usize, String, Option)> = Vec::new(); let mut tier_free_version_receipt_candidates: HashMap = HashMap::new(); let mut delete_marker_purge_candidates = vec![None; objects.len()]; + let mut missing_delete_retry_indices = Vec::new(); + let mut deferred_recovery_indices = Vec::new(); for (i, dobj) in objects.iter().enumerate() { if del_errs[i].is_some() { @@ -8343,7 +8505,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id; let replication_needs_source = replicate_delete && (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name)); - let (goi, authoritative_file_info, gerr) = if object_lock_check_required + let (mut goi, mut authoritative_file_info, mut gerr) = if object_lock_check_required || replication_needs_source || opts.tier_delete_journal_api.is_some() || dobj.expected_identity.is_some() @@ -8356,9 +8518,74 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } else { (ObjectInfo::default(), FileInfo::default(), None) }; + if gerr + .as_ref() + .is_some_and(|error| is_err_object_not_found(error) || is_err_version_not_found(error)) + && !is_meta_bucketname(bucket) + && !opts.data_movement + && !opts.replication_request + && opts.http_preconditions.is_none() + { + let recovery = self + .recover_delete_key( + bucket, + &dobj.object_name, + DeleteRecoveryContext { + dry_run: !opts.no_lock, + namespace_owner: opts.delete_mutation_owner.clone(), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(None, bucket, &dobj.object_name, &opts).is_ok() + }) + }, + ) + .await; + match recovery { + Ok(recovery) if !opts.no_lock && recovery.discovered > 0 && recovery.pending == 0 => { + deferred_recovery_indices.push(i); + continue; + } + Ok(recovery) + if (recovery.pending > 0 + && !(recovery.no_transactions_at_quorum + && dobj.version_id.is_none() + && !versioned + && !version_suspended)) + || (recovery.blocked > 0 && (dobj.version_id.is_some() || versioned || version_suspended)) => + { + del_errs[i] = Some(if recovery.pending > 0 { + StorageError::InsufficientWriteQuorum(bucket.to_owned(), dobj.object_name.clone()) + } else { + StorageError::SlowDown + }); + continue; + } + Ok(recovery) if recovery.rolled_back > 0 || recovery.committed > 0 => { + let (fresh, info, _, error) = self + .get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts) + .await; + goi = fresh; + authoritative_file_info = info; + gerr = error; + } + Ok(_) => {} + Err(error) => { + del_errs[i] = Some(to_object_err(error.into(), vec![bucket, &dobj.object_name])); + continue; + } + } + } let source_missing = gerr .as_ref() .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)); + if source_missing + && dobj.version_id.is_none() + && dobj.expected_identity.is_none() + && !versioned + && !version_suspended + && opts.http_preconditions.is_none() + { + missing_delete_retry_indices.push(i); + } // A pool-local miss keeps the existing idempotent batch-delete // semantics. Exact-version deletes fan out to every pool, and a // retry may also arrive after the intended generation is gone. @@ -8596,10 +8823,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } let rollback_dir = Uuid::new_v4(); + let namespace_owner = (!is_meta_bucketname(bucket)).then(|| { + Arc::new(( + self.ctx.begin_namespace_commit(), + opts.delete_mutation_owner.clone(), + opts.object_lock_config_snapshot.clone(), + opts.scanner_publication_commit_scope.clone(), + opts.write_lock_context.clone(), + )) as Arc + }); - let disks = self.disks.read().await; - - let disks = disks.clone(); + let disks = self.disks.read().await.clone(); if opts .namespace_lock_fence @@ -8668,6 +8902,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // let mut errors = Vec::with_capacity(disks.len()); for disk in disks.iter() { + let disk_namespace_owner = namespace_owner.clone(); let vers = vers.clone(); let namespace_lock_fence = opts.namespace_lock_fence.clone(); let lock_lost_during_commit = Arc::clone(&lock_lost_during_commit); @@ -8677,15 +8912,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { return (0..vers.len()).map(|_| Some(DiskError::DiskOngoingReq)).collect(); } if let Some(disk) = disk { - disk.delete_versions( - bucket, - vers, - DeleteOptions { - old_data_dir: Some(rollback_dir), - ..Default::default() - }, - ) - .await + let options = DeleteOptions { + old_data_dir: Some(rollback_dir), + delete_recovery_bucket_incarnation: opts.expected_bucket_incarnation_id, + ..Default::default() + }; + if is_meta_bucketname(bucket) { + disk.delete_versions(bucket, vers, options).await + } else { + disk.delete_versions_recoverable_with_namespace_owner(bucket, vers, options, disk_namespace_owner) + .await + } } else { let mut errs = Vec::with_capacity(vers.len()); for _ in 0..vers.len() { @@ -8721,7 +8958,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } - let mut has_err = reduce_write_quorum_errs(&disk_err, OBJECT_OP_IGNORED_ERRS, disks.len() / 2 + 1); + let mut has_err = reduce_write_quorum_errs(&disk_err, OBJECT_OP_IGNORED_ERRS, self.set_drive_count / 2 + 1); + if !is_meta_bucketname(bucket) && disk_err.iter().flatten().any(super::delete_recovery::unsupported_recovery) { + has_err = Some(DiskError::UnsupportedDisk); + } if let Some(err) = has_err.clone() { let er = err.into(); if (is_err_object_not_found(&er) || is_err_version_not_found(&er)) && !del_objects[obj_idx].delete_marker { @@ -8765,10 +9005,67 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { #[cfg(test)] pause_delete_before_cleanup(bucket).await; + let mut published_groups = HashSet::new(); + if !is_meta_bucketname(bucket) { + let groups = vers.iter().enumerate().filter_map(|(index, group)| { + group + .versions + .iter() + .all(|version| del_errs[version.idx].is_none()) + .then_some(index) + }); + // Mark all attempted groups before starting publication. A lost + // response can hide a persisted decision and must never roll back. + let groups = groups.collect::>(); + published_groups.extend(groups.iter().map(|index| vers[*index].name.clone())); + let publications = futures::stream::iter(groups.into_iter().map(|index: usize| { + let group = &vers[index]; + let namespace_owner = namespace_owner.clone(); + let disks = &disks; + let opts = &opts; + async move { + let can_commit = || { + ensure_delete_commit_locks_held(None, bucket, &group.name, opts).is_ok() + && opts + .scanner_publication_commit_scope + .as_ref() + .is_none_or(|scope| scope.can_commit()) + }; + let result = self + .publish_delete_commit( + bucket, + &group.name, + rollback_dir, + disks, + self.set_drive_count / 2 + 1, + DeleteRecoveryContext { + namespace_owner, + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &can_commit) + }, + ) + .await; + (index, result) + } + })) + .buffer_unordered(16) + .collect::>() + .await; + for (index, result) in publications { + let group = &vers[index]; + if let Err(error) = result { + for version in &group.versions { + del_errs[version.idx] = Some(to_object_err(error.clone().into(), vec![bucket, &group.name])); + } + } + } + } let mut rollback_futures = Vec::new(); let committed_receipt_indices = committed_tier_free_version_receipt_indices(&vers, &del_errs, &tier_free_version_receipt_candidates); for fi_vers in &vers { + if published_groups.contains(&fi_vers.name) { + continue; + } // delete_versions commits one xl.meta per object group, so rollback must use the same boundary. let should_rollback = fi_vers.versions.iter().any(|fi| del_errs[fi.idx].is_some()); for (disk_idx, disk) in disks.iter().enumerate() { @@ -8788,10 +9085,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let bucket = bucket.to_string(); let object = fi_vers.name.clone(); let versions = fi_vers.clone(); + let disk_namespace_owner = namespace_owner.clone(); rollback_futures.push(async move { if should_rollback { let errs = disk - .delete_versions( + .delete_versions_with_namespace_owner( &bucket, vec![versions], DeleteOptions { @@ -8800,6 +9098,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { old_data_dir: Some(rollback_dir), ..Default::default() }, + disk_namespace_owner, ) .await; if let Some(err) = errs.into_iter().flatten().next() { @@ -8820,6 +9119,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { DeleteOptions { recursive: true, immediate: true, + old_data_dir: Some(rollback_dir), ..Default::default() }, ) @@ -8856,12 +9156,70 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { ); } + // A new explicit DELETE authorizes removal of the exact missing key's + // legacy residue; it does not infer whether the interrupted delete committed. + for index in missing_delete_retry_indices { + if del_errs[index].is_none() { + let object = &objects[index].object_name; + let result = match ensure_delete_commit_locks_held(None, bucket, object, &opts) { + Ok(()) => self.reclaim_missing_delete_residue(bucket, object, &opts, None).await, + Err(err) => Err(err), + }; + if let Err(err) = result { + del_errs[index] = Some(err); + } + } + } + // TODO(backlog): support partial object deletion for multi-part objects if dist_erasure { self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await; } + // Cold missing-key replay uses the same batch contract after the fast + // batch lock is released; renewable key ownership survives settlement. + for index in deferred_recovery_indices { + let object = &objects[index]; + let explicit_delete = object.version_id.is_none() + && !object.synthetic_version_id + && !opts.versioned + && !opts.version_suspended + && opts.http_preconditions.is_none() + && metadata_sys::has_authoritative_never_versioned_state_in(&self.ctx, bucket) + .await + .unwrap_or(false) + && matches!(object_lock_config.state(), ObjectLockConfigState::ConfirmedAbsent); + let recovery = self + .recover_delete_object( + bucket, + &object.object_name, + DeleteRecoveryContext { + explicit_delete, + namespace_owner: Some(Arc::new(( + self.ctx.begin_namespace_commit(), + opts.delete_mutation_owner.clone(), + opts.object_lock_config_snapshot.clone(), + ))), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(None, bucket, &object.object_name, &opts).is_ok() + }) + }, + ) + .await; + match recovery { + Ok(recovery) if recovery.pending == 0 && recovery.blocked == 0 => { + let (mut deleted, mut errors, mut receipts) = + Box::pin(self.delete_objects_with_accounting(bucket, vec![object.clone()], opts.clone())).await; + del_objects[index] = deleted.remove(0); + del_errs[index] = errors.remove(0); + accounting[index] = receipts.remove(0); + } + Ok(_) => del_errs[index] = Some(StorageError::SlowDown), + Err(error) => del_errs[index] = Some(to_object_err(error.into(), vec![bucket, &object.object_name])), + } + } + let purge_submissions = delete_marker_purge_candidates .into_iter() .enumerate() @@ -8918,7 +9276,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // Guard lock for single object delete let _lock_guard = if (!opts.delete_prefix || opts.delete_prefix_object) && !opts.no_lock { - Some(self.acquire_write_lock_diag("delete_object", bucket, object).await?) + Some(Arc::new(self.acquire_write_lock_diag("delete_object", bucket, object).await?)) } else { None }; @@ -8988,7 +9346,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } }; for version in plan { - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; // The lease closes the gap between the durable // reference proof and publishing either a committed // v6 journal or the legacy free-version fallback. @@ -9022,7 +9380,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await?; if let Some((_, deleted_object)) = replication_delete { @@ -9031,7 +9389,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } else { for version in &versions.versions { - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; let mut delete_request = FileInfo { name: object.to_string(), version_id: version.version_id, @@ -9045,12 +9403,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await?; } for version in &versions.free_versions { - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; let mut delete_request = FileInfo { name: object.to_string(), version_id: version.version_id, @@ -9065,7 +9423,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await?; } @@ -9081,7 +9439,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { self.validate_bucket_incarnation(bucket, expected_incarnation_id, &opts) .await?; } - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; self.delete_prefix_with_scanner_publication_lease( bucket, @@ -9132,14 +9490,73 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let mut version_found = true; // delete_object_version below derives its own majority quorum from the // disk array, so the object-derived quorum here is unused. - let (mut goi, authoritative_file_info, _write_quorum, gerr) = + let (mut goi, mut authoritative_file_info, _write_quorum, mut gerr) = self.get_object_info_fileinfo_and_quorum(bucket, object, &opts).await; + if gerr + .as_ref() + .is_some_and(|error| is_err_object_not_found(error) || is_err_version_not_found(error)) + && !is_meta_bucketname(bucket) + && !opts.data_movement + && !opts.replication_request + && opts.http_preconditions.is_none() + { + if let Some(expected) = opts.expected_bucket_incarnation_id { + self.validate_bucket_incarnation(bucket, expected, &opts).await?; + } + let recovery = self + .recover_delete_key( + bucket, + object, + DeleteRecoveryContext { + namespace_owner: Some(Arc::new(( + self.ctx.begin_namespace_commit(), + _lock_guard.clone(), + opts.object_lock_config_snapshot.clone(), + opts.scanner_publication_commit_scope.clone(), + opts.write_lock_context.clone(), + opts.delete_mutation_owner.clone(), + ))), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts).is_ok() + }) + }, + ) + .await?; + if (recovery.pending > 0 + && !(recovery.no_transactions_at_quorum + && opts.version_id.is_none() + && !opts.versioned + && !opts.version_suspended)) + || (recovery.blocked > 0 && (opts.version_id.is_some() || opts.versioned || opts.version_suspended)) + { + return Err(if recovery.pending > 0 { + StorageError::InsufficientWriteQuorum(bucket.to_owned(), object.to_owned()) + } else { + StorageError::SlowDown + }); + } + if recovery.rolled_back > 0 || recovery.committed > 0 { + (goi, authoritative_file_info, _, gerr) = self.get_object_info_fileinfo_and_quorum(bucket, object, &opts).await; + } + } if let Some(err) = &gerr && goi.name.is_empty() { if should_force_delete_marker_for_missing_version(&opts) { version_found = false; } else { + if is_err_object_not_found(err) { + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; + if self + .reclaim_missing_delete_residue(bucket, object, &opts, _lock_guard.as_ref()) + .await? + { + if let Some(scope) = scanner_publication_commit_scope.as_ref() { + let _ = scope.mark_committed(); + } + return Ok(ObjectInfo::default()); + } + } return Err(err.clone()); } } @@ -9173,11 +9590,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { expire_restored: true, ..Default::default() }; - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; - self.delete_object_version_with_purge(bucket, object, &dfi, false, None, opts.expected_bucket_incarnation_id) - .await - .map_err(|e| to_object_err(e, vec![bucket, object]))?; + self.delete_object_version_with_purge( + bucket, + object, + &dfi, + false, + None, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), + ) + .await + .map_err(|e| to_object_err(e, vec![bucket, object]))?; self.invalidate_get_object_metadata_cache(bucket, object).await; if let Some(scope) = scanner_publication_commit_scope.as_ref() { let _ = scope.mark_committed(); @@ -9257,7 +9681,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { None }; - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; let _tier_delete_lease = acquire_single_tier_delete_lease(&opts, &goi).await?; if opts.skip_free_version { @@ -9269,7 +9693,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &fi, should_force_delete_marker_for_missing_version(&opts), delete_marker_purge.clone(), - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -9316,7 +9740,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { dfi.set_delete_marker_incarnation(incarnation); } - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; let _tier_delete_lease = acquire_single_tier_delete_lease(&opts, &goi).await?; if opts.skip_free_version { @@ -9328,7 +9752,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &dfi, opts.delete_marker, delete_marker_purge, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -9959,7 +10383,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { pause_transition_transaction_at(bucket, object, TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit).await; upload_cleanup.disarm(); if let Err(err) = self - .delete_object_version_with_purge(bucket, object, &fi, false, None, opts.expected_bucket_incarnation_id) + .delete_object_version_with_purge(bucket, object, &fi, false, None, DeleteCommitContext::request(opts, None)) .await { warn!( @@ -22994,9 +23418,19 @@ mod marker_purge_receipt_tests { .acquire_write_lock_diag("marker_purge_receipt_test", bucket, object) .await .expect("own the deleted object namespace"); - set.delete_object_version_with_purge(bucket, object, &marker, true, None, Some(incarnation)) - .await - .expect("seed the real marker on every disk"); + set.delete_object_version_with_purge( + bucket, + object, + &marker, + true, + None, + DeleteCommitContext { + incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the real marker on every disk"); let marker = disks[0] .read_version("", bucket, object, &version.to_string(), &ReadOptions::default()) .await @@ -23014,7 +23448,17 @@ mod marker_purge_receipt_tests { set.disks.write().await[0] = None; tokio::time::timeout( Duration::from_secs(10), - set.delete_object_version_with_purge(bucket, object, &request, false, Some(purge.clone()), Some(incarnation)), + set.delete_object_version_with_purge( + bucket, + object, + &request, + false, + Some(purge.clone()), + DeleteCommitContext { + incarnation: Some(incarnation), + ..Default::default() + }, + ), ) .await .expect("the receipt must not reacquire the deleted object's owner") @@ -23202,6 +23646,54 @@ mod single_delete_namespace_owner_tests { ); } + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn batch_delete_uses_one_namespace_owner_across_object_groups() { + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "batch-delete-namespace"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let mut objects = Vec::new(); + for key in ["inline", "external"] { + let version = Uuid::new_v4(); + let body = vec![0x41; if key == "inline" { 4096 } else { 256 * 1024 }]; + seed_version(&set, bucket, key, version, &body).await; + objects.push(ObjectToDelete { + object_name: key.to_owned(), + version_id: Some(version), + ..Default::default() + }); + } + let before = set.ctx.namespace_commit_generation(); + assert!(!set.ctx.namespace_commits_pending()); + let (_, errors, _) = set + .delete_objects_with_accounting( + bucket, + objects, + ObjectOptions { + versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await; + assert!(errors.iter().all(Option::is_none), "batch groups must commit: {errors:?}"); + for dir in &dirs { + for key in ["inline", "external"] { + assert!(!dir.path().join(bucket).join(key).exists(), "all group evidence must converge"); + } + } + assert!(!set.ctx.namespace_commits_pending()); + assert_eq!( + set.ctx.namespace_commit_generation(), + before + 2, + "all batch groups share one complete owner lifetime" + ); + } + #[cfg(not(windows))] async fn assert_single_delete_physical_owner(case: &'static str) { use crate::disk::os::prepared_publication_test_hooks as hooks; @@ -23280,9 +23772,14 @@ mod single_delete_namespace_owner_tests { for (index, path) in metadata_paths.iter().enumerate() { let tx = entered_tx.clone(); let (release, rx) = std::sync::mpsc::channel::<()>(); - if last || cleanup { - let source = if cleanup { &cleanup_sources[index] } else { path }; - destination_guards.push(hooks::observe_rename_destination(source, move |destination| { + if cleanup { + let source = cleanup_sources[index].clone(); + guards.push(hooks::install_at(hooks::Stage::Remove, &source.join("part.1"), move || { + let _ = tx.send((index, source)); + let _ = rx.recv(); + })); + } else if last { + destination_guards.push(hooks::observe_rename_destination(path, move |destination| { let _ = tx.send((index, destination.to_path_buf())); let _ = rx.recv(); })); @@ -23355,6 +23852,12 @@ mod single_delete_namespace_owner_tests { } } } + if cleanup { + for (index, source) in cleanup_sources.iter().enumerate() { + assert_eq!(std::fs::read(source.join("part.1")).expect("paused original shard"), cleanup_parts[index], + "the paused physical cleanup must still own the exact original shard"); + } + } if rollback { tokio::time::pause(); tokio::time::advance(Duration::from_secs(61)).await; @@ -23431,11 +23934,7 @@ mod single_delete_namespace_owner_tests { if cleanup { assert!(!path.exists(), "metadata must remain deleted after cleanup cancellation"); assert!(!cleanup_sources[index].exists(), "physical cleanup must remove the shard directory"); - assert_eq!( - std::fs::read(physical_keys[&index].join("part.1")).expect("actual trashed shard"), - cleanup_parts[index], - "trash must contain the exact original shard" - ); + assert!(!physical_keys[&index].exists(), "the owned physical cleanup must remove its original shard directory"); continue; } if last { @@ -23545,3 +24044,431 @@ mod single_delete_namespace_owner_tests { assert_single_delete_physical_owner("rollback").await; } } + +#[cfg(test)] +mod crash_delete_retry_tests { + use super::*; + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::disk::STORAGE_FORMAT_FILE_BACKUP; + use crate::storage_api_contracts::bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions}; + + async fn stage_interrupted_delete(store: &Arc, dirs: &[tempfile::TempDir], bucket: &str, object: &str, size: usize) { + let mut reader = PutObjReader::from_vec(vec![37; size]); + store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("write the object before simulating an interrupted delete"); + let transaction = Uuid::new_v4(); + for dir in dirs { + let root = dir.path().join(bucket).join(object); + let xl = root.join(STORAGE_FORMAT_FILE); + let bytes = tokio::fs::read(&xl).await.expect("read original object metadata"); + let mut metadata = FileMeta::default(); + metadata.unmarshal_msg(&bytes).expect("decode original object metadata"); + let info = metadata.versions[0] + .into_fileinfo(bucket, object, false) + .expect("read original version"); + let backup = root.join(transaction.to_string()); + tokio::fs::create_dir_all(&backup) + .await + .expect("create transaction directory"); + tokio::fs::write(backup.join(STORAGE_FORMAT_FILE_BACKUP), bytes) + .await + .expect("stage intact rollback backup"); + if let Some(data_dir) = info.data_dir { + let parts = root.join(data_dir.to_string()); + if parts.is_dir() { + tokio::fs::write( + parts.join(format!("{}{}", crate::disk::local::RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, transaction)), + [], + ) + .await + .expect("reserve external deletion before its local commit"); + } + } + tokio::fs::remove_file(&xl) + .await + .expect("simulate crash after removing metadata"); + } + } + + #[tokio::test] + async fn crash_delete_retry_single_reclaims_inline_backup_only() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-single"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + store + .delete_object(bucket, "inline", ObjectOptions::default()) + .await + .expect("retry should reclaim the exact missing key"); + for dir in &dirs { + assert!( + !dir.path().join(bucket).join("inline").exists(), + "inline backup residue must be removed on every disk" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("the empty bucket must be deletable without force"); + } + + #[tokio::test] + async fn crash_delete_retry_batch_reclaims_inline_and_reserved_external() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-batch"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + stage_interrupted_delete(&store, &dirs, bucket, "external", 1024 * 1024).await; + let objects = ["inline", "external"] + .into_iter() + .map(|name| ObjectToDelete { + object_name: name.to_string(), + ..Default::default() + }) + .collect(); + let (_, errors) = store.delete_objects(bucket, objects, ObjectOptions::default()).await; + assert!( + errors.iter().all(Option::is_none), + "retry must complete for both crash shapes: {errors:?}" + ); + for dir in &dirs { + for object in ["inline", "external"] { + assert!(!dir.path().join(bucket).join(object).exists(), "retry must remove {object} on every disk"); + } + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("batch retry must unblock non-force bucket deletion"); + } + + #[tokio::test] + async fn crash_delete_retry_preserves_conditional_and_versioned_residue() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-gates"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + for opts in [ + ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_match: Some("old-etag".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ObjectOptions { + version_id: Some(Uuid::new_v4().to_string()), + ..Default::default() + }, + ] { + let _ = store.delete_object(bucket, "inline", opts).await; + for dir in &dirs { + assert!( + dir.path().join(bucket).join("inline").exists(), + "conditional/version-specific deletes must retain ambiguous backups" + ); + } + } + } + #[tokio::test] + async fn crash_delete_retry_preserves_suspended_bucket_null_backup() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-suspended"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + metadata_sys::update_in(&store.ctx, bucket, crate::bucket::metadata::BUCKET_VERSIONING_CONFIG, + b"Suspended".to_vec()) + .await.expect("persist suspended versioning state"); + let _ = store.delete_object(bucket, "inline", ObjectOptions::default()).await; + for dir in &dirs { + assert!( + dir.path().join(bucket).join("inline").exists(), + "authoritative suspended state must protect a null-version backup" + ); + } + } + + #[tokio::test] + async fn crash_delete_retry_rejects_stale_bucket_incarnation_at_set_boundary() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-incarnation"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + let error = store.pools[0].disk_set[0] + .delete_object( + bucket, + "inline", + ObjectOptions { + expected_bucket_incarnation_id: Some(Uuid::new_v4()), + ..Default::default() + }, + ) + .await + .expect_err("the set must reject a stale authorized bucket identity"); + assert!( + matches!(error, StorageError::BucketNotFound(_)), + "identity mismatch must retain its typed error: {error:?}" + ); + for dir in &dirs { + assert!( + dir.path().join(bucket).join("inline").exists(), + "a stale request must leave every backup intact" + ); + } + } + #[tokio::test] + #[serial_test::serial(shard_integrity_rollout)] + async fn batch_delete_completed_multipart_reclaims_native_part_sidecars() { + assert_completed_multipart_delete_reclaims_sidecars(false).await; + } + + #[tokio::test] + #[serial_test::serial(shard_integrity_rollout)] + async fn batch_delete_completed_multipart_reclaims_protected_part_sidecars() { + assert_completed_multipart_delete_reclaims_sidecars(true).await; + } + + async fn assert_completed_multipart_delete_reclaims_sidecars(protected: bool) { + temp_env::async_with_vars( + [ + (rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some(if protected { "true" } else { "false" })), + ( + rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, + Some(if protected { "true" } else { "false" }), + ), + ], + assert_completed_multipart_delete_reclaims_sidecars_inner(protected), + ) + .await; + } + + async fn assert_completed_multipart_delete_reclaims_sidecars_inner(protected: bool) { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "multipart-delete-cleanup"; + let object = "completed-object"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let upload = store + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("create native multipart upload"); + let mut reader = PutObjReader::from_vec(vec![37; 6 * 1024 * 1024]); + let part = store + .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .expect("write native part"); + store + .clone() + .complete_multipart_upload( + bucket, + object, + &upload.upload_id, + vec![CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + .expect("complete native multipart upload"); + for dir in &dirs { + let bytes = tokio::fs::read(dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE)) + .await + .expect("completed metadata"); + let mut metadata = FileMeta::default(); + metadata.unmarshal_msg(&bytes).expect("decode completed metadata"); + let mut info = metadata.versions[0] + .into_fileinfo(bucket, object, true) + .expect("completed version"); + info.hydrate_shard_integrity() + .expect("decode native completed integrity descriptor"); + let path = dir + .path() + .join(bucket) + .join(object) + .join(info.data_dir.expect("external multipart directory").to_string()); + assert!(path.join("part.1.meta").exists(), "real multipart completion retains its native sidecar"); + let integrity = info.parts[0].integrity.as_ref(); + assert_eq!(integrity.is_some(), protected, "completion must retain the selected native format"); + if let Some(integrity) = integrity { + assert!( + path.join(integrity.file_name()).exists(), + "real protected completion retains its integrity proof" + ); + } + } + let (_, errors) = store + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await; + assert!(errors.iter().all(Option::is_none), "native batch delete must succeed: {errors:?}"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("successful multipart delete must permit ordinary bucket removal"); + for dir in &dirs { + assert!( + !dir.path().join(bucket).exists(), + "bucket and completed part sidecars must be removed on every disk" + ); + } + } +} + +#[cfg(test)] +mod delete_recovery_inventory_tests { + use super::*; + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use std::future::Future; + use std::task::Poll; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial] + async fn batch_delete_completes_with_queued_disk_renewal() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let set = store.pools[0].disk_set[0].clone(); + let disks = set.disk_inventory().await.into_iter().flatten().collect::>(); + let bucket = "batch-delete-renew"; + let object = "existing-inline-version"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative real bucket"); + store + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x73; 4096]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed actual version"); + let endpoint = set.set_endpoints[0].clone(); + let ctx = crate::runtime::global::current_ctx(); + let was_dist = ctx.is_dist_erasure().await; + if was_dist { + ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await; + } + let map = ctx.local_disk_map(); + let previous = map.read().await.get(&endpoint.to_string()).cloned(); + + let barrier = DeleteCleanupBarrier::install(bucket); + let delete_store = store.clone(); + let mut deletion = tokio::spawn(async move { + delete_store + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: object.to_owned(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await + }); + barrier.wait_until_paused().await; + let extra_gate = set.disks.read().await; + let renewal = set.renew_disk(&endpoint); + tokio::pin!(renewal); + tokio::time::timeout( + Duration::from_secs(5), + futures::future::poll_fn(|cx| { + assert!( + std::pin::pin!(tokio::task::unconstrained(renewal.as_mut())) + .poll(cx) + .is_pending(), + "real renewal must reach inventory write" + ); + if set.disks.try_read().is_err() { + Poll::Ready(()) + } else { + Poll::Pending + } + }), + ) + .await + .expect("queue real renewal writer"); + let connected = map + .read() + .await + .get(&endpoint.to_string()) + .cloned() + .flatten() + .expect("renewal registered new handle"); + assert!(!Arc::ptr_eq(&connected, &disks[0])); + drop(extra_gate); + barrier.release(); + let outcome = tokio::time::timeout(Duration::from_secs(3), async { tokio::join!(renewal.as_mut(), &mut deletion) }).await; + if outcome.is_err() { + deletion.abort(); + } + drop(renewal); + if outcome.is_err() { + let _ = deletion.await; + } + { + let mut map = map.write().await; + match previous { + Some(old) => { + map.insert(endpoint.to_string(), old); + } + None => { + map.remove(&endpoint.to_string()); + } + } + } + if was_dist { + ctx.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure) + .await; + } + let (_, result) = + outcome.expect("batch DELETE and real queued disk renewal must finish without nested inventory-read deadlock"); + let (_, errors) = result.expect("delete task must finish"); + assert!(errors.iter().all(Option::is_none), "DELETE result must remain successful: {errors:?}"); + } +} diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 8e20fa184..fa50a54b4 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -270,6 +270,134 @@ impl ECStore { .map_err(|error| Error::other_with_context("bucket heal owner task failed", error))? } + /// Recover hidden delete journals through the lifecycle-owned heal boundary. + pub async fn recover_delete_transactions(self: &Arc, bucket: &str, opts: &HealOpts) -> Result { + Ok(self + .recover_delete_transactions_at_incarnation(bucket, None, None, opts) + .await? + .unwrap_or_else(|| HealResultItem { + heal_item_type: HealItemType::Bucket.to_string(), + bucket: bucket.to_owned(), + detail: "delete recovery: discovered=0".to_owned(), + ..Default::default() + })) + } + + pub async fn recover_delete_transactions_at_incarnation( + self: &Arc, + bucket: &str, + object: Option<&str>, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.recover_delete_transactions_in_scope(bucket, object, None, expected, opts) + .await + } + + pub async fn recover_delete_prefix_at_incarnation( + self: &Arc, + bucket: &str, + prefix: &str, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.recover_delete_transactions_in_scope(bucket, None, Some(prefix), expected, opts) + .await + } + + async fn recover_delete_transactions_in_scope( + self: &Arc, + bucket: &str, + object: Option<&str>, + prefix: Option<&str>, + expected: Option, + opts: &HealOpts, + ) -> Result> { + if opts.no_lock && !opts.dry_run { + return Err(StorageError::InvalidArgument( + "heal".to_string(), + "no_lock".to_string(), + "delete recovery requires namespace locking".to_string(), + )); + } + let expected = match expected { + Some(expected) => expected, + None => self.bucket_incarnation_id_from_disk(bucket).await?, + }; + let object = object.map(str::to_owned); + let prefix = prefix.map(str::to_owned); + self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move { + let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?; + let explicit_delete = opts.remove + && metadata_sys::has_authoritative_never_versioned_state_in(&store.ctx, &bucket).await? + && matches!( + metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket).await?, + metadata_sys::ObjectLockConfigState::ConfirmedAbsent + ); + let movement_gate = store.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.read().await; + let can_commit = || scope.check().is_ok(); + let recovery_context = crate::set_disk::DeleteRecoveryContext { + dry_run: opts.dry_run, + explicit_delete, + namespace_owner: (!opts.dry_run) + .then(|| Arc::new((scope.clone(), store.ctx.begin_namespace_commit())) as Arc), + ..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &can_commit) + }; + let mut summary = crate::set_disk::DeleteRecoverySummary::default(); + for pool in store.get_pools_for_heal_object(&opts)? { + if store.is_suspended(pool.pool_idx).await || store.is_pool_rebalancing(pool.pool_idx).await { + return Err(StorageError::SlowDown); + } + for (set_index, set) in pool.disk_set.iter().enumerate() { + if opts.set.is_some_and(|requested| requested != set_index) { + continue; + } + if let Some(object) = &object { + if !Arc::ptr_eq(set, &pool.get_disks_by_key(object)) { + continue; + } + summary.add(set.recover_delete_object(&bucket, object, recovery_context.clone()).await?); + } else if let Some(prefix) = &prefix { + summary.add(set.recover_delete_prefix(&bucket, prefix, recovery_context.clone()).await?); + } else { + summary.add(set.recover_delete_bucket(&bucket, recovery_context.clone()).await?); + } + } + } + if summary.pending > 0 { + return Err(StorageError::ErasureReadQuorum); + } + if object.is_some() && !opts.dry_run && summary.blocked > 0 { + return Err(StorageError::SlowDown); + } + if summary.discovered == 0 { + return Ok(None); + } + Ok(Some(HealResultItem { + heal_item_type: if object.is_some() { + HealItemType::Object + } else { + HealItemType::Bucket + } + .to_string(), + object: object.or(prefix).unwrap_or_default(), + bucket, + detail: format!( + "delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}", + if opts.dry_run { " (dry run)" } else { "" }, + summary.discovered, + summary.committed, + summary.rolled_back, + summary.reclaimed, + summary.blocked + ), + ..Default::default() + })) + }) + .await + } + pub async fn heal_bucket_at_incarnation( self: &Arc, bucket: &str, diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 4d287a702..9e16d5b90 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -4849,7 +4849,9 @@ impl ECStore { opts: ObjectOptions, guards: impl IntoIterator, ) -> Result { - let guards: Vec<_> = guards.into_iter().collect(); + let guards = Arc::new(guards.into_iter().collect::>()); + let mut opts = opts; + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), guards.clone()))); let store = Arc::clone(self); let bucket = bucket.to_owned(); let object = object.to_owned(); @@ -4861,7 +4863,7 @@ impl ECStore { if result.is_ok() { list_objects::observe_list_objects_mutation(&store, &bucket).await; } - // Keep: the only use of `guards` here, it moves them into the task so the locks outlive a dropped caller. + // Transient options also retain these guards through late disk I/O. drop(guards); result }) @@ -4930,7 +4932,7 @@ impl ECStore { let mut opts = opts; let receipt_sink = install_tier_free_version_receipt_sink(&mut opts); let result = self - .handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api) + .handle_delete_object_with_journal_inner_boxed(bucket, object, opts, tier_journal_api) .await; enqueue_recorded_tier_free_versions(self, receipt_sink).await; result @@ -4938,6 +4940,16 @@ impl ECStore { .await } + fn handle_delete_object_with_journal_inner_boxed<'a>( + &'a self, + bucket: &'a str, + object: &'a str, + opts: ObjectOptions, + tier_journal_api: Option>, + ) -> impl std::future::Future> + Send + 'a { + Box::pin(self.handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api)) + } + async fn handle_delete_object_with_journal_inner( &self, bucket: &str, @@ -4955,9 +4967,9 @@ impl ECStore { if is_meta_bucketname(bucket) || (opts.delete_prefix && opts.bucket_lifecycle_lock_fence.is_some()) { None } else if opts.delete_prefix { - Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?) + Some(Arc::new(self.acquire_bucket_lifecycle_write_lock(bucket).await?)) } else { - Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?) + Some(Arc::new(self.acquire_bucket_lifecycle_read_lock(bucket).await?)) }; let object = if opts.delete_prefix && !opts.delete_prefix_object { object.to_owned() @@ -4975,6 +4987,7 @@ impl ECStore { opts.tier_delete_journal_api = tier_journal_api.clone(); if let Some(guard) = _bucket_lifecycle_guard.as_ref() { opts.add_bucket_lifecycle_lock_guard(guard); + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), guard.clone()))); } if !is_meta_bucketname(bucket) { @@ -5227,6 +5240,40 @@ impl ECStore { return Ok(obj); } Err(err) => { + if (is_err_object_not_found(&err) || is_err_version_not_found(&err)) + && (opts.version_id.is_some() || (!opts.versioned && !opts.version_suspended)) + && opts.http_preconditions.is_none() + && !opts.replication_request + { + let mut reclaimed = None; + for pool in &self.pools { + if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await { + return Err(StorageError::SlowDown); + } + let pool_idx = pool.pool_idx; + let pool = pool.clone(); + match self + .run_external_decommission_capacity_object_delete( + pool_idx, + bucket, + object, + object, + opts.clone(), + |opts| async move { pool.delete_object(bucket, object, opts).await }, + ) + .await + { + Ok(result) => reclaimed = Some(result), + Err(missing) if is_err_object_not_found(&missing) || is_err_version_not_found(&missing) => {} + Err(error) => return Err(error), + } + } + if let Some(mut result) = reclaimed { + result.bucket = bucket.to_owned(); + result.name = decode_dir_object(object); + return Ok(result); + } + } // A folder key (`prefix/`) with no object metadata may still exist on // disk as an orphan empty-directory tree (issue #4189): listings show // it as a common prefix, but no regular delete path can remove it. @@ -5422,12 +5469,13 @@ impl ECStore { None } else { match self.acquire_bucket_lifecycle_read_lock(bucket).await { - Ok(guard) => Some(guard), + Ok(guard) => Some(Arc::new(guard)), Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), } }; if let Some(guard) = _bucket_lifecycle_guard.as_ref() { opts.add_bucket_lifecycle_lock_guard(guard); + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), guard.clone()))); } if opts.delete_replication_config_snapshot.is_none() { match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await { @@ -5493,10 +5541,11 @@ impl ECStore { if current_bucket_incarnation_id.is_some() { pause_delete_after_object_lock_snapshot(bucket).await; } - let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await { + let _object_lock_guards = Arc::new(match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await { Ok(guards) => guards, Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), - }; + }); + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), _object_lock_guards.clone()))); #[cfg(any(test, feature = "test-util"))] if !_object_lock_guards.is_empty() { notify_delete_namespace_acquired(bucket); @@ -9241,8 +9290,31 @@ mod tests { .expect("read original marker"); assert_eq!(info.delete_marker_incarnation(), Some(old_incarnation)); } - // Taking four actual members offline preserves their existing xl.meta. - // All deletion/recreation work below goes through the normal store API. + // Save each stale member's inline versions before converging the old + // bucket's delete receipts, then reintroduce them after recreation. + let mut stale_metadata = Vec::new(); + for disk in before.iter().skip(12).flatten() { + for version in &historical { + let info = disk + .read_version( + "", + bucket, + "history.bin", + &version.to_string(), + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read stale inline data version"); + assert!(info.inline_data() && info.data.is_some(), "fixture payload must be preserved in xl.meta"); + } + stale_metadata.push(( + disk.read_all(bucket, "history.bin/xl.meta").await.unwrap(), + disk.read_all(bucket, &format!("{object}/xl.meta")).await.unwrap(), + )); + } for slot in 12..16 { set.disks.write().await[slot] = None; } @@ -9272,6 +9344,29 @@ mod tests { ) .await .expect("delete exact old marker with four drives offline"); + let pending = store + .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) + .await + .expect_err("offline participants require retained delete receipts"); + assert!(matches!( + pending, + StorageError::BucketNotEmpty(_) | StorageError::BucketNotEmptyWithDetails { .. } + )); + assert!( + !crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation) + .await + .unwrap() + ); + *set.disks.write().await = before.clone(); + store + .recover_delete_transactions_at_incarnation( + bucket, + None, + Some(old_incarnation), + &rustfs_heal_contracts::heal_channel::HealOpts::default(), + ) + .await + .expect("converge old bucket deletes on all participants"); store .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) .await @@ -9281,6 +9376,9 @@ mod tests { .await .expect("durable retirement") ); + for slot in 12..16 { + set.disks.write().await[slot] = None; + } store .handle_make_bucket(bucket, &create) .await @@ -9306,6 +9404,13 @@ mod tests { .expect("write new generation"); new_versions.push((key, info.version_id.unwrap(), payload)); } + for (slot, (history, marker)) in (12..16).zip(stale_metadata) { + for (key, bytes) in [("history.bin", history), (object, marker)] { + let root = dirs[slot].path().join(bucket).join(key); + tokio::fs::create_dir_all(&root).await.unwrap(); + tokio::fs::write(root.join("xl.meta"), bytes).await.unwrap(); + } + } *set.disks.write().await = before.clone(); let heal = rustfs_heal_contracts::heal_channel::HealOpts { remove: true, @@ -11477,3 +11582,160 @@ mod tests { .await; } } + +#[cfg(test)] +mod missing_key_delete_quorum_tests { + use super::*; + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::storage_api_contracts::bucket::{BucketOperations, BucketOptions, MakeBucketOptions}; + + async fn degraded_store(bucket: &str) -> (Vec, Arc) { + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative never-versioned bucket"); + store.pools[0].disk_set[0].disks.write().await[0] = None; + let info = store + .get_bucket_info(bucket, &BucketOptions::default()) + .await + .expect("existing bucket must remain readable at quorum"); + assert_eq!(info.name, bucket); + let lookup = store + .get_object_info_for_delete(bucket, "never-existed", &ObjectOptions::default()) + .await; + assert!( + lookup.as_ref().is_err_and(is_err_object_not_found), + "remaining disks must establish absence through production lookup: {lookup:?}" + ); + (dirs, store) + } + + #[tokio::test] + async fn single_delete_missing_key_preserves_quorum_noop() { + let (_dirs, store) = degraded_store("missing-key-quorum-single").await; + let result = store + .delete_object_with_tier_delete_journal("missing-key-quorum-single", "never-existed", ObjectOptions::default()) + .await; + assert!( + result.as_ref().is_err_and(is_err_object_not_found), + "single missing-key delete must preserve its idempotent not-found outcome at quorum: {result:?}" + ); + } + + #[tokio::test] + async fn batch_delete_missing_key_preserves_quorum_noop() { + let (_dirs, store) = degraded_store("missing-key-quorum-batch").await; + let (_, errors, _) = store + .delete_objects_with_tier_delete_journal_and_accounting( + "missing-key-quorum-batch", + vec![ObjectToDelete { + object_name: "never-existed".to_owned(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await; + assert_eq!(errors.len(), 1, "batch DELETE must return one result per key"); + assert!( + errors.iter().all(Option::is_none), + "batch missing-key delete must preserve its idempotent success at quorum: {errors:?}" + ); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn batch_delete_timeout_retains_physical_key_guard() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + use crate::disk::{DiskAPI, ReadOptions}; + use crate::storage_api_contracts::object::ObjectIO; + temp_env::async_with_vars([("RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS", Some("30"))], async { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "review-batch-late-owner"; + let key = "object"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("fixture bucket"); + let mut reader = PutObjReader::from_vec(vec![0x51; 1024 * 1024]); + store + .put_object( + bucket, + key, + &mut reader, + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed real external object"); + let disks = store.pools[0].disk_set[0].disk_inventory().await; + let disk = disks[0].as_ref().expect("online drive"); + let info = disk + .read_version("", bucket, key, "", &ReadOptions::default()) + .await + .expect("seed metadata"); + assert!(!info.inline_data()); + let crate::disk::Disk::Local(local) = disk.as_ref() else { + panic!("local disk"); + }; + let part = local + .get_disk() + .get_object_path_for_io(bucket, key) + .expect("leased key") + .join(info.data_dir.expect("external data directory").to_string()) + .join("part.1"); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Remove, &part, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let deleting = store.clone(); + let delete = tokio::spawn(async move { + deleting + .delete_objects_with_tier_delete_journal_and_accounting( + bucket, + vec![ObjectToDelete { + object_name: key.to_owned(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .expect("physical cleanup entry") + .expect("entry hook"); + assert!(part.exists(), "payload unlink has not run"); + let (_, errors, _) = tokio::time::timeout(Duration::from_secs(40), delete) + .await + .expect("real settlement timeout must return") + .expect("delete task"); + assert!(errors.iter().all(Option::is_none), "certified batch response: {errors:?}"); + assert!( + store.ctx.namespace_commits_pending(), + "background physical operation must retain its owner" + ); + let ns = store.handle_new_ns_lock(bucket, key).await.expect("same production key lock"); + let probe = ns.get_write_lock(Duration::from_millis(300)).await; + let acquired_while_pending = probe.is_ok(); + drop(probe); + drop(release); + drop(hook); + assert!( + !acquired_while_pending, + "batch released the exact-key guard while physical cleanup was still paused after its real recovery timeout" + ); + let _released = ns + .get_write_lock(Duration::from_secs(10)) + .await + .expect("physical cleanup must eventually release the exact-key guard"); + }) + .await; + } +} diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index 051159c0e..35facd16b 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -574,6 +574,29 @@ pub trait HealStorageAPI: Send + Sync { /// Heal bucket using ecstore async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result; + /// Recover transactions omitted by ordinary listings. Alternate providers + /// may explicitly expose no transaction-recovery capability. + async fn recover_delete_transactions( + &self, + _bucket: &str, + _object: Option<&str>, + _expected: Option, + _opts: &HealOpts, + ) -> Result> { + Ok(None) + } + + /// Recover hidden transactions only within the requested object prefix. + async fn recover_delete_prefix_transactions( + &self, + _bucket: &str, + _prefix: &str, + _expected: Option, + _opts: &HealOpts, + ) -> Result> { + Ok(None) + } + /// Heal format using ecstore async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)>; @@ -1498,6 +1521,32 @@ impl HealStorageAPI for ECStoreHealStorage { .await) } + async fn recover_delete_transactions( + &self, + bucket: &str, + object: Option<&str>, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.ecstore + .recover_delete_transactions_at_incarnation(bucket, object, expected, opts) + .await + .map_err(Error::Storage) + } + + async fn recover_delete_prefix_transactions( + &self, + bucket: &str, + prefix: &str, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.ecstore + .recover_delete_prefix_at_incarnation(bucket, prefix, expected, opts) + .await + .map_err(Error::Storage) + } + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { debug!( target: "rustfs::heal::storage", diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index 73bc20291..cf6b4786f 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -220,6 +220,22 @@ impl HealTask { set: self.options.set_index, }; + let recovery_opts = HealOpts { + remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted, + ..heal_opts + }; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_transactions( + bucket, + None, + self.bucket_incarnation_id, + &recovery_opts, + )) + .await? + { + self.record_result_item(result).await; + } + let heal_result = match self.bucket_incarnation_id { Some(expected) => { self.await_with_control(self.storage.heal_bucket_at_incarnation(bucket, expected, &heal_opts)) @@ -446,6 +462,26 @@ impl HealTask { "Heal prefix started" ); + let opts = HealOpts { + dry_run: self.options.dry_run, + remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted, + no_lock: self.options.no_lock, + pool: self.options.pool_index, + set: self.options.set_index, + ..Default::default() + }; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_prefix_transactions( + bucket, + prefix, + self.bucket_incarnation_id, + &opts, + )) + .await? + { + self.record_result_item(result).await; + } + self.heal_bucket_objects(bucket, prefix).await } diff --git a/crates/heal/src/heal/task/heal_erasure_set.rs b/crates/heal/src/heal/task/heal_erasure_set.rs index df9b852e9..376fc5314 100644 --- a/crates/heal/src/heal/task/heal_erasure_set.rs +++ b/crates/heal/src/heal/task/heal_erasure_set.rs @@ -460,6 +460,21 @@ impl HealTask { set: self.options.set_index, }; + let recovery_opts = HealOpts { + remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted, + ..bucket_heal_opts + }; + // Include metadata-less transactions for both startup and admin heals. + for bucket in &buckets { + self.check_control_flags().await?; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_transactions(bucket, None, None, &recovery_opts)) + .await? + { + self.record_result_item(result).await; + } + } + for bucket in buckets.iter().filter(|_| self.source != HealRequestSource::Admin) { // Check control flags before starting each bucket heal self.check_control_flags().await?; diff --git a/crates/heal/src/heal/task/heal_object.rs b/crates/heal/src/heal/task/heal_object.rs index 0dc4ec298..e20d4f315 100644 --- a/crates/heal/src/heal/task/heal_object.rs +++ b/crates/heal/src/heal/task/heal_object.rs @@ -99,6 +99,40 @@ impl HealTask { // recreated bucket after the replay anchor was captured. return self.heal_mrf_partial_write_object(bucket, object, version_id).await; } + let heal_opts = HealOpts { + recursive: self.options.recursive, + dry_run: self.options.dry_run, + remove: self.options.remove_corrupted, + recreate: self.options.recreate_missing, + scan_mode: self.options.scan_mode, + update_parity: self.options.update_parity, + no_lock: self.options.no_lock, + read_repair: self.source == HealRequestSource::ReadRepair, + pool: self.options.pool_index, + set: self.options.set_index, + }; + let mut recovery_handled = false; + if matches!(self.source, HealRequestSource::Admin | HealRequestSource::Internal) { + let recovery_opts = HealOpts { + remove: heal_opts.remove && version_id.is_none_or(str::is_empty), + ..heal_opts + }; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_transactions( + bucket, + Some(object), + self.bucket_incarnation_id, + &recovery_opts, + )) + .await? + { + recovery_handled = true; + self.record_result_item(result).await; + if self.options.dry_run { + return Ok(()); + } + } + } let mut object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await { Ok(exists) => exists, Err(err @ Error::TransientSkip { .. }) => { @@ -129,7 +163,7 @@ impl HealTask { object }; - if !object_exists { + if !object_exists && !recovery_handled { // Background loops (scanner/MRF/autoheal/read-repair) routinely // race object deletion, so a missing target is per-object noise // for them; only foreground admin/internal requests keep the warn. @@ -197,18 +231,6 @@ impl HealTask { update_parity = self.options.update_parity, "Heal object stage entered" ); - let heal_opts = HealOpts { - recursive: self.options.recursive, - dry_run: self.options.dry_run, - remove: self.options.remove_corrupted, - recreate: self.options.recreate_missing, - scan_mode: self.options.scan_mode, - update_parity: self.options.update_parity, - no_lock: self.options.no_lock, - read_repair: self.source == HealRequestSource::ReadRepair, - pool: self.options.pool_index, - set: self.options.set_index, - }; let mut expected_identity = self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index); expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?; diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 309f485d3..f5c12e5ba 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -527,6 +527,29 @@ pub struct DeletePathsResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +/// Protocol 1 is additive and has no fallback to generic file mutation RPCs. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RecoverDeleteTransactionRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(uint32, tag = "3")] + pub protocol_version: u32, + #[prost(bytes = "bytes", tag = "4")] + pub operation: ::prost::bytes::Bytes, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RecoverDeleteTransactionResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(uint32, tag = "2")] + pub protocol_version: u32, + #[prost(bytes = "bytes", tag = "3")] + pub result: ::prost::bytes::Bytes, + #[prost(message, optional, tag = "4")] + pub error: ::core::option::Option, +} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnapshotLeaseRequest { #[prost(string, tag = "1")] @@ -2082,6 +2105,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "DeleteAtIncarnation")); self.inner.unary(req, path, codec).await } + pub async fn recover_delete_transaction( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RecoverDeleteTransaction"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "RecoverDeleteTransaction")); + self.inner.unary(req, path, codec).await + } pub async fn acquire_snapshot_lease( &mut self, request: impl tonic::IntoRequest, @@ -3463,6 +3501,10 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn recover_delete_transaction( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn acquire_snapshot_lease( &self, request: tonic::Request, @@ -4237,6 +4279,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/RecoverDeleteTransaction" => { + #[allow(non_camel_case_types)] + struct RecoverDeleteTransactionSvc(pub Arc); + impl tonic::server::UnaryService for RecoverDeleteTransactionSvc { + type Response = super::RecoverDeleteTransactionResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::recover_delete_transaction(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RecoverDeleteTransactionSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/AcquireSnapshotLease" => { #[allow(non_camel_case_types)] struct AcquireSnapshotLeaseSvc(pub Arc); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 451afa21a..5b69f697f 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -1259,6 +1259,37 @@ pub fn canonical_prepare_part_transaction_request_body( Ok(body.finish()) } +/// Recovery messages are bounded and must contain exactly one MessagePack value. +pub const DELETE_RECOVERY_MAX_PAYLOAD_BYTES: usize = DEFAULT_GRPC_SERVER_MESSAGE_LEN; +pub const DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES: usize = 1024 * 1024; + +pub fn decode_delete_recovery_payload(payload: &[u8]) -> std::io::Result { + if payload.is_empty() || payload.len() > DELETE_RECOVERY_MAX_PAYLOAD_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid delete recovery payload length", + )); + } + let mut cursor = std::io::Cursor::new(payload); + let value = rmp_serde::from_read(&mut cursor) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid delete recovery payload"))?; + if cursor.position() != u64::try_from(payload.len()).map_err(std::io::Error::other)? { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "trailing delete recovery payload")); + } + Ok(value) +} + +pub fn canonical_recover_delete_transaction_request_body( + request: &proto_gen::node_service::RecoverDeleteTransactionRequest, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-recover-delete-transaction-request-v1\0"); + body.push_str(&request.disk)?; + body.push_str(&request.volume)?; + body.push_u32(request.protocol_version); + body.push_bytes(&request.operation)?; + Ok(body.finish()) +} + pub fn canonical_settle_part_transaction_request_body( request: &proto_gen::node_service::SettlePartTransactionRequest, ) -> Result, std::num::TryFromIntError> { @@ -1313,6 +1344,40 @@ mod disk_mutation_canonical_tests { }; use super::*; + #[test] + fn delete_recovery_canonical_body_binds_all_wire_fields() { + let request = proto_gen::node_service::RecoverDeleteTransactionRequest { + disk: "disk".into(), + volume: "volume".into(), + protocol_version: 1, + operation: vec![0x80].into(), + }; + let mut bodies = vec![canonical_recover_delete_transaction_request_body(&request).expect("canonical body")]; + for mutate in [ + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.disk = "other".into(), + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.volume = "other".into(), + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.protocol_version = 2, + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.operation = vec![0x81].into(), + ] { + let mut changed = request.clone(); + mutate(&mut changed); + bodies.push(canonical_recover_delete_transaction_request_body(&changed).expect("changed canonical body")); + } + assert_all_distinct(&bodies); + } + + #[test] + fn delete_recovery_payload_rejects_trailing_malformed_and_oversized_values() { + let payload = rmp_serde::to_vec_named(&vec![1_u32, 2]).expect("encode test payload"); + assert_eq!(decode_delete_recovery_payload::>(&payload).expect("one payload"), vec![1, 2]); + let mut trailing = payload; + trailing.push(0xc0); + assert!(decode_delete_recovery_payload::>(&trailing).is_err()); + assert!(decode_delete_recovery_payload::>(&[]).is_err()); + assert!(decode_delete_recovery_payload::>(&[0xc1]).is_err()); + assert!(decode_delete_recovery_payload::>(&vec![0xc0; DELETE_RECOVERY_MAX_PAYLOAD_BYTES + 1]).is_err()); + } + fn assert_all_distinct(bodies: &[Vec]) { for (i, a) in bodies.iter().enumerate() { for (j, b) in bodies.iter().enumerate() { @@ -2905,7 +2970,7 @@ pub async fn get_channel_for_class(addr: &str, class: ChannelClass) -> Result Result Result> { /// channels to the same peer be cached independently. async fn build_channel(dial_addr: &str, cache_key: &str) -> Result> { let deadline = tokio::time::Instant::now() + internode_connect_timeout(); - build_channel_until(dial_addr, cache_key, deadline).await + build_channel_until(dial_addr, cache_key, deadline, internode_rpc_timeout()).await +} + +/// A separate channel deadline for slow authenticated recovery work. The cache +/// key keeps this deadline from changing foreground lock, read, and write RPCs. +#[derive(Debug)] +pub enum DeleteRecoveryChannelError { + Connection(Box), +} + +impl std::fmt::Display for DeleteRecoveryChannelError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Connection(error) => write!(formatter, "delete recovery channel connection failed: {error}"), + } + } +} + +impl Error for DeleteRecoveryChannelError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Connection(error) => Some(error.as_ref()), + } + } +} + +pub async fn create_channel_with_timeout(addr: &str, rpc_timeout: Duration) -> Result { + let cache_key = format!("delete-recovery:{}:{addr}", rpc_timeout.as_millis()); + let deadline = tokio::time::Instant::now() + internode_connect_timeout(); + build_channel_until(addr, &cache_key, deadline, rpc_timeout) + .await + .map_err(DeleteRecoveryChannelError::Connection) } async fn build_channel_until( dial_addr: &str, cache_key: &str, deadline: tokio::time::Instant, + rpc_timeout: Duration, ) -> Result> { tokio::time::timeout_at(deadline, async { if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { @@ -2962,7 +3059,7 @@ async fn build_channel_until( if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { return Ok(channel); } - let (channel, generation) = dial_channel(dial_addr, cache_key).await?; + let (channel, generation) = dial_channel(dial_addr, cache_key, rpc_timeout).await?; if cache_dialed_channel(dial_addr, cache_key, &channel, generation).await && let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { @@ -2980,15 +3077,13 @@ async fn build_channel_until( .map_err(channel_acquisition_timeout)? } -async fn dial_channel(dial_addr: &str, cache_key: &str) -> Result<(Channel, Option), Box> { +async fn dial_channel(dial_addr: &str, cache_key: &str, rpc_timeout: Duration) -> Result<(Channel, Option), Box> { debug!("Creating new gRPC channel to: {} (cache key: {})", dial_addr, cache_key); let dial_started_at = Instant::now(); let connect_timeout = internode_connect_timeout(); let tcp_keepalive = internode_tcp_keepalive(); let http2_keepalive_interval = internode_http2_keep_alive_interval(); let http2_keepalive_timeout = internode_http2_keep_alive_timeout(); - let rpc_timeout = internode_rpc_timeout(); - let mut connector = Endpoint::from_shared(dial_addr.to_string())? // Fast connection timeout for dead peer detection .connect_timeout(connect_timeout) @@ -3291,6 +3386,31 @@ mod tests { } } + #[tokio::test] + async fn tls02_delete_recovery_channels_reject_unbound_and_stale_https_cache() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve recovery peer address"); + let addr = format!("https://{}", listener.local_addr().expect("recovery peer address")); + drop(listener); + let rpc_timeout = Duration::from_secs(20); + let cache_key = format!("delete-recovery:{}:{addr}", rpc_timeout.as_millis()); + let generation = runtime_sources::outbound_tls_generation(); + for cached_generation in [None, Some(generation.checked_add(1).unwrap_or(0))] { + cache_connection(cache_key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + if let Some(generation) = cached_generation { + TLS_GENERATION_CACHE.lock().await.insert(cache_key.clone(), generation); + } else { + TLS_GENERATION_CACHE.lock().await.remove(&cache_key); + } + assert!( + create_channel_with_timeout(&addr, rpc_timeout).await.is_err(), + "recovery must dial the unavailable HTTPS peer instead of returning an unbound or stale cached channel" + ); + assert!(!rustfs_common::has_cached_connection(&cache_key).await); + assert!(!TLS_GENERATION_CACHE.lock().await.contains_key(&cache_key)); + } + } + #[tokio::test] async fn tls02_current_https_control_and_bulk_cache_entries_are_reused() { let _test_guard = TLS02_TEST_LOCK.lock().await; @@ -3444,7 +3564,7 @@ mod tests { for _ in 0..8 { tasks.push(tokio::spawn(async move { let deadline = tokio::time::Instant::now() + Duration::from_millis(40); - let result = build_channel_until(addr, addr, deadline).await; + let result = build_channel_until(addr, addr, deadline, internode_rpc_timeout()).await; result.err().is_some_and(|error| { error .downcast_ref::() diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index a99f73185..fd56c543c 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -391,6 +391,21 @@ message DeletePathsResponse { optional Error error = 2; } +// Protocol 1 is additive and has no fallback to generic file mutation RPCs. +message RecoverDeleteTransactionRequest { + string disk = 1; + string volume = 2; + uint32 protocol_version = 3; + bytes operation = 4; +} + +message RecoverDeleteTransactionResponse { + bool success = 1; + uint32 protocol_version = 2; + bytes result = 3; + optional Error error = 4; +} + message SnapshotLeaseRequest { string disk = 1; string volume = 2; @@ -1218,6 +1233,7 @@ service NodeService { rpc CompareAndUpdateFile(CompareAndUpdateFileRequest) returns (CompareAndUpdateFileResponse) {}; // auth-policy: body-bound rpc Delete(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound rpc DeleteAtIncarnation(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound + rpc RecoverDeleteTransaction(RecoverDeleteTransactionRequest) returns (RecoverDeleteTransactionResponse) {}; // auth-policy: body-bound rpc AcquireSnapshotLease(SnapshotLeaseRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound rpc RenewSnapshotLease(SnapshotLeaseRenewRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound rpc ReleaseSnapshotLease(SnapshotLeaseReleaseRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound diff --git a/docs/README.md b/docs/README.md index 2b89a015d..d30e3758a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -47,6 +47,9 @@ For concurrent bucket creation and deletion, see For persisted administrator bucket tasks and bucket recreation, see [Bucket heal recovery](operations/bucket-heal-recovery.md). +For interrupted object deletions and returning drives, see +[Delete transaction recovery](operations/delete-transaction-recovery.md). + For disk replacement across VM restarts and schema 5/6 maintenance migration, see [Replacement generation recovery](operations/replacement-generation-recovery.md). diff --git a/docs/operations/delete-transaction-recovery.md b/docs/operations/delete-transaction-recovery.md new file mode 100644 index 000000000..175eb84f1 --- /dev/null +++ b/docs/operations/delete-transaction-recovery.md @@ -0,0 +1,137 @@ +# Delete transaction recovery + +An interrupted object deletion can leave metadata backups, reserved shards, or +an unpublished target. Normal S3 listings cannot discover all of these files. +Recovery therefore inspects raw transaction records on every configured drive +in the owning erasure set under the bucket incarnation fence and exact-key +write lock. It does not modify GET or PUT behavior. + +## Decisions and replay + +New deletions assemble the backup, versioned prepared journal, and exact target +in the private system temporary area, then atomically publish the complete +transaction directory before modifying metadata. Interrupted preparation cannot +leave an undiscoverable target-only record or temporary file in the object key. After the metadata write quorum, the coordinator publishes durable +commit receipts to a write quorum before returning success. A failed or lost +publication response is indeterminate: recovery must inspect the persisted +records instead of immediately restoring the backup. + +| Evidence | Action | +| --- | --- | +| New prepared transaction, no decision, intact original shards and matching generation | Restore and sync the original metadata before removing preparation evidence. | +| Structured commit receipt | Publish the certified target to matching returning participants and finish deletion. | +| Missing, unavailable, or unsupported participant | Preserve records and retry when every configured participant can be inspected. | +| Newer generation or partially overlapping version history | Preserve records and report blocked recovery. | +| Historical backup or disk-local marker without a structured set decision | Preserve records during automatic recovery. | + +Missing local metadata does not authorize user-bucket residue cleanup, even +when the current object exists on a metadata quorum. Legacy local cleanup is +restricted to internal system-bucket maintenance. User-bucket legacy residue +uses only the guarded set-wide removal described below. Dry-run and execution +share the same source-generation check; an already-absent explicit non-null +version can converge without touching unrelated live versions. + +Rollback preserves native multipart metadata and integrity sidecars. Protected +parts require their declared proof generation; recovery verifies both the proof +records and actual shard payloads before restoring metadata or removing +recovery evidence. Missing or corrupt required proofs leave the transaction +blocked with its backup intact. + +A disk-local `delete-data.` marker does not prove an erasure-set decision. +Returning drives cannot restore stale data after a certified deletion. Commit +receipts remain until every participant and related transaction for the key +has converged. For consecutive certified version deletions, an older decision +can finish its own data cleanup after later deletion removes current metadata. +A prepared copy needs a matching same-transaction decision before promotion; +absence alone cannot certify it. Cross-transaction null-version supersession also requires causal +predecessor evidence; timestamps alone cannot authorize it. Cleanup respects active snapshots and removes only validated +transaction files, with directory removal performed nonrecursively. + +## Entry points + +Unclean-shutdown erasure-set healing performs raw transaction recovery before +ordinary object listing. Administrator bucket, set, prefix, and object heal use the +same recovery core. Existing heal selection, lifecycle admission, namespace +locking, and movement guards still apply. Prefix recovery settles only matching +keys, including transactions hidden from listings. Dry-run inspects and reports without +settling transactions. + +An unconditional DELETE of a missing exact key, or an administrator heal with +`remove` enabled, can authorize historical residue removal only when the +bucket is authoritatively never-versioned and has no Object Lock. Every +configured drive must be inspected, the current object must be absent, and all +residue must match the guarded legacy layout. Versioned/suspended buckets, +retained versions, conditions, unknown entries, symlinks, and prefix children +are not covered by that authorization. The same fresh intent can certify the +absence of an unprotected new prepared null-version transaction when every +configured drive has no current metadata. This is a new authorized decision; +it does not infer an old commit from missing causal evidence. Mixed or +unrecognized historical layouts remain blocked. + +A missing specific-version DELETE first recovers hidden transactions and then +reads the current version again. It must not acknowledge an unresolved hidden +version as an idempotent no-op. + +## Upgrade and failure handling + +Upgrade all participating nodes before relying on the new protocol. Recovery +uses one authenticated additive RPC; unsupported peers cannot silently accept +new flags through the legacy delete endpoint. An online older participant +causes the modern operation to fail before decision publication. Modern nodes +also reject legacy forward transactional user-bucket deletes. A cluster still +controlled by an older majority retains the older behavior. Do not downgrade +while recovery records remain; old binaries cannot interpret the new protocol. + +Keep unavailable drives and transaction directories intact. Bring the full +set online, run a scoped heal in dry-run mode, inspect its recovery result, +and then run the same scoped heal without dry-run. Use `remove` for ambiguous +historical residue only after verifying that the exact objects are intended +to be deleted and the bucket meets the restrictions above. Do not recursively +remove transaction directories by hand to make a bucket appear empty. + +Recovery reports discovered, committed, rolled-back, reclaimed historical, +and blocked transaction counts. Unavailable participants return a retryable +quorum error; blocked evidence requires operator investigation. A successful +S3 delete response does not imply immediate physical cleanup while snapshots +or returning participants still retain necessary proof. + +## Settlement deadline and durability + +`RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS` controls the settlement RPC deadline, +including full shard verification during rollback. Key and bucket lifecycle +ownership stay attached to late local filesystem operations after cancellation +or timeout. The default is 900 seconds; +valid values are 30 through 86400 seconds. Invalid values fail explicitly. +Increase it when intact large objects need more verification time. It does not +change normal object read or delete-apply deadlines. A timeout can conceal a +completed mutation, so rerun inspection and recovery rather than guessing the +outcome from the RPC error. + +The records and restores follow the configured durability policy. Strict mode +syncs the required metadata and directory changes. Process SIGKILL testing +checks restart behavior; it does not establish power-loss behavior on every +filesystem or storage device. Mode `none` intentionally offers weaker sync +guarantees. Do not treat a throughput comparison as a durability substitute. + +## Reproduce the Docker regression suite + +Build the candidate with the test-only barriers enabled: + +```sh +cargo build --locked --release -p rustfs --bin rustfs --no-default-features --features e2e-test-hooks +python3 scripts/test/delete_recovery_docker.py --help +``` + +Pass a native Linux binary with `--binary`, a Linux image containing Python and +curl with `--image`, and a unique Docker namespace. The optional +`--baseline-binary` enables the mixed-version case. The harness uses synthetic +S3 objects, localhost ports, labeled isolated volumes, and a non-root server. +It stops only its own containers and keeps JSON results and logs in the chosen +artifact directory. Production builds must not enable `e2e-test-hooks`. + +Run the preactivation/prepare/apply/decision process-stop cases, interrupted restore, +returning and untouched peers, both generation orderings, version/retention +and prefix protection, administrator dry-run/removal, response-loss, +permission/corruption, and mixed-version cases. Preserve the source identity +of both binaries with the results. A passing process-stop matrix does not +cover hardware power cuts, Windows runtime behavior, or DELETE throughput. diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index c93f7bdd1..f57c4ac6e 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -4167,6 +4167,51 @@ mod tests { ); } + #[tokio::test] + #[serial_test::serial] + async fn cold_delete_recovery_discovery_uses_production_body_auth() { + use storage::DiskAPI; + temp_env::async_with_vars([("RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT", Some("false"))], async { + let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string()); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind recovery RPC peer"); + let addr = listener.local_addr().expect("recovery RPC address"); + let previous_node_name = rustfs_common::get_global_local_node_name().await; + rustfs_common::set_global_local_node_name(&addr.to_string()).await; + let node = InterceptedService::new(NodeServiceServer::new(make_server()), check_auth); + let service = RpcRequestPathService::new(Routes::new(node).prepare()); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.expect("accept recovery RPC connection"); + ConnBuilder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(socket), TowerToHyperService::new(service)) + .await + .expect("serve recovery RPC connection"); + }); + let mut endpoint = + Endpoint::try_from(format!("http://{addr}/cold-recovery-disk").as_str()).expect("remote recovery endpoint"); + endpoint.is_local = false; + let disk = storage::new_disk( + &endpoint, + &storage::DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("production remote disk client"); + // Recovery may race a heal-control peer-state refresh, before another + // authenticated response has repopulated the shared peer state. + let result = disk.discover_delete_transactions("cold-recovery-bucket", None, None, 1).await; + server.abort(); + let _ = server.await; + rustfs_common::set_global_local_node_name(&previous_node_name).await; + assert!( + matches!(result, Err(crate::storage_api::error::DiskError::DiskNotFound)), + "the cold request must pass production authentication and reach the missing-disk handler: {result:?}" + ); + }) + .await; + } + #[tokio::test] #[serial_test::serial] async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() { diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 54f2315b4..255bf1044 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -1536,6 +1536,13 @@ impl Node for NodeService { self.handle_rename_part(request).await } + async fn recover_delete_transaction( + &self, + request: Request, + ) -> Result, Status> { + self.handle_recover_delete_transaction(request).await + } + async fn settle_part_transaction( &self, request: Request, @@ -3073,13 +3080,14 @@ mod tests { LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, - ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, - RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest, - ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, - SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, - StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, - TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, - UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest, + ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest, ReloadPoolMetaRequest, + ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, ScannerActivityRequest, + ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, + ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest, + SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, + StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, TierMutationPeerState, + TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, WriteRequest, heal_control_service_client::HealControlServiceClient, heal_control_service_server::{HealControlService as _, HealControlServiceServer}, node_service_client::NodeServiceClient, @@ -3097,7 +3105,7 @@ mod tests { use tonic::{Request, Response, Status}; use uuid::Uuid; - const DISK_MUTATION_RPC_METHODS: [&str; 24] = [ + const DISK_MUTATION_RPC_METHODS: [&str; 25] = [ "renamedata", "renamedataatincarnation", "writemetadataatincarnation", @@ -3116,6 +3124,7 @@ mod tests { "renamepart", "prepareparttransaction", "settleparttransaction", + "recoverdeletetransaction", "deletevolume", "makevolume", "makevolumes", @@ -4386,6 +4395,177 @@ mod tests { assert!(!signed.into_inner().success, "the unknown test disk cannot resolve"); } + #[tokio::test] + async fn delete_recovery_apply_rejects_excessive_batch_cardinality_before_lookup() { + use crate::storage::storage_api::{DeleteOptions, DeleteRecoveryRpcOperation, FileInfoVersions}; + let service = make_server(); + for versions in [ + vec![FileInfoVersions::default(); 1001], + vec![FileInfoVersions { + name: "object".into(), + versions: vec![Default::default(); 1001], + ..Default::default() + }], + ] { + let operation = DeleteRecoveryRpcOperation::ApplyVersions { + versions, + opts: DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + ..Default::default() + }, + }; + let mut request = Request::new(RecoverDeleteTransactionRequest { + disk: "unknown".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: rmp_serde::to_vec_named(&operation).expect("encode batch").into(), + }); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical batch"); + set_tonic_canonical_body_digest(&mut request, &body).expect("batch digest"); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("oversized batch must reject before disk lookup"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + } + } + + #[tokio::test] + async fn delete_recovery_apply_rejects_legacy_cleanup_options_and_absolute_keys() { + use crate::storage::storage_api::{DeleteOptions, DeleteRecoveryRpcOperation}; + let service = make_server(); + let valid = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + ..Default::default() + }; + let mut invalid = vec![DeleteOptions::default()]; + for mutate in [ + |opts: &mut DeleteOptions| opts.old_data_dir = Some(Uuid::nil()), + |opts: &mut DeleteOptions| opts.undo_delete = true, + |opts: &mut DeleteOptions| opts.undo_write = true, + |opts: &mut DeleteOptions| opts.recursive = true, + |opts: &mut DeleteOptions| opts.immediate = true, + ] { + let mut opts = valid.clone(); + mutate(&mut opts); + invalid.push(opts); + } + let mut operations = Vec::new(); + for opts in invalid { + operations.push(DeleteRecoveryRpcOperation::ApplyVersion { + object: "object".into(), + file_info: Default::default(), + force_del_marker: false, + opts: opts.clone(), + }); + operations.push(DeleteRecoveryRpcOperation::ApplyVersions { + versions: Vec::new(), + opts, + }); + } + operations.push(DeleteRecoveryRpcOperation::ApplyVersion { + object: "/object".into(), + file_info: Default::default(), + force_del_marker: false, + opts: valid.clone(), + }); + operations.push(DeleteRecoveryRpcOperation::ApplyVersions { + versions: vec![crate::storage::storage_api::FileInfoVersions { + name: "/object".into(), + ..Default::default() + }], + opts: valid, + }); + for operation in operations { + let mut request = Request::new(RecoverDeleteTransactionRequest { + disk: "unknown".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: rmp_serde::to_vec_named(&operation).expect("encode apply operation").into(), + }); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical apply"); + set_tonic_canonical_body_digest(&mut request, &body).expect("apply digest"); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("invalid apply must reject before disk lookup"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + } + } + + #[tokio::test] + async fn delete_recovery_requires_digest_and_valid_protocol_payload() { + let service = make_server(); + let operation = crate::storage::storage_api::DeleteRecoveryRpcOperation::Discover { + object: None, + cursor: None, + limit: 1, + }; + let message = RecoverDeleteTransactionRequest { + disk: "unknown".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: rmp_serde::to_vec_named(&operation).expect("encode discovery").into(), + }; + let unsigned = service + .recover_delete_transaction(Request::new(message.clone())) + .await + .expect_err("new RPC must reject unsigned requests"); + assert_eq!(unsigned.code(), tonic::Code::PermissionDenied); + for digest in ["", "UNSIGNED-PAYLOAD"] { + let mut request = Request::new(message.clone()); + request + .metadata_mut() + .insert("x-rustfs-content-sha256", digest.parse().expect("metadata")); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("missing or unsigned digest"); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + } + let mut invalid = message.clone(); + invalid.protocol_version = 2; + let mut malformed = message.clone(); + malformed.operation = vec![0xc1].into(); + let mut trailing = message.clone(); + let mut payload = trailing.operation.to_vec(); + payload.push(0xc0); + trailing.operation = payload.into(); + for (message, expected) in [ + (invalid, tonic::Code::Unimplemented), + (malformed, tonic::Code::InvalidArgument), + (trailing, tonic::Code::InvalidArgument), + ] { + let mut request = Request::new(message); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical recovery"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("invalid recovery input"); + assert_eq!(error.code(), expected); + } + let mut request = Request::new(message); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical recovery"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let response = service + .recover_delete_transaction(request) + .await + .expect("valid request reaches disk lookup") + .into_inner(); + assert!(!response.success); + assert!(response.error.is_some()); + } + /// Per-handler wiring check for every mutating disk RPC. A mismatched digest must be rejected /// (catches a handler that omits its `verify_disk_mutation_digest` gate) and a correctly /// body-bound digest must pass the gate (catches a handler wired to the wrong @@ -4659,6 +4839,16 @@ mod tests { }, rustfs_protos::canonical_prepare_part_transaction_request_body ); + assert_gated!( + recover_delete_transaction, + RecoverDeleteTransactionRequest { + disk: disk.clone(), + volume: "dst".into(), + protocol_version: 1, + operation: vec![0x80].into() + }, + rustfs_protos::canonical_recover_delete_transaction_request_body + ); assert_gated!( settle_part_transaction, SettlePartTransactionRequest { @@ -6597,7 +6787,7 @@ mod tests { }; let rollback_dir = Uuid::new_v4(); remote - .delete_version( + .delete_version_recoverable( bucket, object, marker.clone(), diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index c7e73aecc..22c1a88c1 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -19,7 +19,10 @@ use crate::storage::storage_api::rpc_consumer::node_service::{ validate_batch_read_version_item_count, }; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; -use crate::storage::storage_api::{PartTransactionAction, RenameDataResp, SnapshotLeaseToken, verify_tonic_mutation_body_digest}; +use crate::storage::storage_api::{ + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, PartTransactionAction, RenameDataResp, + SnapshotLeaseToken, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, +}; use bytes::Bytes; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::{ @@ -118,6 +121,16 @@ const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512; const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024; const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1; +fn rejects_legacy_delete_transaction(volume: &str, opts: &DeleteOptions) -> bool { + !(volume == ".rustfs.sys" + || volume.starts_with(".rustfs.sys/") + || volume == ".minio.sys" + || volume.starts_with(".minio.sys/")) + && opts.old_data_dir.is_some() + && !opts.recoverable_delete + && !opts.undo_write +} + fn snapshot_lease_response(result: Result) -> Response { match result { Ok(token) => Response::new(SnapshotLeaseResponse { @@ -715,6 +728,14 @@ impl NodeService { } }; + if rejects_legacy_delete_transaction(&request.volume, &opts) { + return Ok(Response::new(DeleteVersionsResponse { + success: false, + errors: Vec::new(), + error: Some(DiskError::UnsupportedDisk.into()), + item_errors: Vec::new(), + })); + } let (errors, item_errors) = encode_delete_versions_errors(disk.delete_versions(&request.volume, versions, opts).await); @@ -782,6 +803,13 @@ impl NodeService { })); } }; + if rejects_legacy_delete_transaction(&request.volume, &opts) { + return Ok(Response::new(DeleteVersionResponse { + success: false, + raw_file_info: String::new(), + error: Some(DiskError::UnsupportedDisk.into()), + })); + } if require_marker_condition && opts.expected_delete_marker.is_none() { return Err(Status::invalid_argument("retired marker deletion requires a marker precondition")); } @@ -1578,6 +1606,131 @@ impl NodeService { } } + pub(super) async fn handle_recover_delete_transaction( + &self, + request: Request, + ) -> Result, Status> { + if request.get_ref().operation.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES { + return Err(Status::resource_exhausted("delete recovery operation exceeds payload limit")); + } + let body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()) + .map_err(|_| Status::invalid_argument("delete recovery request length cannot be represented"))?; + // This additive protocol has no digestless rolling-upgrade exception. + verify_tonic_canonical_body_digest(&request, &body) + .map_err(|_| Status::permission_denied("delete recovery requires body-bound v2 authentication"))?; + let request = request.into_inner(); + if request.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION { + return Err(Status::unimplemented("incompatible delete recovery protocol")); + } + let operation: DeleteRecoveryRpcOperation = rustfs_protos::decode_delete_recovery_payload(&request.operation) + .map_err(|_| Status::invalid_argument("invalid delete recovery operation"))?; + if request.operation.len() > operation.max_wire_payload_bytes() { + return Err(Status::resource_exhausted("delete recovery operation exceeds payload limit")); + } + match &operation { + DeleteRecoveryRpcOperation::ApplyVersions { versions, .. } + if versions.len() > 1000 + || versions + .iter() + .try_fold(0_usize, |count, versions| count.checked_add(versions.versions.len())) + .is_none_or(|count| count > 1000) => + { + return Err(Status::invalid_argument("recoverable batch delete exceeds 1000 object/version requests")); + } + DeleteRecoveryRpcOperation::Discover { limit, .. } if *limit == 0 || *limit > 256 => { + return Err(Status::invalid_argument("delete recovery page limit must be between 1 and 256")); + } + DeleteRecoveryRpcOperation::Inspect { transaction, .. } | DeleteRecoveryRpcOperation::Settle { transaction, .. } + if transaction.is_nil() => + { + return Err(Status::invalid_argument("delete transaction must be a non-nil UUID")); + } + DeleteRecoveryRpcOperation::ApplyVersion { opts, .. } | DeleteRecoveryRpcOperation::ApplyVersions { opts, .. } + if opts.old_data_dir.is_none_or(|transaction| transaction.is_nil()) + || opts.undo_delete + || opts.undo_write + || opts.recursive + || opts.immediate + || opts.expected_delete_marker.is_some() + || opts + .delete_recovery_bucket_incarnation + .is_some_and(|incarnation| incarnation.is_nil()) => + { + return Err(Status::invalid_argument( + "recoverable delete requires a non-nil transaction and valid apply options", + )); + } + DeleteRecoveryRpcOperation::ApplyVersion { object, .. } if object.starts_with('/') => { + return Err(Status::invalid_argument("recoverable delete requires a relative object key")); + } + DeleteRecoveryRpcOperation::ApplyVersions { versions, .. } + if versions.iter().any(|version| version.name.starts_with('/')) => + { + return Err(Status::invalid_argument("recoverable delete requires relative object keys")); + } + _ => {} + } + let result = if let Some(disk) = self.find_disk(&request.disk).await { + match operation { + DeleteRecoveryRpcOperation::ApplyVersion { + object, + file_info, + force_del_marker, + opts, + } => disk + .delete_version_recoverable(&request.volume, &object, *file_info, force_del_marker, opts) + .await + .map(|()| DeleteRecoveryRpcReply::AppliedVersion), + DeleteRecoveryRpcOperation::ApplyVersions { versions, opts } => { + let errors = disk.delete_versions_recoverable(&request.volume, versions, opts).await; + Ok(DeleteRecoveryRpcReply::AppliedVersions( + errors.into_iter().map(|error| error.map(|error| error.to_u32())).collect(), + )) + } + DeleteRecoveryRpcOperation::Discover { object, cursor, limit } => disk + .discover_delete_transactions(&request.volume, object.as_deref(), cursor, limit) + .await + .map(DeleteRecoveryRpcReply::Discovered), + DeleteRecoveryRpcOperation::Inspect { object, transaction } => disk + .inspect_delete_transaction(&request.volume, &object, transaction) + .await + .map(DeleteRecoveryRpcReply::Inspected), + DeleteRecoveryRpcOperation::Settle { + object, + transaction, + expected_fingerprint, + action, + } => disk + .settle_delete_transaction(&request.volume, &object, transaction, expected_fingerprint, action) + .await + .map(DeleteRecoveryRpcReply::Settled), + } + } else { + Err(DiskError::DiskNotFound) + }; + match result { + Ok(reply) => { + let result = encode_msgpack_named(&reply, "delete recovery reply") + .map_err(|_| Status::internal("cannot encode delete recovery reply"))?; + if result.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES { + return Err(Status::resource_exhausted("delete recovery reply exceeds payload limit")); + } + Ok(Response::new(RecoverDeleteTransactionResponse { + success: true, + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + result: result.into(), + error: None, + })) + } + Err(error) => Ok(Response::new(RecoverDeleteTransactionResponse { + success: false, + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + result: Bytes::new(), + error: Some(error.into()), + })), + } + } + pub(super) async fn handle_settle_part_transaction( &self, request: Request, @@ -2007,9 +2160,10 @@ impl NodeService { #[cfg(test)] mod tests { use super::{ - compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, + DeleteOptions, compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, encode_batch_read_version_response_payloads, encode_delete_versions_errors, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, encode_read_multiple_response_payloads, encode_rename_data_response_payloads, + rejects_legacy_delete_transaction, }; use crate::storage::DiskError; use crate::storage::rpc::node_service::make_server; @@ -2022,6 +2176,7 @@ mod tests { use serde::{Deserialize, Serialize}; use serial_test::serial; use tonic::Request; + use uuid::Uuid; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct SamplePayload { @@ -2029,6 +2184,26 @@ mod tests { count: u32, } + #[test] + fn legacy_delete_transaction_policy_preserves_rollback_and_internal_maintenance() { + let mut opts = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + ..Default::default() + }; + assert!(rejects_legacy_delete_transaction("bucket", &opts)); + assert!(!rejects_legacy_delete_transaction(".rustfs.sys", &opts)); + assert!(!rejects_legacy_delete_transaction(".rustfs.sys/tmp", &opts)); + assert!(!rejects_legacy_delete_transaction(".minio.sys", &opts)); + opts.undo_write = true; + assert!(!rejects_legacy_delete_transaction("bucket", &opts)); + opts.undo_write = false; + opts.recoverable_delete = true; + assert!(!rejects_legacy_delete_transaction("bucket", &opts)); + opts.recoverable_delete = false; + opts.old_data_dir = None; + assert!(!rejects_legacy_delete_transaction("bucket", &opts)); + } + #[test] fn response_compat_send_sites_keep_manifest_json_encoders() { // Rolling-upgrade contract (rustfs-protos compat manifest): every diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index adc83492a..3edbe7d4d 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -478,11 +478,12 @@ pub(crate) mod ecstore_data_usage { #[allow(unused_imports)] pub(crate) mod ecstore_disk { pub(crate) use rustfs_ecstore::api::disk::{ - BUCKET_META_PREFIX, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DeleteOptions, - DiskAPI, DiskInfo, DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, OldCurrentSize, - PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, - validate_batch_read_version_item_count, + BUCKET_META_PREFIX, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, + DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, + DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction, + RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, + UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, validate_batch_read_version_item_count, }; #[cfg(test)] pub(crate) use rustfs_ecstore::api::disk::{DiskOption, new_disk}; @@ -734,6 +735,14 @@ pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq; pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp; pub(crate) type ReadOptions = ecstore_disk::ReadOptions; pub(crate) type OldCurrentSize = ecstore_disk::OldCurrentSize; +pub(crate) type DeleteRecoveryCursor = ecstore_disk::DeleteRecoveryCursor; +pub(crate) type DeleteRecoveryPage = ecstore_disk::DeleteRecoveryPage; +pub(crate) type DeleteRecoveryInspection = ecstore_disk::DeleteRecoveryInspection; +pub(crate) type DeleteRecoveryDigest = ecstore_disk::DeleteRecoveryDigest; +pub(crate) type DeleteRecoveryAction = ecstore_disk::DeleteRecoveryAction; +pub(crate) type DeleteRecoveryRpcOperation = ecstore_disk::DeleteRecoveryRpcOperation; +pub(crate) type DeleteRecoveryRpcReply = ecstore_disk::DeleteRecoveryRpcReply; +pub(crate) const DELETE_RECOVERY_PROTOCOL_VERSION: u32 = ecstore_disk::DELETE_RECOVERY_PROTOCOL_VERSION; pub(crate) type PartTransactionAction = ecstore_disk::PartTransactionAction; pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp; pub(crate) type ReplicationStatusType = ecstore_bucket::replication::ReplicationStatusType; @@ -1444,6 +1453,41 @@ pub(crate) trait StorageDiskRpcExt { dst_path: &str, meta: bytes::Bytes, ) -> DiskResult<()>; + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: rustfs_filemeta::FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> DiskResult<()>; + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec>; + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> DiskResult; + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + ) -> DiskResult; + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> DiskResult; async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()>; async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()>; async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; @@ -1616,6 +1660,53 @@ where ecstore_disk::DiskAPI::prepare_part_transaction(self, src_volume, src_path, dst_volume, dst_path, meta).await } + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: rustfs_filemeta::FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete_version_recoverable(self, volume, object, file_info, force_del_marker, opts).await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + ecstore_disk::DiskAPI::delete_versions_recoverable(self, volume, versions, opts).await + } + + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> DiskResult { + ecstore_disk::DiskAPI::discover_delete_transactions(self, volume, object, cursor, limit).await + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + ) -> DiskResult { + ecstore_disk::DiskAPI::inspect_delete_transaction(self, volume, object, transaction).await + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> DiskResult { + ecstore_disk::DiskAPI::settle_delete_transaction(self, volume, object, transaction, expected_fingerprint, action).await + } + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()> { ecstore_disk::DiskAPI::settle_part_transaction(self, volume, path, action).await } diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 2e51ba37e..ecc668e0d 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -180,6 +180,8 @@ pub(crate) mod server { } pub(crate) mod http { + #[cfg(test)] + pub(crate) use crate::storage::storage_api::ecstore_disk::{DiskAPI, DiskOption, new_disk}; pub(crate) use crate::storage::storage_api::{ ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_tonic_rpc_signature_with_bootstrap, diff --git a/scripts/test/delete_recovery_docker.py b/scripts/test/delete_recovery_docker.py new file mode 100644 index 000000000..c9bb96b32 --- /dev/null +++ b/scripts/test/delete_recovery_docker.py @@ -0,0 +1,1257 @@ +#!/usr/bin/env python3 +"""Run delete recovery regressions in disposable local Docker containers. + +Requires host Python 3.9+, Docker, and curl with --aws-sigv4. Supply a native +Linux binary compiled with e2e-test-hooks and a Linux image containing Python 3 +and that binary's runtime libraries. All S3 requests target localhost with +synthetic credentials. Container processes run as UID 10001. The harness labels +its resources, refuses existing volumes, preserves JSON reports/server logs, +and removes only containers, networks, and volumes it created. + +Example: + python3 scripts/test/delete_recovery_docker.py --binary /absolute/path/to/rustfs-candidate --baseline-binary /absolute/path/to/rustfs-old --image local-rustfs-validation:latest --artifact-dir /absolute/path/to/new-results smoke prepared1 applied1 decision1 restore returning untouched mixed + +Available cases include single/multi-node process termination, inline/empty +objects, legacy explicit cleanup, versioned and retained objects, UUID prefix +children, corrupted journals, real permission errors, lost receipt responses, +admin bucket/object repair, and deep keys. Use --help for runtime options. +Logical disks share the Docker host filesystem; this tests process/RPC failure +and namespace safety, not independent hardware failure or power-loss durability. +""" +from pathlib import Path +import base64, hashlib, ipaddress, json, os, subprocess, tempfile, threading, time, uuid, xml.etree.ElementTree as ET +from urllib.parse import quote +ROOT = None +PREFIX = None +IMAGE = None +PORT = 19010 +CANDIDATE = None +BASELINE = None +ACCESS = 'recovery-test' +SECRET = 'recovery-test-secret' + +def run(*args, check=True, **kwargs): + return subprocess.run(args, check=check, capture_output=True, **kwargs) + +def docker(*args, check=True): + return run('docker', *args, check=check) + +class S3: + + def __init__(self, port): + self.url = f'http://127.0.0.1:{port}' + + def request(self, method, path='', body=None, headers=None, expected=None, timeout=60): + call = ['curl', '--noproxy', '*', '--silent', '--show-error', '--max-time', str(timeout), '--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', ACCESS + ':' + SECRET, '--request', method] + for (key, value) in (headers or {}).items(): + call += ['--header', key + ': ' + value] + if body is not None: + call += ['--data-binary', '@-'] + with tempfile.NamedTemporaryFile(dir=ROOT) as header_file: + response = run(*call, self.url + '/' + path, '--dump-header', header_file.name, '--write-out', '\n%{http_code}', input=body, check=False) + self.last_headers = {} + for line in Path(header_file.name).read_text(errors='replace').splitlines(): + if ':' in line: + (key, value) = line.split(':', 1) + self.last_headers[key.lower()] = value.strip() + if response.returncode: + raise RuntimeError(response.stderr.decode(errors='replace')) + (payload, status) = response.stdout.rsplit(b'\n', 1) + code = int(status) + if expected is not None and code not in ((expected,) if isinstance(expected, int) else expected): + raise AssertionError(f'{method} {path}: {code}, expected {expected}: {payload[:1500]!r}') + return (code, payload) + + def put_bucket(self, bucket, lock=False): + headers = {'x-amz-bucket-object-lock-enabled': 'true'} if lock else {} + return self.request('PUT', bucket, headers=headers, expected=200) + + def put(self, bucket, key, body, headers=None): + return self.request('PUT', bucket + '/' + key, body, headers, 200) + + def get(self, bucket, key): + return self.request('GET', bucket + '/' + key, expected=200)[1] + + def delete(self, bucket, key, expected=(204, 200)): + return self.request('DELETE', bucket + '/' + key, expected=expected) + + def delete_bucket(self, bucket, expected=204): + return self.request('DELETE', bucket, expected=expected) + + def list(self, bucket): + (_, body) = self.request('GET', bucket + '?list-type=2', expected=200) + root = ET.fromstring(body) + return [node.text for node in root.iter() if node.tag.rsplit('}', 1)[-1] == 'Key'] + + def versioning(self, bucket, state='Enabled'): + xml = f'{state}'.encode() + return self.request('PUT', bucket + '?versioning', xml, {'Content-Type': 'application/xml'}, 200) + + def batch_delete(self, bucket, keys): + xml = ET.Element('Delete') + for key in keys: + ET.SubElement(ET.SubElement(xml, 'Object'), 'Key').text = key + data = ET.tostring(xml) + (status, body) = self.request('POST', bucket + '?delete', data, {'Content-Type': 'application/xml', 'Content-MD5': base64.b64encode(hashlib.md5(data).digest()).decode()}, 200) + response = ET.fromstring(body) + errors = [node for node in response.iter() if node.tag.rsplit('}', 1)[-1] == 'Error'] + assert not errors, f'DeleteObjects returned per-key errors: {body!r}' + return (status, body) + + def complete_multipart(self, bucket, key, parts): + path = bucket + '/' + key + (_, response) = self.request('POST', path + '?uploads', expected=200) + upload = next(node.text for node in ET.fromstring(response).iter() if node.tag.rsplit('}', 1)[-1] == 'UploadId') + assert upload, 'native multipart initiation must return an upload ID' + query = '?uploadId=' + quote(upload, safe='') + complete = ET.Element('CompleteMultipartUpload') + for number, body in enumerate(parts, 1): + self.request('PUT', path + query + '&partNumber=' + str(number), body, expected=200) + etag = self.last_headers.get('etag') + assert etag, 'native UploadPart must return its ETag' + part = ET.SubElement(complete, 'Part') + ET.SubElement(part, 'PartNumber').text = str(number) + ET.SubElement(part, 'ETag').text = etag + (_, response) = self.request('POST', path + query, ET.tostring(complete), {'Content-Type': 'application/xml'}, 200) + assert ET.fromstring(response).tag.rsplit('}', 1)[-1] == 'CompleteMultipartUploadResult', response + return upload + +class Cluster: + + def __init__(self, case, nodes=1, drives=4, extra_env=None, binary_by_node=None): + self.case = case + self.nodes = nodes + self.drives = drives + self.port = PORT + self.binary = 'candidate' + self.binary_by_node = binary_by_node or {} + self.names = [f'{PREFIX}-{case}-n{i}' for i in range(1, nodes + 1)] + self.network = f'{PREFIX}-{case}-net' + self.env = extra_env or {} + self.created_names = [] + self.created_network = False + self.clients = [S3(self.port + i) for i in range(nodes)] + self.volumes = [] + self.run_args = {} + self.barriers = [ROOT / 'barriers' / case / f'n{i}' for i in range(1, nodes + 1)] + for directory in self.barriers: + directory.mkdir(parents=True, exist_ok=True) + directory.chmod(511) + + def create(self): + docker('network', 'create', '--label', 'rustfs.delete-recovery-test=' + PREFIX, self.network) + self.created_network = True + configuration = json.loads(docker('network', 'inspect', self.network).stdout)[0]['IPAM']['Config'] + subnet = next(ipaddress.ip_network(value['Subnet']) for value in configuration if ':' not in value['Subnet']) + addresses = [str(subnet[index + 2]) for index in range(self.nodes)] + urls = [] + for name in self.names: + for drive in range(1, self.drives + 1): + urls.append(f'http://{name}:9000/data/d{drive}') + for (idx, name) in enumerate(self.names): + args = ['run', '-d', '--user', '10001:10001', '--name', name, '--network', self.network, '--hostname', name, '-p', f'127.0.0.1:{self.port + idx}:9000', '--label', 'rustfs.delete-recovery-test=' + PREFIX, '-v', str(CANDIDATE) + ':/binaries/candidate:ro', '-v', str(self.barriers[idx]) + ':/barrier'] + # Stopped peers must still resolve while the remaining quorum restarts. + args += ['--ip', addresses[idx]] + for peer, address in zip(self.names, addresses): + args += ['--add-host', peer + ':' + address] + if BASELINE is not None: + args += ['-v', str(BASELINE) + ':/binaries/baseline:ro'] + data_mounts = [] + for drive in range(1, self.drives + 1): + volume = f'{name}-d{drive}' + if docker('volume', 'inspect', volume, check=False).returncode == 0: + raise RuntimeError('refusing existing volume: ' + volume) + docker('volume', 'create', '--label', 'rustfs.delete-recovery-test=' + PREFIX, volume) + self.volumes.append(volume) + args += ['-v', f'{volume}:/data/d{drive}'] + data_mounts += ['-v', f'{volume}:/data/d{drive}'] + docker('run', '--rm', *data_mounts, IMAGE, 'python3', '-c', "from pathlib import Path; import os; [os.chown(p,10001,10001) for p in Path('/data').glob('d*')]") + values = {'RUSTFS_VOLUMES': ' '.join(urls) if self.nodes > 1 else ' '.join((f'/data/d{d}' for d in range(1, self.drives + 1))), 'RUSTFS_ADDRESS': '0.0.0.0:9000', 'RUSTFS_CONSOLE_ENABLE': 'false', 'RUSTFS_ACCESS_KEY': ACCESS, 'RUSTFS_SECRET_KEY': SECRET, 'RUSTFS_OBS_LOG_DIRECTORY': '/tmp/logs', 'RUSTFS_OBS_LOGGER_LEVEL': 'info', 'RUSTFS_OBS_LOG_STDOUT_ENABLED': 'true', 'RUSTFS_DELETE_RECOVERY_BARRIER_DIR': '/barrier', 'RUSTFS_UNSAFE_BYPASS_DISK_CHECK': 'true', 'RUSTFS_DURABILITY_MODE': 'strict', **self.env} + for (key, value) in values.items(): + if value is not None: + args += ['-e', key + '=' + value] + self.run_args[idx] = [*args, IMAGE, '/binaries/' + self.binary_by_node.get(idx, self.binary)] + launch = docker(*self.run_args[idx], check=False) + if launch.returncode: + label = docker('inspect', '--format', '{{index .Config.Labels "rustfs.delete-recovery-test"}}', name, check=False) + if label.returncode == 0 and label.stdout.decode().strip() == PREFIX: + self.created_names.append(name) + raise RuntimeError('Docker launch failed: ' + launch.stderr.decode(errors='replace')) + self.created_names.append(name) + self.ready() + + def ready(self, timeout=180, indices=None): + deadline = time.monotonic() + timeout + pending = set(range(self.nodes) if indices is None else indices) + while pending and time.monotonic() < deadline: + for idx in list(pending): + try: + (code, _) = self.clients[idx].request('GET', '', timeout=2) + if code in (200, 403): + pending.remove(idx) + except Exception: + pass + if pending: + for idx in pending: + state = docker('inspect', '--format', '{{.State.Running}}', self.names[idx], check=False) + if state.returncode == 0 and state.stdout.strip() != b'true': + self.logs() + raise RuntimeError('server exited during startup: ' + self.names[idx]) + time.sleep(0.2) + if pending: + self.logs() + raise TimeoutError('cluster not ready: ' + str(pending)) + + def kill(self, index=0): + docker('kill', '--signal', 'KILL', self.names[index]) + + def restart(self, index=None): + for i in range(self.nodes) if index is None else [index]: + docker('start', self.names[i]) + self.ready() + + def replace_node(self, index, binary): + docker('kill', '--signal', 'KILL', self.names[index], check=False) + docker('rm', self.names[index]) + args = self.run_args[index].copy() + args[-1] = '/binaries/' + binary + docker(*args) + self.run_args[index] = args + self.ready(indices=[index]) + + def logs(self): + for name in self.created_names: + response = docker('logs', name, check=False) + (ROOT / (name + '.log')).write_bytes(response.stdout + response.stderr) + + def inspect_data(self, code, indices=None): + """Access only this cluster's named volumes, including stopped peers.""" + result = [] + for index, name in enumerate(self.names): + if indices is not None and index not in indices: + continue + response = docker('run', '--rm', '--volumes-from', name, IMAGE, 'python3', '-c', code) + result.append(json.loads(response.stdout)) + return result + + def remove(self): + """Keep logs and delete only resources successfully created here.""" + self.logs() + for name in self.created_names: + docker('rm', '-f', name, check=False) + for volume in self.volumes: + docker('volume', 'rm', volume, check=False) + if self.created_network: + docker('network', 'rm', self.network, check=False) + +def smoke(): + cluster = Cluster('smoke', drives=1) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'recovery-smoke' + client.put_bucket(bucket) + for (key, size) in [('zero', 0), ('inline', 4096), ('external', 2 * 1024 * 1024)]: + payload = bytes(range(256)) * (size // 256) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + client.batch_delete(bucket, ['zero', 'inline', 'external']) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': 's3-smoke', 'status': 'passed', 'topology': '1 node / 1 named local volume'} + finally: + cluster.remove() + +def seed_legacy(cluster, bucket, key, shape): + transaction = str(uuid.uuid4()) + if shape == 'alias': + directories = {path.split('/')[0] for node in residue_inventory(cluster, bucket, key) for drive in node for path in drive['files'] if path.endswith('/part.1')} + assert len(directories) == 1, 'alias fixture requires one shared data-directory generation' + transaction = directories.pop() + code = "from pathlib import Path\nimport hashlib,json,os,uuid\nbucket=%r; key=%r; shape=%r; transaction=%r\nresult=[]\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/bucket/key\n original=root/'xl.meta'\n if not original.is_file(): continue\n data=original.read_bytes(); backup=root/transaction/'xl.meta.bkp'\n backup.parent.mkdir(exist_ok=shape=='alias'); backup.write_bytes(data); os.chown(backup.parent,10001,10001); os.chown(backup,10001,10001)\n dirs=[p for p in root.iterdir() if p.is_dir() and (p.name!=transaction or shape=='alias') and any(p.glob('part.*'))]\n if shape in ('reserved','alias'):\n assert dirs, 'external fixture must contain erasure parts'\n for d in dirs:\n marker=d/('reserve-delete-data.'+transaction); marker.write_bytes(b''); os.chown(marker,10001,10001)\n elif shape=='backup-only':\n assert not dirs, 'inline fixture must not contain external parts'\n else: raise AssertionError(shape)\n original.unlink()\n owned=[root,*root.rglob('*')]\n assert all(p.stat().st_uid==10001 and p.stat().st_gid==10001 for p in owned), [(str(p),p.stat().st_uid,p.stat().st_gid) for p in owned]\n result.append({'drive':drive.name,'transaction':transaction,'backup_sha256':hashlib.sha256(data).hexdigest(),'data_dirs':[p.name for p in dirs],'fixture_owner':'10001:10001','fixture_modes':{str(p.relative_to(root)):oct(p.stat().st_mode & 0o777) for p in owned}})\nprint(json.dumps(result))\n" % (bucket, key, shape, transaction) + return cluster.inspect_data(code) + +def residue_inventory(cluster, bucket, key): + return cluster.inspect_data("from pathlib import Path\nimport hashlib,json\nresult=[]\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/%r/%r\n result.append({'drive':drive.name,'exists':root.exists(),'files':{str(p.relative_to(root)):hashlib.sha256(p.read_bytes()).hexdigest() for p in root.rglob('*') if p.is_file()}})\nprint(json.dumps(result))\n" % (bucket, key)) + +def legacy_retry(nodes=1): + case = 'legacy' + str(nodes) + cluster = Cluster(case, nodes=nodes, drives=1 if nodes == 1 else 2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'legacy-recovery' + client.put_bucket(bucket) + original = {key: bytes(range(256)) * (size // 256) for (key, size) in [('inline', 4096), ('reserved', 2 * 1024 * 1024)]} + for (key, payload) in original.items(): + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + for i in range(nodes): + cluster.kill(i) + fixtures = {key: seed_legacy(cluster, bucket, key, 'backup-only' if key == 'inline' else 'reserved') for key in original} + before = {key: residue_inventory(cluster, bucket, key) for key in original} + cluster.restart() + time.sleep(2) + after = {key: residue_inventory(cluster, bucket, key) for key in original} + assert before == after, 'automatic recovery must not guess an unjournaled historical outcome' + client.batch_delete(bucket, list(original)) + for key in original: + current = residue_inventory(cluster, bucket, key) + assert all((not drive['exists'] for node in current for drive in node)), f'current explicit DELETE must reclaim {key}: {current}' + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': case, 'status': 'passed', 'fixtures': fixtures, 'topology': f'{nodes} nodes'} + finally: + cluster.remove() + +def arm_barrier(cluster, index, bucket, key, stage, disk=None): + nonce = str(uuid.uuid4()) + arm = {'nonce': nonce, 'bucket': bucket, 'object': key, 'stage': stage} + if disk is not None: + arm['disk'] = disk + (cluster.barriers[index] / 'arm.json').write_text(json.dumps(arm)) + return nonce + +def reached_barrier(cluster, index, nonce, timeout=45): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + for path in cluster.barriers[index].glob('reached.' + nonce + '.*.json'): + try: + return json.loads(path.read_text()) + except (json.JSONDecodeError, OSError): + pass + time.sleep(0.02) + cluster.logs() + raise TimeoutError('crash stage not reached: ' + nonce) + +def clear_barriers(cluster): + for directory in cluster.barriers: + (directory / 'arm.json').unlink(missing_ok=True) + +def async_delete(client, bucket, key): + result = {} + + def target(): + try: + result['response'] = client.request('DELETE', bucket + '/' + key, timeout=90) + except Exception as error: + result['exception'] = str(error) + thread = threading.Thread(target=target, daemon=True) + thread.start() + return (thread, result) + +def wait_inventory(cluster, bucket, key, predicate, timeout=60): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = residue_inventory(cluster, bucket, key) + if predicate(last): + return last + time.sleep(0.1) + raise AssertionError('recovery did not converge: ' + json.dumps(last)) + +def process_crash(stage, nodes=1, size=2 * 1024 * 1024): + case = ('inline-' if size == 4096 else 'zero-' if size == 0 else 'kill-') + stage.lower() + '-' + str(nodes) + cluster = Cluster(case, nodes=nodes, drives=4 if nodes == 1 else 2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'journal-crash' + key = 'payload' + client.put_bucket(bucket) + payload = bytes(range(256)) * (size // 256) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + original = residue_inventory(cluster, bucket, key) + nonce = arm_barrier(cluster, 0, bucket, key, stage, disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + for index in range(nodes): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + cluster.restart() + if stage == 'DECISION': + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + else: + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': case, 'status': 'passed', 'observation': observation, 'request': {key: str(value) for (key, value) in request.items()}} + finally: + cluster.remove() + +def rollback_crash(): + cluster = Cluster('kill-restore', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'rollback-restart' + key = 'payload' + client.put_bucket(bucket) + payload = bytes(range(256)) * 8192 + client.put(bucket, key, payload) + original = residue_inventory(cluster, bucket, key) + first_nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + first = reached_barrier(cluster, 0, first_nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + second_nonce = arm_barrier(cluster, 0, bucket, key, 'RESTORE', disk='/data/d1') + docker('start', cluster.names[0]) + second = reached_barrier(cluster, 0, second_nonce) + cluster.kill() + clear_barriers(cluster) + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'rollback-second-process-kill', 'status': 'passed', 'observations': [first, second]} + finally: + cluster.remove() + +def versioned_crash(stage): + cluster = Cluster('versions-' + stage.lower(), drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'versioned-recovery' + key = 'history' + client.put_bucket(bucket) + client.versioning(bucket) + first = b'old-version-' * 200000 + second = b'new-version-' * 300 + client.put(bucket, key, first) + v1 = client.last_headers.get('x-amz-version-id') + assert v1 and v1 != 'null' + client.put(bucket, key, second) + v2 = client.last_headers.get('x-amz-version-id') + assert v2 and v2 != v1 + assert client.get(bucket, key + '?versionId=' + v1) == first + assert client.get(bucket, key) == second + original = residue_inventory(cluster, bucket, key) + nonce = arm_barrier(cluster, 0, bucket, key, stage, disk='/data/d1') + (thread, request) = async_delete(client, bucket, key + '?versionId=' + v1) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + if stage == 'APPLIED': + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key + '?versionId=' + v1) == first + client.delete(bucket, key + '?versionId=' + v1) + else: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + (code, _) = client.request('GET', bucket + '/' + key + '?versionId=' + v1) + if code == 404: + break + time.sleep(0.1) + else: + raise AssertionError('committed version deletion was resurrected') + assert client.get(bucket, key) == second + assert client.get(bucket, key + '?versionId=' + v2) == second + client.delete(bucket, key + '?versionId=' + v2) + client.delete_bucket(bucket) + return {'case': 'versioned-' + stage.lower(), 'status': 'passed', 'observation': observation, 'deleted_version': v1, 'retained_version': v2} + finally: + cluster.remove() + +def protected_version(): + import datetime + cluster = Cluster('retention', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'protected-recovery' + key = 'retained' + client.put_bucket(bucket, lock=True) + client.versioning(bucket) + until = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ') + payload = b'compliance-and-legal-hold' * 1024 + client.put(bucket, key, payload, {'x-amz-object-lock-mode': 'COMPLIANCE', 'x-amz-object-lock-retain-until-date': until, 'x-amz-object-lock-legal-hold': 'ON'}) + version = client.last_headers.get('x-amz-version-id') + assert version and version != 'null' + original = residue_inventory(cluster, bucket, key) + client.delete(bucket, key + '?versionId=' + version, expected=403) + assert residue_inventory(cluster, bucket, key) == original + cluster.kill() + cluster.restart() + assert client.get(bucket, key + '?versionId=' + version) == payload + assert residue_inventory(cluster, bucket, key) == original + client.delete_bucket(bucket, expected=409) + return {'case': 'compliance-legal-hold-restart', 'status': 'passed', 'retained_version': version} + finally: + cluster.remove() + +def returning_drive_new_generation(): + cluster = Cluster('returning-new-generation', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'returning-drive' + key = 'payload' + client.put_bucket(bucket) + old = b'old-payload-' * 200000 + new = b'new-payload-' * 180000 + client.put(bucket, key, old) + assert client.get(bucket, key) == old + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + for index in range(4): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + for index in range(3): + docker('start', cluster.names[index]) + cluster.ready(indices=range(3)) + client.put(bucket, key, new) + assert client.get(bucket, key) == new + client.delete(bucket, key) + docker('start', cluster.names[3]) + cluster.ready() + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node)), timeout=90) + client.request('GET', bucket + '/' + key, expected=404) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': 'old-prepared-new-put-new-delete-returning-peer', 'status': 'passed', 'observation': observation} + finally: + cluster.remove() + +def mixed_versions(): + cluster = Cluster('mixed', nodes=4, drives=2, binary_by_node={3: 'baseline'}) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'mixed-recovery' + key = 'payload' + payload = b'mixed-version-data' * 100000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + original = residue_inventory(cluster, bucket, key) + results = [] + for initiator in (0, 3): + (code, body) = cluster.clients[initiator].request('DELETE', bucket + '/' + key) + assert code >= 400, f'mixed initiator{initiator} must fail closed: {code} {body!r}' + after = residue_inventory(cluster, bucket, key) + (ROOT / f'mixed-initiator-{initiator}-inventory.json').write_text(json.dumps({'before': original, 'after': after}, indent=2)) + assert after == original, 'mixed rejection must preserve exact metadata/shards after rollback' + assert client.get(bucket, key) == payload + results.append({'initiator': initiator, 'delete_status': code}) + cluster.replace_node(3, 'candidate') + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'three-current-one-pre-extension-peer-both-initiators', 'status': 'passed', 'results': results} + finally: + cluster.remove() + +def permission_fault(): + cluster = Cluster('permission-error', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'permission-fault' + key = 'payload' + payload = b'permission-test' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + transaction = str(uuid.UUID(observation['transaction'])) + code = "from pathlib import Path; import json,os; p=Path('/data/d1')/%r/%r/%r; os.chmod(p,0o555); print(json.dumps({'path':str(p),'mode':oct(p.stat().st_mode&0o777)}))" % (bucket, key, transaction) + fault = cluster.inspect_data(code) + (cluster.barriers[0] / ('release.' + nonce)).write_bytes(b'') + thread.join(30) + assert not thread.is_alive(), 'permission failure must return rather than hang' + clear_barriers(cluster) + current = residue_inventory(cluster, bucket, key) + assert any((drive['exists'] for node in current for drive in node)) + client.delete_bucket(bucket, expected=409) + repair = "from pathlib import Path; import json,os; p=Path('/data/d1')/%r/%r/%r; os.chmod(p,0o755); print(json.dumps({'repaired':True}))" % (bucket, key, transaction) + cluster.inspect_data(repair) + cluster.kill() + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'actual-eacces-decision-publication', 'status': 'passed', 'fault': fault, 'request': {key: str(value) for (key, value) in request.items()}, 'observation': observation} + finally: + cluster.remove() + +def injected_error(stage): + cluster = Cluster('injected-' + stage.lower(), drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'injected-fault' + key = 'payload' + payload = b'injected-error' * 170000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + nonce = arm_barrier(cluster, 0, bucket, key, stage, disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + (cluster.barriers[0] / ('release.' + nonce)).write_bytes(b'io_error') + thread.join(30) + assert not thread.is_alive(), 'injected error must finish the caller' + clear_barriers(cluster) + cluster.kill() + cluster.restart() + if stage == 'PREPARED': + (code, body) = client.request('GET', bucket + '/' + key) + if code == 200: + assert body == payload + client.delete(bucket, key) + else: + assert code == 404 + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'post-persist-' + stage.lower() + '-io-error', 'status': 'passed', 'observation': observation, 'request': {key: str(value) for (key, value) in request.items()}} + finally: + cluster.remove() + +def missing_version_returning_peer(): + cluster = Cluster('returning-specific-version', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'missing-version-recovery' + key = 'history' + client.put_bucket(bucket) + client.versioning(bucket) + old = b'first-version' * 160000 + newer = b'latest-version' * 1000 + client.put(bucket, key, old) + v1 = client.last_headers['x-amz-version-id'] + client.put(bucket, key, newer) + v2 = client.last_headers['x-amz-version-id'] + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key + '?versionId=' + v1) + observation = reached_barrier(cluster, 0, nonce) + for index in range(4): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + for index in range(3): + docker('start', cluster.names[index]) + cluster.ready(indices=range(3)) + (code, body) = client.request('DELETE', bucket + '/' + key + '?versionId=' + v1) + initial_delete_status = code + assert code >= 400, f'missing version with unresolved offline journal must not acknowledge204: {code} {body!r}' + docker('start', cluster.names[3]) + cluster.ready() + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + (code, body) = client.request('GET', bucket + '/' + key + '?versionId=' + v1) + if code == 200: + assert body == old + break + time.sleep(0.2) + else: + raise AssertionError('undecided historical version did not recover after peer returned') + client.delete(bucket, key + '?versionId=' + v1) + client.request('GET', bucket + '/' + key + '?versionId=' + v1, expected=404) + assert client.get(bucket, key) == newer + client.delete(bucket, key + '?versionId=' + v2) + client.delete_bucket(bucket) + return {'case': 'missing-specific-version-offline-fence', 'status': 'passed', 'observation': observation, 'initial_delete_status': initial_delete_status} + finally: + cluster.remove() + +def untouched_returning_peer(): + cluster = Cluster('untouched-returning', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'untouched-peer' + key = 'payload' + payload = b'untouched-original' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + original = residue_inventory(cluster, bucket, key) + cluster.kill(3) + client.delete(bucket, key) + after = residue_inventory(cluster, bucket, key) + assert after[3] == original[3], 'offline unprepared peer must remain byte-for-byte untouched' + client.delete_bucket(bucket, expected=409) + cluster.restart(3) + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node)), timeout=90) + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'acknowledged-delete-untouched-offline-peer-return', 'status': 'passed'} + finally: + cluster.remove() + +def uuid_prefix_crash(): + cluster = Cluster('uuid-prefix', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'uuid-prefix' + key = 'parent' + payload = b'parent-payload' * 160000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + inventory = residue_inventory(cluster, bucket, key) + part = next((name for name in inventory[0][0]['files'] if name.endswith('/part.1'))) + data_dir = str(uuid.UUID(part.split('/')[0])) + child = key + '/' + data_dir + '/child' + child_payload = b'preserve-live-prefix-child' * 800 + client.put(bucket, child, child_payload) + assert client.get(bucket, child) == child_payload + nonce = arm_barrier(cluster, 0, bucket, key, 'DECISION', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + assert client.get(bucket, child) == child_payload, 'recovery must preserve actual data-dir UUID-prefix descendants' + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket, expected=409) + client.delete(bucket, child) + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'committed-delete-live-uuid-data-prefix', 'status': 'passed', 'observation': observation, 'protected_child': child} + finally: + cluster.remove() + +def corruption_repair(): + cluster = Cluster('corrupt-journal', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'corrupt-journal' + key = 'payload' + payload = b'original-protected-data' * 100000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + transaction = str(uuid.UUID(observation['transaction'])) + code = "from pathlib import Path; import base64,json; p=Path('/data/d1')/%r/%r/%r/'delete-prepared'; old=p.read_bytes(); p.write_bytes(b'corrupt-record'); print(json.dumps({'original':base64.b64encode(old).decode()}))" % (bucket, key, transaction) + saved = cluster.inspect_data(code)[0]['original'] + before = residue_inventory(cluster, bucket, key) + cluster.restart() + time.sleep(2) + assert residue_inventory(cluster, bucket, key) == before, 'corrupt authority must preserve all backup/payload files' + (code, body) = client.request('DELETE', bucket + '/' + key) + assert code >= 400, f'corrupt hidden transaction must not acknowledge cleanup: {code} {body!r}' + assert residue_inventory(cluster, bucket, key) == before + repair = "from pathlib import Path; import base64,json; p=Path('/data/d1')/%r/%r/%r/'delete-prepared'; p.write_bytes(base64.b64decode(%r)); print(json.dumps({'repaired':True}))" % (bucket, key, transaction, saved) + cluster.inspect_data(repair) + cluster.kill() + cluster.restart() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + (code, body) = client.request('GET', bucket + '/' + key) + if code == 200: + assert body == payload + break + time.sleep(0.1) + else: + raise AssertionError('valid original journal must recover original bytes after repair') + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'corrupt-journal-fails-closed-then-repair', 'status': 'passed', 'observation': observation} + finally: + cluster.remove() + +def admin_heal(client, bucket, dry_run=False, remove=False, key=None, recursive=None): + path = 'rustfs/admin/v3/heal/' + bucket + ('/' + key if key else '') + options = {'recursive': key is None if recursive is None else recursive, 'dryRun': dry_run, 'remove': remove, 'recreate': False, 'scanMode': 1, 'updateParity': False, 'nolock': False, 'readRepair': False} + (_, body) = client.request('POST', path, json.dumps(options).encode(), {'Content-Type': 'application/json'}, 200) + token = json.loads(body)['clientToken'] + deadline = time.monotonic() + 90 + + def states(value): + if isinstance(value, dict): + for (k, v) in value.items(): + if k == 'state' and isinstance(v, str): + yield v.lower() + yield from states(v) + elif isinstance(value, list): + for v in value: + yield from states(v) + while time.monotonic() < deadline: + (_, body) = client.request('POST', path + '?clientToken=' + token, headers={'Content-Type': 'application/json'}, expected=200) + result = json.loads(body) + with (ROOT / 'admin-responses.jsonl').open('a') as report: + report.write(json.dumps({'bucket': bucket, 'key': key, 'dry_run': dry_run, 'remove': remove, 'recursive': options['recursive'], 'response': result}) + '\n') + status = set(states(result)) + if status & {'failed', 'completed_with_errors', 'blocked'}: + raise AssertionError('admin heal failed: ' + json.dumps(result)) + if 'completed' in status or result.get('summary') == 'finished': + return result + time.sleep(0.1) + raise TimeoutError('admin heal did not complete: ' + token) + +def admin_legacy_dry_run(direct=False): + cluster = Cluster('admin-legacy', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'admin-legacy' + client.put_bucket(bucket) + for (key, payload) in [('inline', b'i' * 4096), ('external', b'e' * (2 * 1024 * 1024))]: + client.put(bucket, key, payload) + cluster.kill() + seed_legacy(cluster, bucket, 'inline', 'backup-only') + seed_legacy(cluster, bucket, 'external', 'reserved') + before = {key: residue_inventory(cluster, bucket, key) for key in ('inline', 'external')} + cluster.restart() + time.sleep(2) + dry = admin_heal(client, bucket, dry_run=True, remove=True) + assert {key: residue_inventory(cluster, bucket, key) for key in before} == before, 'dry run must never mutate ambiguous residue' + actual = [admin_heal(client, bucket, dry_run=False, remove=True, key=key) for key in before] if direct else admin_heal(client, bucket, dry_run=False, remove=True) + for key in before: + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + client.delete_bucket(bucket) + return {'case': 'admin-legacy-dry-run-and-explicit-cleanup', 'status': 'passed', 'dry_run_report': dry, 'cleanup_report': actual} + finally: + cluster.remove() + +def admin_prefix_scoped(): + cluster = Cluster('admin-prefix', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket, prefix = 'admin-prefix', 'selected/' + selected = {'selected/inline': b'i' * 4096, 'selected/external': b'e' * (2 * 1024 * 1024)} + outside = {'selected-neighbor/hidden': b'n' * 4096, 'outside/hidden': b'o' * 4096} + live_key, live_body = 'selected/live', b'live-prefix-object-must-survive' + client.put_bucket(bucket) + for key, body in {**selected, **outside, live_key: live_body}.items(): + client.put(bucket, key, body) + assert client.get(bucket, key) == body + cluster.kill() + for key in {**selected, **outside}: + seed_legacy(cluster, bucket, key, 'reserved' if key == 'selected/external' else 'backup-only') + before = {key: residue_inventory(cluster, bucket, key) for key in {**selected, **outside}} + cluster.restart() + dry = admin_heal(client, bucket, dry_run=True, remove=True, key=prefix, recursive=True) + assert any('delete recovery (dry run)' in item.get('detail', '') and 'discovered=2' in item.get('detail', '') for item in dry.get('items', [])), 'prefix dry run must inspect the two matching hidden transactions' + assert {key: residue_inventory(cluster, bucket, key) for key in before} == before, 'prefix dry run must preserve every hidden transaction' + actual = admin_heal(client, bucket, remove=True, key=prefix, recursive=True) + after = {key: residue_inventory(cluster, bucket, key) for key in before} + (ROOT / 'prefix-inventory.json').write_text(json.dumps({'before': before, 'after': after}, indent=2)) + for key in selected: + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + for key in outside: + assert residue_inventory(cluster, bucket, key) == before[key], f'prefix recovery must preserve outside transaction {key}' + assert client.get(bucket, live_key) == live_body, 'prefix remove must preserve the live object' + assert client.list(bucket) == [live_key] + client.delete(bucket, live_key) + client.delete_bucket(bucket, expected=409) + for key in outside: + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'admin-prefix-hidden-transactions-only', 'status': 'passed', 'prefix': prefix, 'selected_keys': list(selected), 'protected_keys': list(outside), 'dry_run_report': dry, 'cleanup_report': actual} + finally: + cluster.remove() + +def deep_key_crash(large_stack=False): + extra_env = {'RUSTFS_RUNTIME_THREAD_STACK_SIZE': str(32 * 1024 * 1024)} if large_stack else None + cluster = Cluster('deep-key-large-stack' if large_stack else 'deep-key', drives=4, extra_env=extra_env) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'deep-recovery' + key = '/'.join(['a'] * 300) + '/payload' + payload = b'deep-inline-content' * 200 + client.put_bucket(bucket) + client.put(bucket, key, payload) + original = residue_inventory(cluster, bucket, key) + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'deep-key-recovery-large-stack' if large_stack else 'deep-key-recovery', 'status': 'passed', 'segments': 301, 'runtime_thread_stack_size': 32 * 1024 * 1024 if large_stack else 'default', 'observation': observation} + finally: + cluster.remove() + +def receipt_response_loss(): + cluster = Cluster('receipt-response-loss', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'receipt-response-loss' + key = 'payload' + payload = b'lost-response-data' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + nonces = {index: arm_barrier(cluster, index, bucket, key, 'DECISION') for index in (1, 2)} + (thread, request) = async_delete(client, bucket, key) + observations = [] + for (index, nonce) in nonces.items(): + observations.append(reached_barrier(cluster, index, nonce)) + deadline = time.monotonic() + 40 + while len(list(cluster.barriers[index].glob('reached.' + nonce + '.*.json'))) < 2: + if time.monotonic() > deadline: + raise TimeoutError('both drive receipts must persist before peer kill') + time.sleep(0.02) + for index in (1, 2): + cluster.kill(index) + thread.join(75) + assert not thread.is_alive(), 'DELETE must return after peer response loss' + if 'response' in request: + assert request['response'][0] >= 400, 'four online responses cannot satisfy five-drive receipt quorum' + before = residue_inventory(cluster, bucket, key) + assert any(('delete-quorum-committed' in path for node in before for drive in node for path in drive['files'])), 'stored decisions must remain while peers are offline' + clear_barriers(cluster) + for index in (1, 2): + docker('start', cluster.names[index]) + cluster.ready() + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node)), timeout=90) + client.request('GET', bucket + '/' + key, expected=404) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': 'receipt-publication-response-loss', 'status': 'passed', 'observations': observations, 'request': {key: str(value) for (key, value) in request.items()}} + finally: + cluster.remove() + +def retained_decision_newer_undecided(): + cluster = Cluster('inverse-null-history', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + (bucket, key) = ('inverse-null-history', 'payload') + older = b'older-generation' * 150000 + newer = b'newer-generation' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, older) + cluster.kill(3) + client.delete(bucket, key) + witness = residue_inventory(cluster, bucket, key) + assert any(('delete-quorum-committed' in path for node in witness[:3] for drive in node for path in drive['files'])), 'offline old peer requires retained committed authority' + client.put(bucket, key, newer) + assert client.get(bucket, key) == newer + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + for index in range(3): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + cluster.restart() + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + (status, body) = client.request('GET', bucket + '/' + key) + if status == 200: + assert body == newer, 'older committed nil receipt must never promote a newer undecided generation' + break + time.sleep(0.1) + else: + raise AssertionError('newer undecided generation was not restored after complete peer return') + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'retained-older-decision-newer-undecided-null', 'status': 'passed', 'observation': observation} + finally: + cluster.remove() + + +def completed_multipart_sidecars(protected=False, crash=False): + case = 'multipart-integrity' if protected else 'multipart-sidecars' + if crash: + case += '-applied' + mode = 'true' if protected else 'false' + cluster = Cluster(case, drives=4, extra_env={'RUSTFS_SHARD_INTEGRITY_WRITE': mode, 'RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED': mode}) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'completed-multipart' + key = 'payload' + parts = [bytes(range(256)) * 20480, bytes(reversed(range(256))) * 4096] + payload = b''.join(parts) + client.put_bucket(bucket) + upload = client.complete_multipart(bucket, key, parts) + assert client.get(bucket, key) == payload, 'complete MPU must expose all uploaded bytes' + original = residue_inventory(cluster, bucket, key) + (ROOT / 'multipart-original-inventory.json').write_text(json.dumps(original, indent=2)) + for node in original: + for drive in node: + assert drive['exists'] and 'xl.meta' in drive['files'], 'complete MPU must publish metadata on every drive' + for number in (1, 2): + sidecars = [name for name in drive['files'] if Path(name).name == f'part.{number}.meta'] + assert len(sidecars) == 1, 'native completed MPU must retain each real part metadata sidecar' + prefix = f'part.{number}.integrity.' + proofs = [name for name in drive['files'] if Path(name).name.startswith(prefix)] + assert len(proofs) == (1 if protected else 0), 'native completion must retain the selected integrity format' + for proof in proofs: + assert uuid.UUID(Path(proof).name[len(prefix):]).int != 0, 'integrity proof generation must be non-nil' + observation = None + if crash: + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + restored = residue_inventory(cluster, bucket, key) + assert restored == original, 'rollback must preserve exact metadata, all shards, part sidecars and integrity proofs' + (ROOT / 'multipart-restored-inventory.json').write_text(json.dumps(restored, indent=2)) + assert client.get(bucket, key) == payload, 'rolled-back native MPU must retain the complete readable body' + client.batch_delete(bucket, [key]) + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + client.request('GET', bucket + '/' + key, expected=404) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': case, 'status': 'passed', 'native_upload_id': upload, 'integrity': protected, 'part_sizes': [len(part) for part in parts], 'bytes': len(payload), 'observation': observation, 'bucket_deletion': 'ordinary non-force204'} + finally: + cluster.remove() + +def ordinary_missing_with_offline_peer(): + cluster = Cluster('missing-offline', nodes=4, drives=1) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'missing-offline' + client.put_bucket(bucket) + cluster.kill(3) + client.delete(bucket, 'never-existed-single', expected=204) + client.batch_delete(bucket, ['never-existed-batch-a', 'never-existed-batch-b']) + assert client.list(bucket) == [], 'ordinary missing DELETE must leave the existing bucket valid' + payload = b'bucket-remains-usable' * 200 + client.put(bucket, 'live', payload) + assert client.get(bucket, 'live') == payload + docker('start', cluster.names[3]) + cluster.ready() + assert client.get(bucket, 'live') == payload + client.delete(bucket, 'live') + wait_inventory(cluster, bucket, 'live', lambda inventory: all(not drive['exists'] for node in inventory for drive in node), timeout=90) + client.delete_bucket(bucket) + return {'case': 'ordinary-missing-delete-offline-single-and-batch', 'status': 'passed', 'cleanup_claim': 'no residue recovery was requested'} + finally: + cluster.remove() + +def legacy_transaction_data_dir_alias(): + cluster = Cluster('legacy-data-dir-alias', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'legacy-data-dir-alias', 'payload' + payload = b'legacy-data-dir-content' * 100000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + cluster.kill() + fixture = seed_legacy(cluster, bucket, key, 'alias') + before = residue_inventory(cluster, bucket, key) + cluster.restart() + time.sleep(2) + assert residue_inventory(cluster, bucket, key) == before, 'automatic recovery must preserve ambiguous alias residue' + client.delete(bucket, key) + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + client.delete_bucket(bucket) + return {'case': 'legacy-transaction-equals-data-directory', 'status': 'passed', 'fixture': fixture} + finally: + cluster.remove() + +def admin_smoke(): + cluster = Cluster('admin-smoke', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'admin-smoke', 'payload' + payload = b'healthy-admin-object' * 1000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + bucket_report = admin_heal(client, bucket, dry_run=True) + object_report = admin_heal(client, bucket, key=key) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'admin-healthy-bucket-and-object-client', 'status': 'passed', 'bucket_report': bucket_report, 'object_report': object_report} + finally: + cluster.remove() + +def large_batch_delete(): + cluster = Cluster('batch-1000', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'batch-1000' + client.put_bucket(bucket) + payloads = {f'object-{index:04d}': (bytes(range(256)) * 8192 if index % 250 == 0 else f'content-{index:04d}'.encode()) for index in range(1000)} + for key, payload in payloads.items(): + client.put(bucket, key, payload) + for key, payload in payloads.items(): + assert client.get(bucket, key) == payload + client.batch_delete(bucket, list(payloads)) + assert client.list(bucket) == [] + for key in ('object-0000', 'object-0500', 'object-0999'): + assert all(not drive['exists'] for node in residue_inventory(cluster, bucket, key) for drive in node) + client.delete_bucket(bucket) + return {'case': 'maximum-1000-key-remote-delete-batch', 'status': 'passed', 'exact_gets': 1000, 'keys': 1000} + finally: + cluster.remove() + +def large_version_history(): + cluster = Cluster('versions-3000', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'versions-3000', 'history' + client.put_bucket(bucket) + client.versioning(bucket) + versions = [] + for index in range(3000): + payload = f'version-content-{index:04d}'.encode() + client.put(bucket, key, payload) + versions.append((client.last_headers['x-amz-version-id'], payload)) + for index in (0, 1500, 2999): + version, payload = versions[index] + assert client.request('GET', bucket + '/' + key + '?versionId=' + version, expected=200)[1] == payload + oldest = versions.pop(0)[0] + client.delete(bucket, key + '?versionId=' + oldest) + client.request('GET', bucket + '/' + key + '?versionId=' + oldest, expected=404) + assert client.get(bucket, key) == versions[-1][1] + for offset in range(0, len(versions), 1000): + group = versions[offset:offset + 1000] + xml = ET.Element('Delete') + for version, payload in group: + item = ET.SubElement(xml, 'Object') + ET.SubElement(item, 'Key').text = key + ET.SubElement(item, 'VersionId').text = version + data = ET.tostring(xml) + status, body = client.request('POST', bucket + '?delete', data, {'Content-Type': 'application/xml', 'Content-MD5': base64.b64encode(hashlib.md5(data).digest()).decode()}, 200, timeout=180) + response = ET.fromstring(body) + assert not [node for node in response.iter() if node.tag.rsplit('}', 1)[-1] == 'Error'], body + deleted = [child.text for node in response.iter() if node.tag.rsplit('}', 1)[-1] == 'Deleted' for child in node if child.tag.rsplit('}', 1)[-1] == 'VersionId'] + assert sorted(deleted) == sorted(version for version, payload in group), body + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'large-version-history-remote-single-and-batch-delete', 'status': 'passed', 'versions': 3000, 'survivor_bytes_verified': True} + finally: + cluster.remove() + +def unknown_predecessor_fresh_delete(): + cluster = Cluster('unknown-predecessor', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'unknown-predecessor', 'payload' + client.put_bucket(bucket) + client.put(bucket, key, b'older-inline' * 300) + nonce = arm_barrier(cluster, 3, bucket, key, 'APPLIED', disk='/data/d1') + thread, request = async_delete(cluster.clients[3], bucket, key) + observation = reached_barrier(cluster, 3, nonce) + transaction = str(uuid.UUID(observation['transaction'])) + verify = "from pathlib import Path; import json; result=[];\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/%r/%r; tx=root/%r; result.append(not (root/'xl.meta').exists() and (tx/'xl.meta.bkp').is_file())\nprint(json.dumps(result))" % (bucket, key, transaction) + deadline = time.monotonic() + 30 + while not all(cluster.inspect_data(verify, indices=[3])[0]): + if time.monotonic() >= deadline: + raise TimeoutError('both returning-peer preparations must reach metadata absence') + time.sleep(.05) + cluster.kill(3) + for index in range(3): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + restore = "from pathlib import Path; import json,os; result=[];\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/%r/%r; tx=root/%r; backup=tx/'xl.meta.bkp';\n if backup.exists(): os.replace(backup,root/'xl.meta')\n if tx.exists():\n for path in tx.iterdir():\n assert path.is_file() and path.name in ('delete-prepared','delete-target'), path\n path.unlink()\n tx.rmdir()\n assert (root/'xl.meta').is_file(); result.append(True)\nprint(json.dumps(result))" % (bucket, key, transaction) + cluster.inspect_data(restore, indices=range(3)) + for index in range(3): + docker('start', cluster.names[index]) + cluster.ready(indices=range(3)) + newer = b'newer-inline' * 300 + client.put(bucket, key, newer) + assert client.get(bucket, key) == newer + client.delete(bucket, key) + docker('start', cluster.names[3]) + cluster.ready() + time.sleep(2) + pending = residue_inventory(cluster, bucket, key) + assert all(transaction + '/xl.meta.bkp' in drive['files'] for drive in pending[3]), 'automatic recovery must preserve unknown predecessor evidence' + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket, expected=409) + client.delete(bucket, key, expected=204) + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + client.delete_bucket(bucket) + return {'case': 'unknown-predecessor-preserved-until-fresh-current-delete', 'status': 'passed', 'observation': observation, 'fixture': 'authentic prepared records retained only on stopped returning peer'} + finally: + cluster.remove() + +def main(): + global ROOT, PREFIX, IMAGE, PORT, CANDIDATE, BASELINE + import argparse, re, sys + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--binary', required=True, type=Path, help='Linux candidate binary built with e2e-test-hooks') + parser.add_argument('--baseline-binary', type=Path, help='Older Linux binary, required by mixed case') + parser.add_argument('--image', required=True, help='Local Linux image with Python 3 and the binary runtime libraries') + parser.add_argument('--artifact-dir', required=True, type=Path, help='New directory for logs and stage handshakes') + parser.add_argument('--namespace', default='rustfs-delete-' + uuid.uuid4().hex[:12]) + parser.add_argument('--port', type=int, default=19010, help='First of four sequential localhost ports') + parser.add_argument('cases', nargs='*', default=['smoke']) + options = parser.parse_args() + if not re.fullmatch('[a-z][a-z0-9-]{0,45}', options.namespace): + parser.error('namespace must be a short lowercase Docker name') + if not 1024 <= options.port <= 65532: + parser.error('port must leave room for four localhost ports') + CANDIDATE = options.binary.resolve(strict=True) + if not CANDIDATE.is_file(): + parser.error('binary must be a regular file') + BASELINE = options.baseline_binary.resolve(strict=True) if options.baseline_binary else None + if BASELINE is not None and (not BASELINE.is_file()): + parser.error('baseline binary must be a regular file') + ROOT = options.artifact_dir.resolve() + ROOT.mkdir(parents=True, exist_ok=False) + PREFIX = options.namespace + IMAGE = options.image + PORT = options.port + for resource in ('container', 'volume', 'network'): + existing = docker(resource, 'ls', '-q', '--filter', 'label=rustfs.delete-recovery-test=' + PREFIX).stdout.strip() + if existing: + parser.error('namespace already owns Docker resources; choose a new namespace') + cases = {'unknown-fresh': unknown_predecessor_fresh_delete, 'batch1000': large_batch_delete, 'versions3000': large_version_history, 'admin-smoke': admin_smoke, 'legacy-alias': legacy_transaction_data_dir_alias, 'missing-offline': ordinary_missing_with_offline_peer, 'inverse': retained_decision_newer_undecided, 'deep': deep_key_crash, 'deep-large-stack': lambda : deep_key_crash(True), 'admin-prefix': admin_prefix_scoped, 'response-loss': receipt_response_loss, 'admin-object': lambda : admin_legacy_dry_run(True), 'smoke': smoke, 'legacy1': lambda : legacy_retry(1), 'legacy4': lambda : legacy_retry(4), 'restore': rollback_crash, 'retention': protected_version, 'version-applied': lambda : versioned_crash('APPLIED'), 'version-decision': lambda : versioned_crash('DECISION'), 'returning': returning_drive_new_generation, 'untouched': untouched_returning_peer, 'missing-version': missing_version_returning_peer, 'mixed': mixed_versions, 'admin': admin_legacy_dry_run, 'prefix': uuid_prefix_crash, 'corruption': corruption_repair, 'permission': permission_fault, 'error-prepared': lambda : injected_error('PREPARED'), 'error-applied': lambda : injected_error('APPLIED'), 'error-decision': lambda : injected_error('DECISION')} + cases.update({'multipart-sidecars': completed_multipart_sidecars, 'multipart-integrity': lambda: completed_multipart_sidecars(True), 'multipart-sidecars-applied': lambda: completed_multipart_sidecars(False, True), 'multipart-integrity-applied': lambda: completed_multipart_sidecars(True, True)}) + for stage in ('PREACTIVATION', 'PREPARED', 'APPLIED', 'DECISION'): + for nodes in (1, 4): + cases[f'{stage.lower()}{nodes}'] = lambda stage=stage, nodes=nodes: process_crash(stage, nodes) + cases[f'inline-{stage.lower()}'] = lambda stage=stage: process_crash(stage, 1, 4096) + cases[f'zero-{stage.lower()}'] = lambda stage=stage: process_crash(stage, 1, 0) + if 'mixed' in options.cases and BASELINE is None: + parser.error('mixed requires --baseline-binary') + unknown = set(options.cases) - set(cases) + if unknown: + parser.error('unknown cases: ' + ','.join(sorted(unknown))) + results = [] + for case in options.cases: + started = time.monotonic() + try: + result = cases[case]() + result['elapsed_seconds'] = round(time.monotonic() - started, 3) + except BaseException as error: + result = {'case': case, 'status': 'failed', 'error': str(error), 'elapsed_seconds': round(time.monotonic() - started, 3)} + (ROOT / (case + '.json')).write_text(json.dumps(result, indent=2, sort_keys=True)) + raise + results.append(result) + (ROOT / (case + '.json')).write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, sort_keys=True), flush=True) + (ROOT / 'results.json').write_text(json.dumps(results, indent=2, sort_keys=True)) +if __name__ == '__main__': + main()