diff --git a/.config/nextest.toml b/.config/nextest.toml index 6c083520e..a1c9b4302 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -417,6 +417,12 @@ threads-required = "num-test-threads" filter = 'package(rustfs-heal) & (binary(mrf_partial_write_test) | (binary(mrf_pipeline_test) & test(/^journal_replay_survives_/)) | test(=admin_selector_survives_sigkill_with_original_scope_and_receipts))' threads-required = "num-test-threads" +# Bound the whole recovery fixture even if a synchronous wait prevents its +# per-case async deadline from firing. +[[profile.ci.overrides]] +filter = 'package(rustfs-heal) & binary(heal_integration_test) & test(=serial_tests::test_admin_bucket_directory_recovery_from_task_entry)' +slow-timeout = { period = "60s", terminate-after = 5, grace-period = "10s" } + # Native scanner backlog fixtures include 24-set and pool-expansion cases. # Reserve capacity across test processes while preserving each scenario's fixed # budget, assertions, and internal concurrency. @@ -970,10 +976,10 @@ threads-required = "num-test-threads" filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))' test-group = 'e2e-vault' -# Keep the storage-heavy smoke cases isolated in the full suite too. They -# start multiple server fixtures and issue many sequential S3 operations. +# Keep storage-heavy cases isolated in the full suite too. The large pagination +# fixture alone seeds 2,000 objects and can otherwise starve other servers. [[profile.e2e-full.overrides]] -filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))' +filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))' threads-required = "num-test-threads" # The mc fixture uploads hundreds of objects concurrently before measuring listing. diff --git a/crates/e2e_test/src/reliant/grpc_lock_server.rs b/crates/e2e_test/src/reliant/grpc_lock_server.rs index 954165665..62b6ab513 100644 --- a/crates/e2e_test/src/reliant/grpc_lock_server.rs +++ b/crates/e2e_test/src/reliant/grpc_lock_server.rs @@ -576,6 +576,41 @@ impl NodeService for MinimalLockNodeService { Err(Status::unimplemented("lock-only test server")) } + async fn reserve_object_metadata( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("lock-only test server")) + } + + async fn reserve_object_metadata_short( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("lock-only test server")) + } + + async fn activate_object_metadata_reservation( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("lock-only test server")) + } + + async fn cancel_object_metadata_reservation( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("lock-only test server")) + } + + async fn rename_reserved_data( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("lock-only test server")) + } + async fn rename_data_with_pool_meta_cas( &self, _request: Request, diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index b3a7b1bc4..b7566aeb9 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -67,14 +67,15 @@ use rustfs_protos::proto_gen::node_service::{ BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome, CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest, - ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, - PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, - ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest, - RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest, + ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest, + MetadataReservationCancelRequest, MetadataReservationRequest, + PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest, + ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, + RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest, RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest, - SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, - SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, - node_service_client::NodeServiceClient, + ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, + SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, node_service_client::NodeServiceClient, }; use serde::{Serialize, de::DeserializeOwned}; use std::{ @@ -2452,6 +2453,239 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> { } impl RemoteDisk { + /// Prepare a remote token without retrying the RPC. It owns the bucket + /// publication fence but does not enter the object metadata queue yet. + pub async fn reserve_object_metadata( + &self, + bucket: &str, + object: &str, + incarnation: Uuid, + timeout: Duration, + ) -> Result { + self.reserve_object_metadata_with_protocol(bucket, object, incarnation, timeout, false) + .await + } + + /// Use the distinct short-lease RPC; an old peer rejects it before allocating a token. + pub(crate) async fn reserve_object_metadata_short( + &self, + bucket: &str, + object: &str, + incarnation: Uuid, + timeout: Duration, + ) -> Result { + self.reserve_object_metadata_with_protocol(bucket, object, incarnation, timeout, true) + .await + } + + async fn reserve_object_metadata_with_protocol( + &self, + bucket: &str, + object: &str, + incarnation: Uuid, + timeout: Duration, + short: bool, + ) -> Result { + if timeout.is_zero() { + return Err(DiskError::Timeout); + } + self.execute_with_timeout_for_op_and_health_action( + "reserve_object_metadata", + || async { + let mut client = self.get_client().await?; + let mut request = Request::new(MetadataReservationRequest { + disk: self.endpoint.to_string(), + bucket: bucket.to_owned(), + object: object.to_owned(), + timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?, + bucket_incarnation_id: incarnation.as_bytes().to_vec().into(), + }); + let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()) + .map_err(|_| Error::other("reservation request length overflow"))?; + crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?; + let result = if short { + client.reserve_object_metadata_short(request).await + } else { + client.reserve_object_metadata(request).await + }; + let response = result + .map_err(|status| { + if status.code() == Code::Unimplemented { + DiskError::UnsupportedDisk + } else { + status.into() + } + })? + .into_inner(); + if !response.success { + return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected)); + } + if response.protocol_version != if short { 3 } else { 2 } { + return Err(DiskError::UnsupportedDisk); + } + Uuid::from_slice(&response.token) + .ok() + .filter(|token| !token.is_nil()) + .ok_or(DiskError::Unexpected) + }, + timeout.min(Duration::from_secs(5)), + FailureHealthAction::IgnoreFailure, + ) + .await + } + + /// Enqueue a prepared token. The caller must cancel it on an indeterminate + /// result before allowing any subsequent unprotected mutation. + pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> { + self.update_object_metadata_reservation(bucket, object, token, timeout, false) + .await + } + + pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> { + self.update_object_metadata_reservation(bucket, object, token, timeout, true) + .await + } + + async fn update_object_metadata_reservation( + &self, + bucket: &str, + object: &str, + token: Uuid, + timeout: Duration, + renew_only: bool, + ) -> Result<()> { + if timeout.is_zero() { + return Err(DiskError::Timeout); + } + self.execute_with_timeout_for_op_and_health_action( + "activate_object_metadata_reservation", + || async { + let mut client = self.get_client().await?; + let mut request = Request::new(MetadataReservationActivateRequest { + disk: self.endpoint.to_string(), + bucket: bucket.to_owned(), + object: object.to_owned(), + token: token.as_bytes().to_vec().into(), + renew_only, + }); + let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref()) + .map_err(|_| Error::other("reservation activation request length overflow"))?; + crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?; + let response = client + .activate_object_metadata_reservation(request) + .await + .map_err(|status| { + if status.code() == Code::Unimplemented { + DiskError::UnsupportedDisk + } else { + status.into() + } + })? + .into_inner(); + if !response.success { + return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected)); + } + Ok(()) + }, + timeout.min(Duration::from_secs(5)), + FailureHealthAction::IgnoreFailure, + ) + .await + } + + /// Cancel an unclaimed remote reservation. False cannot prove that an + /// earlier claim did not start a physical mutation. + pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result { + self.execute_with_timeout_for_op( + "cancel_object_metadata_reservation", + || async { + let mut client = self.get_client().await?; + let mut request = Request::new(MetadataReservationCancelRequest { + disk: self.endpoint.to_string(), + token: token.as_bytes().to_vec().into(), + }); + let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref()) + .map_err(|_| Error::other("reservation cancellation request length overflow"))?; + crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?; + let response = client + .cancel_object_metadata_reservation(request) + .await + .map_err(|status| { + if status.code() == Code::Unimplemented { + DiskError::UnsupportedDisk + } else { + status.into() + } + })? + .into_inner(); + if let Some(error) = response.error { + return Err(error.into()); + } + Ok(response.success) + }, + get_max_timeout_duration(), + ) + .await + } + + /// Publish a staged user-object write using a remote one-shot reservation. + /// The caller must retain the bucket lifecycle fence until the tail drains. + pub async fn rename_data_with_metadata_reservation( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + token: Uuid, + ) -> Result { + self.execute_with_timeout_for_op( + "rename_reserved_data", + || async { + let mut client = self.get_client().await?; + let rename = RenameDataRequest { + disk: self.endpoint.to_string(), + src_volume: src_volume.to_owned(), + src_path: src_path.to_owned(), + file_info: compat_json(fi)?, + dst_volume: dst_volume.to_owned(), + dst_path: dst_path.to_owned(), + file_info_bin: encode_file_info_msgpack(fi)?.into(), + scanner_publication_lease_token: Bytes::new(), + bucket_incarnation_id: Bytes::new(), + }; + let mut request = Request::new(ReservedRenameDataRequest { + rename: Some(rename), + token: token.as_bytes().to_vec().into(), + }); + let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref()) + .map_err(|_| Error::other("reserved rename request length overflow"))?; + crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?; + let response = client + .rename_reserved_data(request) + .await + .map_err(|status| { + if status.code() == Code::Unimplemented { + DiskError::UnsupportedDisk + } else { + status.into() + } + })? + .into_inner(); + if !response.success { + return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected)); + } + decode_msgpack_or_json::( + &response.rename_data_resp_bin, + &response.rename_data_resp, + "RenameDataResp", + ) + }, + get_max_timeout_duration(), + ) + .await + } + #[tracing::instrument(level = "trace", skip_all)] pub(crate) async fn rename_data_borrowed( &self, @@ -5109,6 +5343,84 @@ mod tests { peer.stop().await; } + #[tokio::test] + async fn short_metadata_reservation_rejects_old_peer_without_legacy_rpc() { + runtime_sources::ensure_test_rpc_secret(); + let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new()) + .await + .expect("old peer must bind"); + let remote = remote_disk_for_addr(&peer.addr).await; + assert!(matches!( + remote + .reserve_object_metadata_short("bucket", "object", Uuid::new_v4(), Duration::from_secs(5)) + .await, + Err(DiskError::UnsupportedDisk) + )); + assert_eq!( + *peer.peer.unsupported_methods.lock().expect("observed methods lock"), + vec!["/node_service.NodeService/ReserveObjectMetadataShort"], + "an old peer must never receive the longer-lived v2 request" + ); + peer.stop().await; + } + + #[tokio::test] + async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() { + runtime_sources::ensure_test_rpc_secret(); + let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new()) + .await + .expect("old peer must bind"); + let remote = remote_disk_for_addr(&peer.addr).await; + let token = Uuid::new_v4(); + assert!(matches!( + remote + .reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5)) + .await, + Err(DiskError::UnsupportedDisk) + )); + assert!(matches!( + remote + .activate_object_metadata("bucket", "object", token, Duration::from_secs(5)) + .await, + Err(DiskError::UnsupportedDisk) + )); + assert!(matches!( + remote + .renew_object_metadata("bucket", "object", token, Duration::from_secs(5)) + .await, + Err(DiskError::UnsupportedDisk) + )); + assert!(matches!( + remote.cancel_object_metadata_reservation(token).await, + Err(DiskError::UnsupportedDisk) + )); + assert!(matches!( + remote + .rename_data_with_metadata_reservation( + crate::disk::RUSTFS_META_TMP_BUCKET, + "staged", + &sample_rename_data_file_info(), + "bucket", + "object", + token, + ) + .await, + Err(DiskError::UnsupportedDisk) + )); + assert_eq!( + *peer.peer.unsupported_methods.lock().expect("observed methods lock"), + vec![ + "/node_service.NodeService/ReserveObjectMetadata", + "/node_service.NodeService/ActivateObjectMetadataReservation", + "/node_service.NodeService/ActivateObjectMetadataReservation", + "/node_service.NodeService/CancelObjectMetadataReservation", + "/node_service.NodeService/RenameReservedData", + ], + "reservation operations must never downgrade to legacy methods", + ); + peer.stop().await; + } + #[tokio::test] async fn quota_fence_old_peer_rejects_without_legacy_fallback() { runtime_sources::ensure_test_rpc_secret(); diff --git a/crates/ecstore/src/cluster/rpc/remote_locker.rs b/crates/ecstore/src/cluster/rpc/remote_locker.rs index 9e5b9a910..632266221 100644 --- a/crates/ecstore/src/cluster/rpc/remote_locker.rs +++ b/crates/ecstore/src/cluster/rpc/remote_locker.rs @@ -1315,6 +1315,122 @@ impl LockClient for RemoteClient { Ok(resp.success) } + async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result { + let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER; + let mut request = Self::create_unlock_request(lock_id); + request.metadata.tags.insert(marker.to_owned(), marker.to_owned()); + let mut client = self.get_client().await?; + let resource_summary = request.resource.to_string(); + let mut req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&request) + .map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?, + }); + attach_lock_mutation_body_digest(&mut req)?; + let response = self + .execute_rpc( + "refresh", + &resource_summary, + Self::rpc_timeout(), + async move { client.refresh(req).await }, + RpcCleanup::new(None, None, None), + ) + .await? + .into_inner(); + if let Some(error) = response.error_info { + return Err(LockError::internal(error)); + } + // Older nodes refresh normally but return no conversion marker. + Ok(response.success && response.lock_info.as_deref() == Some(marker)) + } + + async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result { + let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER; + let mut request = Self::create_unlock_request(lock_id); + request.metadata.tags.insert(marker.to_owned(), marker.to_owned()); + let mut client = self.get_client().await?; + let resource_summary = request.resource.to_string(); + let mut req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&request) + .map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?, + }); + attach_lock_mutation_body_digest(&mut req)?; + let response = self + .execute_rpc( + "refresh", + &resource_summary, + Self::rpc_timeout(), + async move { client.refresh(req).await }, + RpcCleanup::new(None, None, None), + ) + .await? + .into_inner(); + if let Some(error) = response.error_info { + return Err(LockError::internal(error)); + } + Ok(response.success && response.lock_info.as_deref() == Some(marker)) + } + + async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result { + let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER; + let mut request = Self::create_unlock_request(lock_id); + request.metadata.tags.insert(marker.to_owned(), marker.to_owned()); + let mut client = self.get_client().await?; + let resource_summary = request.resource.to_string(); + let mut req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&request) + .map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?, + }); + attach_lock_mutation_body_digest(&mut req)?; + let response = self + .execute_rpc( + "refresh", + &resource_summary, + Self::rpc_timeout(), + async move { client.refresh(req).await }, + RpcCleanup::new(None, None, None), + ) + .await? + .into_inner(); + if let Some(error) = response.error_info { + return Err(LockError::internal(error)); + } + Ok(response.success && response.lock_info.as_deref() == Some(marker)) + } + + async fn waiting_writer(&self, lock_id: &LockId) -> Result> { + let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER; + let mut request = Self::create_unlock_request(lock_id); + request.metadata.tags.insert(marker.to_owned(), marker.to_owned()); + let mut client = self.get_client().await?; + let resource_summary = request.resource.to_string(); + let mut req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&request) + .map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?, + }); + attach_lock_mutation_body_digest(&mut req)?; + let response = self + .execute_rpc( + "refresh", + &resource_summary, + Self::rpc_timeout(), + async move { client.refresh(req).await }, + RpcCleanup::new(None, None, None), + ) + .await? + .into_inner(); + if let Some(error) = response.error_info { + return Err(LockError::internal(error)); + } + if !response.success { + return Ok(None); + } + Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) { + Some(":0") => Some(false), + Some(":1") => Some(true), + _ => None, + }) + } + async fn force_release(&self, lock_id: &LockId) -> Result { info!("remote force_release for {}", lock_id); let force_request = Self::create_unlock_request(lock_id); diff --git a/crates/ecstore/src/core/pools_test.rs b/crates/ecstore/src/core/pools_test.rs index 4496242ea..2eb82ea10 100644 --- a/crates/ecstore/src/core/pools_test.rs +++ b/crates/ecstore/src/core/pools_test.rs @@ -204,7 +204,7 @@ mod decommission_lock_order_tests { set_decommission_capacity_info_overrides_for_test, }; use crate::data_movement; - use crate::disk::RUSTFS_META_BUCKET; + use crate::disk::{DiskAPI as _, RUSTFS_META_BUCKET, ReadOptions}; use crate::object_api::{ObjectOptions, PutObjReader}; use crate::services::rebalance::{ test_three_pool_stores_with_isolated_node_contexts, @@ -4804,98 +4804,1196 @@ mod decommission_lock_order_tests { #[tokio::test] #[serial_test::serial] - async fn public_put_holds_decommission_capacity_through_early_ack_tail() { + async fn abandoned_reservation_does_not_hold_bucket_fence_behind_slow_peer() { + use crate::set_disk::rename_fanout_barrier::{self, PHASE_RESERVATION_AFTER, PHASE_RESERVATION_BEFORE}; + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { - let (_temp_dirs, store, other_store) = - test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; - let bucket = test_bucket("public-put-tail"); - let object = "public-put-tail.bin"; - let body = vec![0x71; 256 * 1024]; + let (_dirs, store, other_store) = test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("abandoned-preparation"); + let object = "object.bin"; store .make_bucket(&bucket, &MakeBucketOptions::default()) .await - .expect("create public PUT tail bucket"); - - let mut initial = PutObjReader::from_vec(vec![0x69; body.len()]); + .expect("create bucket"); store.pools[2] - .put_object(&bucket, object, &mut initial, &ObjectOptions::default()) + .put_object( + &bucket, + object, + &mut PutObjReader::from_vec(vec![1; 8193]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) .await - .expect("seed the unreserved public PUT target"); + .expect("seed object"); + let slow = rename_fanout_barrier::arm(object, 0, PHASE_RESERVATION_BEFORE); + let ready = rename_fanout_barrier::arm(object, 1, PHASE_RESERVATION_AFTER); + let worker = Arc::clone(&other_store); + let task_bucket = bucket.clone(); + let put = tokio::spawn(async move { + worker + .put_object( + &task_bucket, + object, + &mut PutObjReader::from_vec(vec![2; 8193]), + &ObjectOptions::default(), + ) + .await + }); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + slow.wait_until_paused_before(deadline).await; + ready.wait_until_paused_before(deadline).await; + tokio::time::timeout(Duration::from_secs(3), put) + .await + .expect("PUT fallback finishes") + .expect("PUT task") + .expect("PUT succeeds"); + ready.release(); + drop(ready); + let lock = store + .new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("publication lock"); + let writer = tokio::time::timeout(Duration::from_secs(2), lock.get_write_lock(Duration::from_secs(5))).await; + slow.release(); + drop(slow); + assert!( + matches!(writer, Ok(Ok(_))), + "abandoned completed tokens must not hold publication behind a slow peer" + ); + }) + .await; + } + #[test] + #[serial_test::serial] + fn saturated_budget_keeps_following_same_key_puts_available() { + assert_saturated_budget_keeps_following_same_key_puts_available(None); + } + + #[test] + #[serial_test::serial] + fn slow_short_activation_keeps_following_same_key_puts_available() { + assert_saturated_budget_keeps_following_same_key_puts_available(Some(Duration::from_millis(1200))); + } + + #[test] + #[serial_test::serial] + fn expired_short_activation_keeps_following_same_key_puts_available() { + assert_saturated_budget_keeps_following_same_key_puts_available(Some(Duration::from_millis(2100))); + } + + fn assert_saturated_budget_keeps_following_same_key_puts_available(activation_delay: Option) { + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime") + .block_on(async { + use crate::set_disk::rename_fanout_barrier::{self, PHASE_RENAME}; + + temp_env::async_with_vars( + [(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], + async { + let (_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("speculative-budget-tail"); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + store.pools[2] + .put_object( + &bucket, + object, + &mut PutObjReader::from_vec(vec![1; 8193]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed object"); + let rename_tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, PHASE_RENAME); + let worker = Arc::clone(&other_store); + let task_bucket = bucket.clone(); + let first = tokio::spawn(async move { + worker + .put_object( + &task_bucket, + object, + &mut PutObjReader::from_vec(vec![2; 8193]), + &ObjectOptions::default(), + ) + .await + }); + barrier + .wait_until_paused_before(tokio::time::Instant::now() + Duration::from_secs(10)) + .await; + tokio::time::timeout(Duration::from_secs(5), first) + .await + .expect("first PUT acknowledges") + .expect("first task") + .expect("first PUT succeeds"); + let _permits = (0..crate::runtime::instance::SPECULATIVE_RESERVATION_LIMIT) + .map(|_| other_store.ctx.try_acquire_speculative_reservation().expect("fill budget")) + .collect::>(); + let staging = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + object, + crate::set_disk::PutObjectCommitPause::BeforeBucketSnapshot, + ); + let worker = Arc::clone(&other_store); + let task_bucket = bucket.clone(); + let second_task = tokio::spawn(async move { + worker + .put_object( + &task_bucket, + object, + &mut PutObjReader::from_vec(vec![3; 8193]), + &ObjectOptions::default(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), staging.wait_until_paused()) + .await + .expect("short reservation completes before staging"); + tokio::time::sleep(Duration::from_secs(3)).await; + let activation = activation_delay.map(|_| { + rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RESERVATION_ACTIVATED) + }); + staging.release(); + drop(staging); + if let Some(activation) = activation { + tokio::time::timeout(Duration::from_secs(5), activation.wait_until_paused()) + .await + .expect("short reservation activation must start"); + tokio::time::sleep(activation_delay.expect("activation delay")).await; + activation.disarm(); + activation.release(); + } + let second = tokio::time::timeout(Duration::from_secs(7), second_task) + .await + .map(|result| result.expect("second task")); + let third = if matches!(second, Ok(Ok(_))) { + Some( + tokio::time::timeout( + Duration::from_secs(7), + other_store.put_object( + &bucket, + object, + &mut PutObjReader::from_vec(vec![4; 8193]), + &ObjectOptions::default(), + ), + ) + .await, + ) + } else { + None + }; + barrier.release(); + tokio::task::yield_now().await; + barrier.release(); + drop(barrier); + second.expect("second PUT completes").expect("second PUT succeeds"); + third + .expect("third PUT was attempted") + .expect("third PUT completes") + .expect("third PUT succeeds"); + tokio::time::timeout(Duration::from_secs(30), async { + while rename_tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("all acknowledged rename tails must finish"); + let mut reader = other_store + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("latest PUT must remain readable"); + let etag = reader.object_info.etag.clone().expect("latest object has an ETag"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("read latest PUT"); + assert_eq!(actual, vec![4; 8193], "older tails must not overwrite the latest PUT"); + assert_all_target_shards_current(&other_store, &bucket, object, &etag).await; + }, + ) + .await; + }); + }) + .expect("test thread") + .join() + .expect("test thread must not panic"); + } + + #[tokio::test] + #[serial_test::serial] + async fn saturated_speculative_budget_falls_back_then_readmits() { + use crate::set_disk::rename_fanout_barrier::{self, PHASE_RESERVATION_BEFORE}; + + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let (_dirs, store, other_store) = test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("speculative-budget"); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + store.pools[2] + .put_object( + &bucket, + object, + &mut PutObjReader::from_vec(vec![1; 8193]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed object"); + let mut permits = (0..crate::runtime::instance::SPECULATIVE_RESERVATION_LIMIT) + .map(|_| other_store.ctx.try_acquire_speculative_reservation().expect("fill budget")) + .collect::>(); + let barrier = rename_fanout_barrier::arm(object, 0, PHASE_RESERVATION_BEFORE); + tokio::time::timeout( + Duration::from_secs(3), + other_store.put_object(&bucket, object, &mut PutObjReader::from_vec(vec![2; 8193]), &ObjectOptions::default()), + ) + .await + .expect("saturated preparation must not block") + .expect("fallback PUT succeeds"); + assert!(barrier.is_paused(), "saturation must use the bounded short-lease path"); + drop(barrier); + drop(permits.pop()); + let barrier = rename_fanout_barrier::arm(object, 0, PHASE_RESERVATION_BEFORE); + let worker = Arc::clone(&other_store); + let task_bucket = bucket.clone(); + let put = tokio::spawn(async move { + worker + .put_object( + &task_bucket, + object, + &mut PutObjReader::from_vec(vec![3; 8193]), + &ObjectOptions::default(), + ) + .await + }); + barrier + .wait_until_paused_before(tokio::time::Instant::now() + Duration::from_secs(5)) + .await; + barrier.release(); + drop(barrier); + tokio::time::timeout(Duration::from_secs(5), put) + .await + .expect("readmitted PUT completes") + .expect("PUT task") + .expect("PUT succeeds"); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tentative_reservation_does_not_block_on_bucket_publication_writer() { + let (_dirs, store, other_store) = test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("reservation-bucket-writer"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + for key in ["baseline", "candidate"] { + store.pools[2] + .put_object( + &bucket, + key, + &mut PutObjReader::from_vec(vec![1; 256 * 1024]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed target pool"); + } + let lock = store + .new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("publication lock"); + let guard = lock + .get_write_lock(Duration::from_secs(5)) + .await + .expect("hold publication writer"); + let baseline = + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async { + tokio::time::timeout( + Duration::from_secs(2), + other_store.put_object( + &bucket, + "baseline", + &mut PutObjReader::from_vec(vec![2; 8193]), + &ObjectOptions::default(), + ), + ) + .await + }) + .await; + let baseline = match baseline { + Ok(Ok(result)) => result, + result => { + drop(guard); + panic!("baseline did not complete with publication writer: {result:?}"); + } + }; + assert!(baseline.etag.is_some()); + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let mut data = PutObjReader::from_vec(vec![3; 8193]); + let opts = ObjectOptions::default(); + let candidate = other_store.put_object(&bucket, "candidate", &mut data, &opts); + tokio::pin!(candidate); + let before = tokio::time::timeout(Duration::from_secs(2), &mut candidate).await; + drop(guard); + match before { + Ok(result) => { + result.expect("candidate PUT succeeds"); + } + Err(_) => { + let _ = tokio::time::timeout(Duration::from_secs(15), &mut candidate).await; + panic!("tentative reservations blocked the PUT behind a bucket publication writer"); + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn unrelated_pool_lock_does_not_double_put_selection_wait() { + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let (_dirs, store, other_store) = test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("unrelated-pool-lock"); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + store.pools[2] + .put_object( + &bucket, + object, + &mut PutObjReader::from_vec(vec![1; 256 * 1024]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed healthy pool"); + let unrelated_set = other_store.pools[1].get_disks_by_key(object); + let lock = unrelated_set.new_ns_lock(&bucket, object).await.expect("unrelated pool lock"); + let guard = lock + .get_write_lock(Duration::from_secs(5)) + .await + .expect("hold unrelated pool lock"); + tokio::time::timeout( + Duration::from_secs(1), + other_store.pools[2].get_object_info(&bucket, object, &ObjectOptions::default()), + ) + .await + .expect("healthy pool uses a separate lock") + .expect("healthy object exists"); + let worker = Arc::clone(&other_store); + let put_bucket = bucket.clone(); + let mut put = tokio::spawn(async move { + worker + .put_object( + &put_bucket, + object, + &mut PutObjReader::from_vec(vec![2; 256 * 1024]), + &ObjectOptions::default(), + ) + .await + }); + let before = tokio::time::timeout(Duration::from_secs(7), &mut put).await; + drop(guard); + match before { + Ok(result) => { + result.expect("PUT task").expect("PUT succeeds in healthy pool"); + } + Err(_) => { + let _ = tokio::time::timeout(Duration::from_secs(20), put).await; + panic!("the unrelated pool caused a second lock wait"); + } + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_orders_following_put_without_timeout() { + Box::pin(assert_active_decommission_orders_following_put(false, true, false)).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_orders_cross_node_put_without_timeout() { + Box::pin(assert_active_decommission_orders_following_put(true, true, false)).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_preserves_latest_put_after_tail_drains() { + Box::pin(assert_active_decommission_orders_following_put(true, false, false)).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_releases_object_lock_during_tail_cleanup() { + Box::pin(assert_active_decommission_orders_following_put(true, false, true)).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn retiring_source_multipart_commit_waits_for_migration_read_anchor() { + let (_dirs, store, other_store) = test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("multipart-source-fence"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let fixed = &store.pools[0].disk_set[0]; + let mut object = None; + for i in 0..100 { + let name = format!("source-{i}"); + if !fixed + .shares_namespace_lock_domain(&store.pools[0].get_disks_by_key(&name)) + .await + { + object = Some(name); + break; + } + } + let object = object.expect("fixture needs distinct fixed and source lock domains"); + let body = vec![0x41; 8193]; + let upload = new_multipart_upload(&store, 0, &bucket, &object, ObjectOptions::default()) + .await + .expect("create source multipart upload"); + let source_opts = multipart_options(&store, &bucket, ObjectOptions::default()).await; + let part = store.pools[0] + .put_object_part( + &bucket, + &object, + &upload.upload_id, + 1, + &mut PutObjReader::from_vec(body.clone()), + &source_opts, + ) + .await + .expect("stage source multipart part"); + let parts = vec![crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }]; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, body.len() * 8, body.len() * 8, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, body.len() * 8, body.len() * 8, 0), + ]; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity.clone()]); + set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacity]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("start decommission"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let lock = store.new_ns_lock(&bucket, &object).await.expect("source anchor lock"); + let anchor = lock + .get_read_lock(Duration::from_secs(5)) + .await + .expect("hold migration source anchor"); + let worker = Arc::clone(&other_store); + let worker_bucket = bucket.clone(); + let worker_object = object.clone(); + let mut commit = tokio::spawn(async move { + worker + .complete_multipart_upload(&worker_bucket, &worker_object, &upload.upload_id, parts, &ObjectOptions::default()) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(200), &mut commit).await.is_err(), + "a multipart completion on the retiring source must not pass the migration anchor" + ); + drop(anchor); + tokio::time::timeout(Duration::from_secs(10), commit) + .await + .expect("multipart completion finishes after migration releases the anchor") + .expect("multipart completion task") + .expect("multipart completion succeeds"); + let mut source = store.pools[0] + .get_object_reader(&bucket, &object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read completed source object"); + let mut actual = Vec::new(); + source + .stream + .read_to_end(&mut actual) + .await + .expect("read completed source body"); + assert_eq!(actual, body); + } + + #[tokio::test] + #[serial_test::serial] + async fn active_decommission_distinct_pool_puts_preserve_later_write_and_delete() { + temp_env::async_with_vars( + [(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], + Box::pin(async { + let (_dirs, first_store, second_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("different-pool-writes"); + let object = "object.bin"; + let size = 256 * 1024; + first_store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let capacity = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, size, size), + DecommissionPoolCapacityInfo::for_test(1, layout, size * 8, size * 8, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, size * 8, size * 8, 0), + ]; + set_decommission_capacity_info_overrides_for_test(first_store.id, vec![capacity.clone()]); + set_decommission_capacity_info_overrides_for_test(second_store.id, vec![capacity]); + first_store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("start decommission"); + *second_store.pool_meta.write().await = first_store.pool_meta.read().await.clone(); + let total = 1_u64 << 40; + let available = crate::disk::DiskInfo { + total, + free: total, + ..Default::default() + }; + let full = crate::disk::DiskInfo { + total, + used: total, + free: 0, + ..Default::default() + }; + let _first_space = + crate::store::PoolSpaceForTest::install(first_store.id, &[(1, available.clone()), (2, full.clone())]); + let _second_space = crate::store::PoolSpaceForTest::install(second_store.id, &[(1, full), (2, available)]); + let barrier = DecommissionCapacityLockOrderBarrier::install(first_store.id, first_store.id); + barrier.pause_external_object_commit_phase(); + let first_worker = Arc::clone(&first_store); + let first_bucket = bucket.clone(); + let mut first = tokio::spawn(async move { + first_worker + .put_object( + &first_bucket, + object, + &mut PutObjReader::from_vec(vec![0x41; size]), + &ObjectOptions::default(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(15), barrier.wait_until_external_object_commit_phase_started()) + .await + .expect("first PUT has selected its target"); + let second = tokio::time::timeout( + Duration::from_secs(15), + second_store.put_object( + &bucket, + object, + &mut PutObjReader::from_vec(vec![0x42; size]), + &ObjectOptions::default(), + ), + ) + .await + .expect("second PUT completes") + .expect("second PUT succeeds"); + let pool2 = second_store.pools[2] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("second PUT in pool 2"); + assert_eq!(pool2.etag, second.etag, "second PUT must select pool 2"); + let tail = + crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME); + let tasks = crate::set_disk::rename_fanout_barrier::observe_tasks(object); + barrier.release_external_object_commit_phase(); + tokio::time::timeout(Duration::from_secs(15), tail.wait_until_paused()) + .await + .expect("first PUT tail is parked"); + tail.disarm(); + let first = tokio::time::timeout(Duration::from_secs(15), &mut first) + .await + .expect("first PUT completes") + .expect("first task") + .expect("first PUT succeeds"); + let pool1 = first_store.pools[1] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("first PUT in pool 1"); + assert_eq!(pool1.etag, first.etag, "first PUT must select pool 1"); + let third = tokio::time::timeout( + Duration::from_secs(15), + second_store.put_object( + &bucket, + object, + &mut PutObjReader::from_vec(vec![0x43; size]), + &ObjectOptions::default(), + ), + ) + .await; + tail.release(); + drop(tail); + let third = third.expect("third PUT completes").expect("third PUT succeeds"); + tokio::time::timeout(Duration::from_secs(30), async { + while tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("all tails drain"); + let mut reader = second_store + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read latest"); + assert_eq!(reader.object_info.etag, third.etag, "a later PUT must be current across pools"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("read latest body"); + assert_eq!(body, vec![0x43; size]); + drop(reader); + second_store + .delete_object(&bucket, object, ObjectOptions::default()) + .await + .expect("delete current object"); + let missing = second_store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect_err("delete must not resurrect an older pool copy"); + assert!(crate::error::is_err_object_not_found(&missing), "unexpected delete result: {missing}"); + }), + ) + .await; + } + + async fn assert_active_decommission_orders_following_put(cross_node: bool, hold_tail: bool, hold_cleanup: bool) { + temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], Box::pin(async { + let (_dirs, store, other_store) = test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("active-following-put"); + let object = "object.bin"; + let body_size = if hold_cleanup { 1024 * 1024 } else { 256 * 1024 }; + let body = vec![0x71; body_size]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let source_version = if hold_cleanup { + Some(Box::pin(seed_cleanup_migration_source(&store, &bucket, object, body_size)).await) + } else { + None + }; + store.pools[2] + .put_object( + &bucket, + object, + &mut PutObjReader::from_vec(vec![0x69; body.len()]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed target"); let layout = DecommissionErasureLayout { data: 1, parity: 0 }; - let target_total = body.len().saturating_mul(4); - let capacity_snapshot = vec![ - DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), - DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0), - DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), - ]; - set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot]); + let total = body.len().saturating_mul(4); + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, total, total, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, total, total, 0), + ]], + ); + set_decommission_capacity_info_overrides_for_test( + other_store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, total, total, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, total, total, 0), + ]], + ); store .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) .await - .expect("activate the public PUT tail reservation"); + .expect("activate decommission"); *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); - let reservation_target = store.pool_meta.read().await.pools[0] - .decommission - .as_ref() - .and_then(|info| info.capacity_reservation.as_ref()) - .expect("public PUT tail reservation should exist") - .targets[0] - .pool_index; - assert_eq!(reservation_target, 1, "the public PUT target must remain unreserved"); - - let tail_barrier = - crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME); - let put_store = Arc::clone(&other_store); - let put_bucket = bucket.clone(); - let put_body = body.clone(); - let expected_body = body.clone(); - let mut put = tokio::spawn(async move { - let mut data = PutObjReader::from_vec(put_body); - put_store - .put_object(&put_bucket, object, &mut data, &ObjectOptions::default()) + let rename_tasks = crate::set_disk::rename_fanout_barrier::observe_tasks(object); + let mut barrier = Some(crate::set_disk::rename_fanout_barrier::arm( + object, + 0, + crate::set_disk::rename_fanout_barrier::PHASE_RENAME, + )); + let first_store = Arc::clone(if cross_node { &store } else { &other_store }); + let first_bucket = bucket.clone(); + let mut first = tokio::spawn(async move { + first_store + .put_object(&first_bucket, object, &mut PutObjReader::from_vec(body), &ObjectOptions::default()) .await }); - tail_barrier.wait_until_paused().await; - tokio::time::timeout(Duration::from_secs(30), &mut put) + if tokio::time::timeout(Duration::from_secs(10), barrier.as_ref().expect("tail barrier").wait_until_paused()) .await - .expect("public PUT should return its early quorum acknowledgement") - .expect("public PUT task should not panic") - .expect("public PUT should commit its quorum"); - - let capacity_lock = other_store - .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) - .await - .expect("create the public PUT capacity probe"); - let mut capacity_probe = tokio::spawn(async move { capacity_lock.get_write_lock(Duration::from_secs(30)).await }); - assert!( - tokio::time::timeout(Duration::from_millis(100), &mut capacity_probe) + .is_err() + { + let result = tokio::time::timeout(Duration::from_secs(2), &mut first).await; + panic!("first PUT did not reach the rename barrier: {result:?}"); + } + first.await.expect("first task").expect("first PUT"); + // Keep the first tail parked without trapping a successor on the + // same test-only checkpoint. + barrier.as_ref().expect("tail barrier").disarm(); + let mut cleanup_barrier = hold_cleanup.then(|| { + crate::set_disk::rename_fanout_barrier::arm( + object, + 0, + crate::set_disk::rename_fanout_barrier::PHASE_CLEANUP, + ) + }); + if !hold_tail { + drop(barrier.take()); + } + if let Some(cleanup_gate) = cleanup_barrier.as_ref() { + tokio::time::timeout(Duration::from_secs(10), cleanup_gate.wait_until_paused()) .await - .is_err(), - "public PUT early-ACK tail must retain its staged capacity read fence" - ); - - tail_barrier.release(); - drop(tail_barrier); - let capacity_guard = tokio::time::timeout(Duration::from_secs(30), capacity_probe) - .await - .expect("public PUT capacity probe should finish after its tail") - .expect("public PUT capacity probe should not panic") - .expect("public PUT capacity probe should acquire after its tail"); - drop(capacity_guard); - + .expect("the tail cleanup must start before the next PUT"); + cleanup_gate.disarm(); + Box::pin(assert_decommission_cleanup_capacity_fenced(&other_store)).await; + Box::pin(assert_decommission_cleanup_object_unlocked(&other_store, &bucket, object)).await; + Box::pin(assert_decommission_cleanup_reads_committed_object(&other_store, &bucket, object, body_size, 0x71)).await; + Box::pin(put_during_decommission_cleanup(&other_store, &bucket, object, body_size)).await; + Box::pin(assert_decommission_cleanup_reads_committed_object(&other_store, &bucket, object, body_size, 0x72)).await; + Box::pin(assert_decommission_cleanup_object_unlocked(&other_store, &bucket, object)).await; + let migration = Box::pin(start_cleanup_migration( + &other_store, + &bucket, + object, + source_version.as_deref().expect("cleanup migration source"), + )) + .await; + Box::pin(wait_for_cleanup_migration_anchor(&store, &bucket, object, &migration)).await; + drop(cleanup_barrier.take()); + tokio::time::timeout(Duration::from_secs(30), migration) + .await + .expect("migration should finish after cleanup") + .expect("migration task should join") + .expect("migration should preserve the source version"); + Box::pin(assert_cleanup_migrated_source_readable( + &store, + &bucket, + object, + source_version.as_deref().expect("cleanup migration source"), + body_size, + )) + .await; + Box::pin(assert_decommission_cleanup_reads_committed_object(&other_store, &bucket, object, body_size, 0x72)).await; + } + let commit_phase = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id); + let second_store = Arc::clone(&other_store); + let second_bucket = bucket.clone(); + let final_byte = if hold_cleanup { 0x73 } else { 0x72 }; + let deadline = tokio::time::Instant::now() + Duration::from_secs(7); + let mut second = tokio::spawn(async move { + second_store + .put_object( + &second_bucket, + object, + &mut PutObjReader::from_vec(vec![final_byte; body_size]), + &ObjectOptions::default(), + ) + .await + }); + let reached_commit = + tokio::time::timeout(Duration::from_secs(2), commit_phase.wait_until_external_object_commit_phase_started()) + .await + .is_ok(); + let released_capacity = + tokio::time::timeout(Duration::from_secs(1), commit_phase.wait_until_external_capacity_released()) + .await + .is_ok(); + let result = tokio::time::timeout_at(deadline, &mut second).await; + drop(barrier.take()); + drop(cleanup_barrier); + if result.is_err() { + second.abort(); + } + result + .unwrap_or_else(|_| { + panic!("second PUT timed out, reached commit phase: {reached_commit}, released capacity: {released_capacity}") + }) + .expect("second PUT task") + .unwrap_or_else(|error| { + panic!("second PUT failed, reached commit phase: {reached_commit}, released capacity: {released_capacity}: {error}") + }); + tokio::time::timeout(Duration::from_secs(30), async { + while rename_tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("both rename tails must drain"); let mut reader = other_store .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) .await - .expect("public PUT result should remain readable"); - let mut committed = Vec::new(); - reader - .stream - .read_to_end(&mut committed) + .expect("read the latest object after tail drain"); + let expected_etag = reader.object_info.etag.clone().expect("latest object has an ETag"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("read the latest object body"); + assert_eq!(actual, vec![final_byte; body_size], "the first tail must not restore older data"); + Box::pin(assert_all_target_shards_current(&other_store, &bucket, object, &expected_etag)).await; + })) + .await; + } + + async fn assert_decommission_cleanup_capacity_fenced(store: &crate::store::ECStore) { + let capacity_lock = store + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + .await + .expect("create the capacity probe"); + assert!( + tokio::time::timeout(Duration::from_millis(200), capacity_lock.get_write_lock(Duration::from_secs(5))) .await - .expect("public PUT result body should be readable"); - assert_eq!(committed, expected_body, "public PUT should publish the requested bytes"); + .is_err(), + "cleanup must retain its capacity fence until disk space is reconciled" + ); + } + + async fn assert_all_target_shards_current(store: &crate::store::ECStore, bucket: &str, object: &str, expected_etag: &str) { + let set = store.pools[2].get_disks_by_key(object); + let disks = set.disks.read().await.clone(); + assert!(!disks.is_empty(), "target set must contain disks"); + for disk in disks { + let disk = disk.expect("all target disks must be available"); + let info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("read target disk metadata"); + assert_eq!( + info.get_etag().as_deref(), + Some(expected_etag), + "a late tail must not restore stale metadata" + ); + } + } + + async fn seed_cleanup_migration_source( + store: &Arc, + bucket: &str, + object: &str, + body_size: usize, + ) -> String { + let version = uuid::Uuid::new_v4().to_string(); + store.pools[0] + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x39; body_size]), + &ObjectOptions { + versioned: true, + version_id: Some(version.clone()), + mod_time: Some(time::OffsetDateTime::now_utc() - time::Duration::days(1)), + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed the historical migration source"); + version + } + + async fn start_cleanup_migration( + store: &Arc, + bucket: &str, + object: &str, + version: &str, + ) -> tokio::task::JoinHandle> { + let incarnation = store.bucket_incarnation_id(bucket).await.expect("load bucket incarnation"); + let source = store.pools[0] + .get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("open the historical migration source"); + let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + let store = Arc::clone(store); + let bucket = bucket.to_string(); + let (started, running) = tokio::sync::oneshot::channel(); + let migration = tokio::spawn(async move { + let _ = started.send(()); + data_movement::migrate_decommission_object( + store, + 0, + bucket, + source, + Some(incarnation), + "cleanup_migration", + Some(owner), + ) + .await + }); + running.await.expect("migration task should start"); + migration + } + + async fn wait_for_cleanup_migration_anchor( + store: &crate::store::ECStore, + bucket: &str, + object: &str, + migration: &tokio::task::JoinHandle>, + ) { + let lock = store + .new_ns_lock(bucket, object) + .await + .expect("create the migration anchor probe"); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + assert!(!migration.is_finished(), "migration ended before holding its source anchor"); + match tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(1))).await { + Err(_) => break, + Ok(Ok(guard)) => drop(guard), + Ok(Err(error)) => panic!("migration anchor probe failed: {error}"), + } + tokio::task::yield_now().await; + } }) + .await + .expect("migration must hold its source anchor during cleanup"); + } + + async fn assert_cleanup_migrated_source_readable( + store: &Arc, + bucket: &str, + object: &str, + version: &str, + body_size: usize, + ) { + let opts = ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), + ..Default::default() + }; + for pool in store.pools.iter().skip(1) { + let Ok(mut reader) = pool.get_object_reader(bucket, object, None, HeaderMap::new(), &opts).await else { + continue; + }; + let mut actual = Vec::new(); + tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut actual)) + .await + .expect("reading the migrated source must finish") + .expect("read the migrated source"); + assert_eq!(actual, vec![0x39; body_size]); + return; + } + panic!("the migrated source version must be readable from a target pool"); + } + + async fn assert_decommission_cleanup_object_unlocked(store: &crate::store::ECStore, bucket: &str, object: &str) { + let lock = store.new_ns_lock(bucket, object).await.expect("create the object probe"); + let guard = tokio::time::timeout(Duration::from_secs(2), lock.get_write_lock(Duration::from_secs(5))) + .await + .expect("the finished rename must not hold the object lock through cleanup") + .expect("the object probe must acquire the lock"); + drop(guard); + } + + async fn assert_decommission_cleanup_reads_committed_object( + store: &Arc, + bucket: &str, + object: &str, + body_size: usize, + expected: u8, + ) { + let mut reader = tokio::time::timeout( + Duration::from_secs(5), + store.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()), + ) + .await + .expect("the committed object must be readable during cleanup") + .expect("open the committed object"); + let mut actual = Vec::new(); + tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut actual)) + .await + .expect("reading the committed object must finish during cleanup") + .expect("read the committed object"); + assert_eq!(actual, vec![expected; body_size]); + } + + async fn put_during_decommission_cleanup(store: &Arc, bucket: &str, object: &str, body_size: usize) { + let mut body = PutObjReader::from_vec(vec![0x72; body_size]); + tokio::time::timeout( + Duration::from_secs(5), + store.put_object(bucket, object, &mut body, &ObjectOptions::default()), + ) + .await + .expect("the next PUT must finish while the earlier cleanup is paused") + .expect("the next PUT must succeed"); + } + + #[tokio::test] + #[serial_test::serial] + async fn public_put_holds_decommission_capacity_through_early_ack_tail() { + Box::pin(assert_public_put_holds_decommission_capacity_through_early_ack_tail()).await; + } + + async fn assert_public_put_holds_decommission_capacity_through_early_ack_tail() { + temp_env::async_with_vars( + [(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], + Box::pin(async { + let (_temp_dirs, store, other_store) = + test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; + let bucket = test_bucket("public-put-tail"); + let object = "public-put-tail.bin"; + let body = vec![0x71; 256 * 1024]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create public PUT tail bucket"); + let source_version = Box::pin(seed_cleanup_migration_source(&store, &bucket, object, body.len())).await; + + let mut initial = PutObjReader::from_vec(vec![0x69; body.len()]); + store.pools[2] + .put_object(&bucket, object, &mut initial, &ObjectOptions::default()) + .await + .expect("seed the unreserved public PUT target"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + let target_total = body.len().saturating_mul(4); + let capacity_snapshot = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()), + DecommissionPoolCapacityInfo::for_test(1, layout, target_total, target_total, 0), + DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), + ]; + set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot]); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the public PUT tail reservation"); + *other_store.pool_meta.write().await = store.pool_meta.read().await.clone(); + let reservation_target = store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("public PUT tail reservation should exist") + .targets[0] + .pool_index; + assert_eq!(reservation_target, 1, "the public PUT target must remain unreserved"); + + let tail_barrier = + crate::set_disk::rename_fanout_barrier::arm(object, 0, crate::set_disk::rename_fanout_barrier::PHASE_RENAME); + let put_store = Arc::clone(&other_store); + let put_bucket = bucket.clone(); + let put_body = body.clone(); + let mut put = tokio::spawn(async move { + let mut data = PutObjReader::from_vec(put_body); + put_store + .put_object(&put_bucket, object, &mut data, &ObjectOptions::default()) + .await + }); + tokio::time::timeout(Duration::from_secs(10), tail_barrier.wait_until_paused()) + .await + .expect("public PUT should reach the rename tail"); + tokio::time::timeout(Duration::from_secs(30), &mut put) + .await + .expect("public PUT should return its early quorum acknowledgement") + .expect("public PUT task should not panic") + .expect("public PUT should commit its quorum"); + + let capacity_lock = other_store + .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME) + .await + .expect("create the public PUT capacity probe"); + let mut capacity_probe = tokio::spawn(async move { capacity_lock.get_write_lock(Duration::from_secs(30)).await }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut capacity_probe) + .await + .is_err(), + "public PUT early-ACK tail must retain its staged capacity read fence" + ); + capacity_probe.abort(); + let _ = capacity_probe.await; + + // The target is distinct from the retiring source. A migration + // may read that source while the target tail is ordered, but a + // source mutation must remain excluded until the tail drains. + let source_lock = store + .new_ns_lock(&bucket, object) + .await + .expect("create migration source probe"); + let source_guard = + tokio::time::timeout(Duration::from_secs(2), source_lock.get_read_lock(Duration::from_secs(5))) + .await + .expect("migration source read must be available") + .expect("migration source read guard"); + drop(source_guard); + assert!( + tokio::time::timeout(Duration::from_millis(100), source_lock.get_write_lock(Duration::from_secs(5))) + .await + .is_err(), + "source mutations must wait for the acknowledged PUT tail" + ); + let migration = Box::pin(start_cleanup_migration(&store, &bucket, object, &source_version)).await; + tail_barrier.disarm(); + Box::pin(put_during_decommission_cleanup(&other_store, &bucket, object, body.len())).await; + + tail_barrier.release(); + drop(tail_barrier); + tokio::time::timeout(Duration::from_secs(30), migration) + .await + .expect("migration should finish after the tail") + .expect("migration task should not panic") + .expect("migration should preserve the historical version"); + Box::pin(assert_cleanup_migrated_source_readable( + &store, + &bucket, + object, + &source_version, + body.len(), + )) + .await; + + let mut reader = other_store + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("public PUT result should remain readable"); + let mut committed = Vec::new(); + reader + .stream + .read_to_end(&mut committed) + .await + .expect("public PUT result body should be readable"); + assert_eq!(committed, vec![0x72; body.len()], "the later PUT must remain current after migration"); + }), + ) .await; } diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index febf1d76e..afba434a3 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -655,6 +655,19 @@ impl Sets { .await } + pub(crate) async fn put_object_with_prepared_reservations( + &self, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, + reservations: Option>>, + ) -> Result<(ObjectInfo, Option)> { + self.get_disks_by_key(object) + .put_object_with_prepared_reservations(bucket, object, data, opts, reservations) + .await + } + pub(crate) async fn put_object_with_old_current_size_for_data_movement( &self, bucket: &str, diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 6559c85b1..3b51b949b 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -604,8 +604,69 @@ impl LocalDiskWrapper { let super::RenameDataGuards { external_guard, namespace_owner, + metadata_reservation, + metadata_reservation_token, .. } = guards; + let metadata_reservation = match (metadata_reservation, metadata_reservation_token) { + (Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)), + (Some(reservation), None) => Some(reservation), + (None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) { + Ok(reservation) => Some(reservation), + // A missing token may already have been claimed by an earlier + // request that is still publishing; do not claim it was safe. + Err(DiskError::DiskOngoingReq) => { + return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq)); + } + Err(error) => return super::RenameDataObservation::not_dispatched(error), + }, + (None, None) => None, + }; + // A queued reservation must acquire its metadata lease before disk + // admission. Otherwise a waiting background mutation could hold the + // disk permit needed by the earlier lease owner. + let metadata_reservation = match metadata_reservation { + Some(reservation) => Some(reservation.wait().await), + None => None, + }; + // The claimed reservation stays in this cancellable future while disk + // admission can wait. Only physical work moves into an owned task. + let prechecked_permit = if metadata_reservation.is_some() { + // Background admission is exclusive. Waiting for it while holding + // this object's lease can deadlock an admitted mutation waiting for + // the same lease. An already owned permit does not need to wait. + if current_disk_io_context().is_some_and(|context| { + context.class == DiskIoClass::Background + && !context + .permit + .as_ref() + .is_some_and(|permit| permit.is_for(&self.io_admission)) + }) { + return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq); + } + let preflight = self + .with_native_resources(async { + if self.health.is_faulty() { + return Err(DiskError::FaultyDisk); + } + self.check_disk_stale().await?; + self.acquire_disk_io_permit().await + }) + .await; + match preflight { + Ok(Ok(Some(permit))) => Some(permit), + Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq), + Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error), + } + } else { + None + }; + // The reserved metadata lease must outlive a cancelled RPC waiter. + let external_guard: Option> = if metadata_reservation.is_some() { + Some(Arc::new(external_guard)) + } else { + external_guard + }; let operation = self.clone(); let src_volume = src_volume.to_owned(); let src_path = src_path.to_owned(); @@ -624,27 +685,54 @@ impl LocalDiskWrapper { }; let observed = run_owned_mutation(external_guard, move || async move { let mut preflight_rejection = None; - let result = operation - .track_disk_health_mutation( - "rename_data", - DiskMetricMutation::Write, - || async { - // Preserve the former DiskAPI future's single boxing boundary. - let observed = Box::pin(operation.disk.rename_data_observed( - &src_volume, - &src_path, - &fi, - &dst_volume, - &dst_path, - namespace_owner, + let rename = || async { + // Preserve the former DiskAPI future's single boxing boundary. + let observed = if let Some(reservation) = metadata_reservation { + Box::pin(operation.disk.rename_data_observed_with_reservation( + &src_volume, + &src_path, + &fi, + &dst_volume, + &dst_path, + (namespace_owner, Some(reservation)), + )) + .await + } else { + Box::pin(operation.disk.rename_data_observed( + &src_volume, + &src_path, + &fi, + &dst_volume, + &dst_path, + namespace_owner, + )) + .await + }; + preflight_rejection = observed.preflight_rejection; + observed.result + }; + let result = if let Some(permit) = prechecked_permit { + with_disk_io_permit(Some(permit), async { + operation + .with_native_resources(operation.track_admitted_disk_operation( + "rename_data", + move |io_permit| async move { + let _io_permit = io_permit; + rename().await + }, + Duration::ZERO, + TimeoutHealthAction::MarkFailure, + DiskMetricMutation::Write, + true, )) - .await; - preflight_rejection = observed.preflight_rejection; - observed.result - }, - timeout_duration, - ) - .await; + .await? + }) + .await + } else { + operation + .track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration) + .await + }; // Health tracking must observe the real disk error, not an Ok tuple. Ok(super::RenameDataObservation { result, @@ -1598,6 +1686,81 @@ pub struct LocalDiskWrapper { } impl LocalDiskWrapper { + /// Reserve the metadata mutation lock for one object on this disk instance. + /// Unclaimed reservations expire after 60 seconds. + pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result { + if self.health.is_faulty() { + return Err(DiskError::FaultyDisk); + } + self.check_disk_stale().await?; + self.disk.reserve_object_metadata(bucket, object, timeout).await + } + + /// Register an object metadata reservation in FIFO order on this disk. + /// The returned token may be claimed before its physical lease is available. + pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result { + if self.health.is_faulty() { + return Err(DiskError::FaultyDisk); + } + self.check_disk_stale().await?; + self.disk.enqueue_object_metadata(bucket, object).await + } + + /// Prepare an inert reservation token bound to this disk and object. + pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result { + self.prepare_object_metadata_with_owner(bucket, object, None).await + } + + pub(crate) async fn prepare_object_metadata_with_owner( + &self, + bucket: &str, + object: &str, + owner: Option>, + ) -> Result { + if self.health.is_faulty() { + return Err(DiskError::FaultyDisk); + } + self.check_disk_stale().await?; + self.disk.prepare_object_metadata_with_owner(bucket, object, owner) + } + + pub(crate) async fn prepare_object_metadata_with_owner_ttl( + &self, + bucket: &str, + object: &str, + owner: Option>, + ttl: Duration, + ) -> Result { + if self.health.is_faulty() { + return Err(DiskError::FaultyDisk); + } + self.check_disk_stale().await?; + self.disk.prepare_object_metadata_with_owner_ttl(bucket, object, owner, ttl) + } + + /// Refresh an inert token while the caller is still staging the write. + pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> { + if self.health.is_faulty() { + return Err(DiskError::FaultyDisk); + } + self.check_disk_stale().await?; + self.disk.renew_object_metadata(bucket, object, token) + } + + /// Enqueue a prepared token; a cancelled token cannot be reactivated. + pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> { + if self.health.is_faulty() { + return Err(DiskError::FaultyDisk); + } + self.check_disk_stale().await?; + self.disk.activate_object_metadata(bucket, object, token).await + } + + /// Cancel an unclaimed reservation. A claimed physical operation retains its own lock. + pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool { + self.disk.cancel_object_metadata_reservation(token) + } + /// Create a new LocalDiskWrapper pub fn new(disk: Arc, health_check: bool) -> Self { let admission = DiskIoAdmission::new(disk.endpoint().to_string()); @@ -2581,6 +2744,7 @@ impl LocalDiskWrapper { timeout_duration, timeout_health_action, mutation, + false, )) .await? } @@ -2592,6 +2756,7 @@ impl LocalDiskWrapper { timeout_duration: Duration, timeout_health_action: TimeoutHealthAction, mutation: DiskMetricMutation, + identity_prechecked: bool, ) -> Result where F: FnOnce(Option) -> Fut, @@ -2613,7 +2778,7 @@ impl LocalDiskWrapper { } // Check if disk is stale - if let Err(err) = self.check_disk_stale().await { + if !identity_prechecked && let Err(err) = self.check_disk_stale().await { self.metrics.record_availability_error(); return Err(err); } @@ -5581,6 +5746,115 @@ mod tests { drop(metadata_lease); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn cancelled_reserved_rename_releases_lease_before_disk_admission() { + use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards}; + + let dir = tempfile::tempdir().expect("disk directory"); + let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + let meta = dir.path().join(RUSTFS_META_BUCKET); + tokio::fs::create_dir_all(&meta).await.expect("metadata directory"); + let mut format = FormatV3::new(1, 1); + format.erasure.this = format.erasure.sets[0][0]; + tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes")) + .await + .expect("write format"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk")); + disk.make_volume("bucket").await.expect("bucket volume"); + let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false)); + let token = wrapper + .reserve_object_metadata("bucket", "object", Duration::from_secs(5)) + .await + .expect("reserve key"); + wrapper.set_disk_id_state(Some(format.erasure.this)).await; + disk.format_info.write().await.last_check = None; + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + disk.install_format_probe_hook(false, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv_timeout(Duration::from_secs(30)); + }); + let publisher = Arc::clone(&wrapper); + let rename = tokio::spawn(async move { + publisher + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "staging", + &FileInfo::default(), + "bucket", + "object", + RenameDataGuards { + metadata_reservation_token: Some(token), + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .expect("disk identity preflight starts") + .expect("preflight hook"); + rename.abort(); + assert!(matches!(rename.await, Err(error) if error.is_cancelled())); + let key = disk.get_object_path("bucket", "object").expect("object path"); + let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None)); + let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok(); + release_tx.send(()).expect("release disk identity preflight"); + if !released_before_io { + drop( + tokio::time::timeout(Duration::from_secs(10), successor) + .await + .expect("preflight finishes"), + ); + } + assert!( + released_before_io, + "a cancelled request must not retain the reservation before physical mutation starts" + ); + } + + #[tokio::test] + async fn reserved_background_rename_does_not_wait_for_disk_admission() { + use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards}; + + let dir = tempfile::tempdir().expect("disk directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk")); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let token = wrapper + .reserve_object_metadata("bucket", "object", Duration::from_secs(5)) + .await + .expect("reserve key"); + let held = wrapper + .io_admission + .background(&CancellationToken::new()) + .await + .expect("background permit"); + let fi = FileInfo::default(); + let rename = crate::disk::with_background_disk_io( + CancellationToken::new(), + wrapper.rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "staging", + &fi, + "bucket", + "object", + RenameDataGuards { + metadata_reservation_token: Some(token), + ..Default::default() + }, + ), + ); + let result = tokio::time::timeout(Duration::from_secs(1), rename).await; + drop(held); + let observation = result.expect("reserved mutation must not wait while holding the object lease"); + assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq))); + assert!(observation.rejected_before_publication()); + } + #[tokio::test] async fn delete_versions_counts_faulty_drive_availability_rejection() { let dir = tempfile::tempdir().expect("temp dir should be created"); diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 9b7a6f881..f3b95ecf1 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -6880,6 +6880,8 @@ async fn build_local_io_backend( )] pub struct LocalDisk { pub root: PathBuf, + reservation_instance: Uuid, + metadata_reservations: OnceLock>, publication_root: os::PublicationRoot, multipart_proof_authority: OnceLock, /// I/O root pinned to the mount instance that was opened while the disk @@ -7428,6 +7430,8 @@ impl LocalDisk { root: root.clone(), publication_root, multipart_proof_authority: OnceLock::new(), + reservation_instance: Uuid::new_v4(), + metadata_reservations: OnceLock::new(), io_root: io_root.clone(), #[cfg(target_os = "linux")] mount_lease: mount_lease.clone(), @@ -7995,6 +7999,96 @@ impl LocalDisk { local_disk_object_path(&self.root, bucket, key) } + fn metadata_reservation_registry(&self) -> &Arc { + self.metadata_reservations + .get_or_init(|| os::MetadataReservationRegistry::new(1024)) + } + + pub(in crate::disk) async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result { + if crate::bucket::utils::is_meta_bucketname(bucket) { + return Err(DiskError::DiskOngoingReq); + } + let key = self.get_object_path(bucket, object)?; + self.metadata_reservation_registry() + .reserve(&key, self.reservation_instance, timeout.min(Duration::from_secs(5)), None) + .await + } + + pub(in crate::disk) async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result { + if crate::bucket::utils::is_meta_bucketname(bucket) { + return Err(DiskError::DiskOngoingReq); + } + let key = self.get_object_path(bucket, object)?; + self.metadata_reservation_registry() + .enqueue(&key, self.reservation_instance) + .await + } + + pub(in crate::disk) fn prepare_object_metadata_with_owner( + &self, + bucket: &str, + object: &str, + owner: Option>, + ) -> Result { + if crate::bucket::utils::is_meta_bucketname(bucket) { + return Err(DiskError::InvalidPath); + } + let key = self.get_object_path(bucket, object)?; + self.metadata_reservation_registry() + .prepare_with_owner(&key, self.reservation_instance, owner) + } + + pub(in crate::disk) fn prepare_object_metadata_with_owner_ttl( + &self, + bucket: &str, + object: &str, + owner: Option>, + ttl: Duration, + ) -> Result { + if crate::bucket::utils::is_meta_bucketname(bucket) { + return Err(DiskError::InvalidPath); + } + let key = self.get_object_path(bucket, object)?; + self.metadata_reservation_registry() + .prepare_with_owner_ttl(&key, self.reservation_instance, owner, ttl) + } + + pub(in crate::disk) fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> { + let key = self.get_object_path(bucket, object)?; + self.metadata_reservations + .get() + .ok_or(DiskError::DiskOngoingReq)? + .renew(token, &key, self.reservation_instance) + } + + pub(in crate::disk) async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> { + let key = self.get_object_path(bucket, object)?; + self.metadata_reservations + .get() + .ok_or(DiskError::DiskOngoingReq)? + .activate(token, &key, self.reservation_instance) + .await + } + + pub(in crate::disk) fn claim_object_metadata_reservation( + &self, + token: Uuid, + bucket: &str, + object: &str, + ) -> Result { + let key = self.get_object_path(bucket, object)?; + self.metadata_reservations + .get() + .ok_or(DiskError::DiskOngoingReq)? + .claim(token, &key, self.reservation_instance) + } + + pub(in crate::disk) fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool { + self.metadata_reservations + .get() + .is_some_and(|registry| registry.cancel(token)) + } + // Get the absolute path of a bucket pub fn get_bucket_path(&self, bucket: &str) -> Result { local_disk_bucket_path(&self.root, bucket) @@ -20895,6 +20989,107 @@ mod test { assert_eq!(delete_err, DiskError::VolumeNotEmpty); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn lost_fence_during_publication_cannot_survive_bucket_recreation() { + use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + mpsc, + }; + + struct Fence(AtomicBool); + impl os::MetadataReservationFence for Fence { + fn is_valid(&self) -> bool { + self.0.load(Ordering::Acquire) + } + } + + let dir = tempfile::tempdir().expect("disk directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("disk")); + let bucket = "lost-fence-recreate-bucket"; + let object = "object"; + let temporary = "lost-fence-stage"; + let data_dir = Uuid::new_v4(); + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + let staged = disk + .get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{temporary}/{data_dir}/part.1")) + .expect("staged path"); + fs::create_dir_all(staged.parent().expect("staged parent")) + .await + .expect("create staging"); + fs::write(&staged, b"old generation").await.expect("write staged data"); + + let key = disk.get_object_path(bucket, object).expect("object path"); + let registry = disk.metadata_reservation_registry(); + let fence = Arc::new(Fence(AtomicBool::new(true))); + let token = registry + .prepare_with_owner(&key, disk.reservation_instance, Some(fence.clone())) + .expect("prepare"); + registry + .activate(token, &key, disk.reservation_instance) + .await + .expect("activate"); + let reservation = registry + .claim(token, &key, disk.reservation_instance) + .expect("claim") + .wait() + .await; + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + set_rename_data_after_first_publication(&disk.root, bucket, object, move || { + entered_tx.send(()).expect("publication entered"); + release_rx.recv().expect("release publication"); + }); + let writer = Arc::clone(&disk); + let rename = tokio::spawn(async move { + let fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None); + writer + .rename_data_observed_with_reservation( + RUSTFS_META_TMP_BUCKET, + temporary, + &fi, + bucket, + object, + (None, Some(reservation)), + ) + .await + .result + }); + tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10))) + .await + .expect("wait task") + .expect("publication entered"); + fence.0.store(false, Ordering::Release); + let deleter = Arc::clone(&disk); + let mut delete = tokio::spawn(async move { deleter.delete_volume(bucket, true).await }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut delete).await.is_err(), + "delete waits for physical publication" + ); + assert!( + matches!(disk.make_volume(bucket).await, Err(DiskError::VolumeExists)), + "old volume cannot be recreated while publication runs" + ); + release_tx.send(()).expect("release publication"); + let result = tokio::time::timeout(Duration::from_secs(10), rename) + .await + .expect("rename terminates") + .expect("rename task"); + assert!( + result.is_ok() || matches!(result, Err(DiskError::DiskOngoingReq)), + "unexpected rename result: {result:?}" + ); + tokio::time::timeout(Duration::from_secs(10), delete) + .await + .expect("delete terminates") + .expect("delete task") + .expect("delete volume"); + disk.make_volume(bucket).await.expect("recreate volume"); + assert!(!key.exists(), "the recreated volume must not retain the old object"); + } + #[tokio::test] #[serial_test::serial(rename_data_deleted_bucket)] async fn windows_and_unix_rename_data_non_inline_does_not_recreate_bucket_deleted_before_commit() { @@ -21809,6 +22004,1184 @@ mod test { assert_eq!(resp.old_current_size, Some(OldCurrentSize::Absent)); } + #[tokio::test] + async fn metadata_lease_blocks_only_its_own_disk_publication() { + use std::time::Duration; + + let dirs: Vec<_> = (0..4).map(|_| tempdir().expect("disk directory")).collect(); + let mut disks = Vec::new(); + for dir in &dirs { + disks.push(test_disk(dir).await); + } + let key = disks[0].get_object_path("bucket", "object").expect("object key"); + let reservation = os::acquire_metadata_mutation_lease(&key, None).await; + // A physical operation can retain the lease after its caller drops. + let physical_owner = Arc::clone(&reservation); + let inline_info = || { + let mut fi = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"next"))); + fi.set_inline_data(); + fi + }; + let mut blocked = + Box::pin(disks[0].rename_data(RUSTFS_META_TMP_BUCKET, "tmp-blocked", inline_info(), "bucket", "object")); + assert!(tokio::time::timeout(Duration::from_millis(100), &mut blocked).await.is_err()); + for disk in &disks[1..] { + tokio::time::timeout( + Duration::from_secs(5), + disk.rename_data(RUSTFS_META_TMP_BUCKET, "tmp-ready", inline_info(), "bucket", "object"), + ) + .await + .expect("independent disk publication must not wait") + .expect("independent disk publication succeeds"); + } + drop(reservation); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut blocked).await.is_err(), + "an in-flight physical owner must retain the disk lease" + ); + drop(physical_owner); + tokio::time::timeout(Duration::from_secs(5), blocked) + .await + .expect("publication proceeds after physical owner releases") + .expect("publication succeeds"); + for disk in &disks { + let fi = disk + .read_version( + "", + "bucket", + "object", + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("committed metadata"); + assert_eq!(fi.data.as_deref(), Some(b"next".as_slice())); + } + } + + #[tokio::test] + async fn reserved_metadata_lease_can_publish_and_rejects_wrong_object() { + use crate::disk::{RenameDataGuards, disk_store::LocalDiskWrapper}; + use std::time::Duration; + + let dir = tempdir().expect("disk directory"); + let disk = Arc::new(test_disk(&dir).await); + let key = disk.get_object_path("bucket", "object").expect("object path"); + let reservation = os::reserve_metadata_mutation_lease(&key, disk.reservation_instance, None).await; + let mut state = commit::RenameDataState::with_metadata_reservation(reservation); + let mut fi = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"reserved"))); + fi.set_inline_data(); + tokio::time::timeout( + Duration::from_secs(5), + disk.rename_data_inner(RUSTFS_META_TMP_BUCKET, "reserved", fi, "bucket", "object", &mut state), + ) + .await + .expect("reserved publication must not reacquire its own mutex") + .expect("reserved publication succeeds"); + let stored = disk + .read_version( + "", + "bucket", + "object", + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("published metadata"); + assert_eq!(stored.data.as_deref(), Some(b"reserved".as_slice())); + + let reservation = os::reserve_metadata_mutation_lease(&key, disk.reservation_instance, None).await; + let mut fi = test_file_info("other", Uuid::nil(), None, Some(Bytes::from_static(b"wrong"))); + fi.set_inline_data(); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let observed = wrapper + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "wrong", + &fi, + "bucket", + "other", + RenameDataGuards { + metadata_reservation: Some(reservation), + ..Default::default() + }, + ) + .await; + assert!( + observed.rejected_before_publication(), + "mismatch must remain a proven preflight rejection" + ); + assert!(matches!(observed.result, Err(DiskError::InvalidPath))); + assert!( + matches!( + disk.read_version("", "bucket", "other", "", &ReadOptions::default()).await, + Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) + ), + "mismatched reservation must not publish data" + ); + + let other_dir = tempdir().expect("other disk directory"); + let other_disk = test_disk(&other_dir).await; + let reservation = os::reserve_metadata_mutation_lease(&key, disk.reservation_instance, None).await; + let mut state = commit::RenameDataState::with_metadata_reservation(reservation); + let mut fi = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"wrong-disk"))); + fi.set_inline_data(); + let error = other_disk + .rename_data_inner(RUSTFS_META_TMP_BUCKET, "wrong-disk", fi, "bucket", "object", &mut state) + .await + .expect_err("a reservation from another disk must be rejected"); + assert!(matches!(error, DiskError::InvalidPath)); + assert!( + matches!( + other_disk + .read_version("", "bucket", "object", "", &ReadOptions::default()) + .await, + Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) + ), + "a reservation for another disk must not publish data" + ); + } + + #[tokio::test] + async fn reserved_metadata_lease_rejects_a_reopened_disk_at_the_same_path() { + let dir = tempdir().expect("disk directory"); + let old_disk = test_disk(&dir).await; + let key = old_disk.get_object_path("bucket", "same-path-object").expect("object path"); + let reservation = os::reserve_metadata_mutation_lease(&key, old_disk.reservation_instance, None).await; + let new_disk = test_disk(&dir).await; + let mut state = commit::RenameDataState::with_metadata_reservation(reservation); + let mut fi = test_file_info("same-path-object", Uuid::nil(), None, Some(Bytes::from_static(b"stale"))); + fi.set_inline_data(); + let result = new_disk + .rename_data_inner(RUSTFS_META_TMP_BUCKET, "same-path", fi, "bucket", "same-path-object", &mut state) + .await; + assert!( + matches!(result, Err(DiskError::InvalidPath)), + "old instance reservation must not authorize a reopened disk" + ); + assert!( + matches!( + new_disk + .read_version("", "bucket", "same-path-object", "", &ReadOptions::default()) + .await, + Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) + ), + "rejected reservation must not publish on the reopened disk" + ); + } + + #[tokio::test(start_paused = true)] + async fn metadata_reservation_tokens_are_bounded_single_use_and_expire() { + use crate::disk::{RenameDataGuards, disk_store::LocalDiskWrapper}; + use std::time::Duration; + + let dir = tempdir().expect("disk directory"); + let disk = Arc::new(test_disk(&dir).await); + assert!( + disk.metadata_reservations + .set(os::MetadataReservationRegistry::new(1)) + .is_ok() + ); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let token = wrapper + .reserve_object_metadata("bucket", "object", Duration::from_secs(5)) + .await + .expect("first reservation"); + assert!(matches!( + wrapper + .reserve_object_metadata("bucket", "other", Duration::from_secs(5)) + .await, + Err(DiskError::DiskOngoingReq) + )); + let mut fi = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"reserved"))); + fi.set_inline_data(); + let observed = wrapper + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "token-staging", + &fi, + "bucket", + "object", + RenameDataGuards { + metadata_reservation_token: Some(token), + ..Default::default() + }, + ) + .await; + observed.result.expect("token authorizes one publication"); + assert!(!wrapper.cancel_object_metadata_reservation(token), "consumed token cannot be reused"); + let reused = wrapper + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "reused", + &fi, + "bucket", + "object", + RenameDataGuards { + metadata_reservation_token: Some(token), + ..Default::default() + }, + ) + .await; + assert!( + !reused.rejected_before_publication(), + "a consumed token may still have an in-flight publisher" + ); + assert!(matches!(reused.result, Err(DiskError::DiskOngoingReq))); + + let token = wrapper + .reserve_object_metadata("bucket", "object", Duration::from_secs(5)) + .await + .expect("reservation for path mismatch"); + let mismatch = wrapper + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "mismatch", + &fi, + "bucket", + "other", + RenameDataGuards { + metadata_reservation_token: Some(token), + ..Default::default() + }, + ) + .await; + assert!(mismatch.rejected_before_publication()); + assert!(matches!(mismatch.result, Err(DiskError::InvalidPath))); + + let token = wrapper + .reserve_object_metadata("bucket", "object", Duration::from_secs(5)) + .await + .expect("capacity is released after claim"); + assert!(wrapper.cancel_object_metadata_reservation(token)); + assert!(!wrapper.cancel_object_metadata_reservation(token)); + + let token = wrapper + .reserve_object_metadata("bucket", "object", Duration::from_secs(5)) + .await + .expect("reservation after cancel"); + tokio::time::advance(Duration::from_secs(61)).await; + let expired = wrapper + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "expired", + &fi, + "bucket", + "object", + RenameDataGuards { + metadata_reservation_token: Some(token), + ..Default::default() + }, + ) + .await; + let expired_preflight = expired.rejected_before_publication(); + match expired.result { + Err(DiskError::DiskOngoingReq) => assert!(!expired_preflight), + Err(DiskError::Timeout) => assert!(expired_preflight), + result => panic!("expired reservation returned an unexpected result: {result:?}"), + } + let next = wrapper + .reserve_object_metadata("bucket", "object", Duration::from_secs(5)) + .await + .expect("expired reservation releases the lock and capacity"); + assert!(wrapper.cancel_object_metadata_reservation(next)); + } + + #[tokio::test] + async fn prepared_metadata_token_is_inert_and_cancel_blocks_late_enqueue() { + let dir = tempdir().expect("disk directory"); + let disk = test_disk(&dir).await; + let key = disk.get_object_path("bucket", "object").expect("object path"); + let registry = disk.metadata_reservation_registry(); + let prepared = registry.prepare(&key, disk.reservation_instance).expect("prepare token"); + let other = disk + .enqueue_object_metadata("bucket", "object") + .await + .expect("prepared token holds no lease"); + let held = disk + .claim_object_metadata_reservation(other, "bucket", "object") + .expect("claim") + .wait() + .await; + assert!(registry.cancel(prepared)); + assert!(matches!( + registry.activate(prepared, &key, disk.reservation_instance).await, + Err(DiskError::DiskOngoingReq) + )); + drop(held); + + let token = registry.prepare(&key, disk.reservation_instance).expect("prepare next token"); + let other_key = disk.get_object_path("bucket", "other").expect("other path"); + assert!(matches!( + registry.activate(token, &other_key, disk.reservation_instance).await, + Err(DiskError::InvalidPath) + )); + registry + .activate(token, &key, disk.reservation_instance) + .await + .expect("activate"); + registry + .activate(token, &key, disk.reservation_instance) + .await + .expect("repeat activation"); + let claimed = registry + .claim(token, &key, disk.reservation_instance) + .expect("claim") + .wait() + .await; + assert!(!registry.cancel(token), "claimed token cannot be cancelled"); + drop(claimed); + } + + #[tokio::test(start_paused = true)] + async fn short_unclaimed_reservation_releases_its_owner_after_expiry() { + struct Fence; + impl os::MetadataReservationFence for Fence { + fn is_valid(&self) -> bool { + true + } + } + let registry = os::MetadataReservationRegistry::new(1); + let key = std::path::Path::new("bucket/object"); + let disk_id = Uuid::new_v4(); + let owner = Arc::new(Fence); + let weak = Arc::downgrade(&owner); + let token = registry + .prepare_with_owner_ttl(key, disk_id, Some(owner.clone()), Duration::from_secs(2)) + .expect("short preparation"); + drop(owner); + tokio::time::advance(Duration::from_secs(1)).await; + assert!(weak.upgrade().is_some(), "unexpired reservation owns its fence"); + tokio::time::advance(Duration::from_secs(2)).await; + let next = registry.prepare(key, disk_id).expect("expired reservation frees capacity"); + assert!(weak.upgrade().is_none(), "expired reservation releases its fence"); + assert!(matches!(registry.activate(token, key, disk_id).await, Err(DiskError::DiskOngoingReq))); + assert!(registry.cancel(next)); + + let regular = registry.prepare_with_owner(key, disk_id, None).expect("ordinary preparation"); + tokio::time::advance(Duration::from_secs(3)).await; + registry + .renew(regular, key, disk_id) + .expect("ordinary reservation retains its existing lifetime"); + assert!(registry.cancel(regular)); + } + + #[tokio::test(start_paused = true)] + async fn short_reservation_lost_renew_and_cancel_responses_expires_promptly() { + struct Fence; + impl os::MetadataReservationFence for Fence { + fn is_valid(&self) -> bool { + true + } + } + for queued in [false, true] { + let registry = os::MetadataReservationRegistry::new(1); + let key = std::path::Path::new("bucket/object"); + let disk_id = Uuid::new_v4(); + let owner = Arc::new(Fence); + let weak = Arc::downgrade(&owner); + let token = registry + .prepare_with_owner_ttl(key, disk_id, Some(owner.clone()), Duration::from_secs(2)) + .expect("short preparation"); + drop(owner); + if queued { + registry.activate(token, key, disk_id).await.expect("activate"); + } + tokio::time::advance(Duration::from_secs(1)).await; + registry.renew(token, key, disk_id).expect("server accepted renewal"); + // Model loss of both the renewal response and the subsequent cancel. + tokio::time::advance(Duration::from_secs(3)).await; + let next = registry.prepare(key, disk_id).expect("short lease releases capacity"); + assert!(weak.upgrade().is_none(), "short lease releases its publication fence"); + assert!(matches!(registry.activate(token, key, disk_id).await, Err(DiskError::DiskOngoingReq))); + assert!(registry.cancel(next)); + } + } + + #[tokio::test(start_paused = true)] + async fn short_reservation_can_be_renewed_during_long_staging() { + let registry = os::MetadataReservationRegistry::new(1); + let key = std::path::Path::new("bucket/object"); + let disk_id = Uuid::new_v4(); + let token = registry + .prepare_with_owner_ttl(key, disk_id, None, Duration::from_secs(2)) + .expect("short preparation"); + for _ in 0..10 { + tokio::time::advance(Duration::from_secs(1)).await; + registry.renew(token, key, disk_id).expect("renew while staging"); + } + registry.activate(token, key, disk_id).await.expect("activate after staging"); + assert!(registry.cancel(token)); + } + + #[tokio::test(start_paused = true)] + async fn short_queued_reservation_can_renew_without_reviving_consumed_tokens() { + let registry = os::MetadataReservationRegistry::new(2); + let key = std::path::Path::new("bucket/object"); + let other = std::path::Path::new("bucket/other"); + let disk_id = Uuid::new_v4(); + let token = registry + .prepare_with_owner_ttl(key, disk_id, None, Duration::from_secs(2)) + .expect("prepare short token"); + registry.activate(token, key, disk_id).await.expect("activate"); + tokio::time::advance(Duration::from_secs(1)).await; + assert!(matches!(registry.renew(token, other, disk_id), Err(DiskError::InvalidPath))); + assert!(matches!(registry.renew(token, key, Uuid::new_v4()), Err(DiskError::InvalidPath))); + registry.renew(token, key, disk_id).expect("renew queued short token"); + tokio::time::advance(Duration::from_millis(1100)).await; + let reservation = registry.claim(token, key, disk_id).expect("renewed token remains claimable"); + assert!(matches!(registry.renew(token, key, disk_id), Err(DiskError::DiskOngoingReq))); + drop(reservation); + + let cancelled = registry + .prepare_with_owner_ttl(key, disk_id, None, Duration::from_secs(2)) + .expect("prepare cancellable token"); + registry + .activate(cancelled, key, disk_id) + .await + .expect("activate cancellable token"); + assert!(registry.cancel(cancelled)); + assert!(matches!(registry.renew(cancelled, key, disk_id), Err(DiskError::DiskOngoingReq))); + + let expired = registry + .prepare_with_owner_ttl(key, disk_id, None, Duration::from_secs(2)) + .expect("prepare expiring token"); + registry + .activate(expired, key, disk_id) + .await + .expect("activate expiring token"); + tokio::time::advance(Duration::from_secs(3)).await; + assert!(matches!( + registry.renew(expired, key, disk_id), + Err(DiskError::Timeout | DiskError::DiskOngoingReq) + )); + } + + #[tokio::test] + async fn short_queued_reservation_rejects_lost_owner_before_renew_and_claim() { + struct Fence(std::sync::atomic::AtomicBool); + impl os::MetadataReservationFence for Fence { + fn is_valid(&self) -> bool { + self.0.load(std::sync::atomic::Ordering::Acquire) + } + } + let registry = os::MetadataReservationRegistry::new(1); + let key = std::path::Path::new("bucket/object"); + let disk_id = Uuid::new_v4(); + let fence = Arc::new(Fence(std::sync::atomic::AtomicBool::new(true))); + let token = registry + .prepare_with_owner_ttl(key, disk_id, Some(fence.clone()), Duration::from_secs(2)) + .expect("prepare short token"); + registry.activate(token, key, disk_id).await.expect("activate"); + fence.0.store(false, std::sync::atomic::Ordering::Release); + assert!(matches!(registry.renew(token, key, disk_id), Err(DiskError::DiskOngoingReq))); + assert!(matches!(registry.claim(token, key, disk_id), Err(DiskError::DiskOngoingReq))); + } + + #[tokio::test(start_paused = true)] + async fn prepared_metadata_renewal_never_revives_cancelled_or_activated_tokens() { + let dir = tempdir().expect("disk directory"); + let disk = test_disk(&dir).await; + let key = disk.get_object_path("bucket", "object").expect("object path"); + let other = disk.get_object_path("bucket", "other").expect("other path"); + let registry = disk.metadata_reservation_registry(); + let token = registry.prepare(&key, disk.reservation_instance).expect("prepare"); + tokio::time::advance(Duration::from_secs(40)).await; + assert!(matches!( + registry.renew(token, &other, disk.reservation_instance), + Err(DiskError::InvalidPath) + )); + registry.renew(token, &key, disk.reservation_instance).expect("renew"); + tokio::time::advance(Duration::from_secs(30)).await; + registry + .activate(token, &key, disk.reservation_instance) + .await + .expect("renewed token activates after original expiry"); + assert!(matches!( + registry.renew(token, &key, disk.reservation_instance), + Err(DiskError::DiskOngoingReq) + )); + assert!(registry.cancel(token)); + assert!(matches!( + registry.renew(token, &key, disk.reservation_instance), + Err(DiskError::DiskOngoingReq) + )); + let expired = registry + .prepare(&key, disk.reservation_instance) + .expect("prepare expiring token"); + tokio::time::advance(Duration::from_secs(61)).await; + assert!(matches!( + registry.renew(expired, &key, disk.reservation_instance), + Err(DiskError::Timeout | DiskError::DiskOngoingReq) + )); + } + + #[tokio::test] + async fn prepared_metadata_owner_survives_activation_and_claim() { + struct Owner; + impl os::MetadataReservationFence for Owner { + fn is_valid(&self) -> bool { + true + } + } + let dir = tempdir().expect("disk directory"); + let disk = test_disk(&dir).await; + let key = disk.get_object_path("bucket", "object").expect("object path"); + let registry = disk.metadata_reservation_registry(); + let owner = Arc::new(Owner); + let weak = Arc::downgrade(&owner); + let token = registry + .prepare_with_owner(&key, disk.reservation_instance, Some(owner.clone())) + .expect("prepare owned token"); + drop(owner); + assert!(weak.upgrade().is_some(), "prepared token retains its owner"); + registry + .activate(token, &key, disk.reservation_instance) + .await + .expect("activate"); + let claimed = registry + .claim(token, &key, disk.reservation_instance) + .expect("claim") + .wait() + .await; + assert!(!registry.cancel(token), "claimed token cannot be cancelled"); + assert!(weak.upgrade().is_some(), "claimed lease retains its owner"); + drop(claimed); + assert!(weak.upgrade().is_none(), "owner releases with the claimed lease"); + } + + #[tokio::test] + async fn lost_metadata_fence_rejects_claim_and_commit() { + struct Fence(std::sync::atomic::AtomicBool); + impl os::MetadataReservationFence for Fence { + fn is_valid(&self) -> bool { + self.0.load(std::sync::atomic::Ordering::Acquire) + } + } + let dir = tempdir().expect("disk directory"); + let disk = test_disk(&dir).await; + let key = disk.get_object_path("bucket", "object").expect("object path"); + let registry = disk.metadata_reservation_registry(); + let fence = Arc::new(Fence(std::sync::atomic::AtomicBool::new(true))); + let token = registry + .prepare_with_owner(&key, disk.reservation_instance, Some(fence.clone())) + .expect("prepare"); + registry + .activate(token, &key, disk.reservation_instance) + .await + .expect("activate"); + fence.0.store(false, std::sync::atomic::Ordering::Release); + assert!(matches!( + registry.claim(token, &key, disk.reservation_instance), + Err(DiskError::DiskOngoingReq) + )); + fence.0.store(true, std::sync::atomic::Ordering::Release); + let token = registry + .prepare_with_owner(&key, disk.reservation_instance, Some(fence.clone())) + .expect("prepare again"); + registry + .activate(token, &key, disk.reservation_instance) + .await + .expect("activate again"); + let claimed = registry + .claim(token, &key, disk.reservation_instance) + .expect("claim") + .wait() + .await; + fence.0.store(false, std::sync::atomic::Ordering::Release); + assert!(matches!( + claimed.into_lease_for(&key, disk.reservation_instance), + Err(DiskError::DiskOngoingReq) + )); + } + + #[tokio::test] + async fn queued_metadata_reservation_registers_when_task_budget_is_exhausted() { + use std::{future::Future, task::Poll}; + + let dir = tempdir().expect("disk directory"); + let disk = test_disk(&dir).await; + let first = disk.enqueue_object_metadata("bucket", "object").await.expect("first token"); + let first = disk + .claim_object_metadata_reservation(first, "bucket", "object") + .expect("first claim") + .wait() + .await; + let mut enqueue = Box::pin(disk.enqueue_object_metadata("bucket", "object")); + let second = std::future::poll_fn(|cx| { + let mut exhausted = false; + for _ in 0..1024 { + let mut consume = Box::pin(tokio::task::consume_budget()); + if consume.as_mut().poll(cx).is_pending() { + exhausted = true; + break; + } + } + assert!(exhausted, "test must exhaust the cooperative budget"); + match enqueue.as_mut().poll(cx) { + Poll::Ready(result) => Poll::Ready(result.expect("enqueue must register immediately")), + Poll::Pending => panic!("enqueue must not acknowledge before registering"), + } + }) + .await; + let third = disk.enqueue_object_metadata("bucket", "object").await.expect("third token"); + tokio::task::yield_now().await; + let mut second = Box::pin( + disk.claim_object_metadata_reservation(second, "bucket", "object") + .expect("second") + .wait(), + ); + let mut third = Box::pin( + disk.claim_object_metadata_reservation(third, "bucket", "object") + .expect("third") + .wait(), + ); + assert!(futures::poll!(third.as_mut()).is_pending()); + assert!(futures::poll!(second.as_mut()).is_pending()); + drop(first); + let second = tokio::time::timeout(Duration::from_secs(5), second) + .await + .expect("second acquires first"); + assert!(futures::poll!(third.as_mut()).is_pending(), "third cannot overtake second"); + drop(second); + drop( + tokio::time::timeout(Duration::from_secs(5), third) + .await + .expect("third acquires next"), + ); + } + + #[tokio::test] + async fn queued_metadata_reservations_preserve_order_and_cancel() { + let dir = tempdir().expect("disk directory"); + let disk = test_disk(&dir).await; + let first = disk.enqueue_object_metadata("bucket", "object").await.expect("first token"); + let second = disk.enqueue_object_metadata("bucket", "object").await.expect("second token"); + let third = disk.enqueue_object_metadata("bucket", "object").await.expect("third token"); + let first = disk + .claim_object_metadata_reservation(first, "bucket", "object") + .expect("first claim") + .wait() + .await; + let second = disk + .claim_object_metadata_reservation(second, "bucket", "object") + .expect("second claim"); + let third = disk + .claim_object_metadata_reservation(third, "bucket", "object") + .expect("third claim"); + let mut second = Box::pin(second.wait()); + let mut third = Box::pin(third.wait()); + assert!(futures::poll!(second.as_mut()).is_pending()); + assert!(futures::poll!(third.as_mut()).is_pending()); + drop(first); + let second = tokio::time::timeout(Duration::from_secs(5), second) + .await + .expect("second acquires first"); + assert!(futures::poll!(third.as_mut()).is_pending(), "third cannot overtake second"); + drop(second); + drop( + tokio::time::timeout(Duration::from_secs(5), third) + .await + .expect("third acquires next"), + ); + + let first = disk.enqueue_object_metadata("bucket", "object").await.expect("first token"); + let cancelled = disk + .enqueue_object_metadata("bucket", "object") + .await + .expect("cancelled token"); + let next = disk.enqueue_object_metadata("bucket", "object").await.expect("next token"); + let first = disk + .claim_object_metadata_reservation(first, "bucket", "object") + .expect("first claim") + .wait() + .await; + assert!(disk.cancel_object_metadata_reservation(cancelled)); + let mut next = Box::pin( + disk.claim_object_metadata_reservation(next, "bucket", "object") + .expect("next claim") + .wait(), + ); + assert!(futures::poll!(next.as_mut()).is_pending()); + drop(first); + drop( + tokio::time::timeout(Duration::from_secs(5), next) + .await + .expect("cancelled waiter does not block next"), + ); + } + + #[tokio::test] + async fn cancelled_pending_metadata_reservation_releases_capacity() { + use crate::disk::disk_store::LocalDiskWrapper; + use std::time::Duration; + + let dir = tempdir().expect("disk directory"); + let disk = Arc::new(test_disk(&dir).await); + assert!( + disk.metadata_reservations + .set(os::MetadataReservationRegistry::new(2)) + .is_ok() + ); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let first = wrapper + .reserve_object_metadata("bucket", "object", Duration::from_secs(5)) + .await + .expect("first reservation"); + let mut waiting = Box::pin(wrapper.reserve_object_metadata("bucket", "object", Duration::from_secs(5))); + assert!(futures::poll!(waiting.as_mut()).is_pending(), "same-key reservation must wait"); + assert!(matches!( + wrapper + .reserve_object_metadata("bucket", "other", Duration::from_secs(5)) + .await, + Err(DiskError::DiskOngoingReq) + )); + drop(waiting); + let other = wrapper + .reserve_object_metadata("bucket", "other", Duration::from_secs(5)) + .await + .expect("cancellation releases the waiting reservation's capacity"); + assert!(wrapper.cancel_object_metadata_reservation(other)); + assert!(wrapper.cancel_object_metadata_reservation(first)); + let next = wrapper + .reserve_object_metadata("bucket", "object", Duration::from_secs(5)) + .await + .expect("cancelled waiter did not retain the key"); + assert!(wrapper.cancel_object_metadata_reservation(next)); + } + + #[cfg(not(windows))] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn queued_metadata_rename_publishes_in_admission_order() { + use crate::disk::{ + RenameDataGuards, + disk_store::LocalDiskWrapper, + os::prepared_publication_test_hooks::{Stage, install_at}, + }; + + let dir = tempdir().expect("disk directory"); + let disk = Arc::new(test_disk(&dir).await); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let metadata = disk + .io_get_object_path("bucket", "object") + .expect("object path") + .join(STORAGE_FORMAT_FILE); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let _hook = install_at(Stage::PreparedRename, &metadata, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv_timeout(Duration::from_secs(30)); + }); + let first_token = wrapper + .enqueue_object_metadata("bucket", "object") + .await + .expect("first token"); + let first = wrapper.clone(); + let first_task = tokio::spawn(async move { + let mut fi = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"first"))); + fi.set_inline_data(); + first + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "first", + &fi, + "bucket", + "object", + RenameDataGuards { + metadata_reservation_token: Some(first_token), + ..Default::default() + }, + ) + .await + .result + }); + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .expect("first publication pauses") + .expect("hook"); + let second_token = wrapper + .enqueue_object_metadata("bucket", "object") + .await + .expect("second token"); + let second = wrapper.clone(); + let mut second_task = tokio::spawn(async move { + let mut fi = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"second"))); + fi.set_inline_data(); + second + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "second", + &fi, + "bucket", + "object", + RenameDataGuards { + metadata_reservation_token: Some(second_token), + ..Default::default() + }, + ) + .await + .result + }); + assert!(futures::poll!(&mut second_task).is_pending(), "second write must wait behind first"); + release_tx.send(()).expect("release first publication"); + tokio::time::timeout(Duration::from_secs(10), first_task) + .await + .expect("first finishes") + .expect("first joins") + .expect("first commits"); + tokio::time::timeout(Duration::from_secs(10), second_task) + .await + .expect("second finishes") + .expect("second joins") + .expect("second commits"); + let current = disk + .read_version( + "", + "bucket", + "object", + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read latest object"); + assert_eq!(current.data.as_deref(), Some(b"second".as_slice())); + } + + #[cfg(not(windows))] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn replayed_reservation_is_indeterminate_while_original_publishes() { + use crate::disk::{ + RenameDataGuards, + disk_store::LocalDiskWrapper, + os::prepared_publication_test_hooks::{Stage, install_at}, + }; + use std::time::Duration; + + let dir = tempdir().expect("disk directory"); + let disk = Arc::new(test_disk(&dir).await); + assert!( + disk.metadata_reservations + .set(os::MetadataReservationRegistry::new(1)) + .is_ok() + ); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let token = wrapper + .reserve_object_metadata("bucket", "object", Duration::from_secs(5)) + .await + .expect("reserve object"); + let metadata = disk + .io_get_object_path("bucket", "object") + .expect("physical object path") + .join(STORAGE_FORMAT_FILE); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let _hook = install_at(Stage::PreparedRename, &metadata, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv_timeout(Duration::from_secs(30)); + }); + let mut fi = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"original"))); + fi.set_inline_data(); + let first = wrapper.clone(); + let first_fi = fi.clone(); + let mut publication = tokio::spawn(async move { + first + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "first", + &first_fi, + "bucket", + "object", + RenameDataGuards { + metadata_reservation_token: Some(token), + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("physical rename hook"), + _ = &mut publication => panic!("original publication returned before hook"), + } + }) + .await + .expect("original publisher reaches physical rename"); + let replay = wrapper + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "replay", + &fi, + "bucket", + "object", + RenameDataGuards { + metadata_reservation_token: Some(token), + ..Default::default() + }, + ) + .await; + assert!(!replay.rejected_before_publication(), "the original request may still publish"); + assert!(matches!(replay.result, Err(DiskError::DiskOngoingReq))); + assert!( + matches!( + wrapper + .reserve_object_metadata("bucket", "other", Duration::from_secs(5)) + .await, + Err(DiskError::DiskOngoingReq) + ), + "in-flight physical work must continue to count against the reservation limit" + ); + release_tx.send(()).expect("release original publisher"); + tokio::time::timeout(Duration::from_secs(10), publication) + .await + .expect("original publisher completes") + .expect("publisher task") + .result + .expect("original publication succeeds"); + let next = wrapper + .reserve_object_metadata("bucket", "other", Duration::from_secs(5)) + .await + .expect("capacity is released after physical completion"); + assert!(wrapper.cancel_object_metadata_reservation(next)); + let stored = disk + .read_version( + "", + "bucket", + "object", + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("original metadata"); + assert_eq!(stored.data.as_deref(), Some(b"original".as_slice())); + } + + #[tokio::test] + async fn reserved_metadata_lease_rejects_quota_and_multipart_before_mutation() { + use crate::disk::{RenameDataGuards, disk_store::LocalDiskWrapper}; + use std::time::Duration; + + let dir = tempdir().expect("disk directory"); + let disk = Arc::new(test_disk(&dir).await); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + for quota in [true, false] { + let object = if quota { "quota-object" } else { "multipart-object" }; + let key = disk.get_object_path("bucket", object).expect("object path"); + let reservation = os::reserve_metadata_mutation_lease(&key, disk.reservation_instance, None).await; + let mut fi = test_file_info(object, Uuid::nil(), None, Some(Bytes::from_static(b"data"))); + fi.set_inline_data(); + if quota { + rustfs_utils::http::metadata_compat::insert_str( + &mut fi.metadata, + crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + Uuid::new_v4().to_string(), + ); + } + let source = if quota { + RUSTFS_META_TMP_BUCKET + } else { + RUSTFS_META_MULTIPART_BUCKET + }; + let observed = tokio::time::timeout( + Duration::from_secs(5), + wrapper.rename_data_observed_with_guards( + source, + "staging", + &fi, + "bucket", + object, + RenameDataGuards { + metadata_reservation: Some(reservation), + ..Default::default() + }, + ), + ) + .await + .expect("unsupported lock order must be rejected without waiting"); + assert!(observed.rejected_before_publication()); + assert!(matches!(observed.result, Err(DiskError::DiskOngoingReq))); + assert!( + matches!( + disk.read_version("", "bucket", object, "", &ReadOptions::default()).await, + Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) + ), + "unsupported reservation must not publish" + ); + drop( + tokio::time::timeout(Duration::from_secs(5), os::acquire_metadata_mutation_lease(&key, None)) + .await + .expect("rejected reservation releases its lease"), + ); + } + } + + #[tokio::test] + async fn lost_reservation_while_waiting_for_volume_does_not_publish() { + use crate::disk::{RenameDataGuards, disk_store::LocalDiskWrapper}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + struct Fence { + valid: AtomicBool, + checks: AtomicUsize, + validated: tokio::sync::Notify, + } + impl os::MetadataReservationFence for Fence { + fn is_valid(&self) -> bool { + if self.checks.fetch_add(1, Ordering::AcqRel) == 2 { + self.validated.notify_one(); + } + self.valid.load(Ordering::Acquire) + } + } + let dir = tempdir().expect("disk directory"); + let disk = Arc::new(test_disk(&dir).await); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let key = disk.get_object_path("bucket", "object").expect("object path"); + let registry = disk.metadata_reservation_registry(); + let fence = Arc::new(Fence { + valid: AtomicBool::new(true), + checks: AtomicUsize::new(0), + validated: tokio::sync::Notify::new(), + }); + let token = registry + .prepare_with_owner(&key, disk.reservation_instance, Some(fence.clone())) + .expect("prepare"); + registry + .activate(token, &key, disk.reservation_instance) + .await + .expect("activate"); + let volume_guard = os::disk_volume_mutation_lock(&disk.root, "bucket").write_owned().await; + let publisher = tokio::spawn(async move { + let mut fi = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"stale"))); + fi.set_inline_data(); + wrapper + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "stale", + &fi, + "bucket", + "object", + RenameDataGuards { + metadata_reservation_token: Some(token), + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), fence.validated.notified()) + .await + .expect("metadata lease validated"); + fence.valid.store(false, Ordering::Release); + drop(volume_guard); + let result = tokio::time::timeout(Duration::from_secs(10), publisher) + .await + .expect("publisher drains") + .expect("publisher task"); + assert!( + matches!(&result.result, Err(DiskError::DiskOngoingReq)), + "lost fence must reject before publication: {:?}", + result.result + ); + assert!( + !disk + .io_get_object_path("bucket", "object") + .expect("object path") + .join(STORAGE_FORMAT_FILE) + .exists() + ); + } + + #[cfg(not(windows))] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn reserved_metadata_lease_survives_cancelled_physical_publication() { + use crate::disk::{ + RenameDataGuards, + disk_store::LocalDiskWrapper, + os::prepared_publication_test_hooks::{Stage, install_at}, + }; + use std::time::Duration; + + let dir = tempdir().expect("disk directory"); + let disk = Arc::new(test_disk(&dir).await); + let key = disk.get_object_path("bucket", "object").expect("object path"); + let metadata = disk + .io_get_object_path("bucket", "object") + .expect("physical object path") + .join(STORAGE_FORMAT_FILE); + let reservation = os::reserve_metadata_mutation_lease(&key, disk.reservation_instance, None).await; + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let _hook = install_at(Stage::PreparedRename, &metadata, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv_timeout(Duration::from_secs(30)); + }); + let publisher = LocalDiskWrapper::new(Arc::clone(&disk), false); + let mut rename = tokio::spawn(async move { + let mut fi = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"first"))); + fi.set_inline_data(); + publisher + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "first", + &fi, + "bucket", + "object", + RenameDataGuards { + metadata_reservation: Some(reservation), + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("physical rename hook"), + _ = &mut rename => panic!("publication returned before hook"), + } + }) + .await + .expect("publication must enter physical rename"); + rename.abort(); + assert!(matches!(rename.await, Err(error) if error.is_cancelled()), "publisher must be cancelled"); + let mut successor = Box::pin(os::acquire_metadata_mutation_lease(&key, None)); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut successor) + .await + .is_err(), + "cancelled physical work must retain its pre-acquired metadata lease" + ); + release_tx.send(()).expect("release physical rename"); + let successor = tokio::time::timeout(Duration::from_secs(10), successor) + .await + .expect("successor obtains lease after physical publication"); + drop(successor); + let stored = disk + .read_version( + "", + "bucket", + "object", + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("published metadata"); + assert_eq!(stored.data.as_deref(), Some(b"first".as_slice())); + } + #[tokio::test] async fn inline_rename_data_reports_present_zero_for_delete_marker_latest() { let dir = tempdir().expect("temp dir should be created"); diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs index 1ff322581..d6094c571 100644 --- a/crates/ecstore/src/disk/local/commit.rs +++ b/crates/ecstore/src/disk/local/commit.rs @@ -444,12 +444,29 @@ async fn prepare_rename_commit_and_destination_metadata( #[derive(Debug)] pub(in crate::disk) struct LocalRenamePreflightRejection(()); +impl LocalRenamePreflightRejection { + pub(in crate::disk) fn not_dispatched() -> Self { + Self(()) + } +} + #[derive(Default)] pub(super) struct RenameDataState { namespace_owner: Option>, + metadata_reservation: Option, preflight_rejection: Option, } +#[cfg(test)] +impl RenameDataState { + pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self { + Self { + metadata_reservation: Some(reservation), + ..Default::default() + } + } +} + // Keep optional native MPU state machines out of ordinary rename frames. // Construct the future here as well as pinning it, so its temporary stays off // the caller's poll stack. Call only from the existing native MPU branches. @@ -491,6 +508,15 @@ impl LocalDisk { wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?; } let mut fi = fi; + // Pre-acquired metadata must never reverse quota or multipart-source + // lock order. These paths need their own reservation protocol. + if state.metadata_reservation.is_some() + && (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET + || rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX)) + { + state.preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(DiskError::DiskOngoingReq); + } let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) { Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| { state.preflight_rejection = Some(LocalRenamePreflightRejection(())); @@ -586,7 +612,27 @@ impl LocalDisk { Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))), None => state.namespace_owner.take(), }; - let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await; + // A reserved lease is single-use and bound to the exact disk path. + // Keep the independent namespace/source owner alongside it; a pre-held + // mutex cannot be acquired again without deadlocking. + let reservation_fence = state + .metadata_reservation + .as_ref() + .and_then(|reservation| reservation.fence()); + let metadata_owner: Arc = if let Some(reservation) = state.metadata_reservation.take() { + if multipart_staging.is_some() || quota_fence_claim.is_some() { + state.preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(DiskError::DiskOngoingReq); + } + let lease = reservation + .into_lease_for(&root_destination_object_path, self.reservation_instance) + .inspect_err(|_| { + state.preflight_rejection = Some(LocalRenamePreflightRejection(())); + })?; + Arc::new((lease, metadata_owner)) + } else { + os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await + }; drop(metadata_wait); #[cfg(unix)] let parent_receipt = if native_timing { @@ -600,17 +646,23 @@ impl LocalDisk { }; #[cfg(unix)] let mut receipt_mutation = native_timing - .then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone())); + .then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone())); let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait"); let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner( &self.root, dst_volume, &root_destination_object_path, staging_prefix.as_deref(), - Some(metadata_lease), + Some(metadata_owner), ) .await; drop(namespace_wait); + // The volume read lease prevents deletion of this disk's bucket until + // publication finishes. Recheck after its wait, before any mutation. + if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) { + state.preflight_rejection = Some(LocalRenamePreflightRejection(())); + return Err(DiskError::DiskOngoingReq); + } if let Some(claim) = quota_fence_claim { mutation_lease.attach_external_guard(claim); } @@ -2549,8 +2601,23 @@ impl LocalDisk { dst_path: &str, namespace_owner: Option>, ) -> super::super::RenameDataObservation { + self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None)) + .await + } + + pub(in crate::disk) async fn rename_data_observed_with_reservation( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + owners: (Option>, Option), + ) -> super::super::RenameDataObservation { + let (namespace_owner, metadata_reservation) = owners; let mut state = RenameDataState { namespace_owner, + metadata_reservation, ..Default::default() }; let result = self diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 69c1b35d2..c75103e06 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -106,6 +106,8 @@ pub(crate) struct RenameDataGuards { pub(crate) scanner_publication_lease_token: Option, pub(crate) external_guard: Option>, pub(crate) namespace_owner: Option>, + pub(crate) metadata_reservation: Option, + pub(crate) metadata_reservation_token: Option, } /// Local preflight evidence stays outside DiskAPI and the RPC response format. @@ -122,6 +124,13 @@ impl RenameDataObservation { } } + fn not_dispatched(error: DiskError) -> Self { + Self { + result: Err(error), + preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()), + } + } + pub(crate) fn rejected_before_publication(&self) -> bool { self.result.is_err() && self.preflight_rejection.is_some() } @@ -311,6 +320,82 @@ pub enum Disk { } impl Disk { + /// Prepare an inert single-use reservation on this disk. + /// A lost remote response holds its bucket publication fence until expiration, but does not lock the object. + pub async fn prepare_object_metadata( + &self, + bucket: &str, + object: &str, + incarnation: Uuid, + timeout: std::time::Duration, + ) -> Result { + if timeout.is_zero() { + return Err(DiskError::Timeout); + } + match self { + Disk::Local(disk) => tokio::time::timeout( + timeout.min(std::time::Duration::from_secs(5)), + disk.prepare_object_metadata(bucket, object), + ) + .await + .map_err(|_| DiskError::Timeout)?, + Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await, + } + } + + /// Refresh an inert token while its sender stages the write. + pub async fn renew_object_metadata( + &self, + bucket: &str, + object: &str, + token: Uuid, + timeout: std::time::Duration, + ) -> Result<()> { + if timeout.is_zero() { + return Err(DiskError::Timeout); + } + match self { + Disk::Local(disk) => tokio::time::timeout( + timeout.min(std::time::Duration::from_secs(5)), + disk.renew_object_metadata(bucket, object, token), + ) + .await + .map_err(|_| DiskError::Timeout)?, + Disk::Remote(disk) => disk.renew_object_metadata(bucket, object, token, timeout).await, + } + } + + /// Enqueue a prepared token in object mutation order. + pub async fn activate_object_metadata( + &self, + bucket: &str, + object: &str, + token: Uuid, + timeout: std::time::Duration, + ) -> Result<()> { + if timeout.is_zero() { + return Err(DiskError::Timeout); + } + match self { + Disk::Local(disk) => tokio::time::timeout( + timeout.min(std::time::Duration::from_secs(5)), + disk.activate_object_metadata(bucket, object, token), + ) + .await + .map_err(|_| DiskError::Timeout)?, + Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await, + } + } + + /// Cancel an unclaimed reservation. False does not prove that a claim has + /// not already started a physical mutation. + pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result { + match self { + Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)), + Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await, + } + } + /// Physical ownership belongs to the node hosting the local attachment. pub fn physical_admission(&self) -> Option> { match self { @@ -1251,18 +1336,33 @@ impl Disk { .rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards) .await } - Disk::Remote(remote_disk) => RenameDataObservation::unknown( - remote_disk - .rename_data_borrowed_with_fence( - src_volume, - src_path, - fi, - dst_volume, - dst_path, - guards.scanner_publication_lease_token, - ) - .await, - ), + Disk::Remote(remote_disk) => { + if guards.metadata_reservation.is_some() { + return RenameDataObservation::not_dispatched(DiskError::InvalidPath); + } + if let Some(token) = guards.metadata_reservation_token { + if guards.scanner_publication_lease_token.is_some() { + return RenameDataObservation::not_dispatched(DiskError::InvalidPath); + } + return RenameDataObservation::unknown( + remote_disk + .rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token) + .await, + ); + } + RenameDataObservation::unknown( + remote_disk + .rename_data_borrowed_with_fence( + src_volume, + src_path, + fi, + dst_volume, + dst_path, + guards.scanner_publication_lease_token, + ) + .await, + ) + } } } diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index 2b6839e38..4d6e3ccae 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -22,8 +22,10 @@ use parking_lot::Mutex; use rustfs_utils::path::SLASH_SEPARATOR; use std::{ collections::{HashMap, VecDeque}, + future::Future, io, path::{Component, Path, PathBuf}, + pin::Pin, sync::{Arc, LazyLock, Weak}, time::{Duration, Instant}, }; @@ -2229,6 +2231,375 @@ pub(crate) struct MetadataMutationLease { _owner: Option>, } +pub(crate) trait MetadataReservationFence: Send + Sync { + fn is_valid(&self) -> bool; +} + +/// Single-use ownership of the metadata lock for an exact physical object. +pub(crate) struct MetadataMutationReservation { + key: PathBuf, + disk_instance: uuid::Uuid, + lease: MetadataReservationLease, + capacity: Option, + fence: Option>, +} + +enum MetadataReservationLease { + Held(Arc), + Queued(Pin> + Send>>), +} + +impl MetadataMutationReservation { + pub(crate) fn fence(&self) -> Option> { + self.fence.clone() + } + + /// Wait for a previously enqueued lease without holding an I/O permit. + /// Dropping this future removes an unclaimed waiter from the FIFO queue. + pub(crate) async fn wait(self) -> Self { + let Self { + key, + disk_instance, + lease, + capacity, + fence, + } = self; + let lease = match lease { + MetadataReservationLease::Held(lease) => lease, + MetadataReservationLease::Queued(waiter) => waiter.await, + }; + Self { + key, + disk_instance, + lease: MetadataReservationLease::Held(lease), + capacity, + fence, + } + } + + pub(crate) fn into_lease_for(self, key: &Path, disk_instance: uuid::Uuid) -> Result> { + if self.key != key || self.disk_instance != disk_instance { + return Err(DiskError::InvalidPath); + } + if self.fence.as_ref().is_some_and(|fence| !fence.is_valid()) { + return Err(DiskError::DiskOngoingReq); + } + let MetadataReservationLease::Held(lease) = self.lease else { + return Err(DiskError::DiskOngoingReq); + }; + Ok(match self.capacity { + Some(capacity) => Arc::new((lease, capacity)), + None => lease, + }) + } +} + +pub(crate) async fn reserve_metadata_mutation_lease( + object: &Path, + disk_instance: uuid::Uuid, + owner: Option>, +) -> MetadataMutationReservation { + MetadataMutationReservation { + key: object.to_path_buf(), + disk_instance, + lease: MetadataReservationLease::Held(acquire_metadata_mutation_lease(object, owner).await), + capacity: None, + fence: None, + } +} + +struct MetadataReservationEntry { + state: MetadataReservationState, + expires_at: tokio::time::Instant, + renewal_ttl: Duration, +} + +enum MetadataReservationState { + Prepared { + key: PathBuf, + disk_instance: uuid::Uuid, + capacity: Option, + owner: Option>, + }, + Queued(MetadataMutationReservation), +} + +/// Process-local, bounded one-shot reservations for one disk instance. +pub(crate) struct MetadataReservationRegistry { + entries: Mutex>, + capacity: Arc, +} + +impl MetadataReservationRegistry { + pub(crate) fn new(limit: usize) -> Arc { + let registry = Arc::new(Self { + entries: Mutex::new(HashMap::new()), + capacity: Arc::new(Semaphore::new(limit)), + }); + let weak = Arc::downgrade(®istry); + tokio::spawn(async move { + loop { + tokio::time::sleep(Duration::from_secs(1)).await; + let Some(registry) = weak.upgrade() else { break }; + registry.remove_expired(); + } + }); + registry + } + + pub(crate) async fn reserve( + &self, + key: &Path, + disk_instance: uuid::Uuid, + acquire_timeout: Duration, + owner: Option>, + ) -> Result { + let capacity = match Arc::clone(&self.capacity).try_acquire_owned() { + Ok(capacity) => capacity, + Err(_) => { + self.remove_expired(); + Arc::clone(&self.capacity) + .try_acquire_owned() + .map_err(|_| DiskError::DiskOngoingReq)? + } + }; + let mut reservation = tokio::time::timeout(acquire_timeout, reserve_metadata_mutation_lease(key, disk_instance, owner)) + .await + .map_err(|_| DiskError::Timeout)?; + reservation.capacity = Some(capacity); + let token = uuid::Uuid::new_v4(); + let mut entries = self.entries.lock(); + match entries.entry(token) { + std::collections::hash_map::Entry::Vacant(entry) => entry.insert(MetadataReservationEntry { + state: MetadataReservationState::Queued(reservation), + expires_at: tokio::time::Instant::now() + Duration::from_secs(60), + renewal_ttl: Duration::from_secs(60), + }), + std::collections::hash_map::Entry::Occupied(_) => return Err(DiskError::Unexpected), + }; + Ok(token) + } + + /// Register this mutation in FIFO order before returning its token. + /// The mutex may still be held by an earlier operation when this returns. + pub(crate) async fn enqueue(&self, key: &Path, disk_instance: uuid::Uuid) -> Result { + let capacity = match Arc::clone(&self.capacity).try_acquire_owned() { + Ok(capacity) => capacity, + Err(_) => { + self.remove_expired(); + Arc::clone(&self.capacity) + .try_acquire_owned() + .map_err(|_| DiskError::DiskOngoingReq)? + } + }; + let object = key.to_path_buf(); + let mut waiter: Pin> + Send>> = + Box::pin(async move { acquire_metadata_mutation_lease(&object, None).await }); + // Poll once so Tokio registers this waiter before the token can be + // acknowledged; retaining the future retains its position in the queue. + let lease = match futures::poll!(waiter.as_mut()) { + std::task::Poll::Ready(lease) => MetadataReservationLease::Held(lease), + std::task::Poll::Pending => MetadataReservationLease::Queued(waiter), + }; + let token = uuid::Uuid::new_v4(); + let reservation = MetadataMutationReservation { + key: key.to_path_buf(), + disk_instance, + lease, + capacity: Some(capacity), + fence: None, + }; + let mut entries = self.entries.lock(); + match entries.entry(token) { + std::collections::hash_map::Entry::Vacant(entry) => entry.insert(MetadataReservationEntry { + state: MetadataReservationState::Queued(reservation), + expires_at: tokio::time::Instant::now() + Duration::from_secs(60), + renewal_ttl: Duration::from_secs(60), + }), + std::collections::hash_map::Entry::Occupied(_) => return Err(DiskError::Unexpected), + }; + Ok(token) + } + + /// Allocate an inert token before a client asks to enter the FIFO queue. + /// A lost preparation response cannot hold an object metadata lock. + #[cfg(test)] + pub(crate) fn prepare(&self, key: &Path, disk_instance: uuid::Uuid) -> Result { + self.prepare_with_owner(key, disk_instance, None) + } + + pub(crate) fn prepare_with_owner( + &self, + key: &Path, + disk_instance: uuid::Uuid, + owner: Option>, + ) -> Result { + self.prepare_with_owner_ttl(key, disk_instance, owner, Duration::from_secs(60)) + } + + pub(crate) fn prepare_with_owner_ttl( + &self, + key: &Path, + disk_instance: uuid::Uuid, + owner: Option>, + ttl: Duration, + ) -> Result { + if ttl.is_zero() { + return Err(DiskError::Timeout); + } + let expires_at = tokio::time::Instant::now().checked_add(ttl).ok_or(DiskError::Timeout)?; + let capacity = match Arc::clone(&self.capacity).try_acquire_owned() { + Ok(capacity) => capacity, + Err(_) => { + self.remove_expired(); + Arc::clone(&self.capacity) + .try_acquire_owned() + .map_err(|_| DiskError::DiskOngoingReq)? + } + }; + let token = uuid::Uuid::new_v4(); + let entry = MetadataReservationEntry { + state: MetadataReservationState::Prepared { + key: key.to_path_buf(), + disk_instance, + capacity: Some(capacity), + owner, + }, + expires_at, + renewal_ttl: ttl, + }; + match self.entries.lock().entry(token) { + std::collections::hash_map::Entry::Vacant(slot) => { + slot.insert(entry); + Ok(token) + } + std::collections::hash_map::Entry::Occupied(_) => Err(DiskError::Unexpected), + } + } + + /// Extend an inert token or a queued short token without reviving an expired, + /// cancelled, or claimed entry. Long queued tokens remain non-renewable. + pub(crate) fn renew(&self, token: uuid::Uuid, key: &Path, disk_instance: uuid::Uuid) -> Result<()> { + let mut entries = self.entries.lock(); + let entry = entries.get_mut(&token).ok_or(DiskError::DiskOngoingReq)?; + let now = tokio::time::Instant::now(); + if entry.expires_at <= now { + return Err(DiskError::Timeout); + } + let (stored_key, stored_instance, owner) = match &entry.state { + MetadataReservationState::Prepared { + key, + disk_instance, + owner, + .. + } => (key, disk_instance, owner), + MetadataReservationState::Queued(reservation) if entry.renewal_ttl <= Duration::from_secs(2) => { + (&reservation.key, &reservation.disk_instance, &reservation.fence) + } + MetadataReservationState::Queued(_) => return Err(DiskError::DiskOngoingReq), + }; + if stored_key != key || *stored_instance != disk_instance { + return Err(DiskError::InvalidPath); + } + if owner.as_ref().is_some_and(|fence| !fence.is_valid()) { + return Err(DiskError::DiskOngoingReq); + } + entry.expires_at = now + entry.renewal_ttl; + Ok(()) + } + + /// Atomically enqueue a prepared token. Cancellation and enqueue serialize + /// on the registry, so a delayed enqueue cannot revive a cancelled token. + pub(crate) async fn activate(&self, token: uuid::Uuid, key: &Path, disk_instance: uuid::Uuid) -> Result<()> { + std::future::poll_fn(|cx| { + let mut entries = self.entries.lock(); + let Some(entry) = entries.get_mut(&token) else { + return std::task::Poll::Ready(Err(DiskError::DiskOngoingReq)); + }; + if entry.expires_at <= tokio::time::Instant::now() { + return std::task::Poll::Ready(Err(DiskError::Timeout)); + } + let (stored_key, stored_instance, capacity, owner) = match &mut entry.state { + MetadataReservationState::Prepared { + key, + disk_instance, + capacity, + owner, + } => (key, disk_instance, capacity, owner), + MetadataReservationState::Queued(reservation) => { + return std::task::Poll::Ready(if reservation.key != key || reservation.disk_instance != disk_instance { + Err(DiskError::InvalidPath) + } else if reservation.fence.as_ref().is_some_and(|fence| !fence.is_valid()) { + Err(DiskError::DiskOngoingReq) + } else { + Ok(()) + }); + } + }; + if stored_key != key || *stored_instance != disk_instance { + return std::task::Poll::Ready(Err(DiskError::InvalidPath)); + } + if owner.as_ref().is_some_and(|fence| !fence.is_valid()) { + return std::task::Poll::Ready(Err(DiskError::DiskOngoingReq)); + } + let Some(capacity) = capacity.take() else { + return std::task::Poll::Ready(Err(DiskError::Unexpected)); + }; + let object = stored_key.clone(); + let fence = owner.take(); + let owner: Option> = fence.clone().map(|fence| fence as Arc); + let mut waiter: Pin> + Send>> = + Box::pin(async move { acquire_metadata_mutation_lease(&object, owner).await }); + let lease = match waiter.as_mut().poll(cx) { + std::task::Poll::Ready(lease) => MetadataReservationLease::Held(lease), + std::task::Poll::Pending => MetadataReservationLease::Queued(waiter), + }; + entry.state = MetadataReservationState::Queued(MetadataMutationReservation { + key: stored_key.clone(), + disk_instance, + lease, + capacity: Some(capacity), + fence, + }); + std::task::Poll::Ready(Ok(())) + }) + .await + } + + pub(crate) fn claim(&self, token: uuid::Uuid, key: &Path, disk_instance: uuid::Uuid) -> Result { + let entry = self.entries.lock().remove(&token).ok_or(DiskError::DiskOngoingReq)?; + if entry.expires_at <= tokio::time::Instant::now() { + return Err(DiskError::Timeout); + } + let MetadataReservationState::Queued(reservation) = entry.state else { + return Err(DiskError::DiskOngoingReq); + }; + if reservation.key != key || reservation.disk_instance != disk_instance { + return Err(DiskError::InvalidPath); + } + if reservation.fence.as_ref().is_some_and(|fence| !fence.is_valid()) { + return Err(DiskError::DiskOngoingReq); + } + Ok(reservation) + } + + pub(crate) fn cancel(&self, token: uuid::Uuid) -> bool { + let removed = self.entries.lock().remove(&token); + removed.is_some() + } + + fn remove_expired(&self) { + let now = tokio::time::Instant::now(); + let expired: Vec<_> = self + .entries + .lock() + .extract_if(|_, entry| entry.expires_at <= now) + .map(|(_, entry)| entry) + .collect(); + drop(expired); + } +} + pub(crate) async fn acquire_metadata_mutation_lease( object: &Path, owner: Option>, diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 65580001e..707c8f487 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -28,6 +28,7 @@ use tokio_util::sync::CancellationToken; #[derive(Clone)] pub struct NamespaceLockFence { signals: Arc>>, + local_probes: Arc>, #[cfg(test)] forced_lost: Arc>>, } @@ -44,6 +45,7 @@ impl NamespaceLockFence { pub(crate) fn new() -> Self { Self { signals: Arc::default(), + local_probes: Arc::default(), #[cfg(test)] forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]), } @@ -58,17 +60,28 @@ impl NamespaceLockFence { { return true; } - self.signals.iter().any(|signal| signal.is_lost()) + self.signals.iter().any(|signal| signal.is_lost()) || self.local_probes.iter().any(|probe| probe.is_lost()) } pub(crate) fn add_signal(&mut self, signal: Arc) { Arc::make_mut(&mut self.signals).push(signal); } + fn add_local_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) { + if let rustfs_lock::NamespaceLockGuard::Fast(guard) = guard + && let Some(probe) = guard.release_probe() + { + Arc::make_mut(&mut self.local_probes).push(probe); + } + } + fn extend(&mut self, other: &Self) { if !Arc::ptr_eq(&self.signals, &other.signals) { Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned()); } + if !Arc::ptr_eq(&self.local_probes, &other.local_probes) { + Arc::make_mut(&mut self.local_probes).extend(other.local_probes.iter().cloned()); + } #[cfg(test)] if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) { Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned()); @@ -106,6 +119,7 @@ impl NamespaceLockSignalTestFence { ) -> Self { let fence = NamespaceLockFence { signals: Arc::default(), + local_probes: Arc::default(), forced_lost: Arc::new(vec![loss_handle]), }; let signal_key = Arc::as_ptr(signal) as usize; @@ -160,6 +174,20 @@ pub struct ObjectLockConfigSnapshot { } impl ObjectLockConfigSnapshot { + pub(crate) fn owns_lifecycle_guard(&self) -> bool { + self._lifecycle_guard.is_some() + } + + pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option) -> bool { + self.bucket.as_deref() == Some(bucket) + && incarnation.is_some() + && self.bucket_incarnation_id == incarnation + && self + ._lifecycle_guard + .as_ref() + .is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released()) + } + pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self { Self { store_id: None, @@ -208,6 +236,7 @@ impl ObjectLockConfigSnapshot { if let Some(signal) = lifecycle_guard.lock_lost_signal() { lifecycle_fence.add_signal(signal); } + lifecycle_fence.add_local_guard(&lifecycle_guard); Self { store_id: Some(store_id), bucket: Some(bucket.to_string()), @@ -1353,6 +1382,13 @@ impl ObjectOptions { if let Some(signal) = guard.lock_lost_signal() { self.add_namespace_lock_lost_signal(signal); } + if let rustfs_lock::NamespaceLockGuard::Fast(local) = guard + && !local.is_disabled() + { + self.namespace_lock_fence + .get_or_insert_with(NamespaceLockFence::new) + .add_local_guard(guard); + } } pub fn add_bucket_lifecycle_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) { @@ -1360,6 +1396,7 @@ impl ObjectOptions { if let Some(signal) = guard.lock_lost_signal() { fence.add_signal(signal); } + fence.add_local_guard(guard); } pub fn set_delete_replication_state(&mut self, dsc: ReplicateDecision) { diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index 79e2e5e2d..90b9fd384 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -58,7 +58,7 @@ use std::sync::{ Arc, OnceLock, atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}, }; -use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock}; +use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore}; use tokio::time::Instant; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; @@ -71,6 +71,9 @@ const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2; pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256; +// Bound abandoned speculative RPCs independently for each storage instance. +pub(crate) const SPECULATIVE_RESERVATION_LIMIT: usize = 64; + /// A lease is deliberately short-lived. The coordinator treats expiry as a /// failed publication rather than silently continuing with a peer that may /// have started movement after the lease was abandoned. @@ -101,6 +104,12 @@ pub(crate) struct NamespaceCommitGuard { counted: bool, } +impl NamespaceCommitGuard { + pub(crate) fn covers_instance(&self, ctx: &Arc) -> bool { + Arc::ptr_eq(&self.ctx, ctx) + } +} + impl Drop for NamespaceCommitGuard { fn drop(&mut self) { if self.counted { @@ -332,6 +341,7 @@ pub struct InstanceContext { scanner_publication_state: AtomicU8, namespace_commits: AtomicU64, namespace_commit_generation: AtomicU64, + speculative_reservations: Arc, /// Resolves object-encryption material at the application boundary. object_encryption_resolver: OnceLock>, delete_recovery_hints: DeleteRecoveryHints, @@ -392,6 +402,7 @@ impl InstanceContext { scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN), namespace_commits: AtomicU64::new(0), namespace_commit_generation: AtomicU64::new(0), + speculative_reservations: Arc::new(Semaphore::new(SPECULATIVE_RESERVATION_LIMIT)), object_encryption_resolver: OnceLock::new(), delete_recovery_hints: DeleteRecoveryHints::default(), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), @@ -413,6 +424,11 @@ impl InstanceContext { self.lock_manager.clone() } + /// Skip speculative work when its detached owners already fill the instance budget. + pub(crate) fn try_acquire_speculative_reservation(&self) -> Option { + Arc::clone(&self.speculative_reservations).try_acquire_owned().ok() + } + /// Run a multi-step mutation on its own task and wait for its output. /// /// Dropping the returned future leaves the task running to completion, so @@ -1143,6 +1159,26 @@ mod tests { assert!(ledger.admit(&bucket, &format!("{object_prefix}1024"), (None, false), false)); } + #[test] + fn speculative_reservation_budget_is_bounded_and_instance_local() { + let first = InstanceContext::new(); + let other = InstanceContext::new(); + let mut permits = (0..super::SPECULATIVE_RESERVATION_LIMIT) + .map(|_| first.try_acquire_speculative_reservation().expect("initial permit")) + .collect::>(); + assert!(first.try_acquire_speculative_reservation().is_none()); + assert!( + other.try_acquire_speculative_reservation().is_some(), + "budgets belong to their own instance" + ); + drop(permits.pop()); + let replacement = first + .try_acquire_speculative_reservation() + .expect("dropping an owner frees its slot"); + assert!(first.try_acquire_speculative_reservation().is_none()); + drop(replacement); + } + #[test] fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() { let first = Arc::new(InstanceContext::new()); @@ -1151,6 +1187,8 @@ mod tests { other.set_scanner_publication_state(false); assert!(first.scanner_publication_state_allowed()); let one = first.begin_namespace_commit(); + assert!(one.covers_instance(&first)); + assert!(!one.covers_instance(&other)); let shared_owner = Arc::clone(&one); let two = first.begin_namespace_commit(); assert!(first.namespace_commits_pending()); diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 46423fddf..66f76c0e0 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -2538,6 +2538,8 @@ pub(in crate::set_disk) struct RenameDataCommit { pub(in crate::set_disk) old_current_source: Option, pub(in crate::set_disk) committed_file_info: FileInfo, pub(in crate::set_disk) tail_drain: Option>>, + pub(in crate::set_disk) tail_has_metadata_reservations: bool, + pub(in crate::set_disk) tail_has_owned_bucket_fences: bool, } pub(in crate::set_disk) struct RenameTailCleanup { @@ -3468,6 +3470,8 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> { rollback_receipt: Option, multipart_proof_authority: Option, multipart_commit_owner: Option>, + reservation_ctx: Option>, + prepared_reservations: Option>>, } impl<'a> RenameDataFenceOptions<'a> { @@ -3483,6 +3487,8 @@ impl<'a> RenameDataFenceOptions<'a> { rollback_receipt: None, multipart_proof_authority: None, multipart_commit_owner: None, + reservation_ctx: None, + prepared_reservations: None, } } @@ -3519,6 +3525,415 @@ impl<'a> RenameDataFenceOptions<'a> { self.namespace_commit_guard = namespace_commit_guard; self } + + pub(in crate::set_disk) fn with_reservation_context( + mut self, + ctx: Option>, + reservations: Option>>, + ) -> Self { + self.reservation_ctx = ctx; + self.prepared_reservations = reservations; + self + } +} + +#[derive(Clone)] +struct MetadataRenewalTarget { + disk: DiskStore, + token: Uuid, + renewed_at: Arc>, +} + +struct MetadataRenewalGroup { + targets: Arc>>, + task: tokio::task::JoinHandle<()>, +} + +impl Drop for MetadataRenewalGroup { + fn drop(&mut self) { + self.task.abort(); + } +} + +pub(crate) struct RenameMetadataReservation { + disk: DiskStore, + token: Option, + // Remote reservation protocol v2 and the local owned path both retain + // bucket publication and lifecycle guards until the physical write drains. + owns_bucket_fence: bool, + short: bool, + renewed_at: Arc>, + renewal: Option<(Arc, usize)>, +} + +impl RenameMetadataReservation { + fn stop_renewal(&mut self) { + if let Some((group, index)) = self.renewal.take() { + let mut targets = group.targets.lock(); + targets.remove(&index); + if targets.is_empty() { + group.task.abort(); + } + } + } + + async fn cancel(&mut self) -> disk::error::Result<()> { + self.stop_renewal(); + let Some(token) = self.token else { return Ok(()) }; + match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await { + Ok(Ok(true)) => { + self.token = None; + Ok(()) + } + Ok(Ok(false)) => Err(DiskError::DiskOngoingReq), + Ok(Err(error)) => Err(error), + Err(_) => Err(DiskError::Timeout), + } + } +} + +impl Drop for RenameMetadataReservation { + fn drop(&mut self) { + self.stop_renewal(); + let Some(token) = self.token.take() else { return }; + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + let disk = self.disk.clone(); + runtime.spawn(async move { + let _ = disk.cancel_object_metadata_reservation(token).await; + }); + } + } +} + +#[derive(Clone, Copy)] +pub(crate) enum RenameReservationMode { + Ownerless, + Owned, + ShortOwned, +} + +pub(crate) async fn prepare_rename_metadata_reservations( + disks: &[Option], + bucket: &str, + object: &str, + incarnation: Uuid, + ctx: &Arc, + owns_bucket_fence: bool, +) -> disk::error::Result>>> { + prepare_rename_metadata_reservations_until_abandoned( + disks, + bucket, + object, + incarnation, + ctx, + if owns_bucket_fence { + RenameReservationMode::Owned + } else { + RenameReservationMode::Ownerless + }, + std::future::pending(), + ) + .await +} + +pub(crate) async fn prepare_rename_metadata_reservations_until_abandoned( + disks: &[Option], + bucket: &str, + object: &str, + incarnation: Uuid, + ctx: &Arc, + mode: RenameReservationMode, + abandoned: impl std::future::Future, +) -> disk::error::Result>>> { + let owns_bucket_fence = !matches!(mode, RenameReservationMode::Ownerless); + let short = matches!(mode, RenameReservationMode::ShortOwned); + let mut abandoned = Box::pin(abandoned); + let discarded = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let mut pending: FuturesUnordered<_> = disks + .iter() + .enumerate() + .map(|(_index, disk)| { + let discarded = Arc::clone(&discarded); + async move { + let result: disk::error::Result> = async { + if discarded.load(std::sync::atomic::Ordering::Acquire) { + return Ok(None); + } + let Some(disk) = disk else { return Ok(None) }; + #[cfg(test)] + rename_fanout_barrier::checkpoint(object, _index, rename_fanout_barrier::PHASE_RESERVATION_BEFORE).await; + if disk.is_local() { + crate::store::admit_local_disk(ctx, disk, None, false).await?; + } + if discarded.load(std::sync::atomic::Ordering::Acquire) { + return Ok(None); + } + let prepared_at = tokio::time::Instant::now(); + let token = if disk.is_local() && owns_bucket_fence { + let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx) + .await + .ok_or(DiskError::DiskOngoingReq)?; + if short { + store + .reserve_local_object_metadata_short( + &disk.endpoint().to_string(), + bucket, + object, + incarnation, + Duration::from_millis(50), + ) + .await? + } else { + store + .reserve_local_object_metadata_at_incarnation( + &disk.endpoint().to_string(), + bucket, + object, + incarnation, + Duration::from_secs(5), + ) + .await? + } + } else if short { + let crate::disk::Disk::Remote(remote) = disk.as_ref() else { + return Err(DiskError::InvalidPath); + }; + remote + .reserve_object_metadata_short(bucket, object, incarnation, Duration::from_millis(50)) + .await? + } else { + disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5)) + .await? + }; + // No await separates receipt from installing the cancellation owner. + let reservation = RenameMetadataReservation { + disk: disk.clone(), + token: Some(token), + owns_bucket_fence: !disk.is_local() || owns_bucket_fence, + short, + renewed_at: Arc::new(parking_lot::Mutex::new(prepared_at)), + renewal: None, + }; + if short { + // Install the cancellation owner before extending the short token. + // A dropped renew response still leaves us the token to cancel. + disk.renew_object_metadata(bucket, object, token, Duration::from_millis(50)) + .await?; + } + #[cfg(test)] + rename_fanout_barrier::checkpoint(object, _index, rename_fanout_barrier::PHASE_RESERVATION_AFTER).await; + Ok(Some(reservation)) + } + .await; + (_index, result) + } + }) + .collect(); + let mut reservations: Vec> = (0..disks.len()).map(|_| None).collect(); + let mut discard = false; + while !pending.is_empty() { + tokio::select! { + biased; + _ = &mut abandoned, if !discard => { + discard = true; + discarded.store(true, std::sync::atomic::Ordering::Release); + reservations.clear(); + } + result = pending.next() => { + let Some((index, result)) = result else { break }; + if discard { + drop(result); + } else { + match result { + Ok(reservation) => reservations[index] = reservation, + Err(_) => { + discard = true; + discarded.store(true, std::sync::atomic::Ordering::Release); + reservations.clear(); + } + } + } + } + } + } + if discard { + // Drain all in-flight preparations so late tokens get a cancellation owner. + return Ok(None); + } + let targets = Arc::new(parking_lot::Mutex::new(HashMap::new())); + for (index, reservation) in reservations.iter().enumerate() { + if let Some(reservation) = reservation { + let token = reservation.token.ok_or(DiskError::Unexpected)?; + targets.lock().insert( + index, + MetadataRenewalTarget { + disk: reservation.disk.clone(), + token, + renewed_at: Arc::clone(&reservation.renewed_at), + }, + ); + } + } + if !targets.lock().is_empty() { + let renewal_targets = Arc::clone(&targets); + let renewal_ctx = Arc::clone(ctx); + let renewal_bucket = bucket.to_owned(); + let renewal_object = object.to_owned(); + // Short leases remain short even after renewal, so a lost renewal and + // cancellation cannot strand a bucket fence for the legacy lease time. + let task = tokio::spawn(async move { + let interval = if short { + Duration::from_millis(500) + } else { + Duration::from_secs(20) + }; + let retry = if short { + Duration::from_millis(250) + } else { + Duration::from_secs(5) + }; + let lifetime = if short { + Duration::from_secs(2) + } else { + Duration::from_secs(60) + }; + let mut delay = interval; + loop { + tokio::time::sleep(delay).await; + let current: Vec<_> = renewal_targets.lock().values().cloned().collect(); + if current.is_empty() { + break; + } + // Once any acknowledgment is too old, the complete fanout can + // no longer be safely activated. Stop retrying stale peers; + // their receiver-side tokens still expire on their own. + if current.iter().any(|target| target.renewed_at.lock().elapsed() >= lifetime) { + break; + } + let results = join_all(current.into_iter().map(|target| { + let ctx = Arc::clone(&renewal_ctx); + let bucket = &renewal_bucket; + let object = &renewal_object; + async move { + let started = tokio::time::Instant::now(); + let admitted = !target.disk.is_local() + || crate::store::admit_local_disk(&ctx, &target.disk, None, false).await.is_ok(); + if admitted + && target + .disk + .renew_object_metadata(bucket, object, target.token, retry) + .await + .is_ok() + { + *target.renewed_at.lock() = started; + true + } else { + false + } + } + })) + .await; + delay = if results.into_iter().all(|success| success) { + interval + } else { + retry + }; + } + }); + let group = Arc::new(MetadataRenewalGroup { targets, task }); + for (index, reservation) in reservations.iter_mut().enumerate() { + if let Some(reservation) = reservation { + reservation.renewal = Some((Arc::clone(&group), index)); + } + } + } + Ok(Some(reservations)) +} + +pub(crate) fn align_rename_metadata_reservations( + mut reservations: Vec>, + disks: &[Option], +) -> Option>> { + if reservations.len() != disks.len() { + return None; + } + let mut aligned = Vec::with_capacity(disks.len()); + for disk in disks { + let Some(disk) = disk else { + aligned.push(None); + continue; + }; + let index = reservations + .iter() + .position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?; + aligned.push(reservations[index].take()); + } + // Disks lost while staging need no activation; dropping their still-inert + // reservations schedules cancellation without blocking the commit. + Some(aligned) +} + +async fn activate_rename_metadata_reservations( + mut reservations: Vec>, + bucket: &str, + object: &str, + ctx: &Arc, +) -> disk::error::Result>>> { + // Short tokens continue renewing during activation; an activated token + // can otherwise expire while another disk's activation is still pending. + // Long tokens retain their existing non-renewable queued behavior. + for reservation in reservations.iter_mut().flatten() { + if !reservation.short { + reservation.stop_renewal(); + } + } + if reservations.iter().flatten().any(|reservation| { + let max_age = if reservation.short { + Duration::from_secs(1) + } else { + Duration::from_secs(50) + }; + reservation.renewed_at.lock().elapsed() >= max_age + }) { + return Ok(None); + } + #[cfg(test)] + let test_reservations = &reservations; + let activated = join_all(reservations.iter().flatten().map(|reservation| async move { + if reservation.disk.is_local() { + crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?; + } + let token = reservation.token.ok_or(DiskError::Unexpected)?; + let result = reservation + .disk + .activate_object_metadata(bucket, object, token, Duration::from_secs(5)) + .await; + #[cfg(test)] + if result.is_ok() { + let index = test_reservations + .iter() + .position(|candidate| { + candidate + .as_ref() + .is_some_and(|candidate| std::ptr::eq(candidate, reservation)) + }) + .expect("activated reservation belongs to the fanout"); + rename_fanout_barrier::checkpoint(object, index, rename_fanout_barrier::PHASE_RESERVATION_ACTIVATED).await; + } + result + })) + .await; + for reservation in reservations.iter_mut().flatten() { + reservation.stop_renewal(); + } + if activated.into_iter().any(|result| result.is_err()) { + let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await; + if cancelled.into_iter().any(|result| result.is_err()) { + return Err(DiskError::DiskOngoingReq); + } + return Ok(None); + } + Ok(Some(reservations)) } #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] @@ -3530,7 +3945,7 @@ type RenameDataLegacyTuple = ( Option, ); -fn put_rename_early_ack_enabled() -> bool { +pub(crate) fn put_rename_early_ack_enabled() -> bool { rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true) } @@ -3896,6 +4311,8 @@ impl SetDisks { old_current_source, committed_file_info, tail_drain: None, + tail_has_metadata_reservations: false, + tail_has_owned_bucket_fences: false, }) } @@ -3992,6 +4409,8 @@ impl SetDisks { rollback_receipt, multipart_proof_authority: _, multipart_commit_owner: _, + reservation_ctx, + prepared_reservations, } = fence_options; if let Some(file_info) = disks .iter() @@ -4007,6 +4426,21 @@ impl SetDisks { let disk_count = disks.len(); let fanout_disks = disks.to_vec(); + let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) { + (Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?, + (_, _) => None, + }; + let tail_has_metadata_reservations = reservations.is_some(); + let tail_has_owned_bucket_fences = reservations.as_ref().is_some_and(|reservations| { + reservations.len() == disks.len() + && disks.iter().any(Option::is_some) + && disks.iter().zip(reservations).all(|(disk, reservation)| match disk { + Some(disk) => reservation + .as_ref() + .is_some_and(|owner| owner.owns_bucket_fence && Arc::ptr_eq(&owner.disk, disk)), + None => true, + }) + }); let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?; let coordinator_disks = fanout_disks.clone(); let src_bucket = Arc::new(src_bucket.to_string()); @@ -4039,6 +4473,11 @@ impl SetDisks { let file_info = file_info.clone(); let successful_rename_completion_rank = successful_rename_completion_rank.clone(); let namespace_commit_guard = namespace_commit_guard.clone(); + let reservation = reservations + .as_mut() + .and_then(|entries| entries.get_mut(i)) + .and_then(Option::take); + let reservation_ctx = reservation_ctx.clone(); tasks.spawn(async move { let _namespace_commit_guard = namespace_commit_guard.clone(); let mut dispatch_state = RenameDispatchState::NotDispatched; @@ -4049,6 +4488,17 @@ impl SetDisks { let Some(disk) = disk else { return Err(DiskError::DiskNotFound); }; + let disk_admission = if reservation.is_some() && disk.is_local() { + let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?; + // The fanout owner already covers this instance through + // physical completion; only add a disk owner when it does not. + let covered = namespace_commit_guard + .as_ref() + .is_some_and(|guard| guard.covers_instance(ctx)); + crate::store::admit_local_disk(ctx, &disk, None, !covered).await? + } else { + None + }; let disk_wait_location_stage = if disk.is_local() { rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL } else { @@ -4074,6 +4524,11 @@ impl SetDisks { let disk_wait_started = rustfs_io_metrics::put_stage_timer(); dispatch_state = RenameDispatchState::MayHavePublished; let _foreground_active_guard = rename_foreground_active_guard(i); + let namespace_owner: Option> = if disk_admission.is_some() { + Some(Arc::new((namespace_commit_guard.clone(), disk_admission))) + } else { + namespace_commit_guard.clone().map(|owner| owner as Arc) + }; let observed = disk .rename_data_borrowed_with_fence_observed( &src_bucket, @@ -4083,9 +4538,8 @@ impl SetDisks { &dst_object, disk::RenameDataGuards { scanner_publication_lease_token, - namespace_owner: namespace_commit_guard - .clone() - .map(|owner| owner as Arc), + metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token), + namespace_owner, ..Default::default() }, ) @@ -4363,6 +4817,8 @@ impl SetDisks { let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer(); commit.map(|mut commit| { commit.tail_drain = Some(tail_drain); + commit.tail_has_metadata_reservations = tail_has_metadata_reservations; + commit.tail_has_owned_bucket_fences = tail_has_owned_bucket_fences; rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER, attach_tail_owner_started, @@ -4432,6 +4888,8 @@ impl SetDisks { rollback_receipt, multipart_proof_authority, multipart_commit_owner, + reservation_ctx: _, + prepared_reservations: _, } = fence_options; if let Some(file_info) = disks .iter() @@ -4901,6 +5359,8 @@ impl SetDisks { old_current_source, committed_file_info, tail_drain: None, + tail_has_metadata_reservations: false, + tail_has_owned_bucket_fences: false, }) } @@ -7358,6 +7818,9 @@ pub(crate) mod rename_fanout_barrier { MetadataSlowtailFault, NonInlineHedgeTimer, LateMetadataRefresh, + ReservationBefore, + ReservationAfter, + ReservationActivated, } impl From<&'static str> for Phase { @@ -7380,6 +7843,9 @@ pub(crate) mod rename_fanout_barrier { pub const PHASE_ROLLBACK: Phase = Phase::Rollback; pub const PHASE_READ_VERSION: Phase = Phase::ReadVersion; pub const PHASE_METADATA_SLOWTAIL_FAULT: Phase = Phase::MetadataSlowtailFault; + pub const PHASE_RESERVATION_BEFORE: Phase = Phase::ReservationBefore; + pub const PHASE_RESERVATION_AFTER: Phase = Phase::ReservationAfter; + pub const PHASE_RESERVATION_ACTIVATED: Phase = Phase::ReservationActivated; /// Object-scoped hedge timer checkpoint; slot zero identifies the timer, not a disk. pub const PHASE_NON_INLINE_HEDGE_TIMER: Phase = Phase::NonInlineHedgeTimer; @@ -7454,6 +7920,18 @@ pub(crate) mod rename_fanout_barrier { } impl BarrierHandle { + /// Keep the current task parked without intercepting later tasks. + pub fn disarm(&self) { + let mut reg = lock(); + if reg + .armed + .get(&self.key) + .is_some_and(|armed| Arc::ptr_eq(&armed.release, &self.release)) + { + reg.armed.remove(&self.key); + } + } + /// Await until the armed fan-out task has parked at the checkpoint. Uses a /// stored-permit `Notify`, so this is race-free regardless of whether the /// task reaches the checkpoint before or after this call — no sleeps. @@ -7619,6 +8097,45 @@ mod tests { use tempfile::TempDir; use tokio::io::AsyncReadExt; + #[tokio::test] + async fn renewal_stops_after_the_disk_loses_admission_and_token_expires() { + let (_dir, disk) = read_multiple_test_disk("bucket", &[]).await; + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let endpoint = disk.endpoint().to_string(); + ctx.local_disk_map() + .write() + .await + .insert(endpoint.clone(), Some(disk.clone())); + let reservations = prepare_rename_metadata_reservations(&[Some(disk)], "bucket", "object", Uuid::new_v4(), &ctx, false) + .await + .expect("prepare") + .expect("complete fanout"); + let group = Arc::clone( + &reservations[0] + .as_ref() + .expect("reservation") + .renewal + .as_ref() + .expect("renewal") + .0, + ); + ctx.local_disk_map().write().await.remove(&endpoint); + tokio::task::yield_now().await; + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(21)).await; + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(41)).await; + for _ in 0..10 { + if group.task.is_finished() { + break; + } + tokio::task::yield_now().await; + } + tokio::time::resume(); + assert!(group.task.is_finished(), "stale tokens must not cause indefinite renewal attempts"); + drop(reservations); + } + #[tokio::test] async fn commit_fault_schedule_keeps_multiple_disk_phases_independent() { use rename_fanout_barrier::{PHASE_CLEANUP, PHASE_RENAME, arm, checkpoint}; diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index a895555da..6fd23ba03 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -410,6 +410,10 @@ impl ObjectLockDiagGuard { self } + async fn downgrade_committed_tail(&mut self) -> bool { + self.guard.downgrade_committed_tail().await + } + /// Whether the underlying namespace lock's heartbeat has observed a /// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit /// point on this so a stale lock holder does not race a double-write. @@ -902,6 +906,10 @@ mod core; pub(crate) use core::io_primitives::disk_call_counters; #[cfg(test)] pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier}; +pub(crate) use core::io_primitives::{ + RenameMetadataReservation, RenameReservationMode, align_rename_metadata_reservations, prepare_rename_metadata_reservations, + prepare_rename_metadata_reservations_until_abandoned, put_rename_early_ack_enabled, +}; mod ctx; mod metadata; mod ops; @@ -928,6 +936,8 @@ pub(crate) use ops::object::body_cache_plaintext_len; #[cfg(all(test, feature = "test-util"))] pub(crate) use ops::object::cleanup_rejected_transition_upload_durably; #[cfg(test)] +pub(crate) use ops::object::pause_put_object_commit; +#[cfg(test)] pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier}; #[cfg(any(test, feature = "test-util"))] pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause}; diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index b0d832826..9b33ef07e 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -34,6 +34,7 @@ mod heal_lock_wait_tests; #[derive(Clone, Copy)] struct ExplicitVersionHeal<'a> { opts: &'a HealOpts, + staging_guard: Option<&'a Arc>>, allow_regeneration: bool, preserve_unrepairable: bool, commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>, @@ -41,6 +42,58 @@ struct ExplicitVersionHeal<'a> { retirement: Option<&'a MarkerRetirementContext<'a>>, } +async fn restore_heal_exclusive_guard( + guard: &Arc>, +) -> disk::error::Result<()> { + tokio::time::timeout(get_lock_acquire_timeout(), async { + loop { + { + let mut guard = guard.lock().await; + if guard.is_lock_lost() { + return Err(DiskError::DiskOngoingReq); + } + let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await; + if !matches!(waiting, Ok(Some(false))) { + return Err(DiskError::Timeout); + } + if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail()) + .await + .unwrap_or(false) + { + // A writer may have waited on a peer that upgraded while + // another peer was still waiting for its readers to drain. + let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await; + return if matches!(waiting, Ok(Some(false))) { + Ok(()) + } else { + Err(DiskError::Timeout) + }; + } + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .map_err(|_| DiskError::Timeout)? +} + +async fn wait_for_heal_writer(guard: &Arc>) { + loop { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + let waiting = + tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await; + if !matches!(waiting, Ok(Some(false))) { + return; + } + } +} + +fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) { + drop(tokio::spawn(async move { + let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; + })); +} + // Keep the large heal state machine and its construction temporary out of // each dispatch caller's poll frame. Ordinary object writes do not use this // heal-only allocation boundary. @@ -844,6 +897,7 @@ impl SetDisks { version_id, ExplicitVersionHeal { opts, + staging_guard: None, allow_regeneration: true, preserve_unrepairable: false, commit_fence: None, @@ -994,6 +1048,7 @@ impl SetDisks { ) -> disk::error::Result<(HealResultItem, Option)> { let ExplicitVersionHeal { opts, + staging_guard, allow_regeneration: allow_explicit_version_regen, preserve_unrepairable, commit_fence, @@ -1701,7 +1756,29 @@ impl SetDisks { let dst_data_dir = data_dir; if !latest_meta.deleted && !latest_meta.is_remote() { - let stage_result: disk::error::Result> = async { + let mut shared_staging = false; + if let Some(guard) = staging_guard { + let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async { + guard.lock().await.supports_heal_lock_transition().await + }) + .await + .unwrap_or(false); + if supported { + let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async { + guard.lock().await.downgrade_committed_tail().await + }) + .await + .unwrap_or(false); + if converted { + shared_staging = true; + } else { + // Some peers may have converted. Abort without committing; + // dropping the guard releases each retained lease. + return Err(DiskError::Timeout); + } + } + } + let stage = async { let erasure_info = latest_meta.erasure.clone(); let mut writer_failures = HealWriterFailures::new(disks_to_heal_count); @@ -1926,17 +2003,43 @@ impl SetDisks { writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable"); Ok(None) - } - .await; + }; + let stage_result: disk::error::Result> = if shared_staging { + if let Some(guard) = staging_guard { + tokio::select! { + result = stage => result, + _ = wait_for_heal_writer(guard) => { + // The UUID prefix is private to this attempt. Cleanup must not + // retain the namespace lock while a foreground writer waits; + // the stale-tmp sweeper also covers a cancelled disk operation. + cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone()); + return Err(DiskError::Timeout); + } + } + } else { + stage.await + } + } else { + stage.await + }; match stage_result { Ok(None) => {} failure => { - // Release readers and writers before removing all parts - // staged under this heal's exclusively owned prefix. - let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; + if shared_staging { + cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone()); + } else { + let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; + } return failure.map(|error| (result, error)); } } + if shared_staging + && let Some(guard) = staging_guard + && let Err(error) = restore_heal_exclusive_guard(guard).await + { + cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone()); + return Err(error); + } } let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint { if _read_repair_read_lock_guard @@ -3616,6 +3719,7 @@ impl SetDisks { &version, ExplicitVersionHeal { opts: &opts, + staging_guard: None, allow_regeneration: false, preserve_unrepairable: true, commit_fence: Some(can_commit), @@ -3676,6 +3780,7 @@ impl SetDisks { version_id, ExplicitVersionHeal { opts: &read_opts, + staging_guard: None, allow_regeneration: false, preserve_unrepairable: true, commit_fence: Some(&can_commit), @@ -3702,7 +3807,7 @@ impl SetDisks { .new_ns_lock(bucket, object) .await .map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?; - Some(Arc::new( + Some(Arc::new(tokio::sync::Mutex::new( measure_heal_object_lock( "heal_object", HealObjectLockKind::Write, @@ -3714,7 +3819,7 @@ impl SetDisks { .narrow_to_disk() .unwrap_or_else(DiskError::other) })?, - )) + ))) } else { None }; @@ -3732,10 +3837,18 @@ impl SetDisks { } else { None }; + let write_lock_loss = if let Some(guard) = &_write_lock_guard { + guard.lock().await.lock_lost_signal() + } else { + None + }; let locks_lost = || { - _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()) + write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost()) || transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) }; + // Recovery cleanup can outlive its future. Never permit reads until + // every physical operation holding its recovery owner has finished. + let recovery_tail = Arc::new(()); let mut recovery_removed_version = false; if !opts.read_repair && _write_lock_guard.is_some() @@ -3762,6 +3875,7 @@ impl SetDisks { Arc::new(( scope.clone(), self.ctx.begin_namespace_commit(), + Arc::clone(&recovery_tail), _write_lock_guard.clone(), transaction_guard.clone(), )) as Arc @@ -3885,6 +3999,9 @@ impl SetDisks { version_id, ExplicitVersionHeal { opts: &inner_opts, + staging_guard: (Arc::strong_count(&recovery_tail) == 1) + .then_some(_write_lock_guard.as_ref()) + .flatten(), allow_regeneration: true, preserve_unrepairable: false, commit_fence: Some(&commit_fence), @@ -3909,6 +4026,9 @@ impl SetDisks { version_id, ExplicitVersionHeal { opts: &inner_opts, + staging_guard: (Arc::strong_count(&recovery_tail) == 1) + .then_some(_write_lock_guard.as_ref()) + .flatten(), allow_regeneration: true, preserve_unrepairable: false, commit_fence: Some(&commit_fence), @@ -6047,6 +6167,7 @@ mod heal_result_report_tests { "", ExplicitVersionHeal { opts: &opts, + staging_guard: None, allow_regeneration: true, preserve_unrepairable: false, commit_fence: Some(&can_commit), @@ -10637,6 +10758,369 @@ mod heal_result_report_tests { assert_eq!(actual, expected); } + #[tokio::test] + #[serial_test::serial] + async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() { + use crate::object_api::WriteCompletion; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "background-heal-storage-entry"; + let object = "object.bin"; + let expected = vec![0x37; 1024 * 1024]; + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(expected.clone()), + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("write object"); + let metadata = disks[0] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("read metadata"); + let missing = dirs[0] + .path() + .join(bucket) + .join(object) + .join(metadata.data_dir.expect("external object data").to_string()) + .join("part.1"); + tokio::fs::remove_file(&missing).await.expect("remove shard"); + + let (_pause, arrived, release) = + ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1)); + let heal_set = set.clone(); + let heal = tokio::spawn(async move { + let mut absence = None; + let result = heal_set + .heal_object_with_retirement( + bucket, + object, + "", + &HealOpts { + scan_mode: HealScanMode::Deep, + read_repair: false, + ..Default::default() + }, + &mut absence, + AbsenceProofRequest { + retirement: None, + allow_unversioned: false, + }, + ) + .await; + (result, absence) + }); + tokio::time::timeout(Duration::from_secs(10), arrived.notified()) + .await + .expect("read-repair reached shard reconstruction"); + let mut reader = tokio::time::timeout( + Duration::from_millis(200), + set.get_object_reader( + bucket, + object, + None, + http::HeaderMap::new(), + &ObjectOptions { + suppress_read_repair: true, + ..Default::default() + }, + ), + ) + .await + .expect("background heal reconstruction should allow concurrent readers") + .expect("open object while reconstruction is paused"); + let mut during_heal = Vec::new(); + tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal)) + .await + .expect("read completes while reconstruction is paused") + .expect("read object during reconstruction"); + assert_eq!(during_heal, expected); + drop(reader); + release.notify_one(); + let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal) + .await + .expect("repair finishes") + .expect("repair task joins"); + let (_, error) = result.expect("repair operation succeeds"); + assert!(error.is_none(), "repair failed: {error:?}"); + assert!(absence.is_none()); + assert!(missing.exists(), "missing shard was not restored"); + let mut reader = set + .get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read repaired object"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("read full object"); + assert_eq!(actual, expected); + } + + #[tokio::test] + #[serial_test::serial] + async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() { + use crate::object_api::WriteCompletion; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "background-heal-long-put"; + let object = "object.bin"; + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]), + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("write original object"); + let metadata = disks[0] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("read metadata"); + let missing = dirs[0] + .path() + .join(bucket) + .join(object) + .join(metadata.data_dir.expect("external data").to_string()) + .join("part.1"); + tokio::fs::remove_file(missing).await.expect("remove shard"); + let (_pause, arrived, release) = + ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1)); + let heal_set = set.clone(); + let heal = tokio::spawn(async move { + let mut absence = None; + heal_set + .heal_object_with_retirement( + bucket, + object, + "", + &HealOpts { + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + &mut absence, + AbsenceProofRequest { + retirement: None, + allow_unversioned: false, + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), arrived.notified()) + .await + .expect("reconstruction started"); + let put = tokio::time::timeout( + Duration::from_secs(7), + set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()), + ) + .await; + release.notify_one(); + let heal_result = tokio::time::timeout(Duration::from_secs(20), heal) + .await + .expect("heal finishes after releasing reconstruction") + .expect("heal task joins"); + put.expect("foreground PUT finishes during reconstruction") + .expect("foreground PUT must not time out"); + assert!( + matches!(heal_result, Err(Error::Timeout)), + "interrupted heal must be retryable: {heal_result:?}" + ); + drop(_pause); + let mut absence = None; + let (_, retry_error) = set + .heal_object_with_retirement( + bucket, + object, + "", + &HealOpts { + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + &mut absence, + AbsenceProofRequest { + retirement: None, + allow_unversioned: false, + }, + ) + .await + .expect("a later heal must retry the current object"); + assert!(retry_error.is_none(), "retry must complete: {retry_error:?}"); + let mut reader = set + .get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read foreground write after heal"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("read full object"); + assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write"); + } + + #[tokio::test] + #[serial_test::serial] + async fn background_heal_yields_to_version_delete_without_resurrecting_it() { + use crate::object_api::WriteCompletion; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "background-heal-version-delete"; + let object = "object.bin"; + set.make_bucket( + bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create versioned bucket"); + let original = set + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]), + &ObjectOptions { + versioned: true, + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("write original version"); + let version = original.version_id.expect("version id").to_string(); + let survivor = set + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x52; 1024]), + &ObjectOptions { + versioned: true, + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("write surviving version") + .version_id + .expect("surviving version id"); + let metadata = disks[0] + .read_version("", bucket, object, &version, &ReadOptions::default()) + .await + .expect("read original metadata"); + let missing = dirs[0] + .path() + .join(bucket) + .join(object) + .join(metadata.data_dir.expect("external data").to_string()) + .join("part.1"); + tokio::fs::remove_file(missing).await.expect("remove shard"); + let (_pause, arrived, release) = + ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1)); + let heal_set = set.clone(); + let heal_version = version.clone(); + let heal = tokio::spawn(async move { + heal_set + .heal_object_with_retirement( + bucket, + object, + &heal_version, + &HealOpts { + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + &mut None, + AbsenceProofRequest { + retirement: None, + allow_unversioned: false, + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), arrived.notified()) + .await + .expect("reconstruction started"); + let delete = tokio::time::timeout( + Duration::from_secs(7), + set.delete_object( + bucket, + object, + ObjectOptions { + versioned: true, + version_id: Some(version.clone()), + object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new( + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ), + ) + .await; + release.notify_one(); + let heal_result = tokio::time::timeout(Duration::from_secs(20), heal) + .await + .expect("heal finishes") + .expect("heal task joins"); + delete + .expect("foreground delete finishes") + .expect("foreground delete succeeds"); + assert!( + matches!(heal_result, Err(Error::Timeout)), + "interrupted heal must be retryable: {heal_result:?}" + ); + let result = set + .get_object_info( + bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(version.clone()), + ..Default::default() + }, + ) + .await; + assert!( + matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version), + "stale heal must not resurrect the deleted version: {result:?}" + ); + let mut survivor = set + .get_object_reader( + bucket, + object, + None, + http::HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(survivor.to_string()), + ..Default::default() + }, + ) + .await + .expect("surviving version remains readable"); + let mut body = Vec::new(); + survivor.stream.read_to_end(&mut body).await.expect("read surviving version"); + assert_eq!(body, vec![0x52; 1024]); + for disk in &disks { + let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await; + assert!( + matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)), + "deleted version must remain absent on each disk: {physical:?}" + ); + } + } + #[tokio::test] async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() { let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await; diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 71bcc110f..24ac39e0e 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -3291,6 +3291,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { bucket, object, !write_context.acquires_namespace() || object_lock_guard.is_some(), + false, DecommissionCapacityAdmission::ExistingMultipart, ) .await?; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 7d7712ee7..0b7ec27a4 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -4184,7 +4184,19 @@ impl SetDisks { data: &mut PutObjReader, opts: &ObjectOptions, ) -> Result<(ObjectInfo, Option)> { - self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None) + self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, None) + .await + } + + pub(crate) async fn put_object_with_prepared_reservations( + &self, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, + reservations: Option>>, + ) -> Result<(ObjectInfo, Option)> { + self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, reservations) .await } @@ -4203,7 +4215,7 @@ impl SetDisks { // The owned publication capability supplies the exact target write // domain at commit time; the generic PUT path must not reacquire it. opts.no_lock = true; - self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence)) + self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence), None) .await } @@ -4214,8 +4226,9 @@ impl SetDisks { data: &'a mut PutObjReader, opts: &'a ObjectOptions, publication_fence: Option, + reservations: Option>>, ) -> impl Future)>> + Send + 'a { - Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence)) + Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence, reservations)) } async fn put_object_with_old_current_size_inner( @@ -4225,6 +4238,7 @@ impl SetDisks { data: &mut PutObjReader, opts: &ObjectOptions, mut publication_fence: Option, + reservations: Option>>, ) -> Result<(ObjectInfo, Option)> { let write_context = WriteCommitContext::from_options(opts, bucket, object, publication_fence.is_some())?; let protect_write = opts.shard_integrity_write_enabled(); @@ -4789,6 +4803,83 @@ impl SetDisks { .await?; } + // A new version has no outer bucket snapshot. Prepare its remote + // owners after staging, before taking lifecycle or object locks. + let prepare_new_version = reservations.is_none() + && !opts.overwrites_existing_version() + && !self.ctx.lock_manager().is_disabled() + && crate::set_disk::put_rename_early_ack_enabled() + && write_context.acquires_namespace() + && write_context.allows_early_ack(opts.write_completion) + && publication_fence.is_none() + && opts.scanner_publication_commit_scope.is_none() + && opts.decommission_capacity_admission.is_none() + && opts.decommission_capacity.is_none() + && opts.quota_admission.is_none() + && opts.object_lock_config_snapshot.is_none() + && opts.bucket_lifecycle_lock_fence.is_none() + && opts.namespace_lock_fence.is_none() + && !opts.data_movement + && !is_meta_bucketname(bucket) + && opts + .put_object_cancellation + .as_ref() + .is_none_or(|token| !token.is_cancelled()) + && rustfs_utils::path::object_name_disk_identity(object).as_ref() + != rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH + && crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx) + .await + .is_some(); + #[cfg(any(test, feature = "test-util"))] + if prepare_new_version { + pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await; + } + + // Destructive writes prepare before the bucket snapshot. Local-only + // fixtures without a metadata store can prepare before the object lock. + let prepared_metadata_reservations = if !self.ctx.lock_manager().is_disabled() + && crate::set_disk::put_rename_early_ack_enabled() + && write_context.acquires_namespace() + && write_context.allows_early_ack(opts.write_completion) + && publication_fence.is_none() + && opts.scanner_publication_commit_scope.is_none() + && (opts.decommission_capacity_admission.is_none() || reservations.is_some()) + && !opts.data_movement + && !is_meta_bucketname(bucket) + { + if let Some(prepared) = reservations { + crate::set_disk::align_rename_metadata_reservations(prepared, &shuffle_disks) + } else if prepare_new_version && let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil()) { + crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations( + &shuffle_disks, + bucket, + object, + incarnation, + &self.ctx, + true, + ) + .await? + } else if shuffle_disks.iter().flatten().all(|disk| disk.is_local()) + && crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx) + .await + .is_none() + && let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil()) + { + crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations( + &shuffle_disks, + bucket, + object, + incarnation, + &self.ctx, + false, + ) + .await? + } else { + None + } + } else { + None + }; if let Some(store) = opts.decommission_capacity_admission.as_ref() { #[cfg(test)] { @@ -4801,6 +4892,7 @@ impl SetDisks { bucket, object, !write_context.acquires_namespace() || object_lock_guard.is_some(), + prepared_metadata_reservations.is_some(), DecommissionCapacityAdmission::Mutation, ) .await?; @@ -4812,7 +4904,9 @@ impl SetDisks { || (write_context.acquires_namespace() && object_lock_guard.is_none() && !decommission_target_lock_covered) { #[cfg(any(test, feature = "test-util"))] - pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await; + if !prepare_new_version { + pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await; + } if let Some(expected_incarnation_id) = source_bucket_incarnation_id && opts.bucket_lifecycle_lock_fence.is_none() { @@ -5268,6 +5362,11 @@ impl SetDisks { let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take(); let commit_decommission_capacity_guard = decommission_capacity_guard.take(); let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone(); + let commit_object_lock_snapshot = opts + .object_lock_config_snapshot + .as_ref() + .filter(|snapshot| snapshot.owns_lifecycle_guard()) + .cloned(); // A scanner publication scope owns the movement permit until the // complete rename fan-out drains. Keep this path synchronous so // its terminal state is known before the coordinator releases @@ -5281,6 +5380,26 @@ impl SetDisks { && !(opts.data_movement && opts.has_decommission_capacity_reservation()) && commit_owns_namespace_guard && commit_scanner_publication_scope.is_none(); + let lifecycle_owned = commit_bucket_lifecycle_guard.is_some() + || commit_object_lock_snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.owns_lifecycle_guard_for(bucket, source_bucket_incarnation_id)); + let reservation_ctx = (commit_allows_early_ack + && write_context.acquires_namespace() + && commit_object_lock_guard.is_some() + && (commit_decommission_object_lock_guard.is_none() || prepared_metadata_reservations.is_some()) + && (commit_decommission_capacity_guard.is_none() || prepared_metadata_reservations.is_some()) + && commit_publication_guard.is_none() + && !quota_mutation_fence + && !opts.data_movement + && !is_meta_bucketname(bucket) + && lifecycle_owned) + .then(|| Arc::clone(&self.ctx)); + let prepared_metadata_reservations = if reservation_ctx.is_some() { + prepared_metadata_reservations + } else { + None + }; // Full-tail callers also transfer owned guards to the coordinator: // cancelling their ACK waiter must not cancel an in-flight rename. let detach_commit_owner = commit_scanner_publication_scope.is_some() @@ -5363,7 +5482,7 @@ impl SetDisks { || _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) || _decommission_capacity_guard .as_ref() - .is_some_and(|guard| guard.is_lock_lost()) + .is_some_and(|guard| guard.is_lock_lost() || guard.is_released()) { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "quota_reservation", @@ -5437,7 +5556,7 @@ impl SetDisks { || _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) || _decommission_capacity_guard .as_ref() - .is_some_and(|guard| guard.is_lock_lost()) + .is_some_and(|guard| guard.is_lock_lost() || guard.is_released()) { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "quota_reservation", @@ -5604,6 +5723,7 @@ impl SetDisks { ) .with_publication_scope(commit_scanner_publication_scope.clone()) .with_rollback_receipt(commit_rollback_receipt.clone()) + .with_reservation_context(reservation_ctx, prepared_metadata_reservations) .with_namespace_commit_guard( (!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()), ), @@ -5627,6 +5747,7 @@ impl SetDisks { pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await; } let mut rename_guard_release = None; + let mut committed_tail_guard = None; let mut needs_immediate_heal = false; let mut tail_owns_tmp_cleanup = false; rustfs_io_metrics::record_put_object_stage_duration_from( @@ -5658,11 +5779,27 @@ impl SetDisks { .or_else(|| commit_version_suspended.then(Uuid::nil)) .map(|version_id| version_id.to_string()); request.expected_bucket_incarnation_id = source_bucket_incarnation_id; - let object_lock_guard = _object_lock_guard.take(); + let object_lock_guard = if rename_commit.tail_has_metadata_reservations { + None + } else { + _object_lock_guard + .take() + .map(|guard| Arc::new(tokio::sync::Mutex::new(guard))) + }; + committed_tail_guard = object_lock_guard.clone(); let publication_guard = _publication_guard.take(); let bucket_lifecycle_guard = _bucket_lifecycle_guard.take(); let decommission_object_lock_guard = _decommission_object_lock_guard.take(); let decommission_capacity_guard = _decommission_capacity_guard.take(); + // The quorum committed this snapshot. Receiver-owned + // fences keep its bucket generation protected while + // the remaining ordered disk writes drain. + let object_lock_snapshot = if rename_commit.tail_has_owned_bucket_fences { + drop(commit_object_lock_snapshot); + None + } else { + commit_object_lock_snapshot + }; let cleanup_bucket = commit_bucket.clone(); let cleanup_object = commit_object.clone(); let heal_set = commit_set.clone(); @@ -5683,6 +5820,7 @@ impl SetDisks { write_context, publication_guard, bucket_lifecycle_guard, + object_lock_snapshot, decommission_object_lock_guard, decommission_capacity_guard, ), @@ -5704,6 +5842,7 @@ impl SetDisks { write_context, publication_guard, bucket_lifecycle_guard, + object_lock_snapshot, decommission_object_lock_guard, decommission_capacity_guard, ), @@ -5717,6 +5856,11 @@ impl SetDisks { drop(write_context); drop(publication_guard); drop(bucket_lifecycle_guard); + drop(object_lock_snapshot); + // Every rename has settled; the remaining cleanup + // keeps the capacity fence but no longer publishes + // object metadata. + drop(decommission_object_lock_guard); cleanup_runtime::run(cleanup_reservation, async move { let zero_target_cleanup = targets.is_empty(); rustfs_io_metrics::record_put_object_stage_duration( @@ -5763,7 +5907,6 @@ impl SetDisks { tmp_delete_started, ); let guard_drop_started = rustfs_io_metrics::put_stage_timer(); - drop(decommission_object_lock_guard); drop(decommission_capacity_guard); rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP, @@ -5972,6 +6115,13 @@ impl SetDisks { lock_held_post_rename_started, ); let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer(); + if let Some(guard) = committed_tail_guard.take() { + // The committed tail still owns every physical rename. Convert + // only after metadata invalidation and before acknowledging PUT. + // Unsupported peers keep their exclusive leases and the tail + // retains ownership until its physical work has drained. + let _ = guard.lock().await.downgrade_committed_tail().await; + } if let Some(release) = rename_guard_release.take() { let _ = release.send(true); } @@ -8071,6 +8221,8 @@ fn transaction_fencing_gate_requested_for(requested: bool, fleet_confirmed: bool #[cfg(any(test, feature = "test-util"))] #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum PutObjectCommitPause { + #[cfg(test)] + BeforeBucketSnapshot, BeforeNamespace, AfterNamespace, AfterQuotaReservation, @@ -8165,7 +8317,7 @@ impl Drop for PutObjectCommitBarrier { } #[cfg(any(test, feature = "test-util"))] -async fn pause_put_object_commit(bucket: &str, object: &str, pause: PutObjectCommitPause) { +pub(crate) async fn pause_put_object_commit(bucket: &str, object: &str, pause: PutObjectCommitPause) { let barrier = { let mut slot = PUT_OBJECT_COMMIT_BARRIER .get_or_init(|| std::sync::Mutex::new(Vec::new())) @@ -22181,6 +22333,117 @@ mod put_object_tmp_cleanup_tests { .await; } + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn early_ack_tail_retains_owned_bucket_lifecycle_fence() { + use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey}; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-tail-lifecycle"; + let object = "object"; + for disk in &disks { + disk.make_volume(bucket).await.expect("create volume"); + } + let lock = NamespaceLock::new("tail-lifecycle".to_string(), Arc::new(LocalClient::new())); + let lifecycle = lock + .get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5)) + .await + .expect("lifecycle read lock"); + let metadata = lock + .get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5)) + .await + .expect("metadata read lock"); + let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket( + Uuid::new_v4(), + bucket, + Uuid::new_v4(), + OffsetDateTime::now_utc(), + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + lifecycle, + metadata, + )); + let opts = ObjectOptions { + object_lock_config_snapshot: Some(snapshot), + ..Default::default() + }; + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer_set = Arc::clone(&set); + let put = tokio::spawn(async move { + writer_set + .put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &opts) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("rename tail pauses"); + put.await.expect("PUT joins").expect("PUT acknowledged"); + let mut bucket_delete = + Box::pin(lock.get_write_lock(ObjectKey::new(bucket, "lifecycle"), "delete", Duration::from_secs(10))); + let early = tokio::time::timeout(Duration::from_secs(2), &mut bucket_delete).await; + barrier.release(); + drop(barrier); + let guard = match early { + Ok(result) => { + drop(result.expect("early bucket lock result")); + panic!("bucket lifecycle owner must outlive the acknowledged physical tail"); + } + Err(_) => tokio::time::timeout(Duration::from_secs(15), bucket_delete) + .await + .expect("lifecycle lock released after tail") + .expect("bucket delete acquires lock"), + }; + drop(guard); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn early_ack_tail_drain_allows_read_after_acknowledged_put() { + temp_env::async_with_vars( + [ + (ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")), + (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")), + ], + async { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-early-ack-tail-read"; + let object = "early-ack-tail-read-object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let put_set = Arc::clone(&set_disks); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + put_set + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + }); + tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused()) + .await + .expect("tail disk should pause during rename"); + put.await + .expect("PUT task should join") + .expect("PUT should be acknowledged at quorum"); + let result = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await; + let mut reader = result.expect("acknowledged object should remain readable while tail drains"); + let mut body = Vec::new(); + tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut body)) + .await + .expect("object body should not wait for the tail") + .expect("object body should drain"); + assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]); + rename_barrier.release(); + drop(rename_barrier); + }, + ) + .await; + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes() { @@ -22253,6 +22516,200 @@ mod put_object_tmp_cleanup_tests { .await; } + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn early_ack_tail_does_not_timeout_the_next_put() { + use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey}; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-early-ack-next-writer"; + let object = "object"; + for disk in &disks { + disk.make_volume(bucket).await.expect("create volume"); + } + { + let map = set.ctx.local_disk_map(); + let mut active = map.write().await; + for disk in &disks { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + let lifecycle_lock = NamespaceLock::new("next-put-lifecycle".to_string(), Arc::new(LocalClient::new())); + let lifecycle = lifecycle_lock + .get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5)) + .await + .expect("lifecycle read lock"); + let metadata = lifecycle_lock + .get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5)) + .await + .expect("metadata read lock"); + let incarnation = Uuid::new_v4(); + let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket( + Uuid::new_v4(), + bucket, + incarnation, + OffsetDateTime::now_utc(), + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + lifecycle, + metadata, + )); + let mut opts = ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + object_lock_config_snapshot: Some(snapshot.clone()), + ..Default::default() + }; + snapshot.add_lock_fences(&mut opts); + let rename_tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let first_set = Arc::clone(&set); + let first_opts = opts.clone(); + let first = tokio::spawn(async move { + first_set + .put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &first_opts) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("first rename paused"); + first.await.expect("first PUT joins").expect("first PUT acknowledged"); + let second = tokio::time::timeout( + Duration::from_secs(7), + set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]), &opts), + ) + .await; + barrier.release(); + drop(barrier); + let second = second.expect("second PUT finishes").expect("second PUT must not time out"); + tokio::time::timeout(Duration::from_secs(30), async { + while rename_tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("both rename tails must drain"); + let mut reader = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read newest object"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("read full body"); + assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]); + let expected_etag = second.etag.as_ref().expect("second PUT ETag"); + for disk in &disks { + let info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("each disk must retain the latest version"); + assert_eq!(info.metadata.get("etag"), Some(expected_etag), "late rename must not restore an old copy"); + let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards"); + assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS)); + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn early_ack_tail_cannot_restore_a_deleted_version() { + use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey}; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "put-tail-version-delete"; + let object = "object"; + for disk in &disks { + disk.make_volume(bucket).await.expect("create volume"); + } + { + let map = set.ctx.local_disk_map(); + let mut active = map.write().await; + for disk in &disks { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + let lock = NamespaceLock::new("version-delete-lifecycle".to_string(), Arc::new(LocalClient::new())); + let lifecycle = lock + .get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5)) + .await + .expect("lifecycle read lock"); + let metadata = lock + .get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5)) + .await + .expect("metadata read lock"); + let incarnation = Uuid::new_v4(); + let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket( + Uuid::new_v4(), + bucket, + incarnation, + OffsetDateTime::now_utc(), + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + lifecycle, + metadata, + )); + let mut opts = ObjectOptions { + versioned: true, + expected_bucket_incarnation_id: Some(incarnation), + object_lock_config_snapshot: Some(snapshot.clone()), + ..Default::default() + }; + snapshot.add_lock_fences(&mut opts); + let rename_tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer = Arc::clone(&set); + let put_opts = opts.clone(); + let put = tokio::spawn(async move { + writer + .put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &put_opts) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("rename pauses"); + let version = put + .await + .expect("PUT joins") + .expect("PUT succeeds") + .version_id + .expect("version ID") + .to_string(); + let delete_barrier = DeleteObjectCommitBarrier::install_before_namespace(bucket, object); + let delete_set = Arc::clone(&set); + let mut delete_opts = opts; + delete_opts.version_id = Some(version.clone()); + let delete = tokio::spawn(async move { delete_set.delete_object(bucket, object, delete_opts).await }); + delete_barrier.wait_until_paused().await; + delete_barrier.release(); + tokio::time::timeout(Duration::from_secs(5), async { + while !delete_barrier.namespace_acquired() { + tokio::task::yield_now().await; + } + }) + .await + .expect("version delete acquires namespace while old tail is paused"); + barrier.release(); + drop(barrier); + delete.await.expect("delete joins").expect("version delete succeeds"); + tokio::time::timeout(Duration::from_secs(30), async { + while rename_tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("rename tails drain"); + for disk in &disks { + assert!( + matches!( + disk.read_version("", bucket, object, &version, &ReadOptions::default()).await, + Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) + ), + "late tail must not restore the deleted version" + ); + } + }) + .await; + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn early_ack_successful_tail_reclaims_its_old_non_inline_data_dir() { diff --git a/crates/ecstore/src/set_disk/replication.rs b/crates/ecstore/src/set_disk/replication.rs index 8964a9420..d5402e9f1 100644 --- a/crates/ecstore/src/set_disk/replication.rs +++ b/crates/ecstore/src/set_disk/replication.rs @@ -165,6 +165,7 @@ impl SetDisks { bucket, object, opts.no_lock, + false, DecommissionCapacityAdmission::Mutation, ) .await? @@ -275,6 +276,7 @@ impl SetDisks { bucket, object, opts.no_lock, + false, DecommissionCapacityAdmission::Mutation, ) .await? diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 76bde1755..5e28ca9cf 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -2294,8 +2294,17 @@ mod tests { drop( lock.get_write_lock(Duration::from_secs(30)) .await - .expect("fixture rename tail should finish before checking its generation"), + .expect("fixture object lock should be available"), ); + // A quorum-acknowledged rename can release the object lock while its + // physical tail still holds the instance-wide namespace owner. + tokio::time::timeout(Duration::from_secs(30), async { + while ecstore.ctx.namespace_commits_pending() { + tokio::task::yield_now().await; + } + }) + .await + .expect("fixture namespace owners should drain before checking its generation"); assert_eq!( ecstore.scanner_namespace_mutation_generation(), generation_before_put.saturating_add(3), diff --git a/crates/ecstore/src/store/bucket_fence.rs b/crates/ecstore/src/store/bucket_fence.rs index d5238dc05..2758d544c 100644 --- a/crates/ecstore/src/store/bucket_fence.rs +++ b/crates/ecstore/src/store/bucket_fence.rs @@ -45,7 +45,8 @@ use crate::error::{Result, StorageError}; use std::collections::HashMap; use std::future::Future; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, Weak}; +use std::time::Duration; use rustfs_lock::NamespaceLockGuard; use rustfs_lock::distributed_lock::LockLostSignal; @@ -65,7 +66,7 @@ struct RegisteredGuard { enum LockLossProbe { Distributed(Arc), - Local, + Local(Option), #[cfg(test)] Test(Arc), } @@ -74,14 +75,19 @@ impl LockLossProbe { fn from_guard(guard: &NamespaceLockGuard) -> Self { match guard.lock_lost_signal() { Some(signal) => Self::Distributed(signal), - None => Self::Local, + None => Self::Local(match guard { + NamespaceLockGuard::Fast(guard) => guard.release_probe(), + NamespaceLockGuard::Standard(_) => None, + }), } } fn is_lost(&self) -> bool { + // Registry callers hold entries before checking a local shard; shard + // operations never acquire the bucket fence registry. match self { Self::Distributed(signal) => signal.is_lost(), - Self::Local => false, + Self::Local(probe) => probe.as_ref().is_some_and(|probe| probe.is_lost()), #[cfg(test)] Self::Test(lost) => lost.load(std::sync::atomic::Ordering::SeqCst), } @@ -98,9 +104,74 @@ pub(super) struct FenceRegistration { #[derive(Default)] pub(crate) struct BucketFenceRegistry { entries: Mutex>, + reservation_slots: Mutex>>, +} + +#[derive(Default)] +pub(super) struct ReservationFenceSlot { + pub(super) owner: tokio::sync::Mutex>, +} + +/// Shared by short bursts of reservations, but never kept alive by the cache. +pub(super) struct ReservationFenceGuard { + publication: NamespaceLockGuard, + lifecycle: BucketIncarnationFenceGuard, + created_at: tokio::time::Instant, + _slot: Arc, +} + +impl ReservationFenceGuard { + pub(super) fn new( + publication: NamespaceLockGuard, + lifecycle: BucketIncarnationFenceGuard, + slot: Arc, + ) -> Self { + Self { + publication, + lifecycle, + created_at: tokio::time::Instant::now(), + _slot: slot, + } + } + + pub(super) fn is_reusable(&self) -> bool { + // Stop admitting new reservations through this shared reader so a + // queued bucket writer can progress after the existing tails drain. + self.created_at.elapsed() < Duration::from_secs(1) && self.is_held() + } + + fn is_held(&self) -> bool { + !self.publication.is_lock_lost() + && !self.publication.is_released() + && !self.lifecycle.is_lock_lost() + && self + .lifecycle + .namespace_lock_guard() + .is_some_and(|guard| !guard.is_released()) + } +} + +impl crate::disk::os::MetadataReservationFence for ReservationFenceGuard { + fn is_valid(&self) -> bool { + self.is_held() + } } impl BucketFenceRegistry { + pub(super) fn reservation_slot(&self, bucket: &str, incarnation: Uuid) -> Arc { + let mut slots = self.reservation_slots.lock().expect("reservation fence registry poisoned"); + let key = (bucket.to_owned(), incarnation); + if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) { + return slot; + } + if slots.len() >= 1024 { + slots.retain(|_, slot| slot.strong_count() != 0); + } + let slot = Arc::new(ReservationFenceSlot::default()); + slots.insert(key, Arc::downgrade(&slot)); + slot + } + /// Register a new live guard for `bucket` and return the memoized /// incarnation id if one is valid for the current coverage window. fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration { @@ -234,7 +305,9 @@ impl BucketIncarnationFenceGuard { } pub fn is_lock_lost(&self) -> bool { - self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) + self.inner + .as_ref() + .is_some_and(|guard| guard.is_lock_lost() || guard.is_released()) } pub(crate) fn namespace_lock_guard(&self) -> Option<&NamespaceLockGuard> { @@ -352,6 +425,67 @@ mod tests { reg.exit("b", fourth.token, false); } + #[tokio::test] + async fn disabled_local_lock_does_not_add_a_namespace_fence() { + let manager = Arc::new(rustfs_lock::GlobalLockManager::Disabled( + rustfs_lock::fast_lock::DisabledLockManager::new(), + )); + let lock = NamespaceLock::with_local_manager("disabled-fence".into(), manager); + let guard = lock + .get_read_lock(ObjectKey::new("b", "object"), "owner", Duration::from_secs(1)) + .await + .expect("disabled lock"); + let mut opts = crate::object_api::ObjectOptions::default(); + opts.add_namespace_lock_guard(&guard); + assert!(opts.namespace_lock_fence.is_none()); + } + + #[tokio::test] + async fn forced_local_fence_loss_invalidates_live_incarnation_memo() { + let manager = Arc::new(rustfs_lock::FastObjectLockManager::new()); + let global = Arc::new(rustfs_lock::GlobalLockManager::Enabled(manager.clone())); + let lock = NamespaceLock::with_local_manager("forced-fence".into(), global); + let key = ObjectKey::new("b", "lifecycle"); + let registry = Arc::new(BucketFenceRegistry::default()); + let inner = lock + .get_read_lock(key.clone(), "first", Duration::from_secs(1)) + .await + .expect("first lock"); + let mut first = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true); + first.validate(true, || async { Ok(uuid(7)) }).await.expect("seed memo"); + let mut opts = crate::object_api::ObjectOptions::default(); + first.attach_to_object_options(&mut opts); + opts.add_namespace_lock_guard(first.namespace_lock_guard().expect("local guard")); + assert!( + !opts + .bucket_lifecycle_lock_fence + .as_ref() + .expect("lifecycle fence") + .is_lock_lost() + ); + assert!(!opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost()); + assert_eq!(manager.force_unlock(&key), 1); + assert!( + opts.bucket_lifecycle_lock_fence + .as_ref() + .expect("attached fence") + .is_lock_lost() + ); + assert!(opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost()); + let writer = lock + .get_write_lock(key.clone(), "writer", Duration::from_secs(1)) + .await + .expect("writer"); + drop(writer); + let inner = lock + .get_read_lock(key, "second", Duration::from_secs(1)) + .await + .expect("second lock"); + let mut second = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true); + assert_eq!(second.validate(true, || async { Ok(uuid(8)) }).await.expect("reload"), uuid(8)); + assert!(first.validate(true, || async { Ok(uuid(7)) }).await.is_err()); + } + #[tokio::test] async fn fence_guard_forwards_distributed_lock_loss_to_registry() { let registry = Arc::new(BucketFenceRegistry::default()); diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 549ed536b..268c46894 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -456,6 +456,7 @@ mod rebalance; pub(crate) use rebalance::PoolSpaceForTest; pub(crate) mod utils; +pub(crate) use peer::admit_local_disk; use peer::init_local_peer; pub use peer::{ BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index c72cf70c0..313321be8 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -81,6 +81,7 @@ const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000; const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2; const RESTORE_WORKER_LOCK_PREFIX: &str = "ilm/restore-worker-locks"; const RESTORE_WORKER_LOCK_PROBE_TIMEOUT: Duration = Duration::from_millis(50); +const MULTIPOOL_RESERVATION_HINT_TIMEOUT: Duration = Duration::from_millis(50); const TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY: usize = 16; fn install_tier_free_version_receipt_sink(opts: &mut ObjectOptions) -> Option { @@ -3757,6 +3758,7 @@ impl ECStore { bucket: &str, object: &str, no_lock: bool, + ordered_put: bool, admission: DecommissionCapacityAdmission, ) -> Result<(Option, bool, Option)> { let (capacity_guard, has_active_decommission) = self @@ -3773,6 +3775,8 @@ impl ECStore { return Ok((None, false, Some(capacity_guard))); } drop(capacity_guard); + #[cfg(test)] + crate::core::pools::notify_decommission_external_object_capacity_released(self.id); if no_lock { return Ok((None, false, None)); } @@ -3781,15 +3785,23 @@ impl ECStore { // capacity write fence. Drop the probe and acquire that namespace // before rechecking capacity, so the staged external commit cannot // form capacity-read -> object-write against the migration path. - let object_guard = self - .acquire_object_write_lock("external_capacity_order", bucket, object) - .await?; let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()); let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object)); let target_lock_covered = match (fixed_set, target_set) { (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await, _ => false, }; + // Only a PUT with receiver-owned ordering may share the fixed domain. + // Existing multipart uploads can still publish to a retiring source, + // so all other operations retain the exclusive migration fence. + let object_guard = if ordered_put && !target_lock_covered { + self.acquire_object_read_lock_if_needed("external_capacity_order", bucket, object, &mut ObjectOptions::default()) + .await? + .ok_or_else(|| Error::other("ordered PUT failed to acquire its decommission fence"))? + } else { + self.acquire_object_write_lock("external_capacity_order", bucket, object) + .await? + }; Ok((Some(object_guard), target_lock_covered, None)) } @@ -4814,16 +4826,126 @@ impl ECStore { data: &mut PutObjReader, opts: &ObjectOptions, ) -> Result<(ObjectInfo, Option)> { - let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, opts)).await?; + let mut early_opts = opts.clone(); + // Bucket publication precedes lifecycle and metadata transaction locks. + let prepared = if !self.ctx.lock_manager().is_disabled() + && crate::set_disk::put_rename_early_ack_enabled() + && early_opts.write_completion == crate::object_api::WriteCompletion::Quorum + && early_opts.overwrites_existing_version() + && !is_meta_bucketname(bucket) + && !early_opts.no_lock + && !early_opts.data_movement + && early_opts.write_lock_context.is_none() + && early_opts.object_lock_config_snapshot.is_none() + && early_opts.bucket_lifecycle_lock_fence.is_none() + && early_opts.namespace_lock_fence.is_none() + && early_opts.scanner_publication_commit_scope.is_none() + && early_opts.decommission_capacity_admission.is_none() + && early_opts.decommission_capacity.is_none() + && early_opts.quota_admission.is_none() + && early_opts + .put_object_cancellation + .as_ref() + .is_none_or(|token| !token.is_cancelled()) + && rustfs_utils::path::object_name_disk_identity(object).as_ref() + != rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH + { + check_new_object_args(bucket, object, false)?; + let incarnation = match early_opts.expected_bucket_incarnation_id { + Some(incarnation) => incarnation, + None => self.bucket_incarnation_id(bucket).await?, + }; + early_opts.expected_bucket_incarnation_id = Some(incarnation); + let encoded = encode_dir_object(object); + let tentative_pool = if self.single_pool() { + Some(0) + } else { + // This is only a hint; the locked selection below must agree + // before its reservations can protect a commit. + tokio::time::timeout( + MULTIPOOL_RESERVATION_HINT_TIMEOUT, + self.get_pool_idx_for_reservation_hint(bucket, &encoded, data.size()), + ) + .await + .ok() + .and_then(Result::ok) + }; + if let Some(pool_idx) = tentative_pool { + let set = self.pools[pool_idx].get_disks_by_key(&encoded); + let disks = set.disks.read().await.clone(); + let reservations = if self.single_pool() { + crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, &encoded, incarnation, &self.ctx, true) + .await? + } else if let Some(permit) = self.ctx.try_acquire_speculative_reservation() { + // Keep the preparation owner alive after the hint budget expires. + // A late response must still be received and its token cancelled. + let bucket = bucket.to_owned(); + let object = encoded.clone(); + let ctx = Arc::clone(&self.ctx); + let (sender, receiver) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let _permit = permit; + let mut sender = sender; + let prepared = crate::set_disk::prepare_rename_metadata_reservations_until_abandoned( + &disks, + &bucket, + &object, + incarnation, + &ctx, + crate::set_disk::RenameReservationMode::Owned, + sender.closed(), + ) + .await; + let _ = sender.send(prepared); + }); + tokio::time::timeout(MULTIPOOL_RESERVATION_HINT_TIMEOUT, receiver) + .await + .ok() + .and_then(|result| result.ok()) + .transpose()? + .flatten() + } else { + // When detached owners are saturated, use only the distinct + // short-lease RPC; unknown tokens then expire promptly. + tokio::time::timeout( + MULTIPOOL_RESERVATION_HINT_TIMEOUT, + crate::set_disk::prepare_rename_metadata_reservations_until_abandoned( + &disks, + bucket, + &encoded, + incarnation, + &self.ctx, + crate::set_disk::RenameReservationMode::ShortOwned, + std::future::pending(), + ), + ) + .await + .ok() + .transpose()? + .flatten() + }; + reservations.map(|reservations| (pool_idx, reservations)) + } else { + None + } + } else { + None + }; + #[cfg(test)] + crate::set_disk::pause_put_object_commit(bucket, object, crate::set_disk::PutObjectCommitPause::BeforeBucketSnapshot) + .await; + let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, &early_opts)).await?; let idx = self .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts) .await?; + let prepared = prepared.and_then(|(pool_idx, reservations)| (pool_idx == idx).then_some(reservations)); let mut opts = opts; if !self.single_pool() { opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await; } let receipt_sink = install_tier_free_version_receipt_sink(&mut opts); - let result = Box::pin(self.pools[idx].put_object_with_old_current_size(bucket, object.as_str(), data, &opts)).await; + let result = + Box::pin(self.pools[idx].put_object_with_prepared_reservations(bucket, object.as_str(), data, &opts, prepared)).await; enqueue_recorded_tier_free_versions(self, receipt_sink).await; match result { Ok((object_info, old_current_size)) => { @@ -9938,6 +10060,1289 @@ mod tests { } } + #[test] + #[serial_test::serial] + fn multipool_early_ack_prepares_before_snapshot_and_orders_the_next_put() { + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime") + .block_on(async { + use crate::disk::ReadOptions; + use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier}; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let (_other_dirs, other) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = + Arc::new(new_prepared_reader_test_store_with_ctx(&[original, other], Arc::clone(&ctx)).await); + let set = store.pools[0].get_disks_by_key("object"); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + let other_disks = store.pools[1] + .get_disks_by_key("object") + .disks + .read() + .await + .iter() + .flatten() + .cloned() + .collect::>(); + { + let map = ctx.local_disk_map(); + let mut active = map.write().await; + for disk in disks.iter().chain(&other_disks) { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "ecstore-reservation-order"; + let object = "object"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + store + .handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![b'0'; 1 << 20]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed object in its pool"); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let first_store = Arc::clone(&store); + let first = tokio::spawn(async move { + first_store + .handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![b'1'; 1 << 20]), + &ObjectOptions::default(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("first tail pauses"); + first.await.expect("first task").expect("first PUT"); + let second = tokio::time::timeout( + Duration::from_secs(7), + store.handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![b'2'; 1 << 20]), + &ObjectOptions::default(), + ), + ) + .await; + barrier.release(); + drop(barrier); + let (second, _) = second.expect("second PUT completes").expect("second PUT succeeds"); + tokio::time::timeout(Duration::from_secs(30), async { + while tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("rename tails drain"); + let expected_etag = second.etag.as_ref().expect("second ETag"); + let idx = store.get_pool_idx_no_lock(bucket, object, 1).await.expect("selected pool"); + let selected = store.pools[idx].get_disks_by_key(object); + let selected_disks = selected.disks.read().await.iter().flatten().cloned().collect::>(); + for disk in &selected_disks { + let info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("read latest metadata"); + assert_eq!(info.metadata.get("etag"), Some(expected_etag)); + let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards"); + assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS)); + } + }) + .await; + }); + }) + .expect("test thread") + .join() + .expect("test thread completes"); + } + + #[test] + #[serial_test::serial] + fn changed_pool_selection_discards_tentative_reservations() { + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime") + .block_on(async { + use crate::set_disk::{ + ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, PutObjectCommitBarrier, PutObjectCommitPause, + }; + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_first_dirs, first) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let (_second_dirs, second) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = + Arc::new(new_prepared_reader_test_store_with_ctx(&[first, second], Arc::clone(&ctx)).await); + for pool in &store.pools { + let set = pool.get_disks_by_key("object"); + for disk in set.disks.read().await.iter().flatten() { + ctx.local_disk_map() + .write() + .await + .insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "tentative-pool-changed"; + let object = "object"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + store.pools[0] + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![1; 8193]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed initial pool"); + assert_eq!( + store + .get_pool_idx_no_lock(bucket, object, 8193) + .await + .expect("initial selection"), + 0 + ); + let barrier = + PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeBucketSnapshot); + let worker = Arc::clone(&store); + let put = tokio::spawn(async move { + worker + .handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![3; 8193]), + &ObjectOptions::default(), + ) + .await + }); + barrier.wait_until_paused().await; + store.pools[1] + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![2; 8193]), + &ObjectOptions { + mod_time: Some(OffsetDateTime::now_utc() + time::Duration::seconds(5)), + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("publish a newer object to the other pool"); + assert_eq!(store.get_pool_idx(bucket, object, 8193).await.expect("changed selection"), 1); + barrier.release(); + drop(barrier); + let (result, _) = tokio::time::timeout(Duration::from_secs(20), put) + .await + .expect("PUT completes") + .expect("PUT task") + .expect("PUT succeeds"); + let latest = store.pools[1] + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("read final target"); + assert_eq!(latest.etag, result.etag, "the selected pool must contain the completed PUT"); + let old = store.pools[0] + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("read original pool"); + assert_ne!( + old.etag, result.etag, + "stale tentative reservations must not publish into the original pool" + ); + }) + .await; + }); + }) + .expect("test thread") + .join() + .expect("test thread completes"); + } + + #[test] + #[serial_test::serial] + fn tentative_selection_does_not_wait_for_slow_nonquorum_disk() { + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime") + .block_on(async { + use crate::set_disk::rename_fanout_barrier; + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_first_dirs, first) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let (_second_dirs, second) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[first, second], Arc::clone(&ctx)).await); + for pool in &store.pools { + let set = pool.get_disks_by_key("object"); + for disk in set.disks.read().await.iter().flatten() { + ctx.local_disk_map() + .write() + .await + .insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "slow-selection-disk"; + let object = "object"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + for (i, pool) in store.pools.iter().enumerate() { + pool.put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![u8::try_from(i).expect("pool index"); 8193]), + &ObjectOptions { + mod_time: Some( + OffsetDateTime::now_utc() + + time::Duration::seconds(i64::try_from(i).expect("pool index")), + ), + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed both pools"); + } + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_READ_VERSION); + let locked_store = Arc::clone(&store); + let locked = tokio::spawn(async move { locked_store.get_pool_idx(bucket, object, 8193).await }); + tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused()) + .await + .expect("metadata disk pauses"); + let locked_result = tokio::time::timeout(Duration::from_secs(3), locked).await; + if locked_result.is_err() { + barrier.release(); + drop(barrier); + panic!("the locked baseline did not reach quorum without the slow disk"); + } + locked_result + .expect("locked lookup completes") + .expect("locked task") + .expect("locked selection"); + for pool in &store.pools { + pool.get_disks_by_key(object) + .invalidate_get_object_metadata_cache(bucket, object) + .await; + } + let calls = disk_call_counters::observe(object); + let hint_store = Arc::clone(&store); + let mut hint = + tokio::spawn(async move { hint_store.get_pool_idx_for_reservation_hint(bucket, object, 8193).await }); + let hint_result = tokio::time::timeout(Duration::from_secs(3), &mut hint).await; + barrier.release(); + drop(barrier); + if hint_result.is_err() { + let _ = tokio::time::timeout(Duration::from_secs(20), hint).await; + panic!("tentative pool selection waited for a nonquorum disk"); + } + assert!( + (0..4).any(|index| calls.for_disk(disk_call_counters::KIND_READ_VERSION, index) > 0), + "cold speculative lookup must issue disk reads" + ); + hint_result + .expect("hint lookup completes") + .expect("hint task") + .expect("hint selection"); + }); + }) + .expect("test thread") + .join() + .expect("test thread completes"); + } + + #[test] + #[serial_test::serial] + fn speculative_selection_does_not_double_slow_missing_pool_wait() { + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime") + .block_on(async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_first_dirs, first) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let (_second_dirs, second) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[first, second], Arc::clone(&ctx)).await); + for pool in &store.pools { + let set = pool.get_disks_by_key("object"); + for disk in set.disks.read().await.iter().flatten() { + ctx.local_disk_map() + .write() + .await + .insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "slow-missing-pool"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + for key in ["objects/baseline", "objects/speculative"] { + store.pools[0] + .put_object( + bucket, + key, + &mut PutObjReader::from_vec(vec![1; 8193]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed object in one pool"); + for pool in &store.pools { + pool.get_disks_by_key(key) + .invalidate_get_object_metadata_cache(bucket, key) + .await; + } + } + async fn measured(store: &Arc, bucket: &str, key: &str, early: &'static str) -> (Duration, u64) { + let calls = disk_call_counters::observe(key); + temp_env::async_with_vars( + [ + ("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", Some(early)), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("1200")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("0")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")), + ], + async { + let started = tokio::time::Instant::now(); + tokio::time::timeout( + Duration::from_secs(15), + store.handle_put_object( + bucket, + key, + &mut PutObjReader::from_vec(vec![2; 8193]), + &ObjectOptions::default(), + ), + ) + .await + .expect("PUT completes") + .expect("PUT succeeds"); + (started.elapsed(), calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT)) + }, + ) + .await + } + let (baseline, baseline_faults) = measured(&store, bucket, "objects/baseline", "false").await; + let (candidate, candidate_faults) = measured(&store, bucket, "objects/speculative", "true").await; + assert!(baseline_faults > 0 && candidate_faults > 0, "both PUTs must encounter the delayed disk"); + assert!( + candidate <= baseline + Duration::from_millis(800), + "speculative lookup doubled slow-missing-pool wait: baseline={baseline:?} candidate={candidate:?}" + ); + }); + }) + .expect("test thread") + .join() + .expect("test thread completes"); + } + + #[tokio::test] + #[serial_test::serial] + async fn metadata_snapshot_stays_held_when_reservation_preparation_fails() { + use crate::set_disk::{ + ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, PutObjectCommitBarrier, PutObjectCommitPause, rename_fanout_barrier, + }; + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await); + let set = store.pools[0].get_disks_by_key("object"); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + { + let map = ctx.local_disk_map(); + let mut active = map.write().await; + for disk in &disks { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "tail-owner-fallback"; + let object = "first"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + ctx.local_disk_map().write().await.remove(&disks[1].endpoint().to_string()); + let before_rename = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeQuotaRename); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let first_store = Arc::clone(&store); + let first = tokio::spawn(async move { + first_store + .handle_put_object(bucket, object, &mut PutObjReader::from_vec(vec![1; 1 << 20]), &ObjectOptions::default()) + .await + }); + before_rename.wait_until_paused().await; + ctx.local_disk_map() + .write() + .await + .insert(disks[1].endpoint().to_string(), Some(disks[1].clone())); + before_rename.release(); + drop(before_rename); + barrier.wait_until_paused().await; + tokio::time::timeout(Duration::from_secs(20), first) + .await + .expect("first PUT completes") + .expect("first task") + .expect("first PUT"); + let writer_ctx = Arc::clone(&ctx); + let mut writer = tokio::spawn(async move { + crate::bucket::metadata_sys::update_in( + &writer_ctx, + bucket, + crate::bucket::metadata::BUCKET_TAGGING_CONFIG, + b"phasefallback".to_vec(), + ) + .await + }); + let before = tokio::time::timeout(Duration::from_millis(300), &mut writer).await; + barrier.release(); + drop(barrier); + assert!(before.is_err(), "fallback must retain its snapshot while the physical tail is paused"); + tokio::time::timeout(Duration::from_secs(20), writer) + .await + .expect("writer completes") + .expect("writer task") + .expect("metadata update"); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn different_key_put_not_blocked_by_metadata_writer_behind_committed_tail() { + use crate::disk::ReadOptions; + use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier}; + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await); + let set = store.pools[0].get_disks_by_key("object"); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + { + let map = ctx.local_disk_map(); + let mut active = map.write().await; + for disk in &disks { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "tail-metadata-contention"; + let object = "first"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let first_store = Arc::clone(&store); + let first = tokio::spawn(async move { + first_store + .handle_put_object(bucket, object, &mut PutObjReader::from_vec(vec![1; 1 << 20]), &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + let (first, _) = first.await.expect("first task").expect("first PUT"); + let writer_ctx = Arc::clone(&ctx); + let mut writer = tokio::spawn(async move { + crate::bucket::metadata_sys::update_in( + &writer_ctx, + bucket, + crate::bucket::metadata::BUCKET_TAGGING_CONFIG, + b"phaseafter-ack".to_vec(), + ) + .await + }); + let writer_pending = match tokio::time::timeout(Duration::from_millis(500), &mut writer).await { + Ok(result) => { + result.expect("writer task joins").expect("writer succeeds"); + false + } + Err(_) => true, + }; + let second = tokio::time::timeout( + Duration::from_secs(7), + store.handle_put_object(bucket, "second", &mut PutObjReader::from_vec(vec![2; 8193]), &ObjectOptions::default()), + ) + .await; + let delete_store = Arc::clone(&store); + let mut delete = tokio::spawn(async move { + delete_store + .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut delete).await.is_err(), + "bucket deletion must wait for the reserved physical tail" + ); + barrier.release(); + drop(barrier); + let deletion = tokio::time::timeout(Duration::from_secs(20), delete) + .await + .expect("bucket deletion completes") + .expect("bucket deletion task"); + assert!( + matches!( + deletion, + Err(StorageError::BucketNotEmpty(_) | StorageError::BucketNotEmptyWithDetails { .. }) + ), + "the committed objects must keep the bucket nonempty" + ); + if writer_pending { + tokio::time::timeout(Duration::from_secs(20), writer) + .await + .expect("writer completes") + .expect("writer task joins") + .expect("writer succeeds"); + } + second.expect("second PUT completes").expect("different key PUT succeeds"); + tokio::time::timeout(Duration::from_secs(30), async { + while tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("first tail drains"); + let metadata = crate::bucket::metadata_sys::get_in(&ctx, bucket) + .await + .expect("read changed metadata"); + assert!(metadata.tagging_config.is_some(), "the bucket configuration update must persist"); + for disk in &disks { + let info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("read first metadata"); + assert_eq!(info.metadata.get("etag"), first.etag.as_ref()); + let parts = disk.check_parts(bucket, object, &info).await.expect("check first shards"); + assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS)); + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn object_lock_policy_update_during_reserved_tail_keeps_versions_protected() { + use crate::disk::ReadOptions; + use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier}; + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await); + let set = store.pools[0].get_disks_by_key("object"); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + { + let map = ctx.local_disk_map(); + let mut active = map.write().await; + for disk in &disks { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "tail-object-lock-policy"; + let object = "first"; + store + .handle_make_bucket(bucket, &MakeBucketOptions { lock_enabled: true, ..Default::default() }) + .await + .expect("create bucket"); + crate::bucket::metadata_sys::update_in( + &ctx, bucket, crate::bucket::metadata::OBJECT_LOCK_CONFIG, + b"EnabledGOVERNANCE1".to_vec(), + ).await.expect("set original retention policy"); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let first_store = Arc::clone(&store); + let first = tokio::spawn(async move { + first_store + .handle_put_object(bucket, object, &mut PutObjReader::from_vec(vec![1; 1 << 20]), &ObjectOptions { versioned: true, version_id: Some(Uuid::new_v4().to_string()), ..Default::default() }) + .await + }); + barrier.wait_until_paused().await; + let (first, _) = first.await.expect("first task").expect("first PUT"); + let writer_ctx = Arc::clone(&ctx); + let mut writer = tokio::spawn(async move { + crate::bucket::metadata_sys::update_in( + &writer_ctx, + bucket, + crate::bucket::metadata::OBJECT_LOCK_CONFIG, + b"EnabledCOMPLIANCE1".to_vec(), + ) + .await + }); + let writer_pending = match tokio::time::timeout(Duration::from_millis(500), &mut writer).await { + Ok(result) => { + result.expect("writer task joins").expect("writer succeeds"); + false + } + Err(_) => true, + }; + let second = tokio::time::timeout( + Duration::from_secs(7), + store.handle_put_object(bucket, "second", &mut PutObjReader::from_vec(vec![2; 8193]), &ObjectOptions { versioned: true, version_id: Some(Uuid::new_v4().to_string()), ..Default::default() }), + ) + .await; + let delete_store = Arc::clone(&store); + let mut delete = tokio::spawn(async move { + delete_store + .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut delete).await.is_err(), + "bucket deletion must wait for the reserved physical tail" + ); + barrier.release(); + drop(barrier); + let deletion = tokio::time::timeout(Duration::from_secs(20), delete) + .await + .expect("bucket deletion completes") + .expect("bucket deletion task"); + assert!( + matches!( + deletion, + Err(StorageError::BucketNotEmpty(_) | StorageError::BucketNotEmptyWithDetails { .. }) + ), + "the committed objects must keep the bucket nonempty" + ); + if writer_pending { + tokio::time::timeout(Duration::from_secs(20), writer) + .await + .expect("writer completes") + .expect("writer task joins") + .expect("writer succeeds"); + } + let (second, _) = second.expect("second PUT completes").expect("different key PUT succeeds"); + tokio::time::timeout(Duration::from_secs(30), async { + while tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("first tail drains"); + for (key, info) in [(object, &first), ("second", &second)] { + let err = store.handle_put_object( + bucket, key, &mut PutObjReader::from_vec(vec![3; 8193]), + &ObjectOptions { versioned: true, version_id: Some(info.version_id.expect("committed version ID").to_string()), ..Default::default() }, + ).await.expect_err("updated default retention must protect committed versions"); + assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected retention error: {err:?}"); + } + let metadata = crate::bucket::metadata_sys::get_in(&ctx, bucket) + .await + .expect("read changed metadata"); + assert!(metadata.object_lock_config.is_some(), "the bucket configuration update must persist"); + for disk in &disks { + let info = disk + .read_version("", bucket, object, &first.version_id.expect("first version ID").to_string(), &ReadOptions::default()) + .await + .expect("read first metadata"); + assert_eq!(info.metadata.get("etag"), first.etag.as_ref()); + let parts = disk.check_parts(bucket, object, &info).await.expect("check first shards"); + assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS)); + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ecstore_early_ack_prepares_before_snapshot_and_orders_the_next_put() { + use crate::disk::ReadOptions; + use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier}; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await); + let set = store.pools[0].get_disks_by_key("object"); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + { + let map = ctx.local_disk_map(); + let mut active = map.write().await; + for disk in &disks { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "ecstore-reservation-order"; + let object = "object"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let first_store = Arc::clone(&store); + let first = tokio::spawn(async move { + first_store + .handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![b'1'; 1 << 20]), + &ObjectOptions::default(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("first tail pauses"); + first.await.expect("first task").expect("first PUT"); + let second = tokio::time::timeout( + Duration::from_secs(7), + store.handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![b'2'; 1 << 20]), + &ObjectOptions::default(), + ), + ) + .await; + barrier.release(); + drop(barrier); + let (second, _) = second.expect("second PUT completes").expect("second PUT succeeds"); + tokio::time::timeout(Duration::from_secs(30), async { + while tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("rename tails drain"); + let expected_etag = second.etag.as_ref().expect("second ETag"); + for disk in &disks { + let info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("read latest metadata"); + assert_eq!(info.metadata.get("etag"), Some(expected_etag)); + let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards"); + assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS)); + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn ecstore_new_version_early_ack_orders_the_next_put() { + use crate::disk::ReadOptions; + use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier}; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await); + let set = store.pools[0].get_disks_by_key("object"); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + { + let map = ctx.local_disk_map(); + let mut active = map.write().await; + for disk in &disks { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "ecstore-versioned-reservation-order"; + let object = "object"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let tasks = rename_fanout_barrier::observe_tasks(object); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let first_store = Arc::clone(&store); + let first = tokio::spawn(async move { + first_store + .handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![b'1'; 1 << 20]), + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("first tail pauses"); + first.await.expect("first task").expect("first PUT"); + let second = tokio::time::timeout( + Duration::from_secs(7), + store.handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![b'2'; 1 << 20]), + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ), + ) + .await; + barrier.release(); + drop(barrier); + let (second, _) = second.expect("second PUT completes").expect("second PUT succeeds"); + tokio::time::timeout(Duration::from_secs(30), async { + while tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("rename tails drain"); + let expected_etag = second.etag.as_ref().expect("second ETag"); + for disk in &disks { + let info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("read latest metadata"); + assert_eq!(info.metadata.get("etag"), Some(expected_etag)); + let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards"); + assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS)); + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn local_prepared_reservations_hold_bucket_publication_fence() { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await); + let set = store.pools[0].get_disks_by_key("object"); + let disks = set.disks.read().await.clone(); + { + let map = ctx.local_disk_map(); + let mut active = map.write().await; + for disk in disks.iter().flatten() { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "local-reservation-bucket-fence"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation"); + let reservations = + crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, "object", incarnation, &ctx, true) + .await + .expect("prepare reservations") + .expect("all disks prepared"); + let lock = store + .new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("publication lock"); + let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10))); + assert!(tokio::time::timeout(Duration::from_millis(50), &mut writer).await.is_err()); + drop(reservations); + let guard = tokio::time::timeout(Duration::from_secs(5), writer) + .await + .expect("reservations release fence") + .expect("writer lock"); + drop(guard); + } + + #[tokio::test] + #[serial_test::serial] + async fn long_staged_put_does_not_timeout_the_next_overwrite() { + use crate::set_disk::{ + ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, PutObjectCommitBarrier, PutObjectCommitPause, rename_fanout_barrier, + }; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await); + let set = store.pools[0].get_disks_by_key("object"); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + { + let map = ctx.local_disk_map(); + let mut active = map.write().await; + for disk in &disks { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "long-staged-reservation"; + let object = "object"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let staging_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace); + let tail_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer = Arc::clone(&store); + let first = tokio::spawn(async move { + writer + .handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![b'1'; 1 << 20]), + &ObjectOptions::default(), + ) + .await + }); + staging_barrier.wait_until_paused().await; + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(25)).await; + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(26)).await; + tokio::task::yield_now().await; + tokio::time::resume(); + staging_barrier.release(); + tokio::time::timeout(Duration::from_secs(30), tail_barrier.wait_until_paused()) + .await + .expect("first tail pauses"); + first.await.expect("first task").expect("first PUT"); + drop(staging_barrier); + let second = tokio::time::timeout( + Duration::from_secs(7), + store.handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![b'2'; 1 << 20]), + &ObjectOptions::default(), + ), + ) + .await; + tail_barrier.release(); + drop(tail_barrier); + assert!(matches!(second, Ok(Ok(_))), "second PUT after a long staging interval failed: {second:?}"); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn bucket_reservations_do_not_exhaust_shared_lock_holders() { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await); + let set = store.pools[0].get_disks_by_key("object"); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + { + let map = ctx.local_disk_map(); + let mut active = map.write().await; + for disk in &disks { + active.insert(disk.endpoint().to_string(), Some(disk.clone())); + } + } + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "reservation-reader-capacity"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation"); + let disk = disks[0].endpoint().to_string(); + let mut tokens = Vec::new(); + let mut failure = None; + for index in 0..256 { + match store + .reserve_local_object_metadata_at_incarnation( + &disk, + bucket, + &format!("object-{index}"), + incarnation, + Duration::from_secs(1), + ) + .await + { + Ok(token) => tokens.push(token), + Err(error) => { + failure = Some((index, error)); + break; + } + } + } + for token in tokens { + assert!( + store + .cancel_local_object_metadata_reservation(&disk, token) + .await + .expect("cancel token") + ); + } + assert!(failure.is_none(), "bucket reservations exhausted shared lock holders: {failure:?}"); + } + + #[tokio::test] + #[serial_test::serial] + async fn expired_reservation_fence_does_not_bypass_a_waiting_bucket_writer() { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await); + let set = store.pools[0].get_disks_by_key("object"); + let disk = set.disks.read().await.iter().flatten().next().expect("local disk").clone(); + ctx.local_disk_map() + .write() + .await + .insert(disk.endpoint().to_string(), Some(disk.clone())); + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "reservation-writer-fairness"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation"); + let disk_ref = disk.endpoint().to_string(); + let first = store + .reserve_local_object_metadata_at_incarnation(&disk_ref, bucket, "first", incarnation, Duration::from_secs(5)) + .await + .expect("first reservation"); + let lock = store + .new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("publication lock"); + let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10))); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut writer).await.is_err(), + "writer waits for first token" + ); + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(2)).await; + tokio::time::resume(); + let mut second = Box::pin(store.reserve_local_object_metadata_at_incarnation( + &disk_ref, + bucket, + "second", + incarnation, + Duration::from_secs(5), + )); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(), + "expired cached reader must not bypass writer" + ); + assert!( + store + .cancel_local_object_metadata_reservation(&disk_ref, first) + .await + .expect("cancel first token") + ); + let guard = tokio::time::timeout(Duration::from_secs(2), writer) + .await + .expect("writer proceeds") + .expect("writer lock"); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(), + "new reader waits while writer holds the lock" + ); + drop(guard); + let token = tokio::time::timeout(Duration::from_secs(2), second) + .await + .expect("reader proceeds") + .expect("second reservation"); + assert!( + store + .cancel_local_object_metadata_reservation(&disk_ref, token) + .await + .expect("cancel second token") + ); + } + + #[test] + #[serial_test::serial] + fn forced_capacity_probe_cannot_publish_after_a_new_pool_writer() { + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime") + .block_on(async { + use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause}; + + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, first) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let (_other_dirs, second) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[first, second], Arc::clone(&ctx)).await); + crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "forced-capacity-probe"; + let object = "object"; + store + .handle_make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeQuotaRename); + let put_store = Arc::clone(&store); + let put = tokio::spawn(async move { + put_store + .handle_put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![1; 8193]), + &ObjectOptions::default(), + ) + .await + }); + barrier.wait_until_paused().await; + let manager = ctx.lock_manager().as_fast_lock_manager().expect("local lock manager"); + let key = rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, crate::core::pools::POOL_META_NAME); + assert_eq!(manager.force_unlock(&key), 1, "force release capacity probe"); + let lock = store.pools[0] + .new_ns_lock(RUSTFS_META_BUCKET, crate::core::pools::POOL_META_NAME) + .await + .expect("pool lock"); + let writer = lock.get_write_lock(Duration::from_secs(5)).await.expect("new pool writer"); + barrier.release(); + let result = tokio::time::timeout(Duration::from_secs(30), put) + .await + .expect("PUT finishes") + .expect("PUT task"); + assert!( + matches!(&result, Err(StorageError::NamespaceLockQuorumUnavailable { .. })), + "a PUT must reject a lost capacity probe: {result:?}" + ); + drop(writer); + assert!( + matches!( + store.handle_get_object_info(bucket, object, &ObjectOptions::default()).await, + Err(StorageError::ObjectNotFound(..)) + ), + "the rejected PUT must not publish an object" + ); + }); + }) + .expect("test thread") + .join() + .expect("test completes"); + } + + #[tokio::test] + async fn force_released_local_bucket_fence_cannot_activate_a_prepared_reservation() { + for released in ["publication", "lifecycle"] { + let fast = Arc::new(rustfs_lock::FastObjectLockManager::new()); + let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(fast.clone())); + let lock = rustfs_lock::NamespaceLock::with_local_manager("reservation-force".into(), manager); + let publication_key = rustfs_lock::ObjectKey::new("bucket", "publication"); + let publication = lock + .get_read_lock(publication_key.clone(), "reservation", Duration::from_secs(5)) + .await + .expect("publication lock"); + let lifecycle = lock + .get_read_lock(rustfs_lock::ObjectKey::new("bucket", "lifecycle"), "reservation", Duration::from_secs(5)) + .await + .expect("lifecycle lock"); + let bucket_registry = Arc::new(crate::store::bucket_fence::BucketFenceRegistry::default()); + let lifecycle = + crate::store::bucket_fence::BucketIncarnationFenceGuard::new(bucket_registry.clone(), "bucket", lifecycle, true); + let slot = bucket_registry.reservation_slot("bucket", Uuid::new_v4()); + let owner = Arc::new(crate::store::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot)); + let registry = crate::disk::os::MetadataReservationRegistry::new(1); + let path = std::path::Path::new("bucket/object"); + let disk = Uuid::new_v4(); + let token = registry + .prepare_with_owner(path, disk, Some(owner)) + .expect("prepare reservation"); + let released_key = rustfs_lock::ObjectKey::new("bucket", released); + assert_eq!(fast.force_unlock(&released_key), 1, "force release {released}"); + let replacement = lock + .get_write_lock(released_key, "bucket-delete", Duration::from_secs(5)) + .await + .expect("bucket writer acquires after forced release"); + assert!( + registry.activate(token, path, disk).await.is_err(), + "a released fence cannot activate a write" + ); + drop(replacement); + } + } + + #[tokio::test] + async fn lost_bucket_fence_cannot_activate_a_prepared_reservation() { + let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new( + rustfs_lock::FastObjectLockManager::new(), + ))); + let client = Arc::new(RefreshLossLockClient::with_manager(manager)); + let lock = rustfs_lock::NamespaceLock::with_clients_and_quorum( + "reservation-loss".to_string(), + vec![client.clone() as Arc], + 1, + ); + let acquire = |object: &'static str| { + rustfs_lock::LockRequest::new( + rustfs_lock::ObjectKey::new("bucket", object), + rustfs_lock::LockType::Shared, + "reservation-loss", + ) + .with_ttl(Duration::from_secs(2)) + .with_refresh_interval(Duration::from_millis(20)) + }; + let publication = lock + .acquire_guard(&acquire("publication")) + .await + .expect("publication request") + .expect("publication quorum"); + let signal = publication.lock_lost_signal().expect("distributed loss signal"); + let lifecycle = lock + .acquire_guard(&acquire("lifecycle")) + .await + .expect("lifecycle request") + .expect("lifecycle quorum"); + let bucket_registry = Arc::new(crate::store::bucket_fence::BucketFenceRegistry::default()); + let lifecycle = + crate::store::bucket_fence::BucketIncarnationFenceGuard::new(bucket_registry.clone(), "bucket", lifecycle, true); + let incarnation = Uuid::new_v4(); + let slot = bucket_registry.reservation_slot("bucket", incarnation); + let owner = Arc::new(crate::store::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot)); + let registry = crate::disk::os::MetadataReservationRegistry::new(1); + let key = std::path::Path::new("bucket/object"); + let disk = Uuid::new_v4(); + let token = registry + .prepare_with_owner(key, disk, Some(owner)) + .expect("prepare reservation"); + client.reject_refreshes(); + tokio::time::timeout(Duration::from_secs(5), signal.notified()) + .await + .expect("publication lock loss"); + assert!( + registry.activate(token, key, disk).await.is_err(), + "a lost bucket fence must not enter the mutation queue" + ); + } + #[tokio::test] #[serial_test::serial] async fn borrowed_copy_tail_drained_waits_for_rename_tail() { diff --git a/crates/ecstore/src/store/peer.rs b/crates/ecstore/src/store/peer.rs index 03b69df7e..d4ed3b343 100644 --- a/crates/ecstore/src/store/peer.rs +++ b/crates/ecstore/src/store/peer.rs @@ -85,6 +85,230 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> { } impl ECStore { + /// Prepare an inert user-object reservation on the currently admitted disk. + /// An unclaimed token expires and does not hold the object's metadata lock. + #[cfg(test)] + pub async fn reserve_local_object_metadata( + &self, + disk_ref: &str, + bucket: &str, + object: &str, + timeout: std::time::Duration, + ) -> DiskResult { + self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, None, None) + .await + } + + /// Prepare before the caller acquires bucket lifecycle or object locks. + /// The token holds the bucket publication fence through physical completion. + pub async fn reserve_local_object_metadata_at_incarnation( + &self, + disk_ref: &str, + bucket: &str, + object: &str, + incarnation: Uuid, + timeout: std::time::Duration, + ) -> DiskResult { + self.reserve_local_object_metadata_at_incarnation_with_ttl(disk_ref, bucket, object, incarnation, timeout, None) + .await + } + + /// Prepare a short-lived token for peers that may abandon the response. + pub async fn reserve_local_object_metadata_short( + &self, + disk_ref: &str, + bucket: &str, + object: &str, + incarnation: Uuid, + timeout: std::time::Duration, + ) -> DiskResult { + self.reserve_local_object_metadata_at_incarnation_with_ttl( + disk_ref, + bucket, + object, + incarnation, + timeout, + Some(std::time::Duration::from_secs(2)), + ) + .await + } + + async fn reserve_local_object_metadata_at_incarnation_with_ttl( + &self, + disk_ref: &str, + bucket: &str, + object: &str, + incarnation: Uuid, + timeout: std::time::Duration, + ttl: Option, + ) -> DiskResult { + if self.ctx.lock_manager().is_disabled() + || crate::bucket::utils::check_valid_bucket_name(bucket).is_err() + || is_meta_bucketname(bucket) + || incarnation.is_nil() + || timeout.is_zero() + || rustfs_utils::path::object_name_disk_identity(object).as_ref() + == rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH + { + return Err(DiskError::InvalidPath); + } + tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async { + let slot = self.bucket_fence_registry.reservation_slot(bucket, incarnation); + let mut cached = slot.owner.lock().await; + let owner = match cached.upgrade().filter(|owner| owner.is_reusable()) { + Some(owner) => owner, + None => { + // Bucket deletion takes publication before lifecycle and metadata. + let lock = self + .handle_new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .map_err(|error| DiskError::other(error.to_string()))?; + let publication = lock + .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .await + .map_err(|error| DiskError::other(error.to_string()))?; + // Older peers may acquire lifecycle without publication. + let lifecycle = self + .acquire_bucket_incarnation_fence(bucket, incarnation) + .await + .map_err(|error| DiskError::other(error.to_string()))?; + let owner = Arc::new(super::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot.clone())); + if !owner.is_reusable() { + return Err(DiskError::DiskOngoingReq); + } + *cached = Arc::downgrade(&owner); + owner + } + }; + drop(cached); + self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, Some(owner), ttl) + .await + }) + .await + .map_err(|_| DiskError::Timeout)? + } + + async fn reserve_local_object_metadata_with_owner( + &self, + disk_ref: &str, + bucket: &str, + object: &str, + timeout: std::time::Duration, + owner: Option>, + ttl: Option, + ) -> DiskResult { + if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) { + return Err(DiskError::InvalidPath); + } + if timeout.is_zero() { + return Err(DiskError::Timeout); + } + let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?; + let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?; + let Disk::Local(wrapper) = disk.as_ref() else { + return Err(DiskError::DiskNotFound); + }; + tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async { + match ttl { + Some(ttl) => { + wrapper + .prepare_object_metadata_with_owner_ttl(bucket, object, owner, ttl) + .await + } + None => wrapper.prepare_object_metadata_with_owner(bucket, object, owner).await, + } + }) + .await + .map_err(|_| DiskError::Timeout)? + } + + /// Refresh an inert token on its original disk and object. + pub async fn renew_local_object_metadata_reservation( + &self, + disk_ref: &str, + bucket: &str, + object: &str, + token: Uuid, + ) -> DiskResult<()> { + if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() { + return Err(DiskError::InvalidPath); + } + let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?; + let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?; + let Disk::Local(wrapper) = disk.as_ref() else { + return Err(DiskError::DiskNotFound); + }; + wrapper.renew_object_metadata(bucket, object, token).await + } + + /// Atomically enqueue a prepared token on its original disk and object. + pub async fn activate_local_object_metadata_reservation( + &self, + disk_ref: &str, + bucket: &str, + object: &str, + token: Uuid, + ) -> DiskResult<()> { + if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() { + return Err(DiskError::InvalidPath); + } + let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?; + let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?; + let Disk::Local(wrapper) = disk.as_ref() else { + return Err(DiskError::DiskNotFound); + }; + wrapper.activate_object_metadata(bucket, object, token).await + } + + /// Cancel an unclaimed token on the currently admitted disk. + /// A token already claimed by a physical mutation cannot be cancelled. + pub async fn cancel_local_object_metadata_reservation(&self, disk_ref: &str, token: Uuid) -> DiskResult { + let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?; + let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?; + let Disk::Local(wrapper) = disk.as_ref() else { + return Err(DiskError::DiskNotFound); + }; + Ok(wrapper.cancel_object_metadata_reservation(token)) + } + + /// Publish an ordinary staged user-object write using a one-shot disk reservation. + /// The token is bound to the admitted disk instance and destination. A claimed + /// physical mutation continues to own its lease if the RPC waiter is cancelled. + /// The caller must retain the bucket lifecycle fence until the physical tail + /// finishes; the reservation does not identify a bucket incarnation. + pub async fn rename_local_data_reserved( + &self, + disk_ref: &str, + source: (&str, &str), + fi: &FileInfo, + destination: (&str, &str), + token: Uuid, + ) -> DiskResult { + if token.is_nil() + || source.0 != crate::disk::RUSTFS_META_TMP_BUCKET + || crate::bucket::utils::check_valid_bucket_name(destination.0).is_err() + || is_meta_bucketname(destination.0) + || rustfs_utils::http::metadata_compat::contains_key_str( + &fi.metadata, + crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + ) + { + return Err(DiskError::InvalidPath); + } + rename_local_data_with_ctx( + &self.ctx, + disk_ref, + source, + fi, + destination, + RenameDataGuards { + metadata_reservation_token: Some(token), + ..Default::default() + }, + ) + .await + } + /// Resolve a quota lease against this request's captured storage instance. pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult { let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?; @@ -270,6 +494,7 @@ impl ECStore { scanner_publication_lease_token: scanner_token, external_guard, namespace_owner: None, + ..Default::default() }, ) .await @@ -351,7 +576,7 @@ fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> { Ok(()) } -async fn admit_local_disk( +pub(crate) async fn admit_local_disk( ctx: &Arc, disk: &DiskStore, disk_id: Option, @@ -783,6 +1008,116 @@ mod tests { .expect("seeded metadata bytes") } + #[tokio::test] + async fn reserved_target_rename_is_bound_to_the_admitted_disk() { + let ctx = Arc::new(InstanceContext::new()); + let dir = tempfile::tempdir().expect("disk root"); + let id = Uuid::new_v4(); + let disk = target_disk(&ctx, dir.path(), id).await; + let disk_ref = disk.endpoint().to_string(); + let store = super::super::tests::build_store_with_ctx(ctx.clone()); + let version = Uuid::new_v4(); + let fi = target_file_info("object", version, b"reserved"); + seed_target(&disk, crate::disk::RUSTFS_META_TMP_BUCKET, "staged", fi.clone()).await; + tokio::fs::create_dir_all(dir.path().join("target-bucket")) + .await + .expect("user bucket"); + assert!(matches!( + store + .reserve_local_object_metadata( + &disk_ref, + "target-bucket/../.rustfs.sys", + "object", + std::time::Duration::from_secs(5), + ) + .await, + Err(DiskError::InvalidPath) + )); + assert!(matches!( + store + .reserve_local_object_metadata(&disk_ref, "", ".rustfs.sys/config", std::time::Duration::from_secs(5)) + .await, + Err(DiskError::InvalidPath) + )); + let token = store + .reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5)) + .await + .expect("reserve current disk"); + assert!(matches!( + store + .rename_local_data_reserved(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "object"), token) + .await, + Err(DiskError::InvalidPath) + )); + store + .activate_local_object_metadata_reservation(&disk_ref, "target-bucket", "object", token) + .await + .expect("activate token"); + store + .rename_local_data_reserved( + &disk_ref, + (crate::disk::RUSTFS_META_TMP_BUCKET, "staged"), + &fi, + ("target-bucket", "object"), + token, + ) + .await + .expect("reserved publication"); + assert!( + !store + .cancel_local_object_metadata_reservation(&disk_ref, token) + .await + .expect("consumed token") + ); + let read = disk + .read_version( + "target-bucket", + "target-bucket", + "object", + &version.to_string(), + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read reserved publication"); + assert_eq!(read.data, fi.data); + + let stale = store + .reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5)) + .await + .expect("reserve before disk replacement"); + let replacement = target_disk(&ctx, dir.path(), id).await; + assert!(matches!( + store + .rename_local_data_reserved( + &disk_ref, + (crate::disk::RUSTFS_META_TMP_BUCKET, "staged"), + &fi, + ("target-bucket", "object"), + stale, + ) + .await, + Err(DiskError::DiskOngoingReq) + )); + let after = replacement + .read_version( + "target-bucket", + "target-bucket", + "object", + &version.to_string(), + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("replacement retains publication"); + assert_eq!(after.data, fi.data); + assert!(store.cancel_local_object_metadata_reservation(&disk_ref, stale).await.is_ok()); + } + #[tokio::test] async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() { for warm in [false, true] { diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index ee15a861a..4665a893a 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -604,6 +604,24 @@ impl ECStore { Ok(idx) } + /// A speculative hint only: callers must repeat the locked selection before committing. + pub(super) async fn get_pool_idx_for_reservation_hint(&self, bucket: &str, object: &str, size: i64) -> Result { + let opts = ObjectOptions { + no_lock: true, + metadata_cache_safe: true, + skip_decommissioned: true, + skip_rebalancing: true, + ..Default::default() + }; + match self.get_pool_idx_existing_with_opts(bucket, object, &opts).await { + Ok(idx) => Ok(idx), + Err(err) if is_err_object_not_found(&err) => { + self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull) + } + Err(err) => Err(err), + } + } + pub(super) async fn get_pool_idx_no_lock(&self, bucket: &str, object: &str, size: i64) -> Result { let idx = match self.get_pool_idx_existing_no_lock(bucket, object).await { Ok(res) => res, diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index f22f1b2c4..a28ff0aa6 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -27,6 +27,7 @@ use futures::{StreamExt, stream::FuturesUnordered}; use metrics::{counter, gauge}; use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource, HealScanMode}; use rustfs_madmin::heal_commands::HealResultItem; +use std::collections::HashMap; use std::sync::{ Arc, atomic::{AtomicUsize, Ordering}, @@ -1550,6 +1551,7 @@ impl ErasureSetHealer { let checkpoint = checkpoint_manager.get_checkpoint().await; let page_resume_index = *current_object_index; let semaphore = Arc::new(Semaphore::new(page_concurrency_limit)); + let mut object_locks: HashMap>> = HashMap::new(); let mut page_tasks = FuturesUnordered::new(); let mut completed_in_page = 0usize; @@ -1721,6 +1723,7 @@ impl ErasureSetHealer { let set_label = set_disk_id.to_string(); let heal_opts = self.heal_opts; let semaphore = semaphore.clone(); + let object_lock = object_locks.entry(object_name.clone()).or_default().clone(); let target_endpoints = self.target_endpoints.clone(); let replacement_commit_evidence_required = self.replacement_task_id.is_some(); let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal; @@ -1735,6 +1738,13 @@ impl ErasureSetHealer { .as_ref() .map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone())); let work = async move { + // Versions share one namespace lock. Queue them before taking a + // page permit so a waiting version cannot block other objects. + let _object_guard = tokio::select! { + biased; + _ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None), + guard = object_lock.lock() => guard, + }; let mut retries = 0; let mut admin_identity = admin_identity; let mut admin_task_id = admin_task_id; @@ -4047,6 +4057,64 @@ mod resume_loop_tests { assert_eq!(unique.len(), 50, "no version healed twice"); } + #[tokio::test] + async fn page_serializes_versions_without_blocking_other_objects() { + temp_env::async_with_vars( + [ + (rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("true")), + (rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")), + ], + async { + let mut env = make_env().await; + env.healer.source = HealRequestSource::AutoHeal; + env.healer = env + .healer + .with_replacement_targets(vec!["replacement-a".into()], Some(env.task_id.clone())); + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + *env.storage.heal_gate.lock().unwrap() = Some(gate.clone()); + for (name, version) in [("a", "v1"), ("a", "v2"), ("b", "v1")] { + env.storage + .set_result(name, Some(version), replacement_target_ok_result("replacement-a", name)); + } + env.storage.set_page( + None, + Page { + items: vec![ + item("a", Some("v1"), false), + item("a", Some("v2"), false), + item("b", Some("v1"), false), + ], + next: None, + truncated: false, + }, + ); + let observe = async { + let started = tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let notified = env.storage.heal_started.notified(); + if env.storage.calls().len() >= 2 { + break; + } + notified.await; + } + }) + .await + .is_ok(); + let calls = env.storage.calls(); + gate.add_permits(3); + assert!(started, "different objects should run concurrently"); + assert_eq!(calls.len(), 2); + assert!(calls.iter().any(|(name, _)| name == "a")); + assert!(calls.iter().any(|(name, _)| name == "b")); + }; + let ((processed, successful, failed, skipped, result), ()) = tokio::join!(run(&env), observe); + result.expect("all versions heal"); + assert_eq!((processed, successful, failed, skipped), (3, 3, 0, 0)); + }, + ) + .await; + } + #[tokio::test] async fn auto_replacement_runs_two_object_repairs_within_the_configured_limit() { temp_env::async_with_vars( diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index 4b0a23a37..48de869ca 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -14,13 +14,26 @@ /// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline use super::*; use crate::heal::progress::{add_bytes, increment_counter, stable_generation}; -use crate::heal::storage::HealListItem; +use crate::heal::storage::{HealListItem, HealStorageObjectResult}; use crate::heal::utils::format_set_disk_id; +use futures::{StreamExt, stream::FuturesUnordered}; +use std::collections::HashSet; const MAX_DEFERRED_OBJECTS: usize = 256; const MAX_DEFERRED_BYTES: usize = 256 * 1024; const MAX_DEFERRED_FORWARD_PAGES: u64 = 2; const MAX_DEFERRED_AGE: Duration = Duration::from_secs(30); +const MAX_INFLIGHT_REPAIRS: usize = 8; + +#[cfg(test)] +pub(super) fn deferred_payload_budget_for_test() -> usize { + MAX_DEFERRED_BYTES - MAX_DEFERRED_OBJECTS * size_of::() +} +// A recovered pool can become writable just after the ordinary retry budget. +// Keep this extra attempt inside the existing deferred age and backoff caps. +const MAX_SLOWDOWN_OBJECT_RETRIES: u32 = MAX_BUCKET_OBJECT_HEAL_RETRIES + 1; + +type HealListingPage = (Vec, Option, bool); struct DeferredObject { name: String, @@ -57,7 +70,11 @@ impl DeferredObject { let now = tokio::time::Instant::now(); let first = *self.first_failure.get_or_insert(now); self.attempt += 1; - self.due = (now + delay).min(first + MAX_DEFERRED_AGE); + let remaining = (first + MAX_DEFERRED_AGE).saturating_duration_since(now); + // Scheduling exactly at the deadline guarantees the retry will be + // rejected as expired without making the storage attempt. + let delay = if delay >= remaining { remaining / 2 } else { delay }; + self.due = now + delay; } } @@ -89,16 +106,23 @@ impl DeferredWindow { Ok(()) } - fn pop_due(&mut self) -> Option { + fn pop_due(&mut self, in_flight: &HashSet) -> Option { let now = tokio::time::Instant::now(); - let index = self.objects.iter().position(|item| item.due <= now)?; + let index = self + .objects + .iter() + .position(|item| item.due <= now && !in_flight.contains(&item.name))?; let item = self.objects.remove(index)?; self.bytes -= item.payload_bytes(); Some(item) } - fn next_due(&self) -> Option { - self.objects.iter().map(|item| item.due).min() + fn next_due(&self, in_flight: &HashSet) -> Option { + self.objects + .iter() + .filter(|item| !in_flight.contains(&item.name)) + .map(|item| item.due) + .min() } fn can_advance(&self, page: u64) -> bool { @@ -132,6 +156,32 @@ fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Optio } impl HealTask { + async fn heal_bucket_storage_object( + &self, + bucket: &str, + item: DeferredObject, + incarnation: Option, + opts: &HealOpts, + ) -> (DeferredObject, Result) { + let result = self + .await_with_control(async { + match incarnation { + Some(expected) => { + self.storage + .heal_object_at_incarnation(bucket, &item.name, item.version_id.as_deref(), expected, opts) + .await + } + None => { + self.storage + .heal_object_with_receipt(bucket, &item.name, item.version_id.as_deref(), opts) + .await + } + } + }) + .await; + (item, result) + } + async fn recover_bucket_delete_transactions(&self, bucket: &str, opts: &HealOpts) -> Result { if let Some(result) = self .await_with_control( @@ -649,56 +699,168 @@ impl HealTask { }; let mut continuation_token: Option = None; let mut deferred = DeferredWindow::default(); - let mut inline_retry: Option = None; + // Only already-admitted operations can overflow the deferred byte + // budget. Stop listing and page advancement while these drain. + let mut inline_retry = VecDeque::::new(); let mut page_number = 0_u64; let mut aborted_progress_unknown = false; let mut pending = Vec::::new().into_iter(); let mut listing_finished = false; let mut listing_attempt = 0; let mut listing_due = tokio::time::Instant::now(); + let mut listing_inflight: Option>> = None; + let mut in_flight = HashSet::new(); + let repairs_aborted = Arc::new(AtomicBool::new(false)); + let mut repairs: FuturesUnordered< + futures::future::BoxFuture<'_, (DeferredObject, Result, bool)>, + > = FuturesUnordered::new(); let scope_result: Result<()> = async { loop { self.check_control_flags().await?; - if listing_finished && pending.as_slice().is_empty() && deferred.objects.is_empty() && inline_retry.is_none() + if listing_finished + && pending.as_slice().is_empty() + && deferred.objects.is_empty() + && inline_retry.is_empty() + && repairs.is_empty() { break; } - self.pace_mainline().await?; // Listing and object retries share this safe boundary. A // failed listing never hides an already-due object retry. - let item = deferred.pop_due().or_else(|| { - if inline_retry - .as_ref() - .is_some_and(|item| item.due <= tokio::time::Instant::now()) - { - inline_retry.take() - } else if inline_retry.is_none() { + let item = if repairs.len() < MAX_INFLIGHT_REPAIRS { + let due = inline_retry + .iter() + .position(|item| item.due <= tokio::time::Instant::now() && !in_flight.contains(&item.name)); + due.and_then(|index| inline_retry.remove(index)) + .or_else(|| deferred.pop_due(&in_flight)) + } else { + None + } + .or_else(|| { + if inline_retry.is_empty() && repairs.is_empty() { pending.next().map(|item| DeferredObject::new(item, page_number)) } else { None } }); - let Some(mut item) = item else { - let can_list = !listing_finished && inline_retry.is_none() && deferred.can_advance(page_number); + let (mut item, storage_result, age_exhausted) = if let Some(item) = item { + if item.expired() { + (item, Err(Error::other("heal object retry age exhausted")), true) + } else { + in_flight.insert(item.name.clone()); + let opts = &heal_opts; + let aborted = Arc::clone(&repairs_aborted); + repairs.push(Box::pin(async move { + if aborted.load(Ordering::Acquire) { + return (item, Err(Error::TaskCancelled), false); + } + if let Err(error) = self.pace_mainline().await { + return (item, Err(error), false); + } + if aborted.load(Ordering::Acquire) { + return (item, Err(Error::TaskCancelled), false); + } + let (item, result) = self + .heal_bucket_storage_object(bucket, item, traversal_incarnation_id, opts) + .await; + (item, result, true) + })); + continue; + } + } else if !repairs.is_empty() { + let next_retry = if repairs.len() < MAX_INFLIGHT_REPAIRS { + deferred + .next_due(&in_flight) + .into_iter() + .chain( + inline_retry + .iter() + .filter(|item| !in_flight.contains(&item.name)) + .map(|item| item.due), + ) + .min() + } else { + None + }; + let completed = if let Some(due) = next_retry { + tokio::select! { + biased; + _ = tokio::time::sleep_until(due) => None, + completed = repairs.next() => completed, + } + } else { + repairs.next().await + }; + let Some((item, result, attempted)) = completed else { + continue; + }; + in_flight.remove(&item.name); + if !attempted && item.attempt == 0 { + return Err(result.expect_err("an unstarted repair must have a control error")); + } + (item, result, false) + } else { + self.pace_mainline().await?; + let can_list = !listing_finished && inline_retry.is_empty() && deferred.can_advance(page_number); if can_list && listing_due <= tokio::time::Instant::now() { - let page = if let Some(set_disk_id) = set_disk_id.as_deref() { - self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk( - set_disk_id, - bucket, - prefix, - continuation_token.as_deref(), - false, - )) - .await - } else { - self.await_with_control(self.storage.list_objects_for_heal_page( - bucket, - prefix, - continuation_token.as_deref(), - false, - )) - .await + if listing_inflight.is_none() { + let storage = Arc::clone(&self.storage); + let listing_set = set_disk_id.clone(); + let listing_bucket = bucket.to_owned(); + let listing_prefix = prefix.to_owned(); + let cursor = continuation_token.clone(); + listing_inflight = Some(Box::pin(async move { + if let Some(set_id) = listing_set.as_deref() { + storage + .list_versions_for_heal_page_disk_walk( + set_id, + &listing_bucket, + &listing_prefix, + cursor.as_deref(), + false, + ) + .await + } else { + storage + .list_objects_for_heal_page( + &listing_bucket, + &listing_prefix, + cursor.as_deref(), + false, + ) + .await + } + })); + } + let next_retry = deferred + .next_due(&in_flight) + .into_iter() + .chain(inline_retry.iter().map(|item| item.due)) + .min(); + let Some(listing) = listing_inflight.as_mut() else { + return Err(Error::other("heal listing future is unavailable")); }; + // Keep the same listing future alive while an earlier retry runs. + // Restarting a disk walk here would duplicate work and may change its snapshot. + let result = self + .await_with_control(async { + if let Some(due) = next_retry { + tokio::select! { + biased; + _ = tokio::time::sleep_until(due) => Ok(None), + result = listing => Ok(Some(result)), + } + } else { + Ok(Some(listing.await)) + } + }) + .await; + let page = match result { + Ok(Some(page)) => page, + Ok(None) => continue, + Err(error) => Err(error), + }; + listing_inflight = None; match page { Ok((objects, next_token, is_truncated)) => { page_number = page_number.saturating_add(1); @@ -727,9 +889,9 @@ impl HealTask { continue; } else { let due = deferred - .next_due() + .next_due(&in_flight) .into_iter() - .chain(inline_retry.as_ref().map(|item| item.due)) + .chain(inline_retry.iter().map(|item| item.due)) .chain(can_list.then_some(listing_due)) .min(); if let Some(due) = due { @@ -761,89 +923,74 @@ impl HealTask { } let mut terminal_outcome = true; - let age_exhausted = item.expired(); - let error = if age_exhausted { - Some(Error::other("heal object retry age exhausted")) - } else { - match self - .await_with_control(async { - match traversal_incarnation_id { - Some(expected) => { - self.storage - .heal_object_at_incarnation( - bucket, - object, - item.version_id.as_deref(), - expected, - &heal_opts, - ) - .await - } - None => { - self.storage - .heal_object_with_receipt(bucket, object, item.version_id.as_deref(), &heal_opts) - .await - } - } - }) - .await - { - Ok(storage_result) if storage_result.error.is_none() => { - match unavailable_recreate_error(&storage_result.item, &heal_opts) { - Some(error) => Some(error), - None => { - #[cfg(test)] - super::pause_storage_receipt(&self.id).await; - recorded_authoritative_outcome = - self.record_verified_storage_receipt(&identity, storage_result.receipt).await; - if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() { - Some(Error::TaskCancelled) - } else { - telemetry_unknown |= !increment_counter(&mut healed); - telemetry_unknown |= !add_bytes( - &mut bytes, - u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX), - ); - if !recorded_authoritative_outcome - && !heal_opts.dry_run - && !storage_result.item.detail.is_empty() - { - detail = Some(storage_result.item.detail.clone()); - } - self.record_result_item(storage_result.item).await; - None + let object = item.name.as_str(); + let error = match storage_result { + Ok(storage_result) if storage_result.error.is_none() => { + match unavailable_recreate_error(&storage_result.item, &heal_opts) { + Some(error) => Some(error), + None => { + #[cfg(test)] + super::pause_storage_receipt(&self.id).await; + recorded_authoritative_outcome = + self.record_verified_storage_receipt(&identity, storage_result.receipt).await; + if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() { + Some(Error::TaskCancelled) + } else { + telemetry_unknown |= !increment_counter(&mut healed); + telemetry_unknown |= !add_bytes( + &mut bytes, + u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX), + ); + if !recorded_authoritative_outcome + && !heal_opts.dry_run + && !storage_result.item.detail.is_empty() + { + detail = Some(storage_result.item.detail.clone()); } + self.record_result_item(storage_result.item).await; + None } } } - Ok(storage_result) - if storage_result - .error - .as_ref() - .is_some_and(|err| is_missing_object_dir_heal_result(object, err)) => - { - telemetry_unknown |= !increment_counter(&mut healed); - debug!( - target: "rustfs::heal::task", - event = EVENT_HEAL_BUCKET_RESULT, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - bucket, - object, - result = "object_dir_not_found_skipped", - "Heal bucket object-dir candidate skipped after not-found result" - ); - None - } - Ok(storage_result) => storage_result.error, - Err(err) => Some(err), } + Ok(storage_result) + if storage_result + .error + .as_ref() + .is_some_and(|err| is_missing_object_dir_heal_result(object, err)) => + { + telemetry_unknown |= !increment_counter(&mut healed); + debug!( + target: "rustfs::heal::task", + event = EVENT_HEAL_BUCKET_RESULT, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_TASK, + task_id = %self.id, + bucket, + object, + result = "object_dir_not_found_skipped", + "Heal bucket object-dir candidate skipped after not-found result" + ); + None + } + Ok(storage_result) => storage_result.error, + Err(err) => Some(err), }; if let Some(err) = error { match err { - Error::StaleBucketIncarnation { .. } => return Err(err), + Error::StaleBucketIncarnation { .. } => { + self.outcome.write().await.record(HealObjectOutcome { + deep_check_observed: false, + quorum_selected_data_dir: None, + identity, + disposition: HealObjectDisposition::Unknown, + detail: None, + }); + aborted_progress_unknown |= !increment_counter(&mut scanned); + aborted_progress_unknown |= !increment_counter(&mut skipped); + return Err(err); + } Error::TaskCancelled | Error::TaskTimeout => { let disposition = if matches!(err, Error::TaskCancelled) { HealObjectDisposition::Cancelled @@ -912,7 +1059,12 @@ impl HealTask { ); } else if !age_exhausted && err.is_recoverable_object_heal() - && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES + && retry_attempt + < if matches!(&err, Error::Storage(crate::heal::StorageError::SlowDown)) { + MAX_SLOWDOWN_OBJECT_RETRIES + } else { + MAX_BUCKET_OBJECT_HEAL_RETRIES + } { terminal_outcome = false; debug!( @@ -930,7 +1082,7 @@ impl HealTask { ); item.defer(Self::bucket_object_retry_delay(&self.id, retry_attempt + 1)); if let Err(item) = deferred.push(item) { - inline_retry = Some(item); + inline_retry.push_back(item); } } else { disposition = HealObjectDisposition::Failed(if age_exhausted || err.is_recoverable_object_heal() { @@ -947,17 +1099,24 @@ impl HealTask { first_failed_object.get_or_insert_with(|| object.to_string()); first_error.get_or_insert_with(|| err.to_string()); if take_failure_log_sample(&mut failure_samples_logged) { + let retry_age_ms = item + .first_failure + .map(|first| u64::try_from(first.elapsed().as_millis()).unwrap_or(u64::MAX)); + let retry_due_lag_ms = + age_exhausted.then(|| u64::try_from(item.due.elapsed().as_millis()).unwrap_or(u64::MAX)); warn!( target: "rustfs::heal::task", event = EVENT_HEAL_BUCKET_RESULT, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_TASK, + result = "object_failed", task_id = %self.id, bucket, object, retry_attempt, + retry_age_ms = ?retry_age_ms, + retry_due_lag_ms = ?retry_due_lag_ms, error = %err, - result = "object_failed", "Heal bucket object repair failed" ); } @@ -998,6 +1157,7 @@ impl HealTask { } .await; if let Err(error) = scope_result { + repairs_aborted.store(true, Ordering::Release); let disposition = match error { Error::TaskCancelled => HealObjectDisposition::Cancelled, Error::TaskTimeout => HealObjectDisposition::Deferred { @@ -1006,20 +1166,51 @@ impl HealTask { }, _ => HealObjectDisposition::Unknown, }; + // Keep issued replacement I/O alive until it reaches its storage + // boundary, even when another operation aborts this traversal. + while let Some((item, result, attempted)) = repairs.next().await { + if !attempted && item.attempt == 0 { + continue; + } + let mut identity = + self.outcome_identity(bucket, &item.name, item.version_id.as_deref(), heal_opts.pool, heal_opts.set); + identity.bucket_incarnation_id = bucket_incarnation_id; + let mut recorded = false; + if let Ok(storage_result) = result + && storage_result.error.is_none() + && unavailable_recreate_error(&storage_result.item, &heal_opts).is_none() + { + recorded = self.record_verified_storage_receipt(&identity, storage_result.receipt).await; + if recorded { + aborted_progress_unknown |= !increment_counter(&mut healed); + aborted_progress_unknown |= + !add_bytes(&mut bytes, u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX)); + self.record_result_item(storage_result.item).await; + } + } + if !recorded { + self.outcome.write().await.record(HealObjectOutcome { + deep_check_observed: false, + quorum_selected_data_dir: None, + identity, + disposition: disposition.clone(), + detail: None, + }); + aborted_progress_unknown |= !increment_counter(&mut skipped); + } + aborted_progress_unknown |= !increment_counter(&mut scanned); + } // Only attempted identities have terminal outcomes. Unstarted // page tails remain unprocessed under the task's partial coverage. // No detached sleepers survive abort. for item in deferred.objects.into_iter().chain(inline_retry) { + let mut identity = + self.outcome_identity(bucket, &item.name, item.version_id.as_deref(), heal_opts.pool, heal_opts.set); + identity.bucket_incarnation_id = bucket_incarnation_id; self.outcome.write().await.record(HealObjectOutcome { deep_check_observed: false, quorum_selected_data_dir: None, - identity: self.outcome_identity( - bucket, - &item.name, - item.version_id.as_deref(), - heal_opts.pool, - heal_opts.set, - ), + identity, disposition: disposition.clone(), detail: None, }); diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 6d9a32699..b9afb99f1 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -1658,7 +1658,9 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() { struct MockStorage { retry_test_pages: Option>>, retry_test_delays: HashMap, + retry_test_attempt_delays: Mutex>>, retry_test_listing_delays: Mutex>, + retry_test_disk_walk_pages: bool, retry_test_events: Mutex>, listed: Mutex, list_each_bucket: bool, @@ -1678,6 +1680,7 @@ struct MockStorage { object_heal_detail: Option, object_heal_item: Option, bucket_incarnation_id: Mutex>, + bucket_incarnation_reads: std::sync::atomic::AtomicUsize, bucket_incarnation_after_object_heal: Mutex>, bucket_incarnation_after_listing: Mutex>, bucket_incarnation_unavailable: Mutex, @@ -2427,6 +2430,7 @@ impl HealStorageAPI for MockStorage { } async fn bucket_incarnation_id(&self, _bucket: &str) -> Result> { + self.bucket_incarnation_reads.fetch_add(1, Ordering::Relaxed); if *self.bucket_incarnation_unavailable.lock().unwrap() { return Err(Error::Other("bucket incarnation unavailable".to_string())); } @@ -2459,8 +2463,15 @@ impl HealStorageAPI for MockStorage { ) -> Result<(HealResultItem, Option)> { self.heal_object_calls.lock().unwrap().push(object.to_string()); self.retry_test_events.lock().expect("events").push(format!("heal:{object}")); - if let Some(delay) = self.retry_test_delays.get(object) { - tokio::time::sleep(*delay).await; + let delay = self + .retry_test_attempt_delays + .lock() + .expect("attempt delays") + .get_mut(object) + .and_then(VecDeque::pop_front) + .or_else(|| self.retry_test_delays.get(object).copied()); + if let Some(delay) = delay { + tokio::time::sleep(delay).await; } self.heal_object_version_ids .lock() @@ -2913,6 +2924,11 @@ impl HealStorageAPI for MockStorage { _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { self.disk_walk_calls.lock().unwrap().push(set_disk_id.to_string()); + if self.retry_test_disk_walk_pages { + return self + .list_objects_for_heal_page(_bucket, _prefix, continuation_token, _include_lifecycle_object_info) + .await; + } if continuation_token.is_some() { return Ok((Vec::new(), None, false)); } diff --git a/crates/heal/src/heal/task/tests/deferred_retry.rs b/crates/heal/src/heal/task/tests/deferred_retry.rs index c50c73551..3a5c70d40 100644 --- a/crates/heal/src/heal/task/tests/deferred_retry.rs +++ b/crates/heal/src/heal/task/tests/deferred_retry.rs @@ -52,7 +52,92 @@ fn fail_once(storage: &MockStorage, name: &str) { } #[tokio::test(start_paused = true)] -async fn slow_listing_retry_services_due_object_then_age_before_next_listing() { +async fn slow_listing_does_not_starve_due_object_retry() { + let storage = Arc::new(MockStorage { + recoverable_second_page_failures: Mutex::new(Some(1)), + retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])), + ..Default::default() + }); + fail_once(&storage, "object-a"); + let task = bucket_task(storage.clone()); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err()); + assert_eq!( + storage.retry_test_events.lock().expect("events").as_slice(), + ["list:first", "heal:object-a", "list:second", "heal:object-a"], + "the object retry must run while the slow listing remains in flight" + ); + assert_eq!(task.get_outcome().await.counters.failed, 0); + execution.await.expect("the resumed listing and retried object complete"); + assert_eq!( + storage.retry_test_events.lock().expect("events").as_slice(), + [ + "list:first", + "heal:object-a", + "list:second", + "heal:object-a", + "list:second", + "heal:object-b" + ] + ); +} + +#[tokio::test(start_paused = true)] +async fn slow_set_disk_walk_does_not_starve_due_object_retry() { + let storage = Arc::new(MockStorage { + erasure_set_scopes: Mutex::new(vec![(0, 0)]), + retry_test_disk_walk_pages: true, + retry_test_pages: Some(vec![vec![heal_item("a")], vec![heal_item("b")]]), + retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])), + ..Default::default() + }); + fail_once(&storage, "a"); + let task = bucket_task(storage.clone()); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err()); + assert_eq!( + storage.retry_test_events.lock().expect("events").as_slice(), + ["list:first", "heal:a", "list:1", "heal:a"] + ); + assert_eq!(storage.disk_walk_calls.lock().expect("walks").len(), 2); + execution.await.expect("the same disk walk resumes and both objects heal"); + assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "a", "b"]); + assert_eq!(storage.disk_walk_calls.lock().expect("walks").len(), 2); + assert_eq!(task.get_outcome().await.counters.failed, 0); +} + +#[tokio::test(start_paused = true)] +async fn cancelling_an_inflight_listing_keeps_deferred_outcome_cancelled() { + let incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + recoverable_second_page_failures: Mutex::new(Some(1)), + retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])), + bucket_incarnation_id: Mutex::new(Some(incarnation)), + ..Default::default() + }); + fail_once(&storage, "object-a"); + let task = bucket_task(storage.clone()); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err()); + assert_eq!( + storage.retry_test_events.lock().expect("events").as_slice(), + ["list:first", "heal:object-a", "list:second"] + ); + task.cancel().await.expect("cancel"); + assert!(matches!(execution.await, Err(Error::TaskCancelled))); + tokio::time::advance(Duration::from_secs(60)).await; + assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["object-a"]); + let outcome = task.get_outcome().await; + assert_eq!(outcome.objects.len(), 1); + assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Cancelled); + assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(incarnation)); +} + +#[tokio::test(start_paused = true)] +async fn slow_listing_services_multiple_retries_without_restarting_walk() { let storage = Arc::new(MockStorage { recoverable_second_page_failures: Mutex::new(Some(1)), retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(29)])), @@ -68,22 +153,17 @@ async fn slow_listing_retry_services_due_object_then_age_before_next_listing() { let task = bucket_task(storage.clone()); let execution = task.execute(); tokio::pin!(execution); - assert!( - tokio::time::timeout(Duration::from_millis(30_500), &mut execution) - .await - .is_err() - ); + assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err()); assert_eq!( storage.retry_test_events.lock().expect("events").as_slice(), - ["list:first", "heal:object-a", "list:second", "heal:object-a"] + ["list:first", "heal:object-a", "list:second", "heal:object-a", "heal:object-a"] ); let outcome = task.get_outcome().await; - assert_eq!(outcome.counters.processed, 1); assert_eq!( - outcome.objects[0].disposition, - HealObjectDisposition::Failed(HealFailureClass::RetryExhausted) + (outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures), + (1, 0, 2) ); - execution.await.expect_err("age exhausted object must remain a batch failure"); + execution.await.expect("object and resumed listing complete"); assert_eq!( storage.retry_test_events.lock().expect("events").as_slice(), [ @@ -91,22 +171,25 @@ async fn slow_listing_retry_services_due_object_then_age_before_next_listing() { "heal:object-a", "list:second", "heal:object-a", + "heal:object-a", "list:second", "heal:object-b" ] ); - let outcome = task.get_outcome().await; - assert_eq!((outcome.counters.processed, outcome.counters.attempt_failures), (2, 3)); } #[tokio::test(start_paused = true)] -async fn listing_return_after_age_expires_does_not_start_another_heal_attempt() { +async fn slow_listing_preserves_retry_limit_and_failure() { let storage = Arc::new(MockStorage { recoverable_second_page_failures: Mutex::new(Some(1)), retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])), ..Default::default() }); - fail_once(&storage, "object-a"); + storage + .heal_object_outcomes + .lock() + .expect("outcomes") + .insert("object-a".to_string(), (0..5).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect()); let task = bucket_task(storage.clone()); let execution = task.execute(); tokio::pin!(execution); @@ -115,15 +198,20 @@ async fn listing_return_after_age_expires_does_not_start_another_heal_attempt() .await .is_err() ); + assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["object-a"; 5]); + let outcome = task.get_outcome().await; assert_eq!( - storage.retry_test_events.lock().expect("events").as_slice(), - ["list:first", "heal:object-a", "list:second"] + (outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures), + (1, 1, 6) // Five object failures and one listing failure. ); - assert_eq!(task.get_outcome().await.counters.failed, 1); - execution.await.expect_err("age exhaustion remains a failure"); assert_eq!( - storage.retry_test_events.lock().expect("events").as_slice(), - ["list:first", "heal:object-a", "list:second", "list:second", "heal:object-b"] + outcome.objects[0].disposition, + HealObjectDisposition::Failed(HealFailureClass::RetryExhausted) + ); + execution.await.expect_err("exhausted object must remain a batch failure"); + assert_eq!( + storage.heal_object_calls.lock().expect("calls").last().map(String::as_str), + Some("object-b") ); } @@ -220,6 +308,29 @@ async fn full_window_abort_accounts_inline_once_and_leaves_unstarted_tail_unproc } } +#[tokio::test(start_paused = true)] +async fn slowdown_can_recover_within_existing_deferred_age_budget() { + let storage = Arc::new(pages_storage(&[&["a"]])); + storage + .heal_object_outcomes + .lock() + .expect("outcomes") + .insert("a".to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect()); + let task = bucket_task(storage.clone()); + let started = tokio::time::Instant::now(); + task.execute() + .await + .expect("transient slowdown recovers within the deferred age budget"); + assert!(started.elapsed() < Duration::from_secs(30)); + assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 5); + let outcome = task.get_outcome().await; + assert_eq!( + (outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures), + (1, 0, 4) + ); + assert_eq!(outcome.objects.len(), 1); +} + #[tokio::test(start_paused = true)] async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() { let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]])); @@ -228,7 +339,7 @@ async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() { .heal_object_outcomes .lock() .expect("outcomes") - .insert(name.to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect()); + .insert(name.to_string(), (0..5).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect()); } let task = bucket_task(storage.clone()); let execution = task.execute(); @@ -239,9 +350,9 @@ async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() { let outcome = task.get_outcome().await; assert_eq!( (outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures), - (4, 4, 16) + (4, 4, 20) ); - assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 16); + assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 20); for name in ["a", "b", "c", "d"] { assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == name).count(), 1); } @@ -294,14 +405,360 @@ async fn due_retry_runs_before_next_object_in_a_slow_healthy_page() { } #[tokio::test(start_paused = true)] -async fn expired_retry_is_terminal_without_an_extra_storage_attempt() { - let mut storage = pages_storage(&[&["a"], &["b"]]); +async fn slow_unrelated_object_does_not_starve_due_retry() { + let mut storage = pages_storage(&[&["a", "b"]]); storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31)); fail_once(&storage, "a"); let storage = Arc::new(storage); let task = bucket_task(storage.clone()); - task.execute().await.expect_err("aged pending responsibility is not success"); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err()); + assert_eq!( + storage.heal_object_calls.lock().expect("calls").as_slice(), + ["a", "b", "a"], + "a due retry must run while an unrelated object is still healing" + ); + execution.await.expect("both objects complete"); + let outcome = task.get_outcome().await; + assert_eq!((outcome.counters.processed, outcome.counters.failed), (2, 0)); +} + +#[tokio::test(start_paused = true)] +async fn cancellation_drains_slow_replacement_object_after_a_concurrent_retry() { + let mut storage = pages_storage(&[&["a", "b"]]); + storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31)); + storage + .retry_test_attempt_delays + .lock() + .expect("attempt delays") + .insert("a".to_string(), VecDeque::from([Duration::ZERO, Duration::from_secs(31)])); + fail_once(&storage, "a"); + let storage = Arc::new(storage); + let task = bucket_task(storage.clone()); + let owner = crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new()); + *task.replacement_execution.write().await = Some(owner); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err()); + assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a"]); + assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 1); + task.cancel_token.cancel(); + assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err()); + assert!(matches!(execution.await, Err(Error::TaskCancelled))); + assert!(task.replacement_execution.read().await.is_none()); + assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a"]); + assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 3); +} + +#[tokio::test(start_paused = true)] +async fn stale_bucket_retry_drains_another_inflight_replacement() { + let original = Uuid::new_v4(); + let mut storage = pages_storage(&[&["a", "b"]]); + *storage.bucket_incarnation_id.lock().expect("incarnation") = Some(original); + storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31)); + fail_once(&storage, "a"); + let storage = Arc::new(storage); + let mut task = bucket_task(storage.clone()); + task.bucket_incarnation_id = Some(original); + let owner = crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new()); + *task.replacement_execution.write().await = Some(owner); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err()); assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]); + *storage.bucket_incarnation_id.lock().expect("incarnation") = Some(Uuid::new_v4()); + assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err()); + let result = execution.await; + assert!( + matches!(result, Err(Error::StaleBucketIncarnation { .. })), + "unexpected result: {result:?}" + ); + assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 2); + assert!(task.replacement_execution.read().await.is_none()); + let outcome = task.get_outcome().await; + let stale = outcome + .objects + .iter() + .find(|item| item.identity.object == "a") + .expect("stale retry outcome"); + assert_eq!(stale.identity.bucket_incarnation_id, Some(original)); + assert_eq!(stale.disposition, HealObjectDisposition::Unknown); + let other = outcome + .objects + .iter() + .find(|item| item.identity.object == "b") + .expect("b outcome"); + assert_eq!(other.identity.bucket_incarnation_id, Some(original)); + assert_eq!(other.disposition, HealObjectDisposition::Unknown); +} + +#[tokio::test(start_paused = true)] +async fn stale_retry_preserves_verified_receipt_from_another_inflight_replacement() { + let original = Uuid::new_v4(); + let mut storage = pages_storage(&[&["a", "b"]]); + *storage.bucket_incarnation_id.lock().expect("incarnation") = Some(original); + storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31)); + storage.heal_object_receipts.lock().expect("receipts").insert( + "b".to_string(), + VecDeque::from([object_receipt("b", None, HealObjectDisposition::Repaired, original)]), + ); + fail_once(&storage, "a"); + let storage = Arc::new(storage); + let mut task = bucket_task(storage.clone()); + task.bucket_incarnation_id = Some(original); + *task.replacement_execution.write().await = + Some(crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new())); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err()); + *storage.bucket_incarnation_id.lock().expect("incarnation") = Some(Uuid::new_v4()); + assert!(matches!(execution.await, Err(Error::StaleBucketIncarnation { .. }))); + let outcome = task.get_outcome().await; + let repaired = outcome + .objects + .iter() + .find(|item| item.identity.object == "b") + .expect("b outcome"); + assert_eq!(repaired.identity.bucket_incarnation_id, Some(original)); + assert_eq!(repaired.disposition, HealObjectDisposition::Repaired); +} + +#[tokio::test(start_paused = true)] +async fn stale_retry_does_not_start_other_queued_retries_while_draining() { + let original = Uuid::new_v4(); + let mut storage = pages_storage(&[&["a", "c", "b"]]); + *storage.bucket_incarnation_id.lock().expect("incarnation") = Some(original); + storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31)); + fail_once(&storage, "a"); + fail_once(&storage, "c"); + let storage = Arc::new(storage); + let mut task = bucket_task(storage.clone()); + task.bucket_incarnation_id = Some(original); + *task.replacement_execution.write().await = + Some(crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new())); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err()); + assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "c", "b"]); + let before = storage.bucket_incarnation_reads.load(Ordering::Relaxed); + *storage.bucket_incarnation_id.lock().expect("incarnation") = Some(Uuid::new_v4()); + assert!(matches!(execution.await, Err(Error::StaleBucketIncarnation { .. }))); + assert_eq!(storage.bucket_incarnation_reads.load(Ordering::Relaxed), before + 1); +} + +#[tokio::test(start_paused = true)] +async fn slow_retries_keep_inflight_work_bounded() { + let names: Vec = (0..10).map(|index| format!("retry-{index}")).collect(); + let mut page: Vec<_> = names.iter().map(|name| heal_item(name)).collect(); + page.push(heal_item("slow")); + let mut storage = MockStorage { + retry_test_pages: Some(vec![page]), + ..Default::default() + }; + storage.retry_test_delays.insert("slow".to_string(), Duration::from_secs(31)); + for name in &names { + fail_once(&storage, name); + storage + .retry_test_attempt_delays + .lock() + .expect("attempt delays") + .insert(name.clone(), VecDeque::from([Duration::ZERO, Duration::from_secs(31)])); + } + let storage = Arc::new(storage); + let task = bucket_task(storage.clone()); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err()); + assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 18); + assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 10); + task.cancel_token.cancel(); + assert!(matches!(execution.await, Err(Error::TaskCancelled))); + tokio::time::advance(Duration::from_secs(60)).await; + assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 18); +} + +#[tokio::test(start_paused = true)] +async fn inline_retry_is_not_starved_by_slow_deferred_retries() { + let names: Vec<_> = (0..256).map(|index| format!("slow-{index:03}")).collect(); + let mut page: Vec<_> = names.iter().map(|name| heal_item(name)).collect(); + page.push(heal_item("inline")); + let storage = MockStorage { + retry_test_pages: Some(vec![page]), + ..Default::default() + }; + for name in &names { + fail_once(&storage, name); + storage + .retry_test_attempt_delays + .lock() + .expect("delays") + .insert(name.clone(), VecDeque::from([Duration::ZERO, Duration::from_secs(31)])); + } + fail_once(&storage, "inline"); + let storage = Arc::new(storage); + let task = bucket_task(storage.clone()); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err()); + assert_eq!( + storage + .heal_object_calls + .lock() + .expect("calls") + .iter() + .filter(|name| name.as_str() == "inline") + .count(), + 2, + "the overflow identity must be retried before slow work occupies every slot" + ); + task.cancel_token.cancel(); + assert!(matches!(execution.await, Err(Error::TaskCancelled))); +} + +#[tokio::test(start_paused = true)] +async fn concurrent_retry_overflow_preserves_every_attempted_identity() { + assert_concurrent_retry_overflow_preserves_identities(true).await; +} + +#[tokio::test(start_paused = true)] +async fn concurrent_retry_overflow_completes_every_attempted_identity() { + assert_concurrent_retry_overflow_preserves_identities(false).await; +} + +async fn assert_concurrent_retry_overflow_preserves_identities(cancel: bool) { + let budget = super::super::heal_bucket::deferred_payload_budget_for_test(); + let mut lengths = vec![100; 7]; + let mut remaining = budget - 300 - 700; + while remaining > 1_000 { + lengths.push(1_000); + remaining -= 1_000; + } + if remaining > 0 && remaining < 4 { + *lengths.last_mut().expect("previous key") -= 4 - remaining; + remaining = 4; + } + if remaining != 0 { + lengths.push(remaining); + } + assert!(lengths.len() < 256); + let names: Vec<_> = lengths + .into_iter() + .enumerate() + .map(|(index, length)| format!("{index:04}{}", "x".repeat(length - 4))) + .collect(); + let extra = format!("extra{}", "y".repeat(895)); + let mut page: Vec<_> = names.iter().map(|name| heal_item(name)).collect(); + page.push(heal_item(&extra)); + let storage = MockStorage { + retry_test_pages: Some(vec![page]), + ..Default::default() + }; + for (index, name) in names.iter().enumerate() { + storage.heal_object_outcomes.lock().expect("outcomes").insert( + name.clone(), + VecDeque::from([MockHealObjectOutcome::RetryableLock, MockHealObjectOutcome::RetryableLock]), + ); + let delays = if index < 7 { + vec![Duration::ZERO, Duration::from_millis(1200)] + } else if index == 7 { + vec![Duration::from_millis(1500)] + } else { + Vec::new() + }; + storage + .retry_test_attempt_delays + .lock() + .expect("delays") + .insert(name.clone(), delays.into()); + } + storage.heal_object_outcomes.lock().expect("outcomes").insert( + extra.clone(), + VecDeque::from([MockHealObjectOutcome::RetryableLock, MockHealObjectOutcome::RetryableLock]), + ); + storage.retry_test_attempt_delays.lock().expect("delays").insert( + extra, + VecDeque::from([ + Duration::from_millis(1100), + if cancel { + Duration::from_secs(31) + } else { + Duration::from_secs(2) + }, + ]), + ); + let storage = Arc::new(storage); + let task = bucket_task(storage.clone()); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(5), &mut execution).await.is_err()); + if cancel { + task.cancel_token.cancel(); + assert!(matches!(execution.await, Err(Error::TaskCancelled))); + } else { + execution.await.expect("all attempted identities finish"); + } + assert_eq!( + task.get_outcome().await.counters.processed, + u64::try_from(names.len() + 1).expect("count") + ); +} + +#[tokio::test(start_paused = true)] +async fn due_retry_does_not_overlap_another_version_of_same_object() { + let mut first = heal_item("a"); + first.version_id = Some("first".to_string()); + let mut second = heal_item("a"); + second.version_id = Some("second".to_string()); + let mut storage = MockStorage { + retry_test_pages: Some(vec![vec![first, second]]), + ..Default::default() + }; + storage.retry_test_delays.insert("a".to_string(), Duration::from_secs(31)); + fail_once(&storage, "a"); + let storage = Arc::new(storage); + let task = bucket_task(storage.clone()); + let execution = task.execute(); + tokio::pin!(execution); + assert!(tokio::time::timeout(Duration::from_secs(45), &mut execution).await.is_err()); + assert_eq!( + storage.heal_object_calls.lock().expect("calls").as_slice(), + ["a", "a"], + "a retry must not overlap an in-flight version of the same object" + ); +} + +#[tokio::test(start_paused = true)] +async fn retry_uses_remaining_time_before_age_limit() { + let mut storage = pages_storage(&[&["a", "b"]]); + storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(27)); + storage.heal_object_outcomes.lock().expect("outcomes").insert( + "a".to_string(), + VecDeque::from([MockHealObjectOutcome::RetryableLock, MockHealObjectOutcome::RetryableLock]), + ); + let storage = Arc::new(storage); + let task = bucket_task(storage.clone()); + task.execute().await.expect("the last retry can run before the age limit"); + assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a", "a"]); + assert_eq!(task.get_outcome().await.counters.failed, 0); +} + +#[tokio::test(start_paused = true)] +async fn expired_retry_is_terminal_without_an_extra_storage_attempt() { + let mut first = heal_item("a"); + first.version_id = Some("first".to_string()); + let mut second = heal_item("a"); + second.version_id = Some("second".to_string()); + let mut storage = MockStorage { + retry_test_pages: Some(vec![vec![first, second]]), + ..Default::default() + }; + storage.retry_test_delays.insert("a".to_string(), Duration::from_secs(31)); + fail_once(&storage, "a"); + let storage = Arc::new(storage); + let task = bucket_task(storage.clone()); + task.execute().await.expect_err("aged pending responsibility is not success"); + assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "a"]); let outcome = task.get_outcome().await; assert_eq!(outcome.counters.processed, 2); assert_eq!(outcome.counters.attempt_failures, 1); @@ -310,7 +767,7 @@ async fn expired_retry_is_terminal_without_an_extra_storage_attempt() { outcome .objects .iter() - .find(|item| item.identity.object == "a") + .find(|item| item.identity.version_id.as_deref() == Some("first")) .expect("a outcome") .disposition, HealObjectDisposition::Failed(HealFailureClass::RetryExhausted) @@ -345,13 +802,26 @@ async fn cancellation_drains_owned_retries_once() { #[tokio::test(start_paused = true)] async fn deadline_drains_owned_retries_without_false_completion() { - let storage = Arc::new(pages_storage(&[&["a"], &["b"]])); + let incarnation = Uuid::new_v4(); + let storage = pages_storage(&[&["a"], &["b"]]); + *storage.bucket_incarnation_id.lock().expect("incarnation") = Some(incarnation); + let storage = Arc::new(storage); fail_once(&storage, "a"); let mut task = bucket_task(storage.clone()); task.options.timeout = Some(Duration::from_secs(1)); assert!(matches!(task.execute().await, Err(Error::TaskTimeout))); let outcome = task.get_outcome().await; assert_eq!(outcome.counters.processed, 2); + assert_eq!( + outcome + .objects + .iter() + .find(|item| item.identity.object == "a") + .expect("a") + .identity + .bucket_incarnation_id, + Some(incarnation) + ); assert_eq!( outcome .objects diff --git a/crates/heal/src/heal/task/tests/deferred_retry_window.rs b/crates/heal/src/heal/task/tests/deferred_retry_window.rs index fe93aa1e0..29ccf66cd 100644 --- a/crates/heal/src/heal/task/tests/deferred_retry_window.rs +++ b/crates/heal/src/heal/task/tests/deferred_retry_window.rs @@ -41,7 +41,7 @@ async fn count_cap_and_next_item_preserve_ownership() { assert_eq!(window.objects.len(), MAX_DEFERRED_OBJECTS); assert!(window.bytes <= MAX_DEFERRED_BYTES); assert!(!window.can_advance(1)); - assert!(window.pop_due().is_some()); + assert!(window.pop_due(&HashSet::new()).is_some()); assert!(window.push(rejected).is_ok()); } @@ -56,20 +56,35 @@ async fn byte_cap_counts_key_version_and_reserved_slots() { assert_eq!(window.bytes, MAX_DEFERRED_BYTES); assert!(!window.can_advance(1)); assert!(window.push(item("x".to_string(), None)).is_err()); - assert!(window.pop_due().is_some()); + assert!(window.pop_due(&HashSet::new()).is_some()); assert_eq!(window.bytes, MAX_DEFERRED_OBJECTS * size_of::()); assert!(window.push(item("x".to_string(), None)).is_ok()); } +#[tokio::test(start_paused = true)] +async fn due_retry_excludes_inflight_keys_without_hiding_other_keys() { + let mut window = DeferredWindow::default(); + assert!(window.push(item("same".to_owned(), Some("v1".to_owned()))).is_ok()); + assert!(window.push(item("other".to_owned(), None)).is_ok()); + let mut in_flight = HashSet::from(["same".to_owned()]); + assert_eq!(window.pop_due(&in_flight).expect("unblocked retry").name, "other"); + assert!(window.next_due(&in_flight).is_none()); + assert!(window.pop_due(&in_flight).is_none()); + in_flight.clear(); + assert!(window.next_due(&in_flight).is_some()); + assert_eq!(window.pop_due(&in_flight).expect("released retry").name, "same"); +} + #[tokio::test(start_paused = true)] async fn retry_age_caps_due_time_and_is_not_reset_by_rescheduling() { let mut entry = item("a".to_string(), None); entry.defer(Duration::from_secs(2)); let first = entry.first_failure.expect("first failure"); + assert_eq!(entry.due, first + Duration::from_secs(2)); tokio::time::advance(Duration::from_secs(29)).await; entry.defer(Duration::from_secs(8)); assert_eq!(entry.first_failure, Some(first)); - assert_eq!(entry.due, first + MAX_DEFERRED_AGE); + assert_eq!(entry.due, first + Duration::from_millis(29_500)); assert!(!entry.expired()); tokio::time::advance(Duration::from_secs(1)).await; assert!(entry.expired()); diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index b9b93460c..7167a88cb 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -194,13 +194,21 @@ async fn empty_replacement_with_committed_delete( mod serial_tests { use super::*; + use storage_api::integration::InstanceContext; async fn bucket_directory_loss_task(cluster: bool, missing: usize, dry_run: bool, source: HealRequestSource) { let temp = tempfile::tempdir().expect("isolated bucket recovery fixture"); let env = rustfs_test_utils::TestECStoreEnv::builder() .base_dir(temp.path()) + .instance_context(Arc::new(InstanceContext::new())) .build() .await; + let shutdown = env + .ecstore + .background_cancel_token() + .expect("fixture store must own a shutdown token"); + assert!(!shutdown.is_cancelled(), "a new fixture must not inherit a previous store's shutdown"); + eprintln!("bucket recovery: environment ready"); let bucket = "task-directory-recovery"; env.make_bucket(bucket, false).await; let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone())); @@ -213,6 +221,7 @@ mod serial_tests { .await .expect("remove only an empty bucket directory"); } + eprintln!("bucket recovery: directories removed"); assert_eq!(env.disk_paths.iter().filter(|disk| disk.join(bucket).exists()).count(), 4 - missing); let mut request = HealRequest::new( if cluster { @@ -237,6 +246,7 @@ mod serial_tests { } let task = HealTask::from_request(request, storage.clone()); let result = task.execute().await; + eprintln!("bucket recovery: task returned"); if missing == 4 || source != HealRequestSource::Admin { assert!( result.is_err(), @@ -259,6 +269,7 @@ mod serial_tests { }; assert_eq!(env.disk_paths.iter().filter(|disk| disk.join(bucket).is_dir()).count(), remaining); assert_eq!(storage.admit_bucket_incarnation(bucket).await.expect("retained metadata"), expected); + shutdown.cancel(); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -267,7 +278,13 @@ mod serial_tests { for cluster in [false, true] { // Two surviving directories satisfy ordinary reads; one does not. for missing in [2, 3] { - bucket_directory_loss_task(cluster, missing, false, HealRequestSource::Admin).await; + eprintln!("bucket recovery: cluster={cluster}, missing={missing}"); + tokio::time::timeout( + Duration::from_secs(60), + bucket_directory_loss_task(cluster, missing, false, HealRequestSource::Admin), + ) + .await + .unwrap_or_else(|_| panic!("bucket recovery did not finish: cluster={cluster}, missing={missing}")); } } } diff --git a/crates/heal/tests/storage_api.rs b/crates/heal/tests/storage_api.rs index 047ce6d3d..e60c2995a 100644 --- a/crates/heal/tests/storage_api.rs +++ b/crates/heal/tests/storage_api.rs @@ -30,6 +30,7 @@ pub(crate) mod integration { ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCommitGuard, WriteCompletion, }; pub(crate) use rustfs_ecstore::api::rebalance::{RebalStatus, RebalanceMeta, RebalanceStats}; + pub(crate) use rustfs_ecstore::api::runtime::InstanceContext; pub(crate) use rustfs_ecstore::api::storage::ECStore; pub(crate) use rustfs_storage_api::BucketOperations; pub(crate) use rustfs_storage_api::MakeBucketOptions; diff --git a/crates/lock/src/client/local.rs b/crates/lock/src/client/local.rs index 144db6c72..8c295195d 100644 --- a/crates/lock/src/client/local.rs +++ b/crates/lock/src/client/local.rs @@ -382,6 +382,42 @@ impl LockClient for LocalClient { } } + async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result { + let shard = self.get_shard(lock_id); + let mut guards = shard.write().await; + let Some(entry) = guards.get_mut(lock_id) else { + return Ok(false); + }; + if entry.is_expired() { + return Ok(false); + } + Ok(entry.guard.downgrade_committed_tail()) + } + + async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result { + let shard = self.get_shard(lock_id); + let mut guards = shard.write().await; + let Some(entry) = guards.get_mut(lock_id) else { return Ok(false) }; + if entry.is_expired() { + return Ok(false); + } + Ok(entry.guard.try_upgrade_committed_tail()) + } + + async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result { + self.refresh(lock_id).await + } + + async fn waiting_writer(&self, lock_id: &LockId) -> Result> { + let shard = self.get_shard(lock_id); + let guards = shard.read().await; + let Some(entry) = guards.get(lock_id) else { return Ok(None) }; + if entry.is_expired() { + return Ok(None); + } + Ok(entry.guard.waiting_writer()) + } + async fn force_release(&self, lock_id: &LockId) -> Result { self.release(lock_id).await } @@ -547,6 +583,32 @@ mod tests { assert!(!response.success); } + #[tokio::test] + async fn downgrade_tail_is_idempotent_and_release_preserves_readers() { + let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new())); + let resource = crate::ObjectKey::new("bucket", "tail-downgrade"); + let first = request(resource.clone(), "owner-a", Duration::from_secs(30)); + assert!(client.acquire_lock(&first).await.expect("initial lock").success); + assert!(client.downgrade_committed_tail(&first.lock_id).await.expect("downgrade")); + assert!( + client + .downgrade_committed_tail(&first.lock_id) + .await + .expect("retry downgrade") + ); + let reader = crate::LockRequest::new(resource.clone(), LockType::Shared, "reader") + .with_acquire_timeout(Duration::from_millis(500)); + assert!(client.acquire_lock(&reader).await.expect("reader").success); + let writer = request(resource, "writer", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10)); + assert!(!client.acquire_lock(&writer).await.expect("writer blocked").success); + assert!(client.release(&first.lock_id).await.expect("release tail")); + assert!(!client.acquire_lock(&writer).await.expect("reader excludes writer").success); + assert!(client.release(&reader.lock_id).await.expect("release reader")); + assert!(client.acquire_lock(&writer).await.expect("writer after drain").success); + assert!(client.release(&writer.lock_id).await.expect("release writer")); + assert!(!client.downgrade_committed_tail(&first.lock_id).await.expect("released lease")); + } + #[tokio::test] async fn acquire_rejects_a_lock_id_for_another_resource() { let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new())); diff --git a/crates/lock/src/client/mod.rs b/crates/lock/src/client/mod.rs index bf378c3bf..ee60fec48 100644 --- a/crates/lock/src/client/mod.rs +++ b/crates/lock/src/client/mod.rs @@ -20,6 +20,12 @@ use async_trait::async_trait; use futures::future::join_all; use std::sync::Arc; +/// Marker exchanged in a refresh request and response for a committed-tail conversion. +pub const COMMITTED_TAIL_CONVERSION_MARKER: &str = "rustfs-committed-tail-v1"; +pub const COMMITTED_TAIL_UPGRADE_MARKER: &str = "rustfs-committed-tail-upgrade-v1"; +pub const HEAL_LOCK_TRANSITION_PROBE_MARKER: &str = "rustfs-heal-lock-transition-probe-v1"; +pub const HEAL_WAITING_WRITER_MARKER: &str = "rustfs-heal-waiting-writer-v1"; + /// Lock client trait #[async_trait] pub trait LockClient: Send + Sync + std::fmt::Debug { @@ -48,6 +54,27 @@ pub trait LockClient: Send + Sync + std::fmt::Debug { /// Refresh lock async fn refresh(&self, lock_id: &LockId) -> Result; + /// Convert an exclusive lease to a reader-compatible committed-tail lease. + /// Unsupported peers retain their original lease and report `false`. + async fn downgrade_committed_tail(&self, _lock_id: &LockId) -> Result { + Ok(false) + } + + /// Try to restore exclusive ownership, retaining the writer fence on failure. + async fn try_upgrade_committed_tail(&self, _lock_id: &LockId) -> Result { + Ok(false) + } + + /// Check support without changing the lease mode. + async fn supports_heal_lock_transition(&self, _lock_id: &LockId) -> Result { + Ok(false) + } + + /// Observe foreground writer contention, or return `None` when unsupported. + async fn waiting_writer(&self, _lock_id: &LockId) -> Result> { + Ok(None) + } + /// Force release lock async fn force_release(&self, lock_id: &LockId) -> Result; diff --git a/crates/lock/src/distributed_lock.rs b/crates/lock/src/distributed_lock.rs index 7d9a71cbb..7d6751ad0 100644 --- a/crates/lock/src/distributed_lock.rs +++ b/crates/lock/src/distributed_lock.rs @@ -262,6 +262,8 @@ pub struct DistributedLockGuard { /// All underlying leases that should be refreshed and released with the guard. entries: Vec, lock_type: LockType, + /// Shared and exclusive quorums must intersect before a tail may convert. + committed_tail_quorums_intersect: bool, /// If true, Drop will not try to release (used if user manually released). disarmed: bool, /// Background lease task for renewable leases. @@ -277,7 +279,13 @@ impl DistributedLockGuard { /// - `entries` is the full list of underlying leases that should be refreshed and released /// with this guard. /// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics. - fn new(lock_id: LockId, entries: Vec, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self { + fn new( + lock_id: LockId, + entries: Vec, + lock_type: LockType, + heartbeat: LockHeartbeatConfig, + committed_tail_quorums_intersect: bool, + ) -> Self { record_lock_held_acquire(lock_type); let lock_lost = Arc::new(LockLostSignal::default()); match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) { @@ -296,6 +304,7 @@ impl DistributedLockGuard { lock_id, entries, lock_type, + committed_tail_quorums_intersect, disarmed: false, refresh_task, lock_lost, @@ -412,6 +421,70 @@ impl DistributedLockGuard { &self.lock_id } + /// Convert retained peer leases without ever releasing writer exclusion. + /// A peer without conversion support keeps its exclusive lease. Partial + /// conversion is safe: both lease modes continue to exclude writers. + pub async fn downgrade_committed_tail(&mut self) -> bool { + if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost() + { + return false; + } + let results = futures::future::join_all( + self.entries + .iter() + .map(|entry| entry.client.downgrade_committed_tail(&entry.lock_id)), + ) + .await; + !self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true))) + } + + /// Require every retained lease to confirm exclusivity before committing. + pub async fn try_upgrade_committed_tail(&mut self) -> bool { + if self.disarmed || self.lock_type != LockType::Exclusive || self.is_lock_lost() { + return false; + } + let results = futures::future::join_all( + self.entries + .iter() + .map(|entry| entry.client.try_upgrade_committed_tail(&entry.lock_id)), + ) + .await; + !self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true))) + } + + pub async fn supports_heal_lock_transition(&self) -> bool { + if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost() + { + return false; + } + let results = futures::future::join_all( + self.entries + .iter() + .map(|entry| entry.client.supports_heal_lock_transition(&entry.lock_id)), + ) + .await; + !self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true))) + } + + /// Observe contention on every held lease. Unknown peers prevent a false idle result. + pub async fn waiting_writer(&self) -> Option { + if self.disarmed || self.is_lock_lost() { + return None; + } + let results = + futures::future::join_all(self.entries.iter().map(|entry| entry.client.waiting_writer(&entry.lock_id))).await; + if self.is_lock_lost() { + return None; + } + if results.iter().any(|result| matches!(result, Ok(Some(true)))) { + return Some(true); + } + results + .iter() + .all(|result| matches!(result, Ok(Some(false)))) + .then_some(false) + } + /// Actual granted peer identities, which may differ from the logical guard ID. pub fn granted_lock_ids(&self) -> impl Iterator { self.entries.iter().map(|entry| &entry.lock_id) @@ -599,6 +672,7 @@ impl DistributedLock { individual_locks, request.lock_type, heartbeat, + self.read_quorum().saturating_add(required_quorum) > self.clients.len(), ))) } else { // Check if it's a timeout or quorum failure diff --git a/crates/lock/src/fast_lock/guard.rs b/crates/lock/src/fast_lock/guard.rs index da2ece113..5e628abfb 100644 --- a/crates/lock/src/fast_lock/guard.rs +++ b/crates/lock/src/fast_lock/guard.rs @@ -19,8 +19,8 @@ use crate::fast_lock::{ use rustfs_io_metrics::{ record_read_lock_held_acquire, record_read_lock_held_release, record_write_lock_held_acquire, record_write_lock_held_release, }; -use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Weak}; /// Global counter for guard IDs to prevent double-release static GUARD_ID_COUNTER: AtomicU64 = AtomicU64::new(1); @@ -39,10 +39,25 @@ pub struct FastLockGuard { shard: Option>, // None when locks are disabled released: bool, disabled: bool, // True when locks are disabled globally + committed_tail: bool, + sealed_for_commit: bool, /// Unique ID for this guard instance to prevent double-release guard_id: u64, } +/// A non-owning check of one local acquisition, including forced release. +#[derive(Clone)] +pub struct FastLockGuardProbe { + shard: Weak, + guard_id: u64, +} + +impl FastLockGuardProbe { + pub fn is_lost(&self) -> bool { + self.shard.upgrade().is_none_or(|shard| !shard.is_guard_active(self.guard_id)) + } +} + impl FastLockGuard { pub(crate) fn new(key: ObjectKey, mode: LockMode, owner: Arc, shard: Arc) -> Self { let guard_id = GUARD_ID_COUNTER.fetch_add(1, Ordering::Relaxed); @@ -54,6 +69,8 @@ impl FastLockGuard { shard: Some(shard), released: false, disabled: false, + committed_tail: false, + sealed_for_commit: false, guard_id, } } @@ -68,6 +85,8 @@ impl FastLockGuard { shard: None, released: false, disabled: true, + committed_tail: false, + sealed_for_commit: false, guard_id, } } @@ -87,6 +106,83 @@ impl FastLockGuard { &self.owner } + /// The physical acquisition was force-released before registration. + pub(crate) fn abandon_unregistered(&mut self) { + if !self.released { + self.released = true; + record_lock_held_release(self.mode); + } + } + + /// Keep excluding writers while permitting reads of the committed version. + pub fn downgrade_committed_tail(&mut self) -> bool { + if self.released || self.sealed_for_commit { + return false; + } + if self.committed_tail { + return true; + } + if self.mode != LockMode::Exclusive { + return false; + } + if self.disabled { + self.mode = LockMode::Shared; + self.committed_tail = true; + return true; + } + let Some(shard) = &self.shard else { return false }; + if !shard.downgrade_committed_tail(&self.key, &self.owner, self.guard_id) { + return false; + } + record_lock_held_release(LockMode::Exclusive); + record_lock_held_acquire(LockMode::Shared); + self.mode = LockMode::Shared; + self.committed_tail = true; + true + } + + /// Try to restore exclusive ownership without releasing the writer fence. + pub fn try_upgrade_committed_tail(&mut self) -> bool { + if self.released || (!self.committed_tail && self.mode != LockMode::Exclusive) { + return false; + } + if !self.disabled { + let Some(shard) = &self.shard else { return false }; + if !shard.try_upgrade_committed_tail(&self.key, &self.owner, self.guard_id) { + return false; + } + } + if self.committed_tail { + record_lock_held_release(LockMode::Shared); + record_lock_held_acquire(LockMode::Exclusive); + } + self.mode = LockMode::Exclusive; + self.committed_tail = false; + self.sealed_for_commit = true; + true + } + + /// Observe writers waiting on this guard's exact object key. + pub fn waiting_writer(&self) -> Option { + if self.released { + return None; + } + if self.disabled { + return Some(false); + } + self.shard + .as_ref()? + .waiting_writer_for_guard(&self.key, &self.owner, self.guard_id) + } + + fn release_physical(&self, shard: &LockShard) -> bool { + if self.committed_tail { + shard.release_committed_tail_with_guard(&self.key, &self.owner, self.guard_id) + } else { + shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id) + } + } + /// Manually release the lock early /// /// Returns true if the lock was successfully released, false if it was @@ -106,7 +202,7 @@ impl FastLockGuard { } if let Some(shard) = &self.shard { - let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id); + let success = self.release_physical(shard); if success { self.released = true; record_lock_held_release(self.mode); @@ -138,9 +234,17 @@ impl FastLockGuard { ); } - /// Check if the lock has been released + /// Check if the lock has been released, including by an administrator. pub fn is_released(&self) -> bool { - self.released + self.released || self.shard.as_ref().is_some_and(|shard| !shard.is_guard_active(self.guard_id)) + } + + /// Observe this acquisition without extending the lock's lifetime. + pub fn release_probe(&self) -> Option { + self.shard.as_ref().map(|shard| FastLockGuardProbe { + shard: Arc::downgrade(shard), + guard_id: self.guard_id, + }) } /// Check if this guard represents a disabled lock @@ -169,7 +273,7 @@ impl Drop for FastLockGuard { fn drop(&mut self) { if let Some(shard) = &self.shard { if !self.released && !self.disabled { - let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id); + let success = self.release_physical(shard); if !success { // For high-concurrency scenarios, this is likely due to: // 1. Lock was already released by another thread diff --git a/crates/lock/src/fast_lock/manager.rs b/crates/lock/src/fast_lock/manager.rs index 8bbada477..c62c7d453 100644 --- a/crates/lock/src/fast_lock/manager.rs +++ b/crates/lock/src/fast_lock/manager.rs @@ -96,11 +96,14 @@ impl FastObjectLockManager { /// Acquire object lock pub async fn acquire_lock(&self, request: ObjectLockRequest) -> Result { let shard = self.get_shard(&request.key); - match shard.acquire_lock(&request).await { - Ok(()) => { - let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone()); + match shard.acquire_guarded(&request).await { + Ok(acquired) => { + let mut guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone()); // Register guard to prevent premature cleanup - shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner()); + if !shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired) { + guard.abandon_unregistered(); + return Err(LockResult::Timeout); + } Ok(guard) } Err(err) => Err(err), @@ -201,31 +204,15 @@ impl FastObjectLockManager { let mut all_failed = Vec::new(); let mut guards = Vec::new(); - for (shard_id, requests) in shard_groups { - let shard = self.shards[*shard_id].clone(); - + for (_, requests) in shard_groups { for request in requests { let key = request.key.clone(); - let owner = request.owner.clone(); - let mode = request.mode; - - let acquired = if shard.try_fast_path_only(request) { - true - } else { - match shard.acquire_lock(request).await { - Ok(()) => true, - Err(err) => { - all_failed.push((key.clone(), err)); - false - } + match self.acquire_lock(request.clone()).await { + Ok(guard) => { + all_successful.push(key); + guards.push(guard); } - }; - - if acquired { - let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone()); - shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner()); - all_successful.push(key); - guards.push(guard); + Err(err) => all_failed.push((key, err)), } } } @@ -245,14 +232,10 @@ impl FastObjectLockManager { let mut acquired_guards = Vec::new(); let mut failed_locks = Vec::new(); - 'outer: for (shard_id, requests) in shard_groups { - let shard = self.shards[*shard_id].clone(); - + 'outer: for (_, requests) in shard_groups { for request in requests { - match shard.acquire_lock(request).await { - Ok(()) => { - let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone()); - shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner()); + match self.acquire_lock(request.clone()).await { + Ok(guard) => { acquired_guards.push(guard); } Err(err) => { diff --git a/crates/lock/src/fast_lock/object_pool.rs b/crates/lock/src/fast_lock/object_pool.rs index 779061e1b..63c3867ab 100644 --- a/crates/lock/src/fast_lock/object_pool.rs +++ b/crates/lock/src/fast_lock/object_pool.rs @@ -94,6 +94,7 @@ impl ObjectLockState { pub fn reset_for_reuse(&mut self) { // Reset atomic state self.atomic_state = crate::fast_lock::state::AtomicLockState::new(); + self.force_epoch.store(0, Ordering::Relaxed); // Clear owners *self.current_owner.write() = None; diff --git a/crates/lock/src/fast_lock/shard.rs b/crates/lock/src/fast_lock/shard.rs index aab9299bc..aa43fb5b1 100644 --- a/crates/lock/src/fast_lock/shard.rs +++ b/crates/lock/src/fast_lock/shard.rs @@ -15,6 +15,7 @@ use parking_lot::RwLock; use std::collections::HashMap; use std::sync::Arc; +use std::sync::atomic::Ordering; use std::time::{Duration, Instant, SystemTime}; use tokio::time::timeout; @@ -30,6 +31,12 @@ struct ActiveGuardInfo { key: ObjectKey, mode: LockMode, owner: Arc, + committed_tail: bool, +} + +pub(crate) struct AcquiredLock { + state: Arc, + force_epoch: u64, } #[derive(Debug, PartialEq, Eq, Hash)] @@ -46,6 +53,8 @@ enum TestPausePoint { BeforeFastWrite, FastWriteExisting, SlowLookup, + ForceAfterTailRelease, + ReleaseAfterUnregister, } #[cfg(test)] @@ -69,7 +78,7 @@ fn test_pause(point: TestPausePoint, key: &ObjectKey) { /// Lock shard to reduce global contention. /// -/// When both are needed, lock `objects` before a state's owner metadata. +/// When combined, lock `active_guards`, then `objects`, then a state's owner metadata. #[derive(Debug)] pub struct LockShard { /// Object lock states - using parking_lot for better performance @@ -133,12 +142,16 @@ impl LockShard { /// Acquire lock with fast path optimization pub async fn acquire_lock(&self, request: &ObjectLockRequest) -> Result<(), LockResult> { + self.acquire_guarded(request).await.map(|_| ()) + } + + pub(crate) async fn acquire_guarded(&self, request: &ObjectLockRequest) -> Result { let start_time = Instant::now(); // Try fast path first - if let Some(_state) = self.try_fast_path(request) { + if let Some(acquired) = self.try_fast_path(request) { self.metrics.record_fast_path_success(); - return Ok(()); + return Ok(acquired); } // Slow path with waiting @@ -157,7 +170,7 @@ impl LockShard { } /// Try fast path lock acquisition (lock-free when possible) - fn try_fast_path(&self, request: &ObjectLockRequest) -> Option> { + fn try_fast_path(&self, request: &ObjectLockRequest) -> Option { // First try to get existing state without write lock { let objects = self.objects.read(); @@ -168,14 +181,16 @@ impl LockShard { test_pause(TestPausePoint::FastExisting, &request.key); // Try atomic acquisition - let success = match request.mode { - LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout), - LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout), - }; + let force_epoch = state.force_epoch.load(Ordering::Acquire); + let success = force_epoch != u64::MAX + && match request.mode { + LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout), + LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout), + }; drop(objects); if success { - return Some(state); + return Some(AcquiredLock { state, force_epoch }); } } } @@ -195,10 +210,11 @@ impl LockShard { #[cfg(test)] test_pause(TestPausePoint::FastWriteExisting, &request.key); - let acquired = state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout); + let force_epoch = state.force_epoch.load(Ordering::Acquire); + let acquired = force_epoch != u64::MAX && state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout); drop(objects); if acquired { - return Some(state); + return Some(AcquiredLock { state, force_epoch }); } } else { // Create new state from pool and acquire immediately @@ -206,7 +222,7 @@ impl LockShard { let state = Arc::new(*state_box); if state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout) { objects.insert(request.key.clone(), state.clone()); - return Some(state); + return Some(AcquiredLock { state, force_epoch: 0 }); } } } @@ -215,7 +231,7 @@ impl LockShard { } /// Slow path with async waiting - async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<(), LockResult> { + async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result { // Use adaptive timeout based on current load and request priority let adaptive_timeout = self.calculate_adaptive_timeout(request); let deadline = start_time + adaptive_timeout; @@ -234,7 +250,7 @@ impl LockShard { loop { // Keep the state in the map until its acquisition attempt completes. - let (state, success) = { + let (state, success, force_epoch) = { let mut objects = self.objects.write(); let state = match objects.get(&request.key) { Some(state) => state.clone(), @@ -249,17 +265,19 @@ impl LockShard { #[cfg(test)] test_pause(TestPausePoint::SlowLookup, &request.key); - let success = match request.mode { - LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout), - LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout), - }; + let force_epoch = state.force_epoch.load(Ordering::Acquire); + let success = force_epoch != u64::MAX + && match request.mode { + LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout), + LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout), + }; drop(objects); - (state, success) + (state, success, force_epoch) }; if success { self.metrics.record_slow_path_success(); - return Ok(()); + return Ok(AcquiredLock { state, force_epoch }); } // Check timeout @@ -381,14 +399,10 @@ impl LockShard { /// Release lock with guard ID tracking for double-release prevention pub fn release_lock_with_guard(&self, key: &ObjectKey, owner: &Arc, mode: LockMode, guard_id: u64) -> bool { - // First, try to remove the guard from active set - let guard_was_active = { - let mut guards = self.active_guards.lock(); - guards.remove(&guard_id).is_some() - }; - - // If guard was not active, this is a double-release attempt - if !guard_was_active { + // Keep force-unlock from releasing this holder and admitting a new + // same-owner holder between guard removal and physical release. + let mut guards = self.active_guards.lock(); + if guards.remove(&guard_id).is_none() { tracing::debug!( "Double-release attempt blocked: key={}, owner={}, mode={:?}, guard_id={}", key, @@ -399,6 +413,9 @@ impl LockShard { return false; } + #[cfg(test)] + test_pause(TestPausePoint::ReleaseAfterUnregister, key); + // Proceed with normal release let should_cleanup; let result; @@ -423,6 +440,8 @@ impl LockShard { } } + drop(guards); + if should_cleanup { self.schedule_cleanup(key.clone()); } @@ -430,21 +449,110 @@ impl LockShard { result } + /// Convert an active exclusive guard without opening a writer gap. + pub(crate) fn downgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc, guard_id: u64) -> bool { + let mut guards = self.active_guards.lock(); + let Some(Some(info)) = guards.get_mut(&guard_id) else { return false }; + if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Exclusive { + return false; + } + let objects = self.objects.read(); + let Some(state) = objects.get(key) else { return false }; + if !state.downgrade_committed_tail(owner) { + return false; + } + info.mode = LockMode::Shared; + info.committed_tail = true; + true + } + + pub(crate) fn try_upgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc, guard_id: u64) -> bool { + let mut guards = self.active_guards.lock(); + let Some(Some(info)) = guards.get_mut(&guard_id) else { return false }; + if info.key != *key || info.owner.as_ref() != owner.as_ref() { + return false; + } + if info.mode == LockMode::Exclusive && !info.committed_tail { + return true; + } + if info.mode != LockMode::Shared || !info.committed_tail { + return false; + } + let objects = self.objects.read(); + let Some(state) = objects.get(key) else { return false }; + if !state.try_upgrade_committed_tail(owner) { + return false; + } + info.mode = LockMode::Exclusive; + info.committed_tail = false; + true + } + + pub(crate) fn waiting_writer_for_guard(&self, key: &ObjectKey, owner: &Arc, guard_id: u64) -> Option { + let guards = self.active_guards.lock(); + let info = guards.get(&guard_id)?.as_ref()?; + if info.key != *key || info.owner.as_ref() != owner.as_ref() { + return None; + } + let objects = self.objects.read(); + let state = objects.get(key)?; + Some(state.atomic_state.writer_observed_during_tail()) + } + + /// Release a converted guard and its special reader-admission state. + pub(crate) fn release_committed_tail_with_guard(&self, key: &ObjectKey, owner: &Arc, guard_id: u64) -> bool { + let mut guards = self.active_guards.lock(); + let Some(Some(info)) = guards.get(&guard_id) else { return false }; + if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Shared || !info.committed_tail { + return false; + } + let (success, should_cleanup) = { + let objects = self.objects.read(); + let Some(state) = objects.get(key) else { return false }; + let success = state.release_committed_tail(owner); + (success, success && !state.is_locked() && !state.atomic_state.has_waiters()) + }; + if success { + guards.remove(&guard_id); + self.metrics.record_release(); + } + drop(guards); + if should_cleanup { + self.schedule_cleanup(key.clone()); + } + success + } + /// Register a guard to prevent premature cleanup pub fn register_guard(&self, guard_id: u64) { self.active_guards.lock().insert(guard_id, None); } - pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc) { + pub(crate) fn register_guard_with_info( + &self, + guard_id: u64, + key: &ObjectKey, + mode: LockMode, + owner: &Arc, + acquired: &AcquiredLock, + ) -> bool { let mut guards = self.active_guards.lock(); + let objects = self.objects.read(); + if !objects.get(key).is_some_and(|state| { + Arc::ptr_eq(state, &acquired.state) && state.force_epoch.load(Ordering::Acquire) == acquired.force_epoch + }) { + return false; + } guards.insert( guard_id, Some(ActiveGuardInfo { key: key.clone(), mode, owner: owner.clone(), + committed_tail: false, }), ); + true } /// Unregister a guard (called when guard is dropped) @@ -460,9 +568,8 @@ impl LockShard { guards.len() } - /// Check if a guard is active (for testing) - #[cfg(test)] - pub fn is_guard_active(&self, guard_id: u64) -> bool { + /// Check whether this exact guard still owns its acquisition. + pub(crate) fn is_guard_active(&self, guard_id: u64) -> bool { let guards = self.active_guards.lock(); guards.contains_key(&guard_id) } @@ -719,27 +826,57 @@ impl LockShard { /// Returns the number of owners that were released. Used by the admin /// force-unlock path to clear a stuck resource. pub fn force_release_all(&self, key: &ObjectKey) -> usize { - let owners_modes: Vec<(Arc, LockMode)> = { - let objects = self.objects.read(); - let Some(state) = objects.get(key) else { - return 0; - }; - let mut pairs = Vec::new(); - if let Some(info) = state.current_owner.read().clone() { - pairs.push((info.owner, LockMode::Exclusive)); - } - for entry in state.shared_owners.read().iter() { - pairs.push((entry.owner.clone(), LockMode::Shared)); - } - pairs - }; - + // Serialize tail conversion and prevent new acquisitions while taking + // and releasing the owner snapshot. A new tail must never be mistaken + // for an ordinary shared owner by force unlock. + let mut guards = self.active_guards.lock(); + let objects = self.objects.write(); + let Some(state) = objects.get(key) else { return 0 }; + // Acquisitions completed before this point but not yet registered + // must not later create guards for the released generation. + let _ = state + .force_epoch + .try_update(Ordering::AcqRel, Ordering::Acquire, |epoch| epoch.checked_add(1)); + let tail = guards.iter().find_map(|(&id, info)| { + info.as_ref() + .filter(|info| info.committed_tail && info.key == *key) + .map(|info| (id, info.owner.clone())) + }); let mut released = 0; - for (owner, mode) in owners_modes { - if self.release_lock(key, &owner, mode) { + if let Some((id, owner)) = tail + && state.release_committed_tail(&owner) + { + guards.remove(&id); + self.metrics.record_release(); + released += 1; + } + #[cfg(test)] + test_pause(TestPausePoint::ForceAfterTailRelease, key); + let mut owners_modes = Vec::new(); + if let Some(info) = state.current_owner.read().clone() { + owners_modes.push((info.owner, LockMode::Exclusive, 1)); + } + for entry in state.shared_owners.read().iter() { + owners_modes.push((entry.owner.clone(), LockMode::Shared, entry.count)); + } + for (owner, mode, count) in owners_modes { + let mut owner_released = false; + for _ in 0..count { + let success = match mode { + LockMode::Shared => state.release_shared(&owner), + LockMode::Exclusive => state.release_exclusive(&owner), + }; + if success { + self.metrics.record_release(); + owner_released = true; + } + } + if owner_released { released += 1; } } + // Old guard drops must not release a later holder with the same owner. + guards.retain(|_, info| info.as_ref().is_none_or(|info| info.key != *key)); released } @@ -979,6 +1116,320 @@ mod tests { ) } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn force_unlock_cannot_interleave_an_old_release_with_a_new_holder() { + for reader in [false, true] { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "force-release-gap"); + let old = if reader { + manager.acquire_read_lock(key.clone(), "same-owner").await + } else { + manager.acquire_write_lock(key.clone(), "same-owner").await + } + .expect("old holder"); + let shard = manager.get_shard(&key).clone(); + let (entered, resume) = install_pause(TestPausePoint::ReleaseAfterUnregister, &key); + let old_drop = tokio::task::spawn_blocking(move || drop(old)); + entered.recv_timeout(Duration::from_secs(5)).expect("old release paused"); + + let can_force = shard.active_guards.try_lock().is_some(); + let mut force = { + let manager = manager.clone(); + let key = key.clone(); + tokio::task::spawn_blocking(move || manager.force_unlock(&key)) + }; + let mut new_guard = if can_force { + assert_eq!(force.await.expect("force join"), 1); + let mut guard = manager + .acquire_write_lock(key.clone(), "same-owner") + .await + .expect("new writer"); + if reader { + assert!(guard.downgrade_committed_tail()); + } + resume.send(()).expect("resume old release"); + old_drop.await.expect("old release join"); + guard + } else { + assert!(tokio::time::timeout(Duration::from_millis(100), &mut force).await.is_err()); + resume.send(()).expect("resume old release"); + old_drop.await.expect("old release join"); + assert_eq!(force.await.expect("force join"), 0); + manager + .acquire_write_lock(key.clone(), "same-owner") + .await + .expect("new writer") + }; + if reader && !can_force { + assert!(new_guard.downgrade_committed_tail()); + } + assert!( + tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor")) + .await + .is_err(), + "an old release must not unlock the new writer" + ); + drop(new_guard); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn force_unlock_does_not_capture_a_new_committed_tail() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "force-tail-race"); + let mut old = manager.acquire_write_lock(key.clone(), "old").await.expect("old writer"); + assert!(old.downgrade_committed_tail()); + let (entered, resume) = install_pause(TestPausePoint::ForceAfterTailRelease, &key); + let force = { + let manager = manager.clone(); + let key = key.clone(); + tokio::task::spawn_blocking(move || manager.force_unlock(&key)) + }; + entered.recv_timeout(Duration::from_secs(5)).expect("force unlock paused"); + let mut next = { + let manager = manager.clone(); + let key = key.clone(); + tokio::spawn(async move { manager.acquire_write_lock(key, "new").await }) + }; + assert!(tokio::time::timeout(Duration::from_millis(100), &mut next).await.is_err()); + resume.send(()).expect("resume force unlock"); + assert_eq!(force.await.expect("force unlock task"), 1); + let mut new = next.await.expect("new writer task").expect("new writer"); + assert!(new.downgrade_committed_tail()); + drop(old); + assert!( + tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor")) + .await + .is_err(), + "the new committed tail must still exclude writers" + ); + } + + #[tokio::test] + async fn committed_tail_upgrade_waits_for_readers_and_keeps_writer_fenced() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "upgrade-tail"); + let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock"); + assert!(tail.downgrade_committed_tail()); + let reader = manager + .acquire_read_lock(key.clone(), "reader") + .await + .expect("reader during tail"); + assert!(!tail.try_upgrade_committed_tail(), "an active reader must prevent exclusive commit"); + assert!( + tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "late-reader")) + .await + .is_err(), + "new readers must not starve a pending upgrade" + ); + drop(reader); + assert!(tail.try_upgrade_committed_tail()); + assert!(tail.try_upgrade_committed_tail(), "a repeated request must be idempotent"); + assert!( + !tail.downgrade_committed_tail(), + "a delayed downgrade must not reopen reads after upgrade" + ); + assert!( + tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "reader")) + .await + .is_err() + ); + drop(tail); + let _writer = manager.acquire_write_lock(key, "writer").await.expect("writer after release"); + } + + #[tokio::test] + async fn tail_remembers_a_writer_that_stopped_waiting_between_samples() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "brief-writer-observation"); + let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock"); + assert!(tail.downgrade_committed_tail()); + assert!( + tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer")) + .await + .is_err() + ); + assert_eq!(tail.waiting_writer(), Some(true), "a short wait must not be missed by polling"); + assert!(!tail.try_upgrade_committed_tail(), "observed contention must prevent a later commit"); + drop(tail); + let next = manager.acquire_write_lock(key, "next").await.expect("next lock"); + assert_eq!(next.waiting_writer(), Some(false), "contention must not leak to the next owner"); + } + + #[tokio::test] + async fn upgraded_tail_remembers_a_writer_that_stopped_waiting() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "upgraded-tail-brief-writer"); + let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock"); + assert!(tail.downgrade_committed_tail()); + assert!(tail.try_upgrade_committed_tail()); + assert!( + tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer")) + .await + .is_err() + ); + assert_eq!(tail.waiting_writer(), Some(true), "upgraded leases must retain observed contention"); + drop(tail); + let next = manager.acquire_write_lock(key, "next").await.expect("next lock"); + assert_eq!(next.waiting_writer(), Some(false)); + } + + #[tokio::test] + async fn guard_observes_waiting_writer_and_rejects_stale_identity() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "waiting-writer-observation"); + let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock"); + assert!(tail.downgrade_committed_tail()); + assert_eq!(tail.waiting_writer(), Some(false)); + let writer = { + let manager = manager.clone(); + let key = key.clone(); + tokio::spawn(async move { manager.acquire_write_lock(key, "foreground").await }) + }; + tokio::time::timeout(Duration::from_secs(2), async { + while tail.waiting_writer() != Some(true) { + tokio::task::yield_now().await; + } + }) + .await + .expect("writer becomes observable"); + assert_eq!(manager.force_unlock(&key), 1); + assert_eq!(tail.waiting_writer(), None, "force-released guard must not inspect a replacement owner"); + let _writer = writer.await.expect("writer task").expect("writer succeeds after release"); + } + + #[tokio::test] + async fn dropping_a_pending_upgrade_allows_readers_again() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "cancel-upgrade"); + let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock"); + assert!(tail.downgrade_committed_tail()); + let reader = manager.acquire_read_lock(key.clone(), "reader").await.expect("first reader"); + assert!(!tail.try_upgrade_committed_tail()); + let mut waiting = { + let manager = manager.clone(); + let key = key.clone(); + tokio::spawn(async move { manager.acquire_read_lock(key, "waiting-reader").await }) + }; + assert!(tokio::time::timeout(Duration::from_millis(100), &mut waiting).await.is_err()); + drop(tail); + let second = tokio::time::timeout(Duration::from_secs(2), waiting) + .await + .expect("reader wakes after cancellation") + .expect("reader task joins") + .expect("reader obtains lock"); + drop((reader, second)); + } + + #[tokio::test] + async fn force_released_tail_cannot_claim_an_exclusive_upgrade() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "upgrade-stale"); + let mut tail = manager + .acquire_write_lock(key.clone(), "same-owner") + .await + .expect("write lock"); + assert!(tail.downgrade_committed_tail()); + assert_eq!(manager.force_unlock(&key), 1); + let _new = manager.acquire_write_lock(key, "same-owner").await.expect("new owner"); + assert!(!tail.try_upgrade_committed_tail()); + } + + #[tokio::test] + async fn exclusive_upgrade_ack_fences_a_late_downgrade() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "late-downgrade"); + let mut guard = manager.acquire_write_lock(key, "heal").await.expect("write lock"); + assert!(guard.try_upgrade_committed_tail()); + assert!(!guard.downgrade_committed_tail()); + assert!(guard.try_upgrade_committed_tail()); + } + + #[tokio::test] + async fn force_unlock_old_readers_cannot_release_a_new_tail() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "force-old-readers"); + let old1 = manager + .acquire_read_lock(key.clone(), "same-owner") + .await + .expect("first reader"); + let old2 = manager + .acquire_read_lock(key.clone(), "same-owner") + .await + .expect("second reader"); + assert!(!old1.is_released()); + assert!(!old2.is_released()); + assert_eq!(manager.force_unlock(&key), 1); + let mut tail = manager + .acquire_write_lock(key.clone(), "same-owner") + .await + .expect("new writer"); + assert!(tail.downgrade_committed_tail()); + assert!(old1.is_released()); + assert!(old2.is_released()); + assert!(!tail.is_released()); + drop((old1, old2)); + assert!( + tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor")) + .await + .is_err(), + "old reader drops must not release the new tail" + ); + } + + #[tokio::test] + async fn forced_unregistered_reader_cannot_release_a_new_tail() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "force-unregistered-reader"); + let shard = manager.get_shard(&key).clone(); + let owner: Arc = Arc::from("same-owner"); + let request = ObjectLockRequest::new_read(key.clone(), owner.clone()); + let acquired = shard + .acquire_guarded(&request) + .await + .expect("old reader acquired before registration"); + assert_eq!(manager.force_unlock(&key), 1); + let mut tail = manager + .acquire_write_lock(key.clone(), owner.clone()) + .await + .expect("new writer"); + assert!(tail.downgrade_committed_tail()); + let mut old = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone()); + assert!(!shard.register_guard_with_info(old.guard_id(), old.key(), old.mode(), old.owner(), &acquired)); + old.abandon_unregistered(); + drop(old); + assert!( + tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor")) + .await + .is_err(), + "an old reader must not release a new committed tail" + ); + } + + #[tokio::test] + async fn force_unlock_on_another_key_keeps_unregistered_acquisition_valid() { + let manager = test_manager(); + let key = ObjectKey::new("bucket", "unrelated-acquisition"); + let shard = manager.get_shard(&key).clone(); + let owner: Arc = Arc::from("owner"); + let request = ObjectLockRequest::new_read(key.clone(), owner.clone()); + let acquired = shard.acquire_guarded(&request).await.expect("reader acquired"); + let other = (0..1000) + .map(|index| ObjectKey::new("bucket", format!("other-{index}"))) + .find(|other| Arc::ptr_eq(&shard, manager.get_shard(other))) + .expect("another key on the same shard"); + let other_guard = manager + .acquire_write_lock(other.clone(), "other") + .await + .expect("other writer"); + assert_eq!(manager.force_unlock(&other), 1); + drop(other_guard); + let guard = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone()); + assert!(shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired)); + drop(guard); + assert!(manager.acquire_write_lock(key, "next").await.is_ok()); + } + fn start_acquisition( manager: &Arc, request: ObjectLockRequest, diff --git a/crates/lock/src/fast_lock/state.rs b/crates/lock/src/fast_lock/state.rs index 4ef012248..1648fe3e1 100644 --- a/crates/lock/src/fast_lock/state.rs +++ b/crates/lock/src/fast_lock/state.rs @@ -23,6 +23,11 @@ use crate::fast_lock::types::{LockMode, LockPriority}; /// Optimized atomic lock state encoding in u64 /// Bits: [63:48] reserved | [47:32] writers_waiting | [31:16] readers_waiting | [15:8] readers_count | [7:1] flags | [0] writer_flag const WRITER_FLAG_MASK: u64 = 0x1; +// A committed write tail is a reader-compatible owner that still excludes writers. +const COMMITTED_TAIL_FLAG: u64 = 0x2; +const TAIL_UPGRADE_PENDING_FLAG: u64 = 0x4; +const TAIL_WRITER_OBSERVED_FLAG: u64 = 0x8; +const TAIL_TRACK_WRITERS_FLAG: u64 = 0x10; const READERS_SHIFT: u8 = 8; const READERS_MASK: u64 = 0xFF << READERS_SHIFT; // Support up to 255 concurrent readers const READERS_WAITING_SHIFT: u8 = 16; @@ -31,7 +36,6 @@ const WRITERS_WAITING_SHIFT: u8 = 32; const WRITERS_WAITING_MASK: u64 = 0xFFFF << WRITERS_WAITING_SHIFT; // Fast path check masks -const NO_WRITER_AND_NO_WAITING_WRITERS: u64 = WRITER_FLAG_MASK | WRITERS_WAITING_MASK; const COMPLETELY_UNLOCKED: u64 = 0; /// Fast atomic lock state for single version @@ -66,8 +70,8 @@ impl AtomicLockState { let state = self.state.load(Ordering::Relaxed); // Use Relaxed for better performance match mode { LockMode::Shared => { - // No writer and no waiting writers - (state & NO_WRITER_AND_NO_WAITING_WRITERS) == 0 + (state & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) == 0 + && ((state & WRITERS_WAITING_MASK) == 0 || (state & COMMITTED_TAIL_FLAG) != 0) } LockMode::Exclusive => { // Completely unlocked @@ -83,8 +87,11 @@ impl AtomicLockState { loop { let current = self.state.load(Ordering::Acquire); - // Fast path check - cannot acquire if there's a writer or writers waiting - if (current & NO_WRITER_AND_NO_WAITING_WRITERS) != 0 { + // While a committed tail owns a shared slot, let reads through + // even if a subsequent writer is waiting for that tail to drain. + if (current & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) != 0 + || ((current & WRITERS_WAITING_MASK) != 0 && (current & COMMITTED_TAIL_FLAG) == 0) + { return false; } @@ -137,6 +144,78 @@ impl AtomicLockState { } } + /// Atomically retain writer exclusion while allowing reads of a committed tail. + fn downgrade_committed_tail(&self) -> bool { + loop { + let current = self.state.load(Ordering::Acquire); + if (current & WRITER_FLAG_MASK) == 0 || (current & (READERS_MASK | COMMITTED_TAIL_FLAG)) != 0 { + return false; + } + let next = (current & !WRITER_FLAG_MASK) + | COMMITTED_TAIL_FLAG + | (1 << READERS_SHIFT) + | if current & WRITERS_WAITING_MASK != 0 { + TAIL_WRITER_OBSERVED_FLAG + } else { + 0 + }; + if self + .state + .compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + self.update_access_time(); + return true; + } + } + } + + /// Restore exclusive ownership only after every other reader has left. + fn try_upgrade_committed_tail(&self) -> bool { + loop { + let current = self.state.load(Ordering::Acquire); + if (current & COMMITTED_TAIL_FLAG) == 0 || (current & TAIL_WRITER_OBSERVED_FLAG) != 0 { + return false; + } + let readers_drained = (current & READERS_MASK) == (1 << READERS_SHIFT); + let next = if readers_drained { + (current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG | READERS_MASK)) + | WRITER_FLAG_MASK + | TAIL_TRACK_WRITERS_FLAG + } else { + current | TAIL_UPGRADE_PENDING_FLAG + }; + if self + .state + .compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + self.update_access_time(); + return readers_drained; + } + } + } + + /// Release the committed tail's own shared slot and restore ordinary fairness. + fn release_committed_tail(&self) -> bool { + loop { + let current = self.state.load(Ordering::Acquire); + if (current & COMMITTED_TAIL_FLAG) == 0 || self.readers_count(current) == 0 { + return false; + } + let next = + (current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG)) - (1 << READERS_SHIFT); + if self + .state + .compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + self.update_access_time(); + return true; + } + } + } + /// Release shared lock pub fn release_shared(&self) -> bool { loop { @@ -169,7 +248,7 @@ impl AtomicLockState { return false; // No exclusive lock to release } - let new_state = current & !WRITER_FLAG_MASK; + let new_state = current & !(WRITER_FLAG_MASK | TAIL_TRACK_WRITERS_FLAG | TAIL_WRITER_OBSERVED_FLAG); if self .state @@ -236,7 +315,10 @@ impl AtomicLockState { return false; // Max waiting writers } - let new_state = current + (1 << WRITERS_WAITING_SHIFT); + let mut new_state = current + (1 << WRITERS_WAITING_SHIFT); + if current & (COMMITTED_TAIL_FLAG | TAIL_TRACK_WRITERS_FLAG) != 0 { + new_state |= TAIL_WRITER_OBSERVED_FLAG; + } if self .state @@ -321,10 +403,15 @@ impl AtomicLockState { } #[cfg(test)] - pub fn writers_waiting_count(&self) -> u16 { + pub(crate) fn writers_waiting_count(&self) -> u16 { let state = self.state.load(Ordering::Acquire); self.writers_waiting(state) } + + pub(crate) fn writer_observed_during_tail(&self) -> bool { + let state = self.state.load(Ordering::Acquire); + self.writers_waiting(state) > 0 || state & TAIL_WRITER_OBSERVED_FLAG != 0 + } } /// Object lock state with version support - optimized memory layout @@ -334,6 +421,8 @@ pub struct ObjectLockState { // First cache line: Most frequently accessed data /// Atomic state for fast operations pub atomic_state: AtomicLockState, + /// Changes whenever an administrator force-releases this state. + pub(crate) force_epoch: AtomicU64, // Second cache line: Notification mechanisms /// Notification for readers (traditional) @@ -377,6 +466,7 @@ impl ObjectLockState { pub fn new() -> Self { Self { atomic_state: AtomicLockState::new(), + force_epoch: AtomicU64::new(0), read_notify: Notify::new(), write_notify: Notify::new(), optimized_notify: OptimizedNotify::new(), @@ -425,6 +515,70 @@ impl ObjectLockState { true } + /// Convert the current exclusive owner into the committed-tail owner. + pub(crate) fn downgrade_committed_tail(&self, owner: &Arc) -> bool { + let mut current = self.current_owner.write(); + let Some(info) = current.as_ref().filter(|info| info.owner.as_ref() == owner.as_ref()).cloned() else { + return false; + }; + let mut shared = self.shared_owners.write(); + if !shared.is_empty() || !self.atomic_state.downgrade_committed_tail() { + return false; + } + shared.push(SharedOwnerEntry { + owner: info.owner, + count: 1, + acquired_at: info.acquired_at, + lock_timeout: info.lock_timeout, + }); + *current = None; + drop(shared); + drop(current); + self.optimized_notify.notify_readers(); + true + } + + pub(crate) fn try_upgrade_committed_tail(&self, owner: &Arc) -> bool { + let mut current = self.current_owner.write(); + let mut shared = self.shared_owners.write(); + let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else { + return false; + }; + if current.is_some() || !self.atomic_state.try_upgrade_committed_tail() { + return false; + } + let info = shared.remove(pos); + *current = Some(ExclusiveOwnerInfo { + owner: info.owner, + acquired_at: info.acquired_at, + lock_timeout: info.lock_timeout, + }); + true + } + + /// Release the committed-tail owner's shared slot without releasing readers. + pub(crate) fn release_committed_tail(&self, owner: &Arc) -> bool { + let mut shared = self.shared_owners.write(); + let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else { + return false; + }; + if !self.atomic_state.release_committed_tail() { + return false; + } + if shared[pos].count > 1 { + shared[pos].count -= 1; + } else { + shared.remove(pos); + } + let no_readers = shared.is_empty(); + drop(shared); + if no_readers { + self.optimized_notify.notify_writer(); + } + self.optimized_notify.notify_readers(); + true + } + /// Release shared lock pub fn release_shared(&self, owner: &Arc) -> bool { let mut shared = self.shared_owners.write(); @@ -555,6 +709,24 @@ mod tests { assert!(!state.release_exclusive()); } + #[test] + fn committed_tail_allows_reads_but_preserves_writer_exclusion() { + let state = AtomicLockState::new(); + assert!(state.try_acquire_exclusive()); + assert!(state.inc_writers_waiting()); + assert!(state.downgrade_committed_tail()); + assert!(!state.try_acquire_exclusive(), "the tail must still exclude writers"); + assert!(state.try_acquire_shared(), "a queued writer must not block reads of the committed tail"); + assert!(state.release_committed_tail()); + assert!(!state.try_acquire_shared(), "writer preference resumes when the tail ends"); + assert!(!state.try_acquire_exclusive(), "an outstanding reader must still exclude writers"); + assert!(state.release_shared()); + assert!(state.try_acquire_exclusive()); + assert!(state.release_exclusive()); + state.dec_writers_waiting(); + assert!(state.is_free()); + } + // Regression for the waiter-preserving exclusive CAS. // // The acquisition CAS used to demand a fully-zero state word, which diff --git a/crates/lock/src/fast_lock/tests.rs b/crates/lock/src/fast_lock/tests.rs index 08bbe3639..2a22ee40a 100644 --- a/crates/lock/src/fast_lock/tests.rs +++ b/crates/lock/src/fast_lock/tests.rs @@ -111,6 +111,67 @@ mod fast_lock_tests { assert!(guard.is_released(), "Guard should be marked as released"); } + #[tokio::test] + async fn force_unlock_old_tail_cannot_release_new_tail() { + let manager = create_test_manager(); + let key = ObjectKey::new("test-bucket", "force-tail-generation"); + let owner: Arc = Arc::from("same-owner"); + let mut old = manager + .acquire_write_lock(key.clone(), owner.clone()) + .await + .expect("old writer"); + assert!(old.downgrade_committed_tail()); + assert_eq!(manager.force_unlock(&key), 1); + let mut new = manager + .acquire_write_lock(key.clone(), owner.clone()) + .await + .expect("new writer"); + assert!(new.downgrade_committed_tail()); + drop(old); + let attempt = + tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, Arc::from("competing-writer"))) + .await; + assert!(attempt.is_err(), "dropping the old guard must not release the new tail"); + drop(new); + } + + #[tokio::test] + async fn committed_tail_allows_reader_while_writer_waits() { + let manager = Arc::new(create_test_manager()); + let key = ObjectKey::new("test-bucket", "tail-handoff"); + let mut tail = manager + .acquire_write_lock(key.clone(), Arc::from("first-writer")) + .await + .expect("initial writer"); + let writer = tokio::spawn({ + let manager = Arc::clone(&manager); + let key = key.clone(); + async move { manager.acquire_write_lock(key, Arc::from("second-writer")).await } + }); + tokio::time::timeout(Duration::from_secs(2), async { + while manager.current_lock_counts().1 == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("second writer should be queued"); + assert!(tail.downgrade_committed_tail()); + let reader = tokio::time::timeout(Duration::from_secs(2), manager.acquire_read_lock(key, Arc::from("reader"))) + .await + .expect("reader should not wait behind committed tail") + .expect("reader should acquire"); + assert!(!writer.is_finished(), "tail and reader must exclude the next writer"); + drop(tail); + assert!(!writer.is_finished(), "reader still excludes the next writer"); + drop(reader); + let writer = tokio::time::timeout(Duration::from_secs(2), writer) + .await + .expect("writer should resume") + .expect("writer task should join") + .expect("writer should acquire"); + drop(writer); + } + #[tokio::test] async fn current_lock_counts_report_live_holders_and_waiters() { let manager = Arc::new(create_test_manager()); diff --git a/crates/lock/src/namespace/mod.rs b/crates/lock/src/namespace/mod.rs index 64238295b..18d76eb4b 100644 --- a/crates/lock/src/namespace/mod.rs +++ b/crates/lock/src/namespace/mod.rs @@ -107,6 +107,38 @@ impl NamespaceLockGuard { } } + /// Permit reads while continuing to exclude competing writers. + pub async fn downgrade_committed_tail(&mut self) -> bool { + match self { + Self::Standard(guard) => guard.downgrade_committed_tail().await, + Self::Fast(guard) => guard.downgrade_committed_tail(), + } + } + + /// Try to restore exclusive access while retaining the writer fence. + pub async fn try_upgrade_committed_tail(&mut self) -> bool { + match self { + Self::Standard(guard) => guard.try_upgrade_committed_tail().await, + Self::Fast(guard) => guard.try_upgrade_committed_tail(), + } + } + + /// Check whether the held leases can safely use heal lock transitions. + pub async fn supports_heal_lock_transition(&self) -> bool { + match self { + Self::Standard(guard) => guard.supports_heal_lock_transition().await, + Self::Fast(_) => true, + } + } + + /// Observe whether a writer is waiting on this object's held leases. + pub async fn waiting_writer(&self) -> Option { + match self { + Self::Standard(guard) => guard.waiting_writer().await, + Self::Fast(guard) => guard.waiting_writer(), + } + } + /// Manually release the lock early pub fn release(&mut self) -> bool { match self { diff --git a/crates/lock/src/namespace/tests.rs b/crates/lock/src/namespace/tests.rs index b584f7e23..e5b8e2233 100644 --- a/crates/lock/src/namespace/tests.rs +++ b/crates/lock/src/namespace/tests.rs @@ -154,6 +154,43 @@ impl crate::client::LockClient for DelayedClient { } } +#[derive(Debug)] +struct PendingTailClient(LocalClient); + +#[async_trait::async_trait] +impl crate::client::LockClient for PendingTailClient { + async fn acquire_lock(&self, request: &LockRequest) -> crate::Result { + self.0.acquire_lock(request).await + } + async fn release(&self, lock_id: &LockId) -> crate::Result { + self.0.release(lock_id).await + } + async fn refresh(&self, lock_id: &LockId) -> crate::Result { + self.0.refresh(lock_id).await + } + async fn downgrade_committed_tail(&self, _: &LockId) -> crate::Result { + std::future::pending().await + } + async fn force_release(&self, lock_id: &LockId) -> crate::Result { + self.0.force_release(lock_id).await + } + async fn check_status(&self, lock_id: &LockId) -> crate::Result> { + self.0.check_status(lock_id).await + } + async fn get_stats(&self) -> crate::Result { + self.0.get_stats().await + } + async fn close(&self) -> crate::Result<()> { + self.0.close().await + } + async fn is_online(&self) -> bool { + self.0.is_online().await + } + async fn is_local(&self) -> bool { + self.0.is_local().await + } +} + #[derive(Debug)] struct FlakyAcquireClient { inner: LocalClient, @@ -579,6 +616,286 @@ async fn test_namespace_lock_guard_lock_id() { } } +#[tokio::test] +async fn tail_conversion_rejects_non_intersecting_quorums() { + let clients: Vec> = (0..4) + .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc) + .collect(); + let lock = NamespaceLock::with_clients_and_quorum("non-intersecting".to_owned(), clients, 2); + let resource = ObjectKey::new("bucket", "non-intersecting-tail"); + let mut tail = lock + .get_write_lock(resource, "writer", Duration::from_secs(2)) + .await + .expect("write lease"); + assert!( + !tail.downgrade_committed_tail().await, + "conversion requires intersecting read and write quorums" + ); + assert!(tail.release()); +} + +#[tokio::test] +async fn partial_tail_conversion_keeps_writers_excluded() { + let mut clients: Vec> = Vec::new(); + for index in 0..4 { + let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))); + if index < 2 { + clients.push(inner); + } else { + clients.push(Arc::new(DelayedClient { + inner, + delay: Duration::ZERO, + })); + } + } + let lock = NamespaceLock::with_clients("mixed-peers".to_owned(), clients); + let resource = ObjectKey::new("bucket", "partial-tail"); + let mut tail = lock + .get_write_lock(resource.clone(), "first", Duration::from_secs(2)) + .await + .expect("first writer"); + assert!(!tail.downgrade_committed_tail().await, "mixed peers cannot all confirm conversion"); + assert!( + lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150)) + .await + .is_err(), + "mixed leases still exclude writers" + ); + assert!(tail.release()); + let mut writer = lock + .get_write_lock(resource, "second", Duration::from_secs(3)) + .await + .expect("writer after release"); + assert!(writer.release()); +} + +#[tokio::test] +async fn cancelled_tail_conversion_keeps_writers_excluded() { + let mut clients: Vec> = (0..3) + .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc) + .collect(); + clients.push(Arc::new(PendingTailClient(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))))); + let lock = NamespaceLock::with_clients_and_quorum("cancel-tail".to_owned(), clients, 4); + let resource = ObjectKey::new("bucket", "cancel-tail"); + let mut tail = lock + .get_write_lock(resource.clone(), "first", Duration::from_secs(2)) + .await + .expect("first writer"); + assert!( + tokio::time::timeout(Duration::from_millis(100), tail.downgrade_committed_tail()) + .await + .is_err() + ); + let mut reader = lock + .get_read_lock(resource.clone(), "reader", Duration::from_secs(2)) + .await + .expect("other peers converted before cancellation"); + assert!(reader.release()); + assert!( + lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150)) + .await + .is_err(), + "cancelled conversion must still exclude writers" + ); + assert!(tail.release()); + let mut writer = lock + .get_write_lock(resource, "second", Duration::from_secs(3)) + .await + .expect("writer after release"); + assert!(writer.release()); +} + +#[tokio::test] +async fn unsupported_peer_keeps_exclusive_tail_lease() { + let clients: Vec> = (0..3) + .map(|_| { + let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))); + Arc::new(DelayedClient { + inner, + delay: Duration::ZERO, + }) as Arc + }) + .collect(); + let lock = NamespaceLock::with_clients("old-peers".to_owned(), clients); + let resource = ObjectKey::new("bucket", "old-tail"); + let mut tail = lock + .get_write_lock(resource.clone(), "first", Duration::from_secs(2)) + .await + .expect("first writer"); + assert!(!tail.supports_heal_lock_transition().await, "old peers must not claim support"); + assert_eq!(tail.waiting_writer().await, None, "old peers cannot report contention"); + assert!(!tail.downgrade_committed_tail().await, "old peers must not claim conversion"); + assert!(!tail.try_upgrade_committed_tail().await, "old peers must not claim upgrade support"); + assert!( + lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150)) + .await + .is_err() + ); + assert!( + lock.get_write_lock(resource.clone(), "writer", Duration::from_millis(150)) + .await + .is_err() + ); + assert!(tail.release()); + let mut writer = lock + .get_write_lock(resource, "writer", Duration::from_secs(3)) + .await + .expect("writer after tail"); + assert!(writer.release()); +} + +#[tokio::test] +async fn distributed_committed_tail_allows_reader_and_excludes_writer() { + let clients: Vec> = (0..3) + .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc) + .collect(); + let lock = NamespaceLock::with_clients("tail-test".to_owned(), clients); + let resource = ObjectKey::new("bucket", "committed-tail"); + let mut tail = lock + .get_write_lock(resource.clone(), "first", Duration::from_secs(2)) + .await + .expect("first writer"); + assert!(tail.downgrade_committed_tail().await, "all held peer leases should convert"); + let mut reader = lock + .get_read_lock(resource.clone(), "reader", Duration::from_secs(2)) + .await + .expect("committed reader"); + assert!( + lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150)) + .await + .is_err(), + "tail and reader must exclude writer" + ); + assert!(tail.release()); + assert!( + lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150)) + .await + .is_err(), + "reader still excludes writer" + ); + assert!(reader.release()); + let mut writer = lock + .get_write_lock(resource, "second", Duration::from_secs(3)) + .await + .expect("writer after drain"); + assert!(writer.release()); +} + +#[tokio::test] +async fn distributed_tail_upgrade_requires_all_leases_and_drained_readers() { + let clients: Vec> = (0..3) + .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc) + .collect(); + let lock = NamespaceLock::with_clients("tail-upgrade".to_owned(), clients); + let resource = ObjectKey::new("bucket", "tail-upgrade"); + let mut tail = lock + .get_write_lock(resource.clone(), "heal", Duration::from_secs(2)) + .await + .expect("write lock"); + assert!(tail.supports_heal_lock_transition().await); + assert!(tail.downgrade_committed_tail().await); + let mut reader = lock + .get_read_lock(resource.clone(), "reader", Duration::from_secs(2)) + .await + .expect("reader"); + assert!(!tail.try_upgrade_committed_tail().await, "live readers prevent full upgrade"); + assert!(reader.release()); + tokio::time::timeout(Duration::from_secs(2), async { + while !tail.try_upgrade_committed_tail().await { + tokio::task::yield_now().await; + } + }) + .await + .expect("retry must finish partial upgrades after asynchronous reader release"); + assert!(tail.try_upgrade_committed_tail().await, "upgrade retries must be idempotent"); + assert!( + lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150)) + .await + .is_err() + ); + assert!(tail.release()); + let mut writer = lock + .get_write_lock(resource, "writer", Duration::from_secs(2)) + .await + .expect("writer after release"); + assert!(writer.release()); +} + +#[tokio::test] +async fn distributed_tail_observes_a_waiting_writer() { + let clients: Vec> = (0..3) + .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc) + .collect(); + let lock = Arc::new(NamespaceLock::with_clients("writer-observation".to_owned(), clients)); + let resource = ObjectKey::new("bucket", "writer-observation"); + let mut tail = lock + .get_write_lock(resource.clone(), "heal", Duration::from_secs(2)) + .await + .expect("heal lock"); + assert!(tail.downgrade_committed_tail().await); + assert_eq!(tail.waiting_writer().await, Some(false)); + let writer = { + let lock = Arc::clone(&lock); + let resource = resource.clone(); + tokio::spawn(async move { lock.get_write_lock(resource, "writer", Duration::from_secs(3)).await }) + }; + tokio::time::timeout(Duration::from_secs(2), async { + while tail.waiting_writer().await != Some(true) { + tokio::task::yield_now().await; + } + }) + .await + .expect("waiting writer becomes visible on a held peer"); + assert!(tail.release()); + let mut guard = writer.await.expect("writer task").expect("writer succeeds after release"); + assert!(guard.release()); +} + +#[tokio::test] +async fn distributed_partial_upgrade_remembers_contention_on_upgraded_peer() { + use crate::client::LockClient; + + let clients: Vec> = (0..3) + .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))) + .collect(); + let lock = NamespaceLock::with_clients( + "partial-upgrade-writer".to_owned(), + clients.iter().cloned().map(|client| client as Arc).collect(), + ); + let resource = ObjectKey::new("bucket", "partial-upgrade-writer"); + let mut tail = lock + .get_write_lock(resource.clone(), "heal", Duration::from_secs(2)) + .await + .expect("heal lock"); + let ids = match &tail { + NamespaceLockGuard::Standard(guard) => guard.granted_lock_ids().cloned().collect::>(), + NamespaceLockGuard::Fast(_) => panic!("distributed guard expected"), + }; + let mut holders = Vec::new(); + for client in &clients { + for id in &ids { + if client.check_status(id).await.expect("peer status").is_some() { + holders.push(Arc::clone(client)); + break; + } + } + } + assert!(holders.len() >= 2, "write quorum must hold multiple peers"); + assert!(tail.downgrade_committed_tail().await); + let reader = LockRequest::new(resource.clone(), LockType::Shared, "reader"); + assert!(holders[0].acquire_lock(&reader).await.expect("reader request").success); + assert!(!tail.try_upgrade_committed_tail().await, "reader prevents a full upgrade"); + let writer = LockRequest::new(resource, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_millis(100)); + assert!(!holders[1].acquire_lock(&writer).await.expect("writer request").success); + assert_eq!( + tail.waiting_writer().await, + Some(true), + "partial upgrade must retain the earlier contention" + ); + assert!(holders[0].release(&reader.lock_id).await.expect("release reader")); + assert!(tail.release()); +} + #[tokio::test] async fn test_namespace_lock_distributed_multi_node_simulation() { // Simulate a 3-node distributed environment where each node has its own lock backend diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index f6aad4578..926e73728 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -1,5 +1,6 @@ // This file is @generated by prost-build. /// --- +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Error { #[prost(uint32, tag = "1")] @@ -7,6 +8,7 @@ pub struct Error { #[prost(string, tag = "2")] pub error_info: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PingRequest { #[prost(uint64, tag = "1")] @@ -14,6 +16,7 @@ pub struct PingRequest { #[prost(bytes = "bytes", tag = "2")] pub body: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PingResponse { #[prost(uint64, tag = "1")] @@ -21,6 +24,7 @@ pub struct PingResponse { #[prost(bytes = "bytes", tag = "2")] pub body: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct HealBucketRequest { #[prost(string, tag = "1")] @@ -30,6 +34,7 @@ pub struct HealBucketRequest { #[prost(bytes = "bytes", tag = "3")] pub bucket_incarnation_id: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct HealBucketResponse { #[prost(bool, tag = "1")] @@ -37,11 +42,13 @@ pub struct HealBucketResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListBucketRequest { #[prost(string, tag = "1")] pub options: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListBucketResponse { #[prost(bool, tag = "1")] @@ -51,6 +58,7 @@ pub struct ListBucketResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct MakeBucketRequest { #[prost(string, tag = "1")] @@ -58,6 +66,7 @@ pub struct MakeBucketRequest { #[prost(string, tag = "2")] pub options: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct MakeBucketResponse { #[prost(bool, tag = "1")] @@ -65,6 +74,7 @@ pub struct MakeBucketResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetBucketInfoRequest { #[prost(string, tag = "1")] @@ -72,6 +82,7 @@ pub struct GetBucketInfoRequest { #[prost(string, tag = "2")] pub options: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetBucketInfoResponse { #[prost(bool, tag = "1")] @@ -81,6 +92,7 @@ pub struct GetBucketInfoResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteBucketRequest { #[prost(string, tag = "1")] @@ -88,6 +100,7 @@ pub struct DeleteBucketRequest { #[prost(string, tag = "2")] pub options: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteBucketResponse { #[prost(bool, tag = "1")] @@ -95,6 +108,7 @@ pub struct DeleteBucketResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadAllRequest { /// indicate which one in the disks @@ -105,6 +119,7 @@ pub struct ReadAllRequest { #[prost(string, tag = "3")] pub path: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadAllResponse { #[prost(bool, tag = "1")] @@ -114,6 +129,7 @@ pub struct ReadAllResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WriteAllRequest { /// indicate which one in the disks @@ -126,6 +142,7 @@ pub struct WriteAllRequest { #[prost(bytes = "bytes", tag = "4")] pub data: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WriteAllResponse { #[prost(bool, tag = "1")] @@ -133,6 +150,7 @@ pub struct WriteAllResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CompareAndUpdateFileRequest { /// indicate which one in the disks @@ -147,6 +165,7 @@ pub struct CompareAndUpdateFileRequest { #[prost(bytes = "bytes", optional, tag = "5")] pub replacement: ::core::option::Option<::prost::bytes::Bytes>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CompareAndUpdateFileResponse { #[prost(bool, tag = "1")] @@ -156,6 +175,7 @@ pub struct CompareAndUpdateFileResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteRequest { /// indicate which one in the disks @@ -174,6 +194,7 @@ pub struct DeleteRequest { #[prost(bytes = "bytes", tag = "6")] pub bucket_incarnation_id: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteResponse { #[prost(bool, tag = "1")] @@ -181,6 +202,7 @@ pub struct DeleteResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct VerifyFileRequest { /// indicate which one in the disks @@ -193,6 +215,7 @@ pub struct VerifyFileRequest { #[prost(string, tag = "4")] pub file_info: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct VerifyFileResponse { #[prost(bool, tag = "1")] @@ -202,6 +225,7 @@ pub struct VerifyFileResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadPartsRequest { #[prost(string, tag = "1")] @@ -211,6 +235,7 @@ pub struct ReadPartsRequest { #[prost(string, repeated, tag = "3")] pub paths: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadPartsResponse { #[prost(bool, tag = "1")] @@ -220,6 +245,7 @@ pub struct ReadPartsResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CheckPartsRequest { /// indicate which one in the disks @@ -232,6 +258,7 @@ pub struct CheckPartsRequest { #[prost(string, tag = "4")] pub file_info: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CheckPartsResponse { #[prost(bool, tag = "1")] @@ -241,6 +268,7 @@ pub struct CheckPartsResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RenamePartRequest { #[prost(string, tag = "1")] @@ -256,6 +284,7 @@ pub struct RenamePartRequest { #[prost(bytes = "bytes", tag = "6")] pub meta: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RenamePartResponse { #[prost(bool, tag = "1")] @@ -263,6 +292,7 @@ pub struct RenamePartResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PreparePartTransactionRequest { #[prost(string, tag = "1")] @@ -278,6 +308,7 @@ pub struct PreparePartTransactionRequest { #[prost(bytes = "bytes", tag = "6")] pub meta: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PreparePartTransactionResponse { #[prost(bool, tag = "1")] @@ -285,6 +316,7 @@ pub struct PreparePartTransactionResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SettlePartTransactionRequest { #[prost(string, tag = "1")] @@ -296,6 +328,7 @@ pub struct SettlePartTransactionRequest { #[prost(bool, tag = "4")] pub rollback: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SettlePartTransactionResponse { #[prost(bool, tag = "1")] @@ -303,6 +336,7 @@ pub struct SettlePartTransactionResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RenameFileRequest { #[prost(string, tag = "1")] @@ -319,6 +353,7 @@ pub struct RenameFileRequest { #[prost(bool, tag = "6")] pub durable: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RenameFileResponse { #[prost(bool, tag = "1")] @@ -328,6 +363,7 @@ pub struct RenameFileResponse { #[prost(bool, tag = "3")] pub durability_applied: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WriteRequest { /// indicate which one in the disks @@ -342,6 +378,7 @@ pub struct WriteRequest { #[prost(bytes = "bytes", tag = "5")] pub data: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WriteResponse { #[prost(bool, tag = "1")] @@ -349,6 +386,7 @@ pub struct WriteResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadAtRequest { /// indicate which one in the disks @@ -363,6 +401,7 @@ pub struct ReadAtRequest { #[prost(int64, tag = "5")] pub length: i64, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadAtResponse { #[prost(bool, tag = "1")] @@ -374,6 +413,7 @@ pub struct ReadAtResponse { #[prost(message, optional, tag = "4")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListDirRequest { /// indicate which one in the disks @@ -386,6 +426,7 @@ pub struct ListDirRequest { #[prost(int32, tag = "4")] pub count: i32, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListDirResponse { #[prost(bool, tag = "1")] @@ -395,6 +436,7 @@ pub struct ListDirResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WalkDirRequest { /// indicate which one in the disks @@ -403,6 +445,7 @@ pub struct WalkDirRequest { #[prost(bytes = "bytes", tag = "2")] pub walk_dir_options: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WalkDirResponse { #[prost(bool, tag = "1")] @@ -412,6 +455,7 @@ pub struct WalkDirResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RenameDataRequest { /// indicate which one in the disks @@ -438,6 +482,7 @@ pub struct RenameDataRequest { #[prost(bytes = "bytes", tag = "9")] pub bucket_incarnation_id: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] #[prost(skip_debug)] pub struct RenameDataResponse { @@ -450,8 +495,53 @@ pub struct RenameDataResponse { #[prost(bytes = "bytes", tag = "4")] pub rename_data_resp_bin: ::prost::bytes::Bytes, } +#[allow(clippy::all)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct MetadataReservationRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub bucket: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub object: ::prost::alloc::string::String, + #[prost(uint64, tag = "4")] + pub timeout_ms: u64, + #[prost(bytes = "bytes", tag = "5")] + pub bucket_incarnation_id: ::prost::bytes::Bytes, +} +#[allow(clippy::all)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct MetadataReservationCancelRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(bytes = "bytes", tag = "2")] + pub token: ::prost::bytes::Bytes, +} +#[allow(clippy::all)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct MetadataReservationActivateRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub bucket: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub object: ::prost::alloc::string::String, + #[prost(bytes = "bytes", tag = "4")] + pub token: ::prost::bytes::Bytes, + #[prost(bool, tag = "5")] + pub renew_only: bool, +} +#[allow(clippy::all)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReservedRenameDataRequest { + #[prost(message, optional, tag = "1")] + pub rename: ::core::option::Option, + #[prost(bytes = "bytes", tag = "2")] + pub token: ::prost::bytes::Bytes, +} /// Fixed-scope native pool.bin recovery. The typed MessagePack payload never /// accepts an arbitrary volume or object path. +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PoolMetaNativeRecoveryRequest { #[prost(string, tag = "1")] @@ -459,6 +549,7 @@ pub struct PoolMetaNativeRecoveryRequest { #[prost(bytes = "bytes", tag = "2")] pub payload: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PoolMetaNativeRecoveryResponse { #[prost(bool, tag = "1")] @@ -468,6 +559,7 @@ pub struct PoolMetaNativeRecoveryResponse { #[prost(bytes = "bytes", tag = "3")] pub payload: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct MakeVolumesRequest { /// indicate which one in the disks @@ -476,6 +568,7 @@ pub struct MakeVolumesRequest { #[prost(string, repeated, tag = "2")] pub volumes: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct MakeVolumesResponse { #[prost(bool, tag = "1")] @@ -483,6 +576,7 @@ pub struct MakeVolumesResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct MakeVolumeRequest { /// indicate which one in the disks @@ -491,6 +585,7 @@ pub struct MakeVolumeRequest { #[prost(string, tag = "2")] pub volume: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct MakeVolumeResponse { #[prost(bool, tag = "1")] @@ -498,12 +593,14 @@ pub struct MakeVolumeResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListVolumesRequest { /// indicate which one in the disks #[prost(string, tag = "1")] pub disk: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListVolumesResponse { #[prost(bool, tag = "1")] @@ -513,6 +610,7 @@ pub struct ListVolumesResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StatVolumeRequest { /// indicate which one in the disks @@ -521,6 +619,7 @@ pub struct StatVolumeRequest { #[prost(string, tag = "2")] pub volume: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StatVolumeResponse { #[prost(bool, tag = "1")] @@ -530,6 +629,7 @@ pub struct StatVolumeResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeletePathsRequest { #[prost(string, tag = "1")] @@ -539,6 +639,7 @@ pub struct DeletePathsRequest { #[prost(string, repeated, tag = "3")] pub paths: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeletePathsResponse { #[prost(bool, tag = "1")] @@ -547,6 +648,7 @@ pub struct DeletePathsResponse { pub error: ::core::option::Option, } /// Protocol 1 is additive and has no fallback to generic file mutation RPCs. +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RecoverDeleteTransactionRequest { #[prost(string, tag = "1")] @@ -558,6 +660,7 @@ pub struct RecoverDeleteTransactionRequest { #[prost(bytes = "bytes", tag = "4")] pub operation: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RecoverDeleteTransactionResponse { #[prost(bool, tag = "1")] @@ -570,6 +673,7 @@ pub struct RecoverDeleteTransactionResponse { pub error: ::core::option::Option, } /// Fixed multipart-upload scope; operation bytes never select another volume. +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] #[prost(skip_debug)] pub struct RecoverMultipartSourceRequest { @@ -582,6 +686,7 @@ pub struct RecoverMultipartSourceRequest { #[prost(bytes = "bytes", tag = "4")] pub operation: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] #[prost(skip_debug)] pub struct RecoverMultipartSourceResponse { @@ -594,6 +699,7 @@ pub struct RecoverMultipartSourceResponse { #[prost(message, optional, tag = "4")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnapshotLeaseRequest { #[prost(string, tag = "1")] @@ -605,6 +711,7 @@ pub struct SnapshotLeaseRequest { #[prost(uint64, tag = "4")] pub ttl_ms: u64, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnapshotLeaseRenewRequest { #[prost(string, tag = "1")] @@ -618,6 +725,7 @@ pub struct SnapshotLeaseRenewRequest { #[prost(uint64, tag = "5")] pub ttl_ms: u64, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnapshotLeaseReleaseRequest { #[prost(string, tag = "1")] @@ -629,6 +737,7 @@ pub struct SnapshotLeaseReleaseRequest { #[prost(bytes = "bytes", tag = "4")] pub token: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnapshotLeaseResponse { #[prost(bool, tag = "1")] @@ -640,6 +749,7 @@ pub struct SnapshotLeaseResponse { #[prost(message, optional, tag = "4")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnapshotLeaseMutationResponse { #[prost(bool, tag = "1")] @@ -647,6 +757,7 @@ pub struct SnapshotLeaseMutationResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadMetadataRequest { #[prost(string, tag = "1")] @@ -656,6 +767,7 @@ pub struct ReadMetadataRequest { #[prost(string, tag = "3")] pub path: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadMetadataResponse { #[prost(bool, tag = "1")] @@ -665,6 +777,7 @@ pub struct ReadMetadataResponse { #[prost(bytes = "bytes", tag = "3")] pub data: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct UpdateMetadataRequest { #[prost(string, tag = "1")] @@ -682,6 +795,7 @@ pub struct UpdateMetadataRequest { #[prost(bytes = "bytes", tag = "7")] pub opts_bin: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct UpdateMetadataResponse { #[prost(bool, tag = "1")] @@ -689,6 +803,7 @@ pub struct UpdateMetadataResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WriteMetadataRequest { /// indicate which one in the disks @@ -705,6 +820,7 @@ pub struct WriteMetadataRequest { #[prost(bytes = "bytes", tag = "6")] pub bucket_incarnation_id: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WriteMetadataResponse { #[prost(bool, tag = "1")] @@ -712,6 +828,7 @@ pub struct WriteMetadataResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadVersionRequest { #[prost(string, tag = "1")] @@ -727,6 +844,7 @@ pub struct ReadVersionRequest { #[prost(bytes = "bytes", tag = "6")] pub opts_bin: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadVersionResponse { #[prost(bool, tag = "1")] @@ -738,6 +856,7 @@ pub struct ReadVersionResponse { #[prost(bytes = "bytes", tag = "4")] pub file_info_bin: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BatchReadVersionRequest { #[prost(string, tag = "1")] @@ -747,6 +866,7 @@ pub struct BatchReadVersionRequest { #[prost(bytes = "bytes", tag = "3")] pub batch_read_version_req_bin: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BatchReadVersionResponse { #[prost(bool, tag = "1")] @@ -758,6 +878,7 @@ pub struct BatchReadVersionResponse { #[prost(bytes = "bytes", repeated, tag = "4")] pub batch_read_version_resps_bin: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadXlRequest { #[prost(string, tag = "1")] @@ -769,6 +890,7 @@ pub struct ReadXlRequest { #[prost(bool, tag = "4")] pub read_data: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadXlResponse { #[prost(bool, tag = "1")] @@ -780,6 +902,7 @@ pub struct ReadXlResponse { #[prost(bytes = "bytes", tag = "4")] pub raw_file_info_bin: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteVersionRequest { #[prost(string, tag = "1")] @@ -803,6 +926,7 @@ pub struct DeleteVersionRequest { #[prost(bytes = "bytes", tag = "9")] pub bucket_incarnation_id: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteVersionResponse { #[prost(bool, tag = "1")] @@ -812,6 +936,7 @@ pub struct DeleteVersionResponse { #[prost(message, optional, tag = "3")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteVersionsRequest { #[prost(string, tag = "1")] @@ -830,6 +955,7 @@ pub struct DeleteVersionsRequest { pub opts_bin: ::prost::bytes::Bytes, } /// Additive, fail-closed conditional batch protocol. Probe requests never mutate disk state. +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GuardedDeleteVersionsRequest { #[prost(string, tag = "1")] @@ -841,6 +967,7 @@ pub struct GuardedDeleteVersionsRequest { #[prost(bool, tag = "4")] pub probe_only: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct DeleteVersionsResponse { #[prost(bool, tag = "1")] @@ -854,6 +981,7 @@ pub struct DeleteVersionsResponse { #[prost(message, repeated, tag = "4")] pub item_errors: ::prost::alloc::vec::Vec, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadMultipleRequest { #[prost(string, tag = "1")] @@ -863,6 +991,7 @@ pub struct ReadMultipleRequest { #[prost(bytes = "bytes", tag = "3")] pub read_multiple_req_bin: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReadMultipleResponse { #[prost(bool, tag = "1")] @@ -874,6 +1003,7 @@ pub struct ReadMultipleResponse { #[prost(bytes = "bytes", repeated, tag = "4")] pub read_multiple_resps_bin: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteVolumeRequest { #[prost(string, tag = "1")] @@ -886,6 +1016,7 @@ pub struct DeleteVolumeRequest { #[prost(bool, tag = "3")] pub force: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteVolumeResponse { #[prost(bool, tag = "1")] @@ -893,6 +1024,7 @@ pub struct DeleteVolumeResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DiskInfoRequest { #[prost(string, tag = "1")] @@ -900,6 +1032,7 @@ pub struct DiskInfoRequest { #[prost(string, tag = "2")] pub opts: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DiskInfoResponse { #[prost(bool, tag = "1")] @@ -910,11 +1043,13 @@ pub struct DiskInfoResponse { pub error: ::core::option::Option, } /// lock api have same argument type +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GenerallyLockRequest { #[prost(string, tag = "1")] pub args: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GenerallyLockResponse { #[prost(bool, tag = "1")] @@ -925,11 +1060,13 @@ pub struct GenerallyLockResponse { #[prost(string, optional, tag = "3")] pub lock_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BatchGenerallyLockRequest { #[prost(string, repeated, tag = "1")] pub args: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GenerallyLockResult { #[prost(bool, tag = "1")] @@ -940,21 +1077,25 @@ pub struct GenerallyLockResult { #[prost(string, optional, tag = "3")] pub lock_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BatchGenerallyLockResponse { #[prost(message, repeated, tag = "1")] pub results: ::prost::alloc::vec::Vec, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Mss { #[prost(map = "string, string", tag = "1")] pub value: ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct LocalStorageInfoRequest { #[prost(bool, tag = "1")] pub metrics: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LocalStorageInfoResponse { #[prost(bool, tag = "1")] @@ -966,11 +1107,13 @@ pub struct LocalStorageInfoResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "4")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ServerInfoRequest { #[prost(bool, tag = "1")] pub metrics: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ServerInfoResponse { #[prost(bool, tag = "1")] @@ -980,8 +1123,10 @@ pub struct ServerInfoResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetCpusRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetCpusResponse { #[prost(bool, tag = "1")] @@ -991,8 +1136,10 @@ pub struct GetCpusResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetNetInfoRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetNetInfoResponse { #[prost(bool, tag = "1")] @@ -1002,8 +1149,10 @@ pub struct GetNetInfoResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetPartitionsRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetPartitionsResponse { #[prost(bool, tag = "1")] @@ -1013,8 +1162,10 @@ pub struct GetPartitionsResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetOsInfoRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetOsInfoResponse { #[prost(bool, tag = "1")] @@ -1024,8 +1175,10 @@ pub struct GetOsInfoResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetSeLinuxInfoRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetSeLinuxInfoResponse { #[prost(bool, tag = "1")] @@ -1035,8 +1188,10 @@ pub struct GetSeLinuxInfoResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetSysConfigRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetSysConfigResponse { #[prost(bool, tag = "1")] @@ -1046,8 +1201,10 @@ pub struct GetSysConfigResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetSysErrorsRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetSysErrorsResponse { #[prost(bool, tag = "1")] @@ -1057,8 +1214,10 @@ pub struct GetSysErrorsResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetMemInfoRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetMemInfoResponse { #[prost(bool, tag = "1")] @@ -1068,6 +1227,7 @@ pub struct GetMemInfoResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetMetricsRequest { #[prost(bytes = "bytes", tag = "1")] @@ -1075,6 +1235,7 @@ pub struct GetMetricsRequest { #[prost(bytes = "bytes", tag = "2")] pub opts: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetMetricsResponse { #[prost(bool, tag = "1")] @@ -1084,8 +1245,10 @@ pub struct GetMetricsResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetProcInfoRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetProcInfoResponse { #[prost(bool, tag = "1")] @@ -1095,11 +1258,13 @@ pub struct GetProcInfoResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StartProfilingRequest { #[prost(string, tag = "1")] pub profiler: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StartProfilingResponse { #[prost(bool, tag = "1")] @@ -1107,8 +1272,10 @@ pub struct StartProfilingResponse { #[prost(string, optional, tag = "2")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct DownloadProfileDataRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct DownloadProfileDataResponse { #[prost(bool, tag = "1")] @@ -1118,11 +1285,13 @@ pub struct DownloadProfileDataResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetBucketStatsDataRequest { #[prost(string, tag = "1")] pub bucket: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetBucketStatsDataResponse { #[prost(bool, tag = "1")] @@ -1132,8 +1301,10 @@ pub struct GetBucketStatsDataResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetSrMetricsDataRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetSrMetricsDataResponse { #[prost(bool, tag = "1")] @@ -1143,8 +1314,10 @@ pub struct GetSrMetricsDataResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetAllBucketStatsRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetAllBucketStatsResponse { #[prost(bool, tag = "1")] @@ -1154,6 +1327,7 @@ pub struct GetAllBucketStatsResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadBucketMetadataRequest { #[prost(string, tag = "1")] @@ -1161,6 +1335,7 @@ pub struct LoadBucketMetadataRequest { #[prost(bool, tag = "2")] pub scanner_maintenance_change: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadBucketMetadataResponse { #[prost(bool, tag = "1")] @@ -1170,11 +1345,13 @@ pub struct LoadBucketMetadataResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteBucketMetadataRequest { #[prost(string, tag = "1")] pub bucket: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteBucketMetadataResponse { #[prost(bool, tag = "1")] @@ -1182,11 +1359,13 @@ pub struct DeleteBucketMetadataResponse { #[prost(string, optional, tag = "2")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeletePolicyRequest { #[prost(string, tag = "1")] pub policy_name: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeletePolicyResponse { #[prost(bool, tag = "1")] @@ -1196,11 +1375,13 @@ pub struct DeletePolicyResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadPolicyRequest { #[prost(string, tag = "1")] pub policy_name: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadPolicyResponse { #[prost(bool, tag = "1")] @@ -1210,6 +1391,7 @@ pub struct LoadPolicyResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadPolicyMappingRequest { #[prost(string, tag = "1")] @@ -1219,6 +1401,7 @@ pub struct LoadPolicyMappingRequest { #[prost(bool, tag = "3")] pub is_group: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadPolicyMappingResponse { #[prost(bool, tag = "1")] @@ -1228,11 +1411,13 @@ pub struct LoadPolicyMappingResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteUserRequest { #[prost(string, tag = "1")] pub access_key: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteUserResponse { #[prost(bool, tag = "1")] @@ -1242,11 +1427,13 @@ pub struct DeleteUserResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteServiceAccountRequest { #[prost(string, tag = "1")] pub access_key: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct DeleteServiceAccountResponse { #[prost(bool, tag = "1")] @@ -1256,6 +1443,7 @@ pub struct DeleteServiceAccountResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadUserRequest { #[prost(string, tag = "1")] @@ -1263,6 +1451,7 @@ pub struct LoadUserRequest { #[prost(bool, tag = "2")] pub temp: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadUserResponse { #[prost(bool, tag = "1")] @@ -1272,11 +1461,13 @@ pub struct LoadUserResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadServiceAccountRequest { #[prost(string, tag = "1")] pub access_key: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadServiceAccountResponse { #[prost(bool, tag = "1")] @@ -1286,11 +1477,13 @@ pub struct LoadServiceAccountResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadGroupRequest { #[prost(string, tag = "1")] pub group: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadGroupResponse { #[prost(bool, tag = "1")] @@ -1300,8 +1493,10 @@ pub struct LoadGroupResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReloadSiteReplicationConfigRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReloadSiteReplicationConfigResponse { #[prost(bool, tag = "1")] @@ -1311,11 +1506,13 @@ pub struct ReloadSiteReplicationConfigResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SignalServiceRequest { #[prost(message, optional, tag = "1")] pub vars: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SignalServiceResponse { #[prost(bool, tag = "1")] @@ -1331,6 +1528,7 @@ pub struct SignalServiceResponse { #[prost(string, optional, tag = "4")] pub config_fingerprint: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerActivityRequest { #[prost(bytes = "bytes", tag = "1")] @@ -1342,6 +1540,7 @@ pub struct ScannerActivityRequest { #[prost(uint64, tag = "4")] pub acknowledge_dirty_usage_generation: u64, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerActivityResponse { #[prost(string, tag = "1")] @@ -1369,6 +1568,7 @@ pub struct ScannerActivityResponse { #[prost(bool, optional, tag = "11")] pub publication_blocked: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerDirtyUsageBucket { #[prost(string, tag = "1")] @@ -1378,6 +1578,7 @@ pub struct ScannerDirtyUsageBucket { #[prost(bytes = "bytes", tag = "3")] pub bucket_incarnation: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerDirtyUsageSnapshotRequest { #[prost(bytes = "bytes", tag = "1")] @@ -1385,6 +1586,7 @@ pub struct ScannerDirtyUsageSnapshotRequest { #[prost(uint32, tag = "2")] pub protocol_version: u32, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ScannerDirtyUsageSnapshotResponse { #[prost(string, tag = "1")] @@ -1407,6 +1609,7 @@ pub struct ScannerDirtyUsageSnapshotResponse { } /// Receiver-only protocol. Producers must retain whole-cycle ACK until they /// have a durable per-bucket publication proof. +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerScopedDirtyUsageEntry { #[prost(string, tag = "1")] @@ -1416,6 +1619,7 @@ pub struct ScannerScopedDirtyUsageEntry { #[prost(uint64, tag = "3")] pub generation: u64, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ScannerScopedDirtyUsageAckRequest { #[prost(bytes = "bytes", tag = "1")] @@ -1434,6 +1638,7 @@ pub struct ScannerScopedDirtyUsageAckRequest { #[prost(message, repeated, tag = "7")] pub entries: ::prost::alloc::vec::Vec, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerScopedDirtyUsageAckResponse { #[prost(uint32, tag = "1")] @@ -1457,6 +1662,7 @@ pub struct ScannerScopedDirtyUsageAckResponse { /// scanner metadata publication. It is intentionally separate from the /// ScannerActivity observation wire so v6/v7 rolling compatibility remains /// unchanged. +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerPublicationLeaseRequest { #[prost(bytes = "bytes", tag = "1")] @@ -1480,6 +1686,7 @@ pub struct ScannerPublicationLeaseRequest { #[prost(bool, tag = "6")] pub observational_only: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerPublicationLeaseResponse { #[prost(bool, tag = "1")] @@ -1501,6 +1708,7 @@ pub struct ScannerPublicationLeaseResponse { #[prost(bool, tag = "9")] pub observational_only: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerPublicationLeaseReleaseRequest { #[prost(bytes = "bytes", tag = "1")] @@ -1512,6 +1720,7 @@ pub struct ScannerPublicationLeaseReleaseRequest { #[prost(string, tag = "4")] pub session_id: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ScannerPublicationLeaseReleaseResponse { #[prost(bool, tag = "1")] @@ -1521,11 +1730,13 @@ pub struct ScannerPublicationLeaseReleaseResponse { #[prost(bytes = "bytes", tag = "3")] pub response_proof: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BackgroundHealStatusRequest { #[prost(uint32, tag = "1")] pub protocol_version: u32, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BackgroundHealStatusResponse { #[prost(bool, tag = "1")] @@ -1537,8 +1748,10 @@ pub struct BackgroundHealStatusResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "4")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReplacementRecoveryStatusRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReplacementRecoveryStatusResponse { #[prost(bool, tag = "1")] @@ -1552,12 +1765,14 @@ pub struct ReplacementRecoveryStatusResponse { } /// Asks a peer how it currently sees the drives served by `host`, so a /// restarting node can wait for re-admission before publishing readiness. +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PeerDriveAdmissionRequest { /// Topology host:port of the drives' owner, as written in the endpoint list. #[prost(string, tag = "1")] pub host: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PeerDriveAdmission { #[prost(int32, tag = "1")] @@ -1574,6 +1789,7 @@ pub struct PeerDriveAdmission { #[prost(bool, tag = "6")] pub host_online: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct PeerDriveAdmissionResponse { #[prost(bool, tag = "1")] @@ -1585,6 +1801,7 @@ pub struct PeerDriveAdmissionResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "4")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct HealControlRequest { #[prost(uint32, tag = "1")] @@ -1594,6 +1811,7 @@ pub struct HealControlRequest { #[prost(bytes = "bytes", tag = "3")] pub command: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct HealControlResponse { #[prost(bool, tag = "1")] @@ -1605,11 +1823,13 @@ pub struct HealControlResponse { #[prost(bytes = "bytes", tag = "4")] pub response_proof: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetMetacacheListingRequest { #[prost(bytes = "bytes", tag = "1")] pub opts: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetMetacacheListingResponse { #[prost(bool, tag = "1")] @@ -1619,11 +1839,13 @@ pub struct GetMetacacheListingResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct UpdateMetacacheListingRequest { #[prost(bytes = "bytes", tag = "1")] pub metacache: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct UpdateMetacacheListingResponse { #[prost(bool, tag = "1")] @@ -1633,8 +1855,10 @@ pub struct UpdateMetacacheListingResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReloadPoolMetaRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReloadPoolMetaResponse { #[prost(bool, tag = "1")] @@ -1644,11 +1868,13 @@ pub struct ReloadPoolMetaResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StopRebalanceRequest { #[prost(string, tag = "1")] pub expected_rebalance_id: ::prost::alloc::string::String, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StopRebalanceResponse { #[prost(bool, tag = "1")] @@ -1658,11 +1884,13 @@ pub struct StopRebalanceResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadRebalanceMetaRequest { #[prost(bool, tag = "1")] pub start_rebalance: bool, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadRebalanceMetaResponse { #[prost(bool, tag = "1")] @@ -1672,11 +1900,13 @@ pub struct LoadRebalanceMetaResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StartDecommissionRequest { #[prost(uint32, repeated, tag = "1")] pub pool_indices: ::prost::alloc::vec::Vec, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StartDecommissionResponse { #[prost(bool, tag = "1")] @@ -1686,11 +1916,13 @@ pub struct StartDecommissionResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CancelDecommissionRequest { #[prost(uint32, tag = "1")] pub pool_index: u32, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CancelDecommissionResponse { #[prost(bool, tag = "1")] @@ -1700,11 +1932,13 @@ pub struct CancelDecommissionResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ClearDecommissionRequest { #[prost(uint32, tag = "1")] pub pool_index: u32, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ClearDecommissionResponse { #[prost(bool, tag = "1")] @@ -1714,8 +1948,10 @@ pub struct ClearDecommissionResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadTransitionTierConfigRequest {} +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct LoadTransitionTierConfigResponse { #[prost(bool, tag = "1")] @@ -1725,6 +1961,7 @@ pub struct LoadTransitionTierConfigResponse { #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")] pub error_code: ::core::option::Option, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct TierDailyStatsRequest {} /// One node's own rolling-day transition counters, per remote tier. @@ -1735,6 +1972,7 @@ pub struct TierDailyStatsRequest {} /// obtain a cluster total. A peer that does not implement this RPC answers /// UNIMPLEMENTED, which the caller reports as a non-reporting node rather than /// as zero activity. +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct TierDailyStatsResponse { #[prost(bool, tag = "1")] @@ -1744,6 +1982,7 @@ pub struct TierDailyStatsResponse { #[prost(string, optional, tag = "3")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct TierMutationPrepareRequest { #[prost(uint32, tag = "1")] @@ -1753,6 +1992,7 @@ pub struct TierMutationPrepareRequest { #[prost(bytes = "bytes", tag = "3")] pub canonical_payload: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct TierMutationCommitRequest { #[prost(uint32, tag = "1")] @@ -1762,6 +2002,7 @@ pub struct TierMutationCommitRequest { #[prost(bytes = "bytes", tag = "3")] pub canonical_payload: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct TierMutationAbortRequest { #[prost(uint32, tag = "1")] @@ -1771,6 +2012,7 @@ pub struct TierMutationAbortRequest { #[prost(bytes = "bytes", tag = "3")] pub canonical_payload: ::prost::bytes::Bytes, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct TierMutationControlResponse { #[prost(bool, tag = "1")] @@ -1786,6 +2028,7 @@ pub struct TierMutationControlResponse { #[prost(enumeration = "TierMutationFailureClass", tag = "6")] pub failure_class: i32, } +#[allow(clippy::all)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetLiveEventsRequest { #[prost(uint64, tag = "1")] @@ -1793,6 +2036,7 @@ pub struct GetLiveEventsRequest { #[prost(uint32, tag = "2")] pub limit: u32, } +#[allow(clippy::all)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetLiveEventsResponse { #[prost(bool, tag = "1")] @@ -1810,6 +2054,7 @@ pub struct GetLiveEventsResponse { /// error_info string on control-plane responses. Rolling-upgrade compat: /// old peers ignore the field and keep reading error_info. /// RUSTFS_COMPAT_TODO(not-initialized-error-code-v1): legacy string dual-write. Remove after the minimum supported RustFS peer version always sends error_code. +#[allow(clippy::all)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum ControlPlaneErrorCode { @@ -1843,6 +2088,7 @@ impl ControlPlaneErrorCode { } } } +#[allow(clippy::all)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum CompareAndUpdateFileOutcome { @@ -1875,6 +2121,7 @@ impl CompareAndUpdateFileOutcome { } } } +#[allow(clippy::all)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum TierMutationPeerState { @@ -1907,6 +2154,7 @@ impl TierMutationPeerState { } } } +#[allow(clippy::all)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum TierMutationFailureClass { @@ -2497,6 +2745,81 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "RenameData")); self.inner.unary(req, path, codec).await } + pub async fn reserve_object_metadata( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReserveObjectMetadata"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ReserveObjectMetadata")); + self.inner.unary(req, path, codec).await + } + pub async fn reserve_object_metadata_short( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReserveObjectMetadataShort"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ReserveObjectMetadataShort")); + self.inner.unary(req, path, codec).await + } + pub async fn activate_object_metadata_reservation( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ActivateObjectMetadataReservation"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ActivateObjectMetadataReservation")); + self.inner.unary(req, path, codec).await + } + pub async fn cancel_object_metadata_reservation( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/CancelObjectMetadataReservation"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "CancelObjectMetadataReservation")); + self.inner.unary(req, path, codec).await + } + pub async fn rename_reserved_data( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameReservedData"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "RenameReservedData")); + self.inner.unary(req, path, codec).await + } /// The method itself requires source-preserving multipart publication on every peer. pub async fn rename_multipart_data( &mut self, @@ -3836,6 +4159,26 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn reserve_object_metadata( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn reserve_object_metadata_short( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn activate_object_metadata_reservation( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn cancel_object_metadata_reservation( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn rename_reserved_data( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; /// The method itself requires source-preserving multipart publication on every peer. async fn rename_multipart_data( &self, @@ -5125,6 +5468,152 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/ReserveObjectMetadata" => { + #[allow(non_camel_case_types)] + struct ReserveObjectMetadataSvc(pub Arc); + impl tonic::server::UnaryService for ReserveObjectMetadataSvc { + type Response = super::SnapshotLeaseResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::reserve_object_metadata(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ReserveObjectMetadataSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/ReserveObjectMetadataShort" => { + #[allow(non_camel_case_types)] + struct ReserveObjectMetadataShortSvc(pub Arc); + impl tonic::server::UnaryService for ReserveObjectMetadataShortSvc { + type Response = super::SnapshotLeaseResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::reserve_object_metadata_short(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ReserveObjectMetadataShortSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/ActivateObjectMetadataReservation" => { + #[allow(non_camel_case_types)] + struct ActivateObjectMetadataReservationSvc(pub Arc); + impl tonic::server::UnaryService + for ActivateObjectMetadataReservationSvc + { + type Response = super::SnapshotLeaseMutationResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = + async move { ::activate_object_metadata_reservation(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ActivateObjectMetadataReservationSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/CancelObjectMetadataReservation" => { + #[allow(non_camel_case_types)] + struct CancelObjectMetadataReservationSvc(pub Arc); + impl tonic::server::UnaryService + for CancelObjectMetadataReservationSvc + { + type Response = super::SnapshotLeaseMutationResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = + async move { ::cancel_object_metadata_reservation(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = CancelObjectMetadataReservationSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/RenameReservedData" => { + #[allow(non_camel_case_types)] + struct RenameReservedDataSvc(pub Arc); + impl tonic::server::UnaryService for RenameReservedDataSvc { + type Response = super::RenameDataResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::rename_reserved_data(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RenameReservedDataSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/RenameMultipartData" => { #[allow(non_camel_case_types)] struct RenameMultipartDataSvc(pub Arc); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index e75b24a5c..8240c81f3 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -1092,6 +1092,56 @@ impl_canonical_mutation_body!( // its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force // the JSON fallback decode path, breaks the signature-bound digest. +pub fn canonical_metadata_reservation_request_body( + request: &proto_gen::node_service::MetadataReservationRequest, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-request-v1\0"); + body.push_str(&request.disk)?; + body.push_str(&request.bucket)?; + body.push_str(&request.object)?; + body.push_u64(request.timeout_ms); + body.push_bytes(&request.bucket_incarnation_id)?; + Ok(body.finish()) +} + +pub fn canonical_metadata_reservation_cancel_request_body( + request: &proto_gen::node_service::MetadataReservationCancelRequest, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-cancel-v1\0"); + body.push_str(&request.disk)?; + body.push_bytes(&request.token)?; + Ok(body.finish()) +} + +pub fn canonical_metadata_reservation_activate_request_body( + request: &proto_gen::node_service::MetadataReservationActivateRequest, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0"); + body.push_str(&request.disk)?; + body.push_str(&request.bucket)?; + body.push_str(&request.object)?; + body.push_bytes(&request.token)?; + // Keep ordinary activations byte-compatible with existing peers. Renewal + // adds a distinct signed marker so an older peer cannot treat it as activation. + if request.renew_only { + body.push_bool(true); + } + Ok(body.finish()) +} + +pub fn canonical_reserved_rename_data_request_body( + request: &proto_gen::node_service::ReservedRenameDataRequest, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-reserved-rename-data-v1\0"); + if let Some(rename) = request.rename.as_ref() { + body.push_bytes(&canonical_rename_data_request_body(rename)?)?; + } else { + body.push_bytes(&[])?; + } + body.push_bytes(&request.token)?; + Ok(body.finish()) +} + pub fn canonical_rename_data_request_body( request: &proto_gen::node_service::RenameDataRequest, ) -> Result, std::num::TryFromIntError> { @@ -1722,6 +1772,96 @@ mod disk_mutation_canonical_tests { } } + #[test] + fn metadata_reservation_canonical_bodies_bind_scope_and_token() { + use proto_gen::node_service::{ + MetadataReservationActivateRequest, MetadataReservationCancelRequest, MetadataReservationRequest, + ReservedRenameDataRequest, + }; + + let request = MetadataReservationRequest { + disk: "disk".into(), + bucket: "bucket".into(), + object: "object".into(), + timeout_ms: 5000, + bucket_incarnation_id: vec![1; 16].into(), + }; + let mut bodies = vec![canonical_metadata_reservation_request_body(&request).expect("reservation body")]; + for mutate in [ + |r: &mut MetadataReservationRequest| r.disk = "other".into(), + |r: &mut MetadataReservationRequest| r.bucket = "other".into(), + |r: &mut MetadataReservationRequest| r.object = "other".into(), + |r: &mut MetadataReservationRequest| r.timeout_ms = 1, + |r: &mut MetadataReservationRequest| r.bucket_incarnation_id = vec![2; 16].into(), + ] { + let mut changed = request.clone(); + mutate(&mut changed); + bodies.push(canonical_metadata_reservation_request_body(&changed).expect("changed body")); + } + assert_all_distinct(&bodies); + let activation = MetadataReservationActivateRequest { + disk: "disk".into(), + bucket: "bucket".into(), + object: "object".into(), + token: vec![1; 16].into(), + renew_only: false, + }; + let mut legacy_activation = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0"); + legacy_activation.push_str(&activation.disk).expect("disk"); + legacy_activation.push_str(&activation.bucket).expect("bucket"); + legacy_activation.push_str(&activation.object).expect("object"); + legacy_activation.push_bytes(&activation.token).expect("token"); + assert_eq!( + canonical_metadata_reservation_activate_request_body(&activation).expect("ordinary activation"), + legacy_activation.finish(), + "ordinary activation must retain its previous body digest" + ); + let mut bodies = vec![canonical_metadata_reservation_activate_request_body(&activation).expect("activation body")]; + for mutate in [ + |r: &mut MetadataReservationActivateRequest| r.disk = "other".into(), + |r: &mut MetadataReservationActivateRequest| r.bucket = "other".into(), + |r: &mut MetadataReservationActivateRequest| r.object = "other".into(), + |r: &mut MetadataReservationActivateRequest| r.token = vec![2; 16].into(), + |r: &mut MetadataReservationActivateRequest| r.renew_only = true, + ] { + let mut changed = activation.clone(); + mutate(&mut changed); + bodies.push(canonical_metadata_reservation_activate_request_body(&changed).expect("changed activation")); + } + assert_all_distinct(&bodies); + let cancellation = MetadataReservationCancelRequest { + disk: "disk".into(), + token: vec![1; 16].into(), + }; + let mut changed = cancellation.clone(); + changed.token = vec![2; 16].into(); + assert_ne!( + canonical_metadata_reservation_cancel_request_body(&cancellation).expect("cancel body"), + canonical_metadata_reservation_cancel_request_body(&changed).expect("changed cancel body") + ); + let rename = ReservedRenameDataRequest { + rename: Some(RenameDataRequest { + dst_path: "object".into(), + ..Default::default() + }), + token: vec![1; 16].into(), + }; + let mut changed_token = rename.clone(); + changed_token.token = vec![2; 16].into(); + let mut changed_object = rename.clone(); + changed_object.rename.as_mut().expect("rename").dst_path = "other".into(); + assert_all_distinct(&[ + canonical_reserved_rename_data_request_body(&rename).expect("rename body"), + canonical_reserved_rename_data_request_body(&changed_token).expect("changed token"), + canonical_reserved_rename_data_request_body(&changed_object).expect("changed object"), + canonical_reserved_rename_data_request_body(&ReservedRenameDataRequest { + rename: None, + token: rename.token, + }) + .expect("missing rename"), + ]); + } + #[test] fn rename_data_canonical_body_binds_every_field() { let baseline = RenameDataRequest { diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index e03598b99..dc16791da 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -339,6 +339,32 @@ message RenameDataResponse { bytes rename_data_resp_bin = 4; } +message MetadataReservationRequest { + string disk = 1; + string bucket = 2; + string object = 3; + uint64 timeout_ms = 4; + bytes bucket_incarnation_id = 5; +} + +message MetadataReservationCancelRequest { + string disk = 1; + bytes token = 2; +} + +message MetadataReservationActivateRequest { + string disk = 1; + string bucket = 2; + string object = 3; + bytes token = 4; + bool renew_only = 5; +} + +message ReservedRenameDataRequest { + RenameDataRequest rename = 1; + bytes token = 2; +} + // Fixed-scope native pool.bin recovery. The typed MessagePack payload never // accepts an arbitrary volume or object path. message PoolMetaNativeRecoveryRequest { @@ -1314,6 +1340,11 @@ service NodeService { rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound + rpc ReserveObjectMetadata(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound + rpc ReserveObjectMetadataShort(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound + rpc ActivateObjectMetadataReservation(MetadataReservationActivateRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound + rpc CancelObjectMetadataReservation(MetadataReservationCancelRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound + rpc RenameReservedData(ReservedRenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound // The method itself requires source-preserving multipart publication on every peer. rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound diff --git a/crates/test-utils/src/ecstore_test_compat.rs b/crates/test-utils/src/ecstore_test_compat.rs index 6ed8bfaa1..f184c1979 100644 --- a/crates/test-utils/src/ecstore_test_compat.rs +++ b/crates/test-utils/src/ecstore_test_compat.rs @@ -27,9 +27,10 @@ pub(crate) mod fixture { pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; pub(crate) use rustfs_ecstore::api::object::{PutObjReader, SelectObjectSnapshot}; + pub(crate) use rustfs_ecstore::api::runtime::InstanceContext; #[cfg(feature = "put-object-commit-barrier")] pub(crate) use rustfs_ecstore::api::set_disk as ecstore_set_disk; - pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; + pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks, init_local_disks_with_instance_ctx}; pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions, ObjectIO}; #[cfg(test)] diff --git a/crates/test-utils/src/lib.rs b/crates/test-utils/src/lib.rs index 354716dc0..187fb7ddb 100644 --- a/crates/test-utils/src/lib.rs +++ b/crates/test-utils/src/lib.rs @@ -35,8 +35,9 @@ use std::sync::{Arc, Once}; #[cfg(feature = "put-object-commit-barrier")] use ecstore_test_compat::fixture::ecstore_set_disk; use ecstore_test_compat::fixture::{ - BucketOperations as _, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _, - PoolEndpoints, PutObjReader, SelectObjectSnapshot, init_bucket_metadata_sys, init_local_disks, + BucketOperations as _, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, MakeBucketOptions, + ObjectIO as _, PoolEndpoints, PutObjReader, SelectObjectSnapshot, init_bucket_metadata_sys, init_local_disks, + init_local_disks_with_instance_ctx, }; use tokio_util::sync::CancellationToken; @@ -90,9 +91,8 @@ pub struct TestECStoreEnv { pub temp_root: PathBuf, /// The per-disk directories (`disk1`..`diskN`) under `temp_root`. pub disk_paths: Vec, - /// The store, bootstrapped exactly like the historical test setups: - /// `init_local_disks` + `ECStore::new` on `127.0.0.1:0` (random port keeps - /// nextest's process-per-test parallelism safe). + /// The store, using the historical bootstrap context by default or the + /// explicit instance supplied to the builder, on `127.0.0.1:0`. pub ecstore: Arc, /// The single-pool, single-set topology the store was built from. /// @@ -155,6 +155,7 @@ pub struct TestECStoreEnvBuilder { prefix: String, base_dir: Option, init_bucket_metadata: bool, + instance_context: Option>, } impl Default for TestECStoreEnvBuilder { @@ -164,6 +165,7 @@ impl Default for TestECStoreEnvBuilder { prefix: "rustfs_test_utils".to_string(), base_dir: None, init_bucket_metadata: true, + instance_context: None, } } } @@ -196,6 +198,12 @@ impl TestECStoreEnvBuilder { self } + /// Use an explicit instance instead of the process bootstrap context. + pub fn instance_context(mut self, context: Arc) -> Self { + self.instance_context = Some(context); + self + } + /// Build the environment. Panics on any bootstrap failure — this is test /// scaffolding, and a broken environment must fail the test loudly. pub async fn build(self) -> TestECStoreEnv { @@ -237,14 +245,22 @@ impl TestECStoreEnvBuilder { }; let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]); - init_local_disks(endpoint_pools.clone()).await.expect("init local disks"); - // Port 0 keeps ECStore-backed integration binaries parallel-safe under // nextest: no fixed peer port is ever shared between test processes. let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().expect("parse test addr"); - let ecstore = ECStore::new(server_addr, endpoint_pools.clone(), CancellationToken::new()) - .await - .expect("build test ECStore"); + let ecstore = if let Some(context) = self.instance_context { + init_local_disks_with_instance_ctx(&context, endpoint_pools.clone()) + .await + .expect("init isolated local disks"); + ECStore::new_with_instance_ctx(server_addr, endpoint_pools.clone(), CancellationToken::new(), context) + .await + .expect("build isolated test ECStore") + } else { + init_local_disks(endpoint_pools.clone()).await.expect("init local disks"); + ECStore::new(server_addr, endpoint_pools.clone(), CancellationToken::new()) + .await + .expect("build test ECStore") + }; // The production bootstrap only persists pool.bin from the elected // first cluster node. Test stores intentionally have no cluster diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 4bb1952d9..a9e0f3dc7 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -1826,6 +1826,41 @@ impl Node for NodeService { self.handle_rename_data(request, false, true, false).await } + async fn reserve_object_metadata( + &self, + request: Request, + ) -> Result, Status> { + self.handle_reserve_object_metadata(request, false).await + } + + async fn reserve_object_metadata_short( + &self, + request: Request, + ) -> Result, Status> { + self.handle_reserve_object_metadata(request, true).await + } + + async fn cancel_object_metadata_reservation( + &self, + request: Request, + ) -> Result, Status> { + self.handle_cancel_object_metadata_reservation(request).await + } + + async fn activate_object_metadata_reservation( + &self, + request: Request, + ) -> Result, Status> { + self.handle_activate_object_metadata_reservation(request).await + } + + async fn rename_reserved_data( + &self, + request: Request, + ) -> Result, Status> { + self.handle_rename_reserved_data(request).await + } + async fn undo_pool_meta_write( &self, request: Request, @@ -3245,15 +3280,17 @@ mod tests { HealControlRequest, HealControlResponse, ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, - MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest, PreparePartTransactionRequest, + MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest, MetadataReservationCancelRequest, + MetadataReservationRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, - RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest, - ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, - SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, - StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, - TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, - UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest, + RenameFileRequest, RenamePartRequest, ReservedRenameDataRequest, ScannerActivityRequest, + ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, + ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest, + SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, + StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, TierMutationPeerState, + TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, WriteRequest, heal_control_service_client::HealControlServiceClient, heal_control_service_server::{HealControlService as _, HealControlServiceServer}, node_service_client::NodeServiceClient, @@ -3271,8 +3308,13 @@ mod tests { use tonic::{Request, Response, Status}; use uuid::Uuid; - const DISK_MUTATION_RPC_METHODS: [&str; 34] = [ + const DISK_MUTATION_RPC_METHODS: [&str; 39] = [ "renamedata", + "reserveobjectmetadata", + "reserveobjectmetadatashort", + "activateobjectmetadatareservation", + "cancelobjectmetadatareservation", + "renamereserveddata", "renamemultipartdata", "recovermultipartsource", "renamedataatincarnation", @@ -4970,6 +5012,55 @@ mod tests { }, rustfs_protos::canonical_rename_data_request_body ); + assert_gated!( + reserve_object_metadata, + MetadataReservationRequest { + disk: disk.clone(), + bucket: "bucket".into(), + object: "object".into(), + timeout_ms: 5000, + bucket_incarnation_id: vec![1; 16].into(), + }, + rustfs_protos::canonical_metadata_reservation_request_body + ); + assert_gated!( + reserve_object_metadata_short, + MetadataReservationRequest { + disk: disk.clone(), + bucket: "bucket".into(), + object: "object".into(), + timeout_ms: 5000, + bucket_incarnation_id: vec![1; 16].into(), + }, + rustfs_protos::canonical_metadata_reservation_request_body + ); + assert_gated!( + activate_object_metadata_reservation, + MetadataReservationActivateRequest { + disk: disk.clone(), + bucket: "bucket".into(), + object: "object".into(), + token: Uuid::new_v4().as_bytes().to_vec().into(), + renew_only: false, + }, + rustfs_protos::canonical_metadata_reservation_activate_request_body + ); + assert_gated!( + cancel_object_metadata_reservation, + MetadataReservationCancelRequest { + disk: disk.clone(), + token: Uuid::new_v4().as_bytes().to_vec().into(), + }, + rustfs_protos::canonical_metadata_reservation_cancel_request_body + ); + assert_gated!( + rename_reserved_data, + ReservedRenameDataRequest { + rename: Some(RenameDataRequest::default()), + token: Uuid::new_v4().as_bytes().to_vec().into(), + }, + rustfs_protos::canonical_reserved_rename_data_request_body + ); assert_gated!( recover_multipart_source, RecoverMultipartSourceRequest::default(), @@ -7475,13 +7566,26 @@ mod tests { } async fn target_rpc_fixture() -> TargetRpcFixture { + target_rpc_fixture_with_bucket_metadata(false).await + } + + async fn target_rpc_fixture_with_bucket_metadata(initialize_bucket_metadata: bool) -> TargetRpcFixture { super::timeout(Duration::from_secs(90), async { let root = tempfile::tempdir().expect("target RPC root"); - let env = rustfs_test_utils::TestECStoreEnv::builder() + let instance = if initialize_bucket_metadata { + crate::storage::storage_api::new_instance_ctx() + } else { + crate::storage::storage_api::bootstrap_instance_ctx() + }; + let builder = rustfs_test_utils::TestECStoreEnv::builder() .base_dir(root.path()) - .init_bucket_metadata(false) - .build() - .await; + .init_bucket_metadata(initialize_bucket_metadata); + let builder = if initialize_bucket_metadata { + builder.instance_context(instance.clone()) + } else { + builder + }; + let env = builder.build().await; ObjectStore::new(env.ecstore.clone()) .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX)) .await @@ -7494,10 +7598,9 @@ mod tests { iam.clone(), Arc::new(KmsServiceManager::new()), )); - let instance = crate::storage::storage_api::bootstrap_instance_ctx(); assert!( super::BootstrapLocalTarget::new(instance.clone()).is_for_store(&env.ecstore), - "the standard builder must use this exact instance context" + "the fixture must use this exact instance context" ); super::timeout(Duration::from_secs(10), async { while env.ecstore.namespace_commits_pending_for_test() { @@ -8146,6 +8249,13 @@ mod tests { } async fn stage_target_rpc(fixture: &TargetRpcFixture) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec) { + stage_target_rpc_inner(fixture, true).await + } + + async fn stage_target_rpc_inner( + fixture: &TargetRpcFixture, + create_volume: bool, + ) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec) { use crate::storage::storage_api::ecstore_disk::{DiskAPI, ReadOptions}; let set = fixture .env @@ -8168,7 +8278,9 @@ mod tests { }]; fi.data = Some(Bytes::from_static(b"target-rpc-inline")); fi.set_inline_data(); - disk.make_volume("target-rpc").await.expect("target volume"); + if create_volume { + disk.make_volume("target-rpc").await.expect("target volume"); + } disk.write_metadata("target-rpc", "target-rpc", "staged", fi.clone()) .await .expect("stage real inline body"); @@ -8220,6 +8332,740 @@ mod tests { request } + #[tokio::test] + #[serial_test::serial] + async fn isolated_reservation_fixture_does_not_capture_gating_store() { + let fixture = target_rpc_fixture_with_bucket_metadata(true).await; + fixture.env.make_bucket("rpc-isolated-order", false).await; + let (_store, bucket) = tokio::time::timeout( + Duration::from_secs(20), + crate::app::gating_test_env::durable_quota_test_bucket("rpc-isolated-order", 1024), + ) + .await + .expect("gating fixture initializes after isolated RPC fixture"); + assert!(!bucket.is_empty()); + } + + #[tokio::test] + #[serial_test::serial] + async fn lost_short_reservation_rpc_response_releases_the_bucket_fence() { + use crate::storage::storage_api::contract::namespace::NamespaceLocking as _; + + let fixture = target_rpc_fixture_with_bucket_metadata(true).await; + fixture.env.make_bucket("target-rpc", false).await; + let bucket = "target-rpc-short-lost"; + fixture.env.make_bucket(bucket, false).await; + let incarnation = fixture + .env + .ecstore + .bucket_incarnation_id(bucket) + .await + .expect("bucket incarnation"); + let (disk, _, _) = stage_target_rpc_inner(&fixture, false).await; + let lock = fixture + .env + .ecstore + .new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("publication lock"); + let service = make_server_for_context(Some(fixture.context.clone())); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind RPC listener"); + let addr = listener.local_addr().expect("RPC listener address"); + let (shutdown, stopped) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(NodeServiceServer::new(service)) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async { + let _ = stopped.await; + }) + .await + .expect("serve RPC target"); + }); + let mut client = NodeServiceClient::connect(format!("http://{addr}")) + .await + .expect("connect RPC client"); + let mut request = Request::new(MetadataReservationRequest { + disk: disk.endpoint().to_string(), + bucket: bucket.into(), + object: "destination".into(), + timeout_ms: 5000, + bucket_incarnation_id: incarnation.as_bytes().to_vec().into(), + }); + let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()).expect("reservation body"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let mut pause = + super::disk::short_response_test_hook::install(bucket, super::disk::short_response_test_hook::Phase::Reserve); + let call = tokio::spawn(async move { client.reserve_object_metadata_short(request).await }); + tokio::time::timeout(Duration::from_secs(5), &mut pause.entered) + .await + .expect("server allocated the token") + .expect("server signalled allocation"); + assert!( + tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(5))) + .await + .is_err(), + "the token owns the bucket fence before response delivery" + ); + call.abort(); + let _ = call.await; + drop(pause); + let writer = tokio::time::timeout(Duration::from_secs(4), lock.get_write_lock(Duration::from_secs(4))) + .await + .expect("the lost response does not strand the bucket fence") + .expect("publication writer acquires"); + drop(writer); + + let mut client = NodeServiceClient::connect(format!("http://{addr}")) + .await + .expect("reconnect RPC client"); + let mut request = Request::new(MetadataReservationRequest { + disk: disk.endpoint().to_string(), + bucket: bucket.into(), + object: "destination".into(), + timeout_ms: 5000, + bucket_incarnation_id: incarnation.as_bytes().to_vec().into(), + }); + let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()).expect("reservation body"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let prepared = client + .reserve_object_metadata_short(request) + .await + .expect("short reservation") + .into_inner(); + assert!(prepared.success, "short reservation must succeed"); + let mut renew = Request::new(MetadataReservationActivateRequest { + disk: disk.endpoint().to_string(), + bucket: bucket.into(), + object: "destination".into(), + token: prepared.token, + renew_only: true, + }); + let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(renew.get_ref()).expect("renewal body"); + set_tonic_canonical_body_digest(&mut renew, &body).expect("renewal digest"); + mark_v2_authenticated(&mut renew); + let mut pause = + super::disk::short_response_test_hook::install(bucket, super::disk::short_response_test_hook::Phase::Renew); + let call = tokio::spawn(async move { client.activate_object_metadata_reservation(renew).await }); + tokio::time::timeout(Duration::from_secs(5), &mut pause.entered) + .await + .expect("server accepted renewal") + .expect("server signalled renewal"); + assert!( + tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(5))) + .await + .is_err(), + "the renewed token must retain the bucket fence" + ); + call.abort(); + let _ = call.await; + drop(pause); // Model loss of the renewal response and subsequent cancel. + let writer = tokio::time::timeout(Duration::from_secs(4), lock.get_write_lock(Duration::from_secs(4))) + .await + .expect("lost renewal and cancel do not strand the bucket fence") + .expect("publication writer acquires after renewal expiry"); + drop(writer); + + let mut client = NodeServiceClient::connect(format!("http://{addr}")) + .await + .expect("reconnect RPC client"); + let mut request = Request::new(MetadataReservationRequest { + disk: disk.endpoint().to_string(), + bucket: bucket.into(), + object: "destination".into(), + timeout_ms: 5000, + bucket_incarnation_id: incarnation.as_bytes().to_vec().into(), + }); + let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()).expect("reservation body"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let prepared = client + .reserve_object_metadata_short(request) + .await + .expect("short reservation") + .into_inner(); + assert!(prepared.success, "short reservation must succeed"); + let mut activate = Request::new(MetadataReservationActivateRequest { + disk: disk.endpoint().to_string(), + bucket: bucket.into(), + object: "destination".into(), + token: prepared.token, + renew_only: false, + }); + let body = + rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activation body"); + set_tonic_canonical_body_digest(&mut activate, &body).expect("activation digest"); + mark_v2_authenticated(&mut activate); + let mut pause = + super::disk::short_response_test_hook::install(bucket, super::disk::short_response_test_hook::Phase::Activate); + let call = tokio::spawn(async move { client.activate_object_metadata_reservation(activate).await }); + tokio::time::timeout(Duration::from_secs(5), &mut pause.entered) + .await + .expect("server activated reservation") + .expect("server signalled activation"); + assert!( + tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(5))) + .await + .is_err(), + "the activated token must retain the bucket fence" + ); + call.abort(); + let _ = call.await; + drop(pause); // Model loss of the activation response and subsequent cancel. + let writer = tokio::time::timeout(Duration::from_secs(4), lock.get_write_lock(Duration::from_secs(4))) + .await + .expect("lost activation and cancel do not strand the bucket fence") + .expect("publication writer acquires after activation expiry"); + drop(writer); + + let mut client = NodeServiceClient::connect(format!("http://{addr}")) + .await + .expect("reconnect RPC client"); + let mut request = Request::new(MetadataReservationRequest { + disk: disk.endpoint().to_string(), + bucket: bucket.into(), + object: "destination".into(), + timeout_ms: 5000, + bucket_incarnation_id: incarnation.as_bytes().to_vec().into(), + }); + let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()).expect("reservation body"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let prepared = client + .reserve_object_metadata_short(request) + .await + .expect("short reservation") + .into_inner(); + assert!(prepared.success, "short reservation must succeed"); + let mut activate = Request::new(MetadataReservationActivateRequest { + disk: disk.endpoint().to_string(), + bucket: bucket.into(), + object: "destination".into(), + token: prepared.token.clone(), + renew_only: false, + }); + let body = + rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activation body"); + set_tonic_canonical_body_digest(&mut activate, &body).expect("activation digest"); + mark_v2_authenticated(&mut activate); + assert!( + client + .activate_object_metadata_reservation(activate) + .await + .expect("activation RPC") + .into_inner() + .success, + "short token must activate" + ); + let mut renew = Request::new(MetadataReservationActivateRequest { + disk: disk.endpoint().to_string(), + bucket: bucket.into(), + object: "destination".into(), + token: prepared.token, + renew_only: true, + }); + let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(renew.get_ref()).expect("renewal body"); + set_tonic_canonical_body_digest(&mut renew, &body).expect("renewal digest"); + mark_v2_authenticated(&mut renew); + let mut pause = + super::disk::short_response_test_hook::install(bucket, super::disk::short_response_test_hook::Phase::Renew); + let call = tokio::spawn(async move { client.activate_object_metadata_reservation(renew).await }); + tokio::time::timeout(Duration::from_secs(5), &mut pause.entered) + .await + .expect("server accepted queued renewal") + .expect("server signalled queued renewal"); + call.abort(); + let _ = call.await; + drop(pause); + let writer = tokio::time::timeout(Duration::from_secs(4), lock.get_write_lock(Duration::from_secs(4))) + .await + .expect("lost queued renewal and cancel do not strand the bucket fence") + .expect("publication writer acquires after queued renewal expiry"); + drop(writer); + let _ = shutdown.send(()); + server.await.expect("server task"); + } + + #[tokio::test] + #[serial_test::serial] + async fn short_metadata_reservation_expires_and_preserves_legacy_lifetime() { + use crate::storage::storage_api::contract::namespace::NamespaceLocking as _; + + let fixture = target_rpc_fixture_with_bucket_metadata(true).await; + fixture.env.make_bucket("target-rpc", false).await; + let incarnation = fixture + .env + .ecstore + .bucket_incarnation_id("target-rpc") + .await + .expect("bucket incarnation"); + let (disk, _, _) = stage_target_rpc_inner(&fixture, false).await; + let service = make_server_for_context(Some(fixture.context.clone())); + let lock = fixture + .env + .ecstore + .new_ns_lock("target-rpc", rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("publication lock"); + let make_request = || { + let mut request = Request::new(MetadataReservationRequest { + disk: disk.endpoint().to_string(), + bucket: "target-rpc".into(), + object: "destination".into(), + timeout_ms: 5000, + bucket_incarnation_id: incarnation.as_bytes().to_vec().into(), + }); + let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()).expect("reservation body"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + request + }; + let writer = lock + .get_write_lock(Duration::from_secs(1)) + .await + .expect("hold publication writer"); + let refused = tokio::time::timeout(Duration::from_millis(500), service.reserve_object_metadata_short(make_request())) + .await + .expect("short acquisition is bounded") + .expect("short handler") + .into_inner(); + assert!(!refused.success, "an occupied publication lock must not issue a token"); + assert_eq!(refused.protocol_version, 3); + drop(writer); + + let response = service + .reserve_object_metadata_short(make_request()) + .await + .expect("short handler") + .into_inner(); + assert!(response.success); + assert_eq!(response.protocol_version, 3); + drop(response); // Model a successful server response that never reaches its caller. + assert!( + tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(5))) + .await + .is_err(), + "live token retains the fence" + ); + let writer = tokio::time::timeout(Duration::from_secs(4), lock.get_write_lock(Duration::from_secs(4))) + .await + .expect("short token releases after expiry") + .expect("publication writer acquires"); + drop(writer); + + let legacy = service + .reserve_object_metadata(make_request()) + .await + .expect("legacy handler") + .into_inner(); + assert!(legacy.success); + assert_eq!(legacy.protocol_version, 2); + tokio::time::sleep(Duration::from_secs(3)).await; + assert!( + tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(5))) + .await + .is_err(), + "legacy token remains live" + ); + let mut cancel = Request::new(MetadataReservationCancelRequest { + disk: disk.endpoint().to_string(), + token: legacy.token, + }); + let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(cancel.get_ref()).expect("cancel body"); + set_tonic_canonical_body_digest(&mut cancel, &body).expect("cancel digest"); + mark_v2_authenticated(&mut cancel); + assert!( + service + .cancel_object_metadata_reservation(cancel) + .await + .expect("cancel handler") + .into_inner() + .success + ); + lock.get_write_lock(Duration::from_secs(1)) + .await + .expect("cancel releases legacy fence"); + } + + #[tokio::test] + #[serial_test::serial] + async fn metadata_reservation_handlers_bind_and_publish_on_the_target_disk() { + use crate::storage::storage_api::ecstore_disk::{DiskAPI, RUSTFS_META_TMP_BUCKET, ReadOptions}; + + let fixture = target_rpc_fixture_with_bucket_metadata(true).await; + fixture.env.make_bucket("target-rpc", false).await; + let incarnation = fixture + .env + .ecstore + .bucket_incarnation_id("target-rpc") + .await + .expect("bucket incarnation"); + let (disk, fi, _) = stage_target_rpc_inner(&fixture, false).await; + disk.write_metadata(RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET, "reserved-stage", fi.clone()) + .await + .expect("stage reserved body"); + let service = make_server_for_context(Some(fixture.context.clone())); + let mut reserve = Request::new(MetadataReservationRequest { + disk: disk.endpoint().to_string(), + bucket: "target-rpc".into(), + object: "destination".into(), + timeout_ms: 5000, + bucket_incarnation_id: incarnation.as_bytes().to_vec().into(), + }); + let mut stale = Request::new(reserve.get_ref().clone()); + stale.get_mut().bucket_incarnation_id = Uuid::new_v4().as_bytes().to_vec().into(); + let stale_body = rustfs_protos::canonical_metadata_reservation_request_body(stale.get_ref()).expect("stale body"); + set_tonic_canonical_body_digest(&mut stale, &stale_body).expect("stale digest"); + mark_v2_authenticated(&mut stale); + assert!( + !service + .reserve_object_metadata(stale) + .await + .expect("stale handler") + .into_inner() + .success, + "a token must not be issued for a different bucket incarnation" + ); + let mut self_lock = Request::new(reserve.get_ref().clone()); + self_lock.get_mut().object = rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH.to_owned(); + let self_lock_body = + rustfs_protos::canonical_metadata_reservation_request_body(self_lock.get_ref()).expect("self-lock body"); + set_tonic_canonical_body_digest(&mut self_lock, &self_lock_body).expect("self-lock digest"); + mark_v2_authenticated(&mut self_lock); + assert!( + !service + .reserve_object_metadata(self_lock) + .await + .expect("self-lock handler") + .into_inner() + .success, + "a token cannot hold the same lock that its object commit must acquire" + ); + let body = rustfs_protos::canonical_metadata_reservation_request_body(reserve.get_ref()).expect("reserve body"); + set_tonic_canonical_body_digest(&mut reserve, &body).expect("reserve digest"); + mark_v2_authenticated(&mut reserve); + let response = service + .reserve_object_metadata(reserve) + .await + .expect("reserve RPC") + .into_inner(); + assert!(response.success, "reservation must succeed: {:?}", response.error); + assert_eq!(response.protocol_version, 2); + let token = response.token; + let mut tampered_renew = Request::new(MetadataReservationActivateRequest { + disk: disk.endpoint().to_string(), + bucket: "target-rpc".into(), + object: "destination".into(), + token: token.clone(), + renew_only: false, + }); + let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(tampered_renew.get_ref()) + .expect("ordinary activation body"); + set_tonic_canonical_body_digest(&mut tampered_renew, &body).expect("ordinary activation digest"); + mark_v2_authenticated(&mut tampered_renew); + tampered_renew.get_mut().renew_only = true; + assert_eq!( + service + .activate_object_metadata_reservation(tampered_renew) + .await + .expect_err("changing activation into renewal must invalidate the body digest") + .code(), + tonic::Code::PermissionDenied + ); + let mut renew = Request::new(MetadataReservationActivateRequest { + disk: disk.endpoint().to_string(), + bucket: "target-rpc".into(), + object: "destination".into(), + token: token.clone(), + renew_only: true, + }); + let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(renew.get_ref()).expect("renew body"); + set_tonic_canonical_body_digest(&mut renew, &body).expect("renew digest"); + mark_v2_authenticated(&mut renew); + assert!( + service + .activate_object_metadata_reservation(renew) + .await + .expect("renew RPC") + .into_inner() + .success, + "prepared reservation should renew without activation" + ); + let mut activate = Request::new(MetadataReservationActivateRequest { + disk: disk.endpoint().to_string(), + bucket: "target-rpc".into(), + object: "destination".into(), + token: token.clone(), + renew_only: false, + }); + let body = + rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activate body"); + set_tonic_canonical_body_digest(&mut activate, &body).expect("activate digest"); + mark_v2_authenticated(&mut activate); + let activated = service + .activate_object_metadata_reservation(activate) + .await + .expect("activate RPC") + .into_inner(); + assert!(activated.success, "activation must succeed: {:?}", activated.error); + let message = ReservedRenameDataRequest { + rename: Some(RenameDataRequest { + disk: disk.endpoint().to_string(), + src_volume: RUSTFS_META_TMP_BUCKET.into(), + src_path: "reserved-stage".into(), + file_info: serde_json::to_string(&fi).expect("encode metadata"), + dst_volume: "target-rpc".into(), + dst_path: "destination".into(), + ..Default::default() + }), + token, + }; + let body = rustfs_protos::canonical_reserved_rename_data_request_body(&message).expect("rename body"); + let mut tampered = Request::new(message.clone()); + set_tonic_canonical_body_digest(&mut tampered, &body).expect("rename digest"); + mark_v2_authenticated(&mut tampered); + tampered.get_mut().rename.as_mut().expect("rename").dst_path = "other".into(); + assert_eq!( + service + .rename_reserved_data(tampered) + .await + .expect_err("tampered object") + .code(), + tonic::Code::PermissionDenied + ); + let mut rename = Request::new(message); + set_tonic_canonical_body_digest(&mut rename, &body).expect("rename digest"); + mark_v2_authenticated(&mut rename); + let response = service.rename_reserved_data(rename).await.expect("rename RPC").into_inner(); + assert!(response.success, "reserved rename must succeed: {:?}", response.error); + let read = disk + .read_version( + "target-rpc", + "target-rpc", + "destination", + &fi.version_id.expect("version").to_string(), + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read committed object"); + assert_eq!(read.data, fi.data); + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial] + async fn reserved_rename_rpc_cancellation_does_not_stop_physical_publication() { + assert_reserved_rename_rpc_retains_fence_after_cancellation(false).await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial] + async fn short_reserved_rename_rpc_retains_fence_past_expiry_after_cancellation() { + assert_reserved_rename_rpc_retains_fence_after_cancellation(true).await; + } + + #[cfg(not(windows))] + async fn assert_reserved_rename_rpc_retains_fence_after_cancellation(short: bool) { + use crate::storage::storage_api::{ + LocalPublicationPause, LocalPublicationStage, + contract::namespace::NamespaceLocking as _, + ecstore_disk::{DiskAPI, RUSTFS_META_TMP_BUCKET, ReadOptions}, + }; + + let fixture = target_rpc_fixture_with_bucket_metadata(true).await; + fixture.env.make_bucket("target-rpc", false).await; + let incarnation = fixture + .env + .ecstore + .bucket_incarnation_id("target-rpc") + .await + .expect("bucket incarnation"); + let (disk, fi, _) = stage_target_rpc_inner(&fixture, false).await; + disk.write_metadata(RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET, "reserved-cancel", fi.clone()) + .await + .expect("stage reserved body"); + let service = make_server_for_context(Some(fixture.context.clone())); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind RPC listener"); + let addr = listener.local_addr().expect("RPC listener address"); + let (shutdown, stopped) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(NodeServiceServer::new(service)) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async { + let _ = stopped.await; + }) + .await + .expect("serve RPC target"); + }); + let mut client = NodeServiceClient::connect(format!("http://{addr}")) + .await + .expect("connect RPC client"); + let mut prepare = Request::new(MetadataReservationRequest { + disk: disk.endpoint().to_string(), + bucket: "target-rpc".into(), + object: "destination".into(), + timeout_ms: 5000, + bucket_incarnation_id: incarnation.as_bytes().to_vec().into(), + }); + let body = rustfs_protos::canonical_metadata_reservation_request_body(prepare.get_ref()).expect("prepare body"); + set_tonic_canonical_body_digest(&mut prepare, &body).expect("prepare digest"); + mark_v2_authenticated(&mut prepare); + let prepared = if short { + client.reserve_object_metadata_short(prepare).await + } else { + client.reserve_object_metadata(prepare).await + } + .expect("prepare handler") + .into_inner(); + assert!(prepared.success, "prepare: {:?}", prepared.error); + let mut renew = Request::new(MetadataReservationActivateRequest { + disk: disk.endpoint().to_string(), + bucket: "target-rpc".into(), + object: "destination".into(), + token: prepared.token.clone(), + renew_only: true, + }); + let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(renew.get_ref()).expect("renew body"); + set_tonic_canonical_body_digest(&mut renew, &body).expect("renew digest"); + mark_v2_authenticated(&mut renew); + assert!( + client + .activate_object_metadata_reservation(renew) + .await + .expect("renew handler") + .into_inner() + .success, + "renewal over the real transport must retain the prepared token" + ); + let mut activate = Request::new(MetadataReservationActivateRequest { + disk: disk.endpoint().to_string(), + bucket: "target-rpc".into(), + object: "destination".into(), + token: prepared.token.clone(), + renew_only: false, + }); + let body = + rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activate body"); + set_tonic_canonical_body_digest(&mut activate, &body).expect("activate digest"); + mark_v2_authenticated(&mut activate); + let activated = client + .activate_object_metadata_reservation(activate) + .await + .expect("activate handler") + .into_inner(); + assert!(activated.success, "activate: {:?}", activated.error); + let mut rename = Request::new(ReservedRenameDataRequest { + rename: Some(RenameDataRequest { + disk: disk.endpoint().to_string(), + src_volume: RUSTFS_META_TMP_BUCKET.into(), + src_path: "reserved-cancel".into(), + dst_volume: "target-rpc".into(), + dst_path: "destination".into(), + file_info: serde_json::to_string(&fi).expect("encode metadata"), + ..Default::default() + }), + token: prepared.token, + }); + let body = rustfs_protos::canonical_reserved_rename_data_request_body(rename.get_ref()).expect("rename body"); + set_tonic_canonical_body_digest(&mut rename, &body).expect("rename digest"); + mark_v2_authenticated(&mut rename); + let mut pause = + LocalPublicationPause::install(&disk, "target-rpc", "destination/xl.meta", LocalPublicationStage::PreparedRename) + .expect("install physical pause"); + let mut handler = Box::pin(client.rename_reserved_data(rename)); + super::timeout(Duration::from_secs(10), async { + tokio::select! { + result = &mut handler => panic!("handler completed before physical entry: {result:?}"), + entered = pause.entered() => entered.expect("physical executor entered"), + } + }) + .await + .expect("bounded physical entry"); + drop(handler); + assert!( + fixture.env.ecstore.namespace_commits_pending_for_test(), + "physical publication remains active" + ); + if short { + // Once claimed, the physical write owns the fence independently + // of the now-expired unclaimed-token lease. + tokio::time::sleep(Duration::from_secs(3)).await; + } + let lifecycle = fixture + .env + .ecstore + .new_ns_lock("target-rpc", rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("bucket publication lock"); + let mut bucket_writer = Box::pin(lifecycle.get_write_lock(Duration::from_secs(10))); + let early = super::timeout(Duration::from_millis(500), &mut bucket_writer).await; + let old_lifecycle = fixture + .env + .ecstore + .new_ns_lock("target-rpc", "") + .await + .expect("bucket lifecycle lock"); + let mut old_writer = Box::pin(old_lifecycle.get_write_lock(Duration::from_secs(10))); + let old_early = super::timeout(Duration::from_millis(500), &mut old_writer).await; + drop(pause); + match early { + Ok(result) => { + drop(result.expect("early lifecycle result")); + panic!("bucket publication writer acquired while the remote physical write was still paused"); + } + Err(_) => drop( + super::timeout(Duration::from_secs(10), bucket_writer) + .await + .expect("lifecycle owner drains") + .expect("bucket writer acquires"), + ), + } + match old_early { + Ok(result) => { + drop(result.expect("early lifecycle result")); + panic!("bucket lifecycle writer acquired while the remote physical write was still paused"); + } + Err(_) => drop( + super::timeout(Duration::from_secs(10), old_writer) + .await + .expect("lifecycle owner drains") + .expect("bucket lifecycle writer acquires"), + ), + } + super::timeout(Duration::from_secs(10), async { + while fixture.env.ecstore.namespace_commits_pending_for_test() { + tokio::task::yield_now().await; + } + }) + .await + .expect("physical publication drains"); + let read = disk + .read_version( + "target-rpc", + "target-rpc", + "destination", + &fi.version_id.expect("version").to_string(), + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read published object"); + assert_eq!(read.data, fi.data); + drop(client); + let _ = shutdown.send(()); + super::timeout(Duration::from_secs(10), server) + .await + .expect("RPC server shuts down") + .expect("RPC server task"); + } + #[cfg(not(windows))] #[tokio::test] async fn quota_fence_handlers_commit_real_data_and_drain_before_revocation() { @@ -8974,28 +9820,47 @@ mod tests { )) } - async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap>> { + async fn internal_snapshot_once( + root: &std::path::Path, + ) -> std::io::Result>>> { let mut snapshot = std::collections::BTreeMap::new(); let mut directories = (0..4) .map(|index| std::path::PathBuf::from(format!("disk{index}/{RUSTFS_META_BUCKET}"))) .collect::>(); while let Some(relative) = directories.pop() { - let mut entries = tokio::fs::read_dir(root.join(&relative)) - .await - .expect("read internal snapshot directory"); + let mut entries = tokio::fs::read_dir(root.join(&relative)).await?; snapshot.insert(relative.clone(), None); - while let Some(entry) = entries.next_entry().await.expect("read internal snapshot entry") { + while let Some(entry) = entries.next_entry().await? { let path = relative.join(entry.file_name()); - let file_type = entry.file_type().await.expect("read internal snapshot entry type"); + let file_type = entry.file_type().await?; if file_type.is_dir() { directories.push(path); } else { assert!(file_type.is_file(), "fixture snapshot must contain only directories and regular files"); - snapshot.insert(path, Some(tokio::fs::read(entry.path()).await.expect("read snapshot file bytes"))); + snapshot.insert(path, Some(tokio::fs::read(entry.path()).await?)); } } } - snapshot + Ok(snapshot) + } + + async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap>> { + // Startup cleanup can remove temporary paths after the namespace + // commit drains. Capture the complete tree only once it is stable. + super::timeout(Duration::from_secs(10), async { + let mut previous = None; + loop { + match internal_snapshot_once(root).await { + Ok(snapshot) if previous.as_ref() == Some(&snapshot) => return snapshot, + Ok(snapshot) => previous = Some(snapshot), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => previous = None, + Err(error) => panic!("read internal snapshot: {error}"), + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("internal snapshot must stabilize within ten seconds") } fn file_info(object: &str, version: Uuid, body: Bytes) -> FileInfo { diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index cc101b7b4..368793efc 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -13,6 +13,76 @@ // limitations under the License. use super::{LocalMutationTarget, NodeService}; + +#[cfg(test)] +pub(super) mod short_response_test_hook { + use std::sync::{LazyLock, Mutex}; + use tokio::sync::oneshot; + use uuid::Uuid; + + #[derive(Clone, Copy, PartialEq, Eq)] + pub(crate) enum Phase { + Reserve, + Renew, + Activate, + } + + struct Hook { + id: Uuid, + bucket: String, + phase: Phase, + entered: oneshot::Sender<()>, + release: oneshot::Receiver<()>, + } + + static HOOK: LazyLock>> = LazyLock::new(|| Mutex::new(None)); + + pub(crate) struct Pause { + id: Uuid, + pub(crate) entered: oneshot::Receiver<()>, + release: Option>, + } + + pub(crate) fn install(bucket: &str, phase: Phase) -> Pause { + let id = Uuid::new_v4(); + let (entered_tx, entered) = oneshot::channel(); + let (release, release_rx) = oneshot::channel(); + let mut hook = HOOK.lock().expect("short response hook"); + assert!(hook.is_none(), "short response hook already installed"); + *hook = Some(Hook { + id, + bucket: bucket.to_owned(), + phase, + entered: entered_tx, + release: release_rx, + }); + Pause { + id, + entered, + release: Some(release), + } + } + + impl Drop for Pause { + fn drop(&mut self) { + HOOK.lock().expect("short response hook").take_if(|hook| hook.id == self.id); + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + } + } + + pub(super) async fn wait(bucket: &str, phase: Phase) { + let hook = HOOK + .lock() + .expect("short response hook") + .take_if(|hook| hook.bucket == bucket && hook.phase == phase); + if let Some(hook) = hook { + let _ = hook.entered.send(()); + let _ = hook.release.await; + } + } +} use crate::storage::storage_api::ecstore_disk::{ MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartSourceOperation, MultipartSourceRequest, POOL_META_NATIVE_CAS_METADATA_SUFFIX, PoolMetaNativeRecoveryRequest as NativeRecoveryRequest, QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, @@ -507,6 +577,190 @@ impl NodeService { Ok(store) } + pub(super) async fn handle_reserve_object_metadata( + &self, + request: Request, + short: bool, + ) -> Result, Status> { + let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()) + .map_err(|_| Status::invalid_argument("invalid reservation body"))?; + verify_tonic_canonical_body_digest(&request, &body) + .map_err(|_| Status::permission_denied("metadata reservation requires body-bound authentication"))?; + let request = request.into_inner(); + if request.timeout_ms == 0 { + return Err(Status::invalid_argument("reservation timeout must be positive")); + } + let incarnation = Uuid::from_slice(&request.bucket_incarnation_id) + .ok() + .filter(|id| !id.is_nil()) + .ok_or_else(|| Status::invalid_argument("invalid bucket incarnation"))?; + let LocalMutationTarget::Ready(store) = self.local_mutation_target() else { + return Err(Status::failed_precondition("storage instance is unavailable")); + }; + // A cancelled caller may never receive the token. Keep late v3 + // acquisition bounded before the short unclaimed-token lease begins. + let max_timeout_ms = if short { 50 } else { 5000 }; + let timeout = std::time::Duration::from_millis(request.timeout_ms.min(max_timeout_ms)); + let result = if short { + store + .reserve_local_object_metadata_short(&request.disk, &request.bucket, &request.object, incarnation, timeout) + .await + } else { + store + .reserve_local_object_metadata_at_incarnation( + &request.disk, + &request.bucket, + &request.object, + incarnation, + timeout, + ) + .await + }; + #[cfg(test)] + if short && result.is_ok() { + short_response_test_hook::wait(&request.bucket, short_response_test_hook::Phase::Reserve).await; + } + let protocol_version = if short { 3 } else { 2 }; + Ok(Response::new(match result { + Ok(token) => SnapshotLeaseResponse { + success: true, + token: token.as_bytes().to_vec().into(), + protocol_version, + error: None, + }, + Err(error) => SnapshotLeaseResponse { + success: false, + token: Bytes::new(), + protocol_version, + error: Some(error.into()), + }, + })) + } + + pub(super) async fn handle_activate_object_metadata_reservation( + &self, + request: Request, + ) -> Result, Status> { + let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref()) + .map_err(|_| Status::invalid_argument("invalid reservation activation body"))?; + verify_tonic_canonical_body_digest(&request, &body) + .map_err(|_| Status::permission_denied("reservation activation requires body-bound authentication"))?; + let request = request.into_inner(); + let token = Uuid::from_slice(&request.token) + .ok() + .filter(|token| !token.is_nil()) + .ok_or_else(|| Status::invalid_argument("invalid reservation token"))?; + let LocalMutationTarget::Ready(store) = self.local_mutation_target() else { + return Err(Status::failed_precondition("storage instance is unavailable")); + }; + let result = if request.renew_only { + store + .renew_local_object_metadata_reservation(&request.disk, &request.bucket, &request.object, token) + .await + } else { + store + .activate_local_object_metadata_reservation(&request.disk, &request.bucket, &request.object, token) + .await + }; + #[cfg(test)] + if result.is_ok() { + let phase = if request.renew_only { + short_response_test_hook::Phase::Renew + } else { + short_response_test_hook::Phase::Activate + }; + short_response_test_hook::wait(&request.bucket, phase).await; + } + Ok(Response::new(match result { + Ok(()) => SnapshotLeaseMutationResponse { + success: true, + error: None, + }, + Err(error) => SnapshotLeaseMutationResponse { + success: false, + error: Some(error.into()), + }, + })) + } + + pub(super) async fn handle_cancel_object_metadata_reservation( + &self, + request: Request, + ) -> Result, Status> { + let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref()) + .map_err(|_| Status::invalid_argument("invalid reservation cancellation body"))?; + verify_tonic_canonical_body_digest(&request, &body) + .map_err(|_| Status::permission_denied("reservation cancellation requires body-bound authentication"))?; + let request = request.into_inner(); + let token = Uuid::from_slice(&request.token) + .ok() + .filter(|token| !token.is_nil()) + .ok_or_else(|| Status::invalid_argument("invalid reservation token"))?; + let LocalMutationTarget::Ready(store) = self.local_mutation_target() else { + return Err(Status::failed_precondition("storage instance is unavailable")); + }; + Ok(Response::new( + match store.cancel_local_object_metadata_reservation(&request.disk, token).await { + Ok(cancelled) => SnapshotLeaseMutationResponse { + success: cancelled, + error: None, + }, + Err(error) => SnapshotLeaseMutationResponse { + success: false, + error: Some(error.into()), + }, + }, + )) + } + + pub(super) async fn handle_rename_reserved_data( + &self, + request: Request, + ) -> Result, Status> { + let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref()) + .map_err(|_| Status::invalid_argument("invalid reserved rename body"))?; + verify_tonic_canonical_body_digest(&request, &body) + .map_err(|_| Status::permission_denied("reserved rename requires body-bound authentication"))?; + let request = request.into_inner(); + let token = Uuid::from_slice(&request.token) + .ok() + .filter(|token| !token.is_nil()) + .ok_or_else(|| Status::invalid_argument("invalid reservation token"))?; + let rename = request.rename.ok_or_else(|| Status::invalid_argument("missing rename"))?; + if !rename.scanner_publication_lease_token.is_empty() || !rename.bucket_incarnation_id.is_empty() { + return Err(Status::invalid_argument("reserved rename does not support scanner or heal scope")); + } + let decoded = decode_rename_data_request_file_info(&rename.file_info_bin, &rename.file_info) + .map_err(|_| Status::invalid_argument("invalid reserved rename FileInfo"))?; + let LocalMutationTarget::Ready(store) = self.local_mutation_target() else { + return Err(Status::failed_precondition("storage instance is unavailable")); + }; + let result = store + .rename_local_data_reserved( + &rename.disk, + (&rename.src_volume, &rename.src_path), + &decoded.value, + (&rename.dst_volume, &rename.dst_path), + token, + ) + .await; + let encoded = result.and_then(|value| encode_rename_data_response_payloads(&value, decoded.from_msgpack)); + Ok(Response::new(match encoded { + Ok((rename_data_resp, rename_data_resp_bin)) => RenameDataResponse { + success: true, + rename_data_resp, + rename_data_resp_bin: rename_data_resp_bin.into(), + error: None, + }, + Err(error) => RenameDataResponse { + success: false, + rename_data_resp: String::new(), + rename_data_resp_bin: Bytes::new(), + error: Some(error.into()), + }, + })) + } + pub(super) async fn handle_acquire_snapshot_lease( &self, request: Request, diff --git a/rustfs/src/storage/rpc/node_service/lock.rs b/rustfs/src/storage/rpc/node_service/lock.rs index 74283a584..23dc30123 100644 --- a/rustfs/src/storage/rpc/node_service/lock.rs +++ b/rustfs/src/storage/rpc/node_service/lock.rs @@ -49,6 +49,91 @@ async fn refresh_lock( lock_client: &std::sync::Arc, args: &LockRequest, ) -> GenerallyLockResponse { + let writer_marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER; + if args + .metadata + .tags + .get(writer_marker) + .is_some_and(|value| value == writer_marker) + { + return match lock_client.waiting_writer(&args.lock_id).await { + Ok(Some(waiting)) => GenerallyLockResponse { + success: true, + error_info: None, + lock_info: Some(format!("{writer_marker}:{}", u8::from(waiting))), + }, + Ok(None) => GenerallyLockResponse { + success: false, + error_info: None, + lock_info: None, + }, + Err(error) => GenerallyLockResponse { + success: false, + error_info: Some(error.to_string()), + lock_info: None, + }, + }; + } + let probe_marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER; + if args + .metadata + .tags + .get(probe_marker) + .is_some_and(|value| value == probe_marker) + { + return match lock_client.supports_heal_lock_transition(&args.lock_id).await { + Ok(success) => GenerallyLockResponse { + success, + error_info: None, + lock_info: success.then(|| probe_marker.to_owned()), + }, + Err(error) => GenerallyLockResponse { + success: false, + error_info: Some(error.to_string()), + lock_info: None, + }, + }; + } + let upgrade_marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER; + if args + .metadata + .tags + .get(upgrade_marker) + .is_some_and(|value| value == upgrade_marker) + { + return match lock_client.try_upgrade_committed_tail(&args.lock_id).await { + Ok(success) => GenerallyLockResponse { + success, + error_info: None, + lock_info: success.then(|| upgrade_marker.to_owned()), + }, + Err(error) => GenerallyLockResponse { + success: false, + error_info: Some(error.to_string()), + lock_info: None, + }, + }; + } + let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER; + if args.metadata.tags.get(marker).is_some_and(|value| value == marker) { + return match lock_client.downgrade_committed_tail(&args.lock_id).await { + Ok(true) => GenerallyLockResponse { + success: true, + error_info: None, + lock_info: Some(marker.to_owned()), + }, + Ok(false) => GenerallyLockResponse { + success: false, + error_info: None, + lock_info: None, + }, + Err(error) => GenerallyLockResponse { + success: false, + error_info: Some(error.to_string()), + lock_info: None, + }, + }; + } match lock_client.refresh(&args.lock_id).await { Ok(true) => GenerallyLockResponse { success: true, @@ -347,6 +432,115 @@ mod tests { assert!(result.lock_info.is_none()); } + #[tokio::test] + async fn refresh_reports_waiting_writer_for_held_lease() { + use super::refresh_lock; + use rustfs_lock::client::{HEAL_WAITING_WRITER_MARKER, LockClient}; + use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey}; + use std::{sync::Arc, time::Duration}; + + let client: Arc = Arc::new(LocalClient::new()); + let key = ObjectKey::new("bucket", "writer-observation-rpc"); + let mut held = LockRequest::new(key.clone(), LockType::Exclusive, "heal"); + assert!(client.acquire_lock(&held).await.expect("acquire heal lease").success); + held.metadata + .tags + .insert(HEAL_WAITING_WRITER_MARKER.into(), HEAL_WAITING_WRITER_MARKER.into()); + let writer = { + let client = Arc::clone(&client); + tokio::spawn(async move { + let request = LockRequest::new(key, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_secs(3)); + let response = client.acquire_lock(&request).await.expect("writer request"); + (request, response) + }) + }; + let expected = format!("{HEAL_WAITING_WRITER_MARKER}:1"); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let observed = refresh_lock(&client, &held).await; + if observed.lock_info.as_deref() == Some(expected.as_str()) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("waiting writer reported"); + assert!(client.release(&held.lock_id).await.expect("release heal lease")); + let (request, response) = writer.await.expect("writer task"); + assert!(response.success); + assert!(client.release(&request.lock_id).await.expect("release writer")); + } + + #[tokio::test] + async fn refresh_upgrade_seals_lease_against_late_conversion() { + use super::refresh_lock; + use rustfs_lock::client::{ + COMMITTED_TAIL_CONVERSION_MARKER, COMMITTED_TAIL_UPGRADE_MARKER, HEAL_LOCK_TRANSITION_PROBE_MARKER, + HEAL_WAITING_WRITER_MARKER, LockClient, + }; + use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey}; + use std::sync::Arc; + + let client: Arc = Arc::new(LocalClient::new()); + let mut request = LockRequest::new(ObjectKey::new("bucket", "upgrade-rpc"), LockType::Exclusive, "owner"); + assert!(client.acquire_lock(&request).await.expect("acquire lease").success); + request + .metadata + .tags + .insert(HEAL_LOCK_TRANSITION_PROBE_MARKER.into(), HEAL_LOCK_TRANSITION_PROBE_MARKER.into()); + let probe = refresh_lock(&client, &request).await; + assert!(probe.success); + assert_eq!(probe.lock_info.as_deref(), Some(HEAL_LOCK_TRANSITION_PROBE_MARKER)); + request.metadata.tags.clear(); + request + .metadata + .tags + .insert(HEAL_WAITING_WRITER_MARKER.into(), HEAL_WAITING_WRITER_MARKER.into()); + let observation = refresh_lock(&client, &request).await; + assert!(observation.success); + assert_eq!(observation.lock_info.as_deref(), Some(format!("{HEAL_WAITING_WRITER_MARKER}:0").as_str())); + request.metadata.tags.clear(); + request + .metadata + .tags + .insert(COMMITTED_TAIL_UPGRADE_MARKER.into(), COMMITTED_TAIL_UPGRADE_MARKER.into()); + let response = refresh_lock(&client, &request).await; + assert!(response.success); + assert_eq!(response.lock_info.as_deref(), Some(COMMITTED_TAIL_UPGRADE_MARKER)); + request.metadata.tags.clear(); + request + .metadata + .tags + .insert(COMMITTED_TAIL_CONVERSION_MARKER.into(), COMMITTED_TAIL_CONVERSION_MARKER.into()); + assert!(!refresh_lock(&client, &request).await.success, "late conversion must be rejected"); + assert!(client.release(&request.lock_id).await.expect("release lease")); + } + + #[tokio::test] + async fn refresh_can_convert_committed_tail_and_acknowledges_protocol_version() { + use super::refresh_lock; + use rustfs_lock::client::{COMMITTED_TAIL_CONVERSION_MARKER, LockClient}; + use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey}; + use std::{sync::Arc, time::Duration}; + + let client: Arc = Arc::new(LocalClient::new()); + let resource = ObjectKey::new("bucket", "tail-conversion-rpc"); + let mut request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner"); + assert!(client.acquire_lock(&request).await.expect("acquire write lease").success); + request + .metadata + .tags + .insert(COMMITTED_TAIL_CONVERSION_MARKER.into(), COMMITTED_TAIL_CONVERSION_MARKER.into()); + let response = refresh_lock(&client, &request).await; + assert!(response.success); + assert_eq!(response.lock_info.as_deref(), Some(COMMITTED_TAIL_CONVERSION_MARKER)); + let read = LockRequest::new(resource, LockType::Shared, "reader").with_acquire_timeout(Duration::from_secs(1)); + assert!(client.acquire_lock(&read).await.expect("read committed tail").success); + assert!(client.release(&read.lock_id).await.expect("release reader")); + assert!(client.release(&request.lock_id).await.expect("release tail")); + } + // B1 -- refresh_lock delegates to LocalClient::refresh for a held lock, returns // success=true, and actually extends expires_at (inverts the "parse then drop" stub). #[tokio::test]