mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-01 16:50:45 +00:00
Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3c56d09c08 | |||
| 97d1a39536 | |||
| a354771c45 | |||
| 837d805d7c | |||
| 6d3b882aa1 | |||
| e399112044 | |||
| 039a728213 | |||
| a1ee666c4f | |||
| 9518a5d1e2 | |||
| 2bcc4895ae | |||
| f36154a8aa | |||
| 21cb36c8be | |||
| e76648b5e3 | |||
| 5cc296314e | |||
| 01d43729aa | |||
| 0a32ad9109 | |||
| d0b26195c2 | |||
| 7a740fe5e7 | |||
| 2aaef5451d | |||
| a213ce597e | |||
| c749cf9da5 | |||
| 1ab0612c9c | |||
| 6538f5ca36 | |||
| 7040cb65c1 | |||
| 5427bd8f84 | |||
| 3c2b54aeb5 | |||
| 0de223d8e6 | |||
| 351bd12031 | |||
| a2381c8421 | |||
| 5eee702e88 |
@@ -970,10 +970,10 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# Keep the storage-heavy smoke cases isolated in the full suite too. They
|
||||
# start multiple server fixtures and issue many sequential S3 operations.
|
||||
# Keep storage-heavy cases isolated in the full suite too. The large pagination
|
||||
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Fixed, small contract for every executable server change. These existing
|
||||
|
||||
@@ -576,6 +576,34 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn activate_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationActivateRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn cancel_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationCancelRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_reserved_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ReservedRenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_data_with_pool_meta_cas(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -67,14 +67,15 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest,
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest,
|
||||
MetadataReservationCancelRequest, MetadataReservationRequest,
|
||||
PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest,
|
||||
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -2452,6 +2453,194 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
/// Prepare a remote token without retrying the RPC. It owns the bucket
|
||||
/// publication fence but does not enter the object metadata queue yet.
|
||||
pub async fn reserve_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: Duration,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"reserve_object_metadata",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?,
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.reserve_object_metadata(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
if response.protocol_version != 2 {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
Uuid::from_slice(&response.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or(DiskError::Unexpected)
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token. The caller must cancel it on an indeterminate
|
||||
/// result before allowing any subsequent unprotected mutation.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"activate_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationActivateRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation activation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.activate_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed remote reservation. False cannot prove that an
|
||||
/// earlier claim did not start a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"cancel_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationCancelRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation cancellation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.cancel_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error {
|
||||
return Err(error.into());
|
||||
}
|
||||
Ok(response.success)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Publish a staged user-object write using a remote one-shot reservation.
|
||||
/// The caller must retain the bucket lifecycle fence until the tail drains.
|
||||
pub async fn rename_data_with_metadata_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
token: Uuid,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_reserved_data",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let rename = RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
scanner_publication_lease_token: Bytes::new(),
|
||||
bucket_incarnation_id: Bytes::new(),
|
||||
};
|
||||
let mut request = Request::new(ReservedRenameDataRequest {
|
||||
rename: Some(rename),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reserved rename request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.rename_reserved_data(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
@@ -5109,6 +5298,56 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let token = Uuid::new_v4();
|
||||
assert!(matches!(
|
||||
remote
|
||||
.reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.activate_object_metadata("bucket", "object", token, Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote.cancel_object_metadata_reservation(token).await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.rename_data_with_metadata_reservation(
|
||||
crate::disk::RUSTFS_META_TMP_BUCKET,
|
||||
"staged",
|
||||
&sample_rename_data_file_info(),
|
||||
"bucket",
|
||||
"object",
|
||||
token,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
|
||||
vec![
|
||||
"/node_service.NodeService/ReserveObjectMetadata",
|
||||
"/node_service.NodeService/ActivateObjectMetadataReservation",
|
||||
"/node_service.NodeService/CancelObjectMetadataReservation",
|
||||
"/node_service.NodeService/RenameReservedData",
|
||||
],
|
||||
"reservation operations must never downgrade to legacy methods",
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
|
||||
@@ -1315,6 +1315,122 @@ impl LockClient for RemoteClient {
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
// Older nodes refresh normally but return no conversion marker.
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
|
||||
let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
if !response.success {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) {
|
||||
Some(":0") => Some(false),
|
||||
Some(":1") => Some(true),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote force_release for {}", lock_id);
|
||||
let force_request = Self::create_unlock_request(lock_id);
|
||||
|
||||
@@ -655,6 +655,19 @@ impl Sets {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_prepared_reservations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
self.get_disks_by_key(object)
|
||||
.put_object_with_prepared_reservations(bucket, object, data, opts, reservations)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -604,8 +604,69 @@ impl LocalDiskWrapper {
|
||||
let super::RenameDataGuards {
|
||||
external_guard,
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
metadata_reservation_token,
|
||||
..
|
||||
} = guards;
|
||||
let metadata_reservation = match (metadata_reservation, metadata_reservation_token) {
|
||||
(Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)),
|
||||
(Some(reservation), None) => Some(reservation),
|
||||
(None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) {
|
||||
Ok(reservation) => Some(reservation),
|
||||
// A missing token may already have been claimed by an earlier
|
||||
// request that is still publishing; do not claim it was safe.
|
||||
Err(DiskError::DiskOngoingReq) => {
|
||||
return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq));
|
||||
}
|
||||
Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
},
|
||||
(None, None) => None,
|
||||
};
|
||||
// A queued reservation must acquire its metadata lease before disk
|
||||
// admission. Otherwise a waiting background mutation could hold the
|
||||
// disk permit needed by the earlier lease owner.
|
||||
let metadata_reservation = match metadata_reservation {
|
||||
Some(reservation) => Some(reservation.wait().await),
|
||||
None => None,
|
||||
};
|
||||
// The claimed reservation stays in this cancellable future while disk
|
||||
// admission can wait. Only physical work moves into an owned task.
|
||||
let prechecked_permit = if metadata_reservation.is_some() {
|
||||
// Background admission is exclusive. Waiting for it while holding
|
||||
// this object's lease can deadlock an admitted mutation waiting for
|
||||
// the same lease. An already owned permit does not need to wait.
|
||||
if current_disk_io_context().is_some_and(|context| {
|
||||
context.class == DiskIoClass::Background
|
||||
&& !context
|
||||
.permit
|
||||
.as_ref()
|
||||
.is_some_and(|permit| permit.is_for(&self.io_admission))
|
||||
}) {
|
||||
return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let preflight = self
|
||||
.with_native_resources(async {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.acquire_disk_io_permit().await
|
||||
})
|
||||
.await;
|
||||
match preflight {
|
||||
Ok(Ok(Some(permit))) => Some(permit),
|
||||
Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// The reserved metadata lease must outlive a cancelled RPC waiter.
|
||||
let external_guard: Option<Arc<dyn Send + Sync>> = if metadata_reservation.is_some() {
|
||||
Some(Arc::new(external_guard))
|
||||
} else {
|
||||
external_guard
|
||||
};
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -624,27 +685,54 @@ impl LocalDiskWrapper {
|
||||
};
|
||||
let observed = run_owned_mutation(external_guard, move || async move {
|
||||
let mut preflight_rejection = None;
|
||||
let result = operation
|
||||
.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
let rename = || async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = if let Some(reservation) = metadata_reservation {
|
||||
Box::pin(operation.disk.rename_data_observed_with_reservation(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
(namespace_owner, Some(reservation)),
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await
|
||||
};
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
};
|
||||
let result = if let Some(permit) = prechecked_permit {
|
||||
with_disk_io_permit(Some(permit), async {
|
||||
operation
|
||||
.with_native_resources(operation.track_admitted_disk_operation(
|
||||
"rename_data",
|
||||
move |io_permit| async move {
|
||||
let _io_permit = io_permit;
|
||||
rename().await
|
||||
},
|
||||
Duration::ZERO,
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::Write,
|
||||
true,
|
||||
))
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
},
|
||||
timeout_duration,
|
||||
)
|
||||
.await;
|
||||
.await?
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
operation
|
||||
.track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration)
|
||||
.await
|
||||
};
|
||||
// Health tracking must observe the real disk error, not an Ok tuple.
|
||||
Ok(super::RenameDataObservation {
|
||||
result,
|
||||
@@ -1598,6 +1686,58 @@ pub struct LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
/// Reserve the metadata mutation lock for one object on this disk instance.
|
||||
/// Unclaimed reservations expire after 60 seconds.
|
||||
pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.reserve_object_metadata(bucket, object, timeout).await
|
||||
}
|
||||
|
||||
/// Register an object metadata reservation in FIFO order on this disk.
|
||||
/// The returned token may be claimed before its physical lease is available.
|
||||
pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.enqueue_object_metadata(bucket, object).await
|
||||
}
|
||||
|
||||
/// Prepare an inert reservation token bound to this disk and object.
|
||||
pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
self.prepare_object_metadata_with_owner(bucket, object, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_object_metadata_with_owner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
|
||||
) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.prepare_object_metadata_with_owner(bucket, object, owner)
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token; a cancelled token cannot be reactivated.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. A claimed physical operation retains its own lock.
|
||||
pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool {
|
||||
self.disk.cancel_object_metadata_reservation(token)
|
||||
}
|
||||
|
||||
/// Create a new LocalDiskWrapper
|
||||
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
|
||||
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
|
||||
@@ -2577,6 +2717,7 @@ impl LocalDiskWrapper {
|
||||
timeout_duration,
|
||||
timeout_health_action,
|
||||
mutation,
|
||||
false,
|
||||
))
|
||||
.await?
|
||||
}
|
||||
@@ -2588,6 +2729,7 @@ impl LocalDiskWrapper {
|
||||
timeout_duration: Duration,
|
||||
timeout_health_action: TimeoutHealthAction,
|
||||
mutation: DiskMetricMutation,
|
||||
identity_prechecked: bool,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
@@ -2609,7 +2751,7 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
|
||||
// Check if disk is stale
|
||||
if let Err(err) = self.check_disk_stale().await {
|
||||
if !identity_prechecked && let Err(err) = self.check_disk_stale().await {
|
||||
self.metrics.record_availability_error();
|
||||
return Err(err);
|
||||
}
|
||||
@@ -5577,6 +5719,115 @@ mod tests {
|
||||
drop(metadata_lease);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancelled_reserved_rename_releases_lease_before_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let meta = dir.path().join(RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
|
||||
.await
|
||||
.expect("write format");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
|
||||
disk.make_volume("bucket").await.expect("bucket volume");
|
||||
let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false));
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
wrapper.set_disk_id_state(Some(format.erasure.this)).await;
|
||||
disk.format_info.write().await.last_check = None;
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
disk.install_format_probe_hook(false, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv_timeout(Duration::from_secs(30));
|
||||
});
|
||||
let publisher = Arc::clone(&wrapper);
|
||||
let rename = tokio::spawn(async move {
|
||||
publisher
|
||||
.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&FileInfo::default(),
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), entered_rx)
|
||||
.await
|
||||
.expect("disk identity preflight starts")
|
||||
.expect("preflight hook");
|
||||
rename.abort();
|
||||
assert!(matches!(rename.await, Err(error) if error.is_cancelled()));
|
||||
let key = disk.get_object_path("bucket", "object").expect("object path");
|
||||
let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None));
|
||||
let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok();
|
||||
release_tx.send(()).expect("release disk identity preflight");
|
||||
if !released_before_io {
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), successor)
|
||||
.await
|
||||
.expect("preflight finishes"),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
released_before_io,
|
||||
"a cancelled request must not retain the reservation before physical mutation starts"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_background_rename_does_not_wait_for_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
let held = wrapper
|
||||
.io_admission
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("background permit");
|
||||
let fi = FileInfo::default();
|
||||
let rename = crate::disk::with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
wrapper.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&fi,
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
);
|
||||
let result = tokio::time::timeout(Duration::from_secs(1), rename).await;
|
||||
drop(held);
|
||||
let observation = result.expect("reserved mutation must not wait while holding the object lease");
|
||||
assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq)));
|
||||
assert!(observation.rejected_before_publication());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_versions_counts_faulty_drive_availability_rejection() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -439,12 +439,29 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
#[derive(Debug)]
|
||||
pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
|
||||
impl LocalRenamePreflightRejection {
|
||||
pub(in crate::disk) fn not_dispatched() -> Self {
|
||||
Self(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl RenameDataState {
|
||||
pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self {
|
||||
Self {
|
||||
metadata_reservation: Some(reservation),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keep optional native MPU state machines out of ordinary rename frames.
|
||||
// Construct the future here as well as pinning it, so its temporary stays off
|
||||
// the caller's poll stack. Call only from the existing native MPU branches.
|
||||
@@ -486,6 +503,15 @@ impl LocalDisk {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let mut fi = fi;
|
||||
// Pre-acquired metadata must never reverse quota or multipart-source
|
||||
// lock order. These paths need their own reservation protocol.
|
||||
if state.metadata_reservation.is_some()
|
||||
&& (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX))
|
||||
{
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) {
|
||||
Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
@@ -579,7 +605,27 @@ impl LocalDisk {
|
||||
Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))),
|
||||
None => state.namespace_owner.take(),
|
||||
};
|
||||
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await;
|
||||
// A reserved lease is single-use and bound to the exact disk path.
|
||||
// Keep the independent namespace/source owner alongside it; a pre-held
|
||||
// mutex cannot be acquired again without deadlocking.
|
||||
let reservation_fence = state
|
||||
.metadata_reservation
|
||||
.as_ref()
|
||||
.and_then(|reservation| reservation.fence());
|
||||
let metadata_owner: Arc<dyn Send + Sync> = if let Some(reservation) = state.metadata_reservation.take() {
|
||||
if multipart_staging.is_some() || quota_fence_claim.is_some() {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let lease = reservation
|
||||
.into_lease_for(&root_destination_object_path, self.reservation_instance)
|
||||
.inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
Arc::new((lease, metadata_owner))
|
||||
} else {
|
||||
os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await
|
||||
};
|
||||
drop(metadata_wait);
|
||||
#[cfg(unix)]
|
||||
let parent_receipt = if native_timing {
|
||||
@@ -593,17 +639,23 @@ impl LocalDisk {
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let mut receipt_mutation = native_timing
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone()));
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone()));
|
||||
let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait");
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&root_destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
Some(metadata_owner),
|
||||
)
|
||||
.await;
|
||||
drop(namespace_wait);
|
||||
// The volume read lease prevents deletion of this disk's bucket until
|
||||
// publication finishes. Recheck after its wait, before any mutation.
|
||||
if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -2135,8 +2187,23 @@ impl LocalDisk {
|
||||
dst_path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::super::RenameDataObservation {
|
||||
self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed_with_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
owners: (Option<Arc<dyn Send + Sync>>, Option<os::MetadataMutationReservation>),
|
||||
) -> super::super::RenameDataObservation {
|
||||
let (namespace_owner, metadata_reservation) = owners;
|
||||
let mut state = RenameDataState {
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
..Default::default()
|
||||
};
|
||||
let result = self
|
||||
|
||||
@@ -106,6 +106,8 @@ pub(crate) struct RenameDataGuards {
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
pub(crate) metadata_reservation_token: Option<Uuid>,
|
||||
}
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
@@ -122,6 +124,13 @@ impl RenameDataObservation {
|
||||
}
|
||||
}
|
||||
|
||||
fn not_dispatched(error: DiskError) -> Self {
|
||||
Self {
|
||||
result: Err(error),
|
||||
preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rejected_before_publication(&self) -> bool {
|
||||
self.result.is_err() && self.preflight_rejection.is_some()
|
||||
}
|
||||
@@ -311,6 +320,60 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// Prepare an inert single-use reservation on this disk.
|
||||
/// A lost remote response holds its bucket publication fence until expiration, but does not lock the object.
|
||||
pub async fn prepare_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.prepare_object_metadata(bucket, object),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token in object mutation order.
|
||||
pub async fn activate_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.activate_object_metadata(bucket, object, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. False does not prove that a claim has
|
||||
/// not already started a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
match self {
|
||||
Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)),
|
||||
Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Physical ownership belongs to the node hosting the local attachment.
|
||||
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
|
||||
match self {
|
||||
@@ -1244,18 +1307,33 @@ impl Disk {
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
Disk::Remote(remote_disk) => {
|
||||
if guards.metadata_reservation.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
if let Some(token) = guards.metadata_reservation_token {
|
||||
if guards.scanner_publication_lease_token.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
return RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,8 +22,10 @@ use parking_lot::Mutex;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
future::Future,
|
||||
io,
|
||||
path::{Component, Path, PathBuf},
|
||||
pin::Pin,
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
@@ -2202,6 +2204,326 @@ pub(crate) struct MetadataMutationLease {
|
||||
_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
pub(crate) trait MetadataReservationFence: Send + Sync {
|
||||
fn is_valid(&self) -> bool;
|
||||
}
|
||||
|
||||
/// Single-use ownership of the metadata lock for an exact physical object.
|
||||
pub(crate) struct MetadataMutationReservation {
|
||||
key: PathBuf,
|
||||
disk_instance: uuid::Uuid,
|
||||
lease: MetadataReservationLease,
|
||||
capacity: Option<OwnedSemaphorePermit>,
|
||||
fence: Option<Arc<dyn MetadataReservationFence>>,
|
||||
}
|
||||
|
||||
enum MetadataReservationLease {
|
||||
Held(Arc<MetadataMutationLease>),
|
||||
Queued(Pin<Box<dyn Future<Output = Arc<MetadataMutationLease>> + Send>>),
|
||||
}
|
||||
|
||||
impl MetadataMutationReservation {
|
||||
pub(crate) fn fence(&self) -> Option<Arc<dyn MetadataReservationFence>> {
|
||||
self.fence.clone()
|
||||
}
|
||||
|
||||
/// Wait for a previously enqueued lease without holding an I/O permit.
|
||||
/// Dropping this future removes an unclaimed waiter from the FIFO queue.
|
||||
pub(crate) async fn wait(self) -> Self {
|
||||
let Self {
|
||||
key,
|
||||
disk_instance,
|
||||
lease,
|
||||
capacity,
|
||||
fence,
|
||||
} = self;
|
||||
let lease = match lease {
|
||||
MetadataReservationLease::Held(lease) => lease,
|
||||
MetadataReservationLease::Queued(waiter) => waiter.await,
|
||||
};
|
||||
Self {
|
||||
key,
|
||||
disk_instance,
|
||||
lease: MetadataReservationLease::Held(lease),
|
||||
capacity,
|
||||
fence,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn into_lease_for(self, key: &Path, disk_instance: uuid::Uuid) -> Result<Arc<dyn Send + Sync>> {
|
||||
if self.key != key || self.disk_instance != disk_instance {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if self.fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let MetadataReservationLease::Held(lease) = self.lease else {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
};
|
||||
Ok(match self.capacity {
|
||||
Some(capacity) => Arc::new((lease, capacity)),
|
||||
None => lease,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reserve_metadata_mutation_lease(
|
||||
object: &Path,
|
||||
disk_instance: uuid::Uuid,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> MetadataMutationReservation {
|
||||
MetadataMutationReservation {
|
||||
key: object.to_path_buf(),
|
||||
disk_instance,
|
||||
lease: MetadataReservationLease::Held(acquire_metadata_mutation_lease(object, owner).await),
|
||||
capacity: None,
|
||||
fence: None,
|
||||
}
|
||||
}
|
||||
|
||||
struct MetadataReservationEntry {
|
||||
state: MetadataReservationState,
|
||||
expires_at: tokio::time::Instant,
|
||||
}
|
||||
|
||||
enum MetadataReservationState {
|
||||
Prepared {
|
||||
key: PathBuf,
|
||||
disk_instance: uuid::Uuid,
|
||||
capacity: Option<OwnedSemaphorePermit>,
|
||||
owner: Option<Arc<dyn MetadataReservationFence>>,
|
||||
},
|
||||
Queued(MetadataMutationReservation),
|
||||
}
|
||||
|
||||
/// Process-local, bounded one-shot reservations for one disk instance.
|
||||
pub(crate) struct MetadataReservationRegistry {
|
||||
entries: Mutex<HashMap<uuid::Uuid, MetadataReservationEntry>>,
|
||||
capacity: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl MetadataReservationRegistry {
|
||||
pub(crate) fn new(limit: usize) -> Arc<Self> {
|
||||
let registry = Arc::new(Self {
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
capacity: Arc::new(Semaphore::new(limit)),
|
||||
});
|
||||
let weak = Arc::downgrade(®istry);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
let Some(registry) = weak.upgrade() else { break };
|
||||
registry.remove_expired();
|
||||
}
|
||||
});
|
||||
registry
|
||||
}
|
||||
|
||||
pub(crate) async fn reserve(
|
||||
&self,
|
||||
key: &Path,
|
||||
disk_instance: uuid::Uuid,
|
||||
acquire_timeout: Duration,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<uuid::Uuid> {
|
||||
let capacity = match Arc::clone(&self.capacity).try_acquire_owned() {
|
||||
Ok(capacity) => capacity,
|
||||
Err(_) => {
|
||||
self.remove_expired();
|
||||
Arc::clone(&self.capacity)
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| DiskError::DiskOngoingReq)?
|
||||
}
|
||||
};
|
||||
let mut reservation = tokio::time::timeout(acquire_timeout, reserve_metadata_mutation_lease(key, disk_instance, owner))
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?;
|
||||
reservation.capacity = Some(capacity);
|
||||
let token = uuid::Uuid::new_v4();
|
||||
let mut entries = self.entries.lock();
|
||||
match entries.entry(token) {
|
||||
std::collections::hash_map::Entry::Vacant(entry) => entry.insert(MetadataReservationEntry {
|
||||
state: MetadataReservationState::Queued(reservation),
|
||||
expires_at: tokio::time::Instant::now() + Duration::from_secs(60),
|
||||
}),
|
||||
std::collections::hash_map::Entry::Occupied(_) => return Err(DiskError::Unexpected),
|
||||
};
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// Register this mutation in FIFO order before returning its token.
|
||||
/// The mutex may still be held by an earlier operation when this returns.
|
||||
pub(crate) async fn enqueue(&self, key: &Path, disk_instance: uuid::Uuid) -> Result<uuid::Uuid> {
|
||||
let capacity = match Arc::clone(&self.capacity).try_acquire_owned() {
|
||||
Ok(capacity) => capacity,
|
||||
Err(_) => {
|
||||
self.remove_expired();
|
||||
Arc::clone(&self.capacity)
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| DiskError::DiskOngoingReq)?
|
||||
}
|
||||
};
|
||||
let object = key.to_path_buf();
|
||||
let mut waiter: Pin<Box<dyn Future<Output = Arc<MetadataMutationLease>> + Send>> =
|
||||
Box::pin(async move { acquire_metadata_mutation_lease(&object, None).await });
|
||||
// Poll once so Tokio registers this waiter before the token can be
|
||||
// acknowledged; retaining the future retains its position in the queue.
|
||||
let lease = match futures::poll!(waiter.as_mut()) {
|
||||
std::task::Poll::Ready(lease) => MetadataReservationLease::Held(lease),
|
||||
std::task::Poll::Pending => MetadataReservationLease::Queued(waiter),
|
||||
};
|
||||
let token = uuid::Uuid::new_v4();
|
||||
let reservation = MetadataMutationReservation {
|
||||
key: key.to_path_buf(),
|
||||
disk_instance,
|
||||
lease,
|
||||
capacity: Some(capacity),
|
||||
fence: None,
|
||||
};
|
||||
let mut entries = self.entries.lock();
|
||||
match entries.entry(token) {
|
||||
std::collections::hash_map::Entry::Vacant(entry) => entry.insert(MetadataReservationEntry {
|
||||
state: MetadataReservationState::Queued(reservation),
|
||||
expires_at: tokio::time::Instant::now() + Duration::from_secs(60),
|
||||
}),
|
||||
std::collections::hash_map::Entry::Occupied(_) => return Err(DiskError::Unexpected),
|
||||
};
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// Allocate an inert token before a client asks to enter the FIFO queue.
|
||||
/// A lost preparation response cannot hold an object metadata lock.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn prepare(&self, key: &Path, disk_instance: uuid::Uuid) -> Result<uuid::Uuid> {
|
||||
self.prepare_with_owner(key, disk_instance, None)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_with_owner(
|
||||
&self,
|
||||
key: &Path,
|
||||
disk_instance: uuid::Uuid,
|
||||
owner: Option<Arc<dyn MetadataReservationFence>>,
|
||||
) -> Result<uuid::Uuid> {
|
||||
let capacity = match Arc::clone(&self.capacity).try_acquire_owned() {
|
||||
Ok(capacity) => capacity,
|
||||
Err(_) => {
|
||||
self.remove_expired();
|
||||
Arc::clone(&self.capacity)
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| DiskError::DiskOngoingReq)?
|
||||
}
|
||||
};
|
||||
let token = uuid::Uuid::new_v4();
|
||||
let entry = MetadataReservationEntry {
|
||||
state: MetadataReservationState::Prepared {
|
||||
key: key.to_path_buf(),
|
||||
disk_instance,
|
||||
capacity: Some(capacity),
|
||||
owner,
|
||||
},
|
||||
expires_at: tokio::time::Instant::now() + Duration::from_secs(60),
|
||||
};
|
||||
match self.entries.lock().entry(token) {
|
||||
std::collections::hash_map::Entry::Vacant(slot) => {
|
||||
slot.insert(entry);
|
||||
Ok(token)
|
||||
}
|
||||
std::collections::hash_map::Entry::Occupied(_) => Err(DiskError::Unexpected),
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomically enqueue a prepared token. Cancellation and enqueue serialize
|
||||
/// on the registry, so a delayed enqueue cannot revive a cancelled token.
|
||||
pub(crate) async fn activate(&self, token: uuid::Uuid, key: &Path, disk_instance: uuid::Uuid) -> Result<()> {
|
||||
std::future::poll_fn(|cx| {
|
||||
let mut entries = self.entries.lock();
|
||||
let Some(entry) = entries.get_mut(&token) else {
|
||||
return std::task::Poll::Ready(Err(DiskError::DiskOngoingReq));
|
||||
};
|
||||
if entry.expires_at <= tokio::time::Instant::now() {
|
||||
return std::task::Poll::Ready(Err(DiskError::Timeout));
|
||||
}
|
||||
let (stored_key, stored_instance, capacity, owner) = match &mut entry.state {
|
||||
MetadataReservationState::Prepared {
|
||||
key,
|
||||
disk_instance,
|
||||
capacity,
|
||||
owner,
|
||||
} => (key, disk_instance, capacity, owner),
|
||||
MetadataReservationState::Queued(reservation) => {
|
||||
return std::task::Poll::Ready(if reservation.key != key || reservation.disk_instance != disk_instance {
|
||||
Err(DiskError::InvalidPath)
|
||||
} else if reservation.fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
Err(DiskError::DiskOngoingReq)
|
||||
} else {
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
};
|
||||
if stored_key != key || *stored_instance != disk_instance {
|
||||
return std::task::Poll::Ready(Err(DiskError::InvalidPath));
|
||||
}
|
||||
if owner.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
return std::task::Poll::Ready(Err(DiskError::DiskOngoingReq));
|
||||
}
|
||||
let Some(capacity) = capacity.take() else {
|
||||
return std::task::Poll::Ready(Err(DiskError::Unexpected));
|
||||
};
|
||||
let object = stored_key.clone();
|
||||
let fence = owner.take();
|
||||
let owner: Option<Arc<dyn Send + Sync>> = fence.clone().map(|fence| fence as Arc<dyn Send + Sync>);
|
||||
let mut waiter: Pin<Box<dyn Future<Output = Arc<MetadataMutationLease>> + Send>> =
|
||||
Box::pin(async move { acquire_metadata_mutation_lease(&object, owner).await });
|
||||
let lease = match waiter.as_mut().poll(cx) {
|
||||
std::task::Poll::Ready(lease) => MetadataReservationLease::Held(lease),
|
||||
std::task::Poll::Pending => MetadataReservationLease::Queued(waiter),
|
||||
};
|
||||
entry.state = MetadataReservationState::Queued(MetadataMutationReservation {
|
||||
key: stored_key.clone(),
|
||||
disk_instance,
|
||||
lease,
|
||||
capacity: Some(capacity),
|
||||
fence,
|
||||
});
|
||||
std::task::Poll::Ready(Ok(()))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn claim(&self, token: uuid::Uuid, key: &Path, disk_instance: uuid::Uuid) -> Result<MetadataMutationReservation> {
|
||||
let entry = self.entries.lock().remove(&token).ok_or(DiskError::DiskOngoingReq)?;
|
||||
if entry.expires_at <= tokio::time::Instant::now() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
let MetadataReservationState::Queued(reservation) = entry.state else {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
};
|
||||
if reservation.key != key || reservation.disk_instance != disk_instance {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if reservation.fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
Ok(reservation)
|
||||
}
|
||||
|
||||
pub(crate) fn cancel(&self, token: uuid::Uuid) -> bool {
|
||||
let removed = self.entries.lock().remove(&token);
|
||||
removed.is_some()
|
||||
}
|
||||
|
||||
fn remove_expired(&self) {
|
||||
let now = tokio::time::Instant::now();
|
||||
let expired: Vec<_> = self
|
||||
.entries
|
||||
.lock()
|
||||
.extract_if(|_, entry| entry.expires_at <= now)
|
||||
.map(|(_, entry)| entry)
|
||||
.collect();
|
||||
drop(expired);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_metadata_mutation_lease(
|
||||
object: &Path,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
|
||||
@@ -160,6 +160,20 @@ pub struct ObjectLockConfigSnapshot {
|
||||
}
|
||||
|
||||
impl ObjectLockConfigSnapshot {
|
||||
pub(crate) fn owns_lifecycle_guard(&self) -> bool {
|
||||
self._lifecycle_guard.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option<Uuid>) -> bool {
|
||||
self.bucket.as_deref() == Some(bucket)
|
||||
&& incarnation.is_some()
|
||||
&& self.bucket_incarnation_id == incarnation
|
||||
&& self
|
||||
._lifecycle_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released())
|
||||
}
|
||||
|
||||
pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self {
|
||||
Self {
|
||||
store_id: None,
|
||||
|
||||
@@ -2538,6 +2538,7 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
||||
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
|
||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
|
||||
pub(in crate::set_disk) tail_has_metadata_reservations: bool,
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) struct RenameTailCleanup {
|
||||
@@ -3468,6 +3469,8 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: Option<RenameRollbackReceipt>,
|
||||
multipart_proof_authority: Option<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
multipart_commit_owner: Option<Arc<MultipartLogicalGuards>>,
|
||||
reservation_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
prepared_reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
}
|
||||
|
||||
impl<'a> RenameDataFenceOptions<'a> {
|
||||
@@ -3483,6 +3486,8 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: None,
|
||||
multipart_proof_authority: None,
|
||||
multipart_commit_owner: None,
|
||||
reservation_ctx: None,
|
||||
prepared_reservations: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3519,6 +3524,165 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
self.namespace_commit_guard = namespace_commit_guard;
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_reservation_context(
|
||||
mut self,
|
||||
ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
) -> Self {
|
||||
self.reservation_ctx = ctx;
|
||||
self.prepared_reservations = reservations;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameMetadataReservation {
|
||||
disk: DiskStore,
|
||||
token: Option<Uuid>,
|
||||
prepared_at: tokio::time::Instant,
|
||||
}
|
||||
|
||||
impl RenameMetadataReservation {
|
||||
async fn cancel(&mut self) -> disk::error::Result<()> {
|
||||
let Some(token) = self.token else { return Ok(()) };
|
||||
match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await {
|
||||
Ok(Ok(true)) => {
|
||||
self.token = None;
|
||||
Ok(())
|
||||
}
|
||||
Ok(Ok(false)) => Err(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) => Err(error),
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameMetadataReservation {
|
||||
fn drop(&mut self) {
|
||||
let Some(token) = self.token.take() else { return };
|
||||
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
|
||||
let disk = self.disk.clone();
|
||||
runtime.spawn(async move {
|
||||
let _ = disk.cancel_object_metadata_reservation(token).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_rename_metadata_reservations(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
owns_bucket_fence: bool,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
let prepared: Vec<disk::error::Result<Option<RenameMetadataReservation>>> = join_all(disks.iter().map(|disk| async move {
|
||||
let Some(disk) = disk else { return Ok(None) };
|
||||
if disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, disk, None, false).await?;
|
||||
}
|
||||
let prepared_at = tokio::time::Instant::now();
|
||||
let token = if disk.is_local() && owns_bucket_fence {
|
||||
let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx)
|
||||
.await
|
||||
.ok_or(DiskError::DiskOngoingReq)?;
|
||||
store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&disk.endpoint().to_string(),
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5))
|
||||
.await?
|
||||
};
|
||||
// No await separates receipt from installing the cancellation owner.
|
||||
Ok(Some(RenameMetadataReservation {
|
||||
disk: disk.clone(),
|
||||
token: Some(token),
|
||||
prepared_at,
|
||||
}))
|
||||
}))
|
||||
.await;
|
||||
let mut reservations = Vec::with_capacity(prepared.len());
|
||||
let mut failed = false;
|
||||
for result in prepared {
|
||||
match result {
|
||||
Ok(reservation) => reservations.push(reservation),
|
||||
Err(_) => {
|
||||
failed = true;
|
||||
reservations.push(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
// Prepared tokens are inert; dropping them cannot leave a queued writer.
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
pub(crate) fn align_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> Option<Vec<Option<RenameMetadataReservation>>> {
|
||||
if reservations.len() != disks.len() {
|
||||
return None;
|
||||
}
|
||||
let mut aligned = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
let Some(disk) = disk else {
|
||||
aligned.push(None);
|
||||
continue;
|
||||
};
|
||||
let index = reservations
|
||||
.iter()
|
||||
.position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?;
|
||||
aligned.push(reservations[index].take());
|
||||
}
|
||||
// Disks lost while staging need no activation; dropping their still-inert
|
||||
// reservations schedules cancellation without blocking the commit.
|
||||
Some(aligned)
|
||||
}
|
||||
|
||||
async fn activate_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
// Tokens expire after 60 seconds. Before sending any activation, fall
|
||||
// back if a long upload consumed the safety margin for its RPC deadline.
|
||||
if reservations
|
||||
.iter()
|
||||
.flatten()
|
||||
.any(|reservation| reservation.prepared_at.elapsed() >= Duration::from_secs(50))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let activated = join_all(reservations.iter().flatten().map(|reservation| async move {
|
||||
if reservation.disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?;
|
||||
}
|
||||
let token = reservation.token.ok_or(DiskError::Unexpected)?;
|
||||
reservation
|
||||
.disk
|
||||
.activate_object_metadata(bucket, object, token, Duration::from_secs(5))
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
if activated.into_iter().any(|result| result.is_err()) {
|
||||
let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await;
|
||||
if cancelled.into_iter().any(|result| result.is_err()) {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
@@ -3530,7 +3694,7 @@ type RenameDataLegacyTuple = (
|
||||
Option<OldCurrentSize>,
|
||||
);
|
||||
|
||||
fn put_rename_early_ack_enabled() -> bool {
|
||||
pub(crate) fn put_rename_early_ack_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
|
||||
}
|
||||
|
||||
@@ -3896,6 +4060,7 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3992,6 +4157,8 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority: _,
|
||||
multipart_commit_owner: _,
|
||||
reservation_ctx,
|
||||
prepared_reservations,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -4007,6 +4174,11 @@ impl SetDisks {
|
||||
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) {
|
||||
(Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?,
|
||||
(_, _) => None,
|
||||
};
|
||||
let tail_has_metadata_reservations = reservations.is_some();
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
|
||||
let coordinator_disks = fanout_disks.clone();
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
@@ -4039,6 +4211,11 @@ impl SetDisks {
|
||||
let file_info = file_info.clone();
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let reservation = reservations
|
||||
.as_mut()
|
||||
.and_then(|entries| entries.get_mut(i))
|
||||
.and_then(Option::take);
|
||||
let reservation_ctx = reservation_ctx.clone();
|
||||
tasks.spawn(async move {
|
||||
let _namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
@@ -4049,6 +4226,12 @@ impl SetDisks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let disk_admission = if reservation.is_some() && disk.is_local() {
|
||||
let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?;
|
||||
crate::store::admit_local_disk(ctx, &disk, None, true).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let disk_wait_location_stage = if disk.is_local() {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
|
||||
} else {
|
||||
@@ -4074,6 +4257,11 @@ impl SetDisks {
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
dispatch_state = RenameDispatchState::MayHavePublished;
|
||||
let _foreground_active_guard = rename_foreground_active_guard(i);
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> = if disk_admission.is_some() {
|
||||
Some(Arc::new((namespace_commit_guard.clone(), disk_admission)))
|
||||
} else {
|
||||
namespace_commit_guard.clone().map(|owner| owner as Arc<dyn Send + Sync>)
|
||||
};
|
||||
let observed = disk
|
||||
.rename_data_borrowed_with_fence_observed(
|
||||
&src_bucket,
|
||||
@@ -4083,9 +4271,8 @@ impl SetDisks {
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token),
|
||||
namespace_owner,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -4363,6 +4550,7 @@ impl SetDisks {
|
||||
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
|
||||
commit.map(|mut commit| {
|
||||
commit.tail_drain = Some(tail_drain);
|
||||
commit.tail_has_metadata_reservations = tail_has_metadata_reservations;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
|
||||
attach_tail_owner_started,
|
||||
@@ -4432,6 +4620,8 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority,
|
||||
multipart_commit_owner,
|
||||
reservation_ctx: _,
|
||||
prepared_reservations: _,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -4901,6 +5091,7 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -410,6 +410,10 @@ impl ObjectLockDiagGuard {
|
||||
self
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
self.guard.downgrade_committed_tail().await
|
||||
}
|
||||
|
||||
/// Whether the underlying namespace lock's heartbeat has observed a
|
||||
/// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
|
||||
/// point on this so a stale lock holder does not race a double-write.
|
||||
@@ -902,6 +906,10 @@ mod core;
|
||||
pub(crate) use core::io_primitives::disk_call_counters;
|
||||
#[cfg(test)]
|
||||
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
pub(crate) use core::io_primitives::{
|
||||
RenameMetadataReservation, align_rename_metadata_reservations, prepare_rename_metadata_reservations,
|
||||
put_rename_early_ack_enabled,
|
||||
};
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
|
||||
@@ -34,6 +34,7 @@ mod heal_lock_wait_tests;
|
||||
#[derive(Clone, Copy)]
|
||||
struct ExplicitVersionHeal<'a> {
|
||||
opts: &'a HealOpts,
|
||||
staging_guard: Option<&'a Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>>,
|
||||
allow_regeneration: bool,
|
||||
preserve_unrepairable: bool,
|
||||
commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>,
|
||||
@@ -41,6 +42,58 @@ struct ExplicitVersionHeal<'a> {
|
||||
retirement: Option<&'a MarkerRetirementContext<'a>>,
|
||||
}
|
||||
|
||||
async fn restore_heal_exclusive_guard(
|
||||
guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>,
|
||||
) -> disk::error::Result<()> {
|
||||
tokio::time::timeout(get_lock_acquire_timeout(), async {
|
||||
loop {
|
||||
{
|
||||
let mut guard = guard.lock().await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail())
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
// A writer may have waited on a peer that upgraded while
|
||||
// another peer was still waiting for its readers to drain.
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
return if matches!(waiting, Ok(Some(false))) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::Timeout)
|
||||
};
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn wait_for_heal_writer(guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>) {
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
let waiting =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) {
|
||||
drop(tokio::spawn(async move {
|
||||
let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}));
|
||||
}
|
||||
|
||||
// Keep the large heal state machine and its construction temporary out of
|
||||
// each dispatch caller's poll frame. Ordinary object writes do not use this
|
||||
// heal-only allocation boundary.
|
||||
@@ -844,6 +897,7 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: None,
|
||||
@@ -994,6 +1048,7 @@ impl SetDisks {
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
let ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard,
|
||||
allow_regeneration: allow_explicit_version_regen,
|
||||
preserve_unrepairable,
|
||||
commit_fence,
|
||||
@@ -1701,7 +1756,29 @@ impl SetDisks {
|
||||
let dst_data_dir = data_dir;
|
||||
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = async {
|
||||
let mut shared_staging = false;
|
||||
if let Some(guard) = staging_guard {
|
||||
let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.supports_heal_lock_transition().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if supported {
|
||||
let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.downgrade_committed_tail().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if converted {
|
||||
shared_staging = true;
|
||||
} else {
|
||||
// Some peers may have converted. Abort without committing;
|
||||
// dropping the guard releases each retained lease.
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
let stage = async {
|
||||
let erasure_info = latest_meta.erasure.clone();
|
||||
let mut writer_failures = HealWriterFailures::new(disks_to_heal_count);
|
||||
|
||||
@@ -1926,17 +2003,43 @@ impl SetDisks {
|
||||
|
||||
writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable");
|
||||
Ok(None)
|
||||
}
|
||||
.await;
|
||||
};
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = if shared_staging {
|
||||
if let Some(guard) = staging_guard {
|
||||
tokio::select! {
|
||||
result = stage => result,
|
||||
_ = wait_for_heal_writer(guard) => {
|
||||
// The UUID prefix is private to this attempt. Cleanup must not
|
||||
// retain the namespace lock while a foreground writer waits;
|
||||
// the stale-tmp sweeper also covers a cancelled disk operation.
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
};
|
||||
match stage_result {
|
||||
Ok(None) => {}
|
||||
failure => {
|
||||
// Release readers and writers before removing all parts
|
||||
// staged under this heal's exclusively owned prefix.
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
if shared_staging {
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
} else {
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}
|
||||
return failure.map(|error| (result, error));
|
||||
}
|
||||
}
|
||||
if shared_staging
|
||||
&& let Some(guard) = staging_guard
|
||||
&& let Err(error) = restore_heal_exclusive_guard(guard).await
|
||||
{
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
|
||||
if _read_repair_read_lock_guard
|
||||
@@ -3616,6 +3719,7 @@ impl SetDisks {
|
||||
&version,
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(can_commit),
|
||||
@@ -3676,6 +3780,7 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &read_opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -3702,7 +3807,7 @@ impl SetDisks {
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
|
||||
Some(Arc::new(
|
||||
Some(Arc::new(tokio::sync::Mutex::new(
|
||||
measure_heal_object_lock(
|
||||
"heal_object",
|
||||
HealObjectLockKind::Write,
|
||||
@@ -3714,7 +3819,7 @@ impl SetDisks {
|
||||
.narrow_to_disk()
|
||||
.unwrap_or_else(DiskError::other)
|
||||
})?,
|
||||
))
|
||||
)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -3732,10 +3837,18 @@ impl SetDisks {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let write_lock_loss = if let Some(guard) = &_write_lock_guard {
|
||||
guard.lock().await.lock_lost_signal()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let locks_lost = || {
|
||||
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost())
|
||||
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
};
|
||||
// Recovery cleanup can outlive its future. Never permit reads until
|
||||
// every physical operation holding its recovery owner has finished.
|
||||
let recovery_tail = Arc::new(());
|
||||
let mut recovery_removed_version = false;
|
||||
if !opts.read_repair
|
||||
&& _write_lock_guard.is_some()
|
||||
@@ -3762,6 +3875,7 @@ impl SetDisks {
|
||||
Arc::new((
|
||||
scope.clone(),
|
||||
self.ctx.begin_namespace_commit(),
|
||||
Arc::clone(&recovery_tail),
|
||||
_write_lock_guard.clone(),
|
||||
transaction_guard.clone(),
|
||||
)) as Arc<dyn Send + Sync>
|
||||
@@ -3885,6 +3999,9 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -3909,6 +4026,9 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -6047,6 +6167,7 @@ mod heal_result_report_tests {
|
||||
"",
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -10637,6 +10758,369 @@ mod heal_result_report_tests {
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-storage-entry";
|
||||
let object = "object.bin";
|
||||
let expected = vec![0x37; 1024 * 1024];
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external object data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(&missing).await.expect("remove shard");
|
||||
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
let result = heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
read_repair: false,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
(result, absence)
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("read-repair reached shard reconstruction");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_millis(200),
|
||||
set.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("background heal reconstruction should allow concurrent readers")
|
||||
.expect("open object while reconstruction is paused");
|
||||
let mut during_heal = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal))
|
||||
.await
|
||||
.expect("read completes while reconstruction is paused")
|
||||
.expect("read object during reconstruction");
|
||||
assert_eq!(during_heal, expected);
|
||||
drop(reader);
|
||||
release.notify_one();
|
||||
let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("repair finishes")
|
||||
.expect("repair task joins");
|
||||
let (_, error) = result.expect("repair operation succeeds");
|
||||
assert!(error.is_none(), "repair failed: {error:?}");
|
||||
assert!(absence.is_none());
|
||||
assert!(missing.exists(), "missing shard was not restored");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read repaired object");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-long-put";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let put = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes after releasing reconstruction")
|
||||
.expect("heal task joins");
|
||||
put.expect("foreground PUT finishes during reconstruction")
|
||||
.expect("foreground PUT must not time out");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
drop(_pause);
|
||||
let mut absence = None;
|
||||
let (_, retry_error) = set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a later heal must retry the current object");
|
||||
assert!(retry_error.is_none(), "retry must complete: {retry_error:?}");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read foreground write after heal");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_yields_to_version_delete_without_resurrecting_it() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-version-delete";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create versioned bucket");
|
||||
let original = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original version");
|
||||
let version = original.version_id.expect("version id").to_string();
|
||||
let survivor = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x52; 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write surviving version")
|
||||
.version_id
|
||||
.expect("surviving version id");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, &version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("read original metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal_version = version.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
&heal_version,
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut None,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let delete = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes")
|
||||
.expect("heal task joins");
|
||||
delete
|
||||
.expect("foreground delete finishes")
|
||||
.expect("foreground delete succeeds");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
let result = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version),
|
||||
"stale heal must not resurrect the deleted version: {result:?}"
|
||||
);
|
||||
let mut survivor = set
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(survivor.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("surviving version remains readable");
|
||||
let mut body = Vec::new();
|
||||
survivor.stream.read_to_end(&mut body).await.expect("read surviving version");
|
||||
assert_eq!(body, vec![0x52; 1024]);
|
||||
for disk in &disks {
|
||||
let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await;
|
||||
assert!(
|
||||
matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
|
||||
"deleted version must remain absent on each disk: {physical:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() {
|
||||
let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
|
||||
@@ -4184,7 +4184,19 @@ impl SetDisks {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None)
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_prepared_reservations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, reservations)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -4203,7 +4215,7 @@ impl SetDisks {
|
||||
// The owned publication capability supplies the exact target write
|
||||
// domain at commit time; the generic PUT path must not reacquire it.
|
||||
opts.no_lock = true;
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence))
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence), None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -4214,8 +4226,9 @@ impl SetDisks {
|
||||
data: &'a mut PutObjReader,
|
||||
opts: &'a ObjectOptions,
|
||||
publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> impl Future<Output = Result<(ObjectInfo, Option<OldCurrentSize>)>> + Send + 'a {
|
||||
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence))
|
||||
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence, reservations))
|
||||
}
|
||||
|
||||
async fn put_object_with_old_current_size_inner(
|
||||
@@ -4225,6 +4238,7 @@ impl SetDisks {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
mut publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
let write_context = WriteCommitContext::from_options(opts, bucket, object, publication_fence.is_some())?;
|
||||
let protect_write = opts.shard_integrity_write_enabled();
|
||||
@@ -4789,6 +4803,41 @@ impl SetDisks {
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Remote publication owners were prepared before bucket metadata
|
||||
// locks. Local-only fixtures can prepare before the object lock.
|
||||
let prepared_metadata_reservations = if !self.ctx.lock_manager().is_disabled()
|
||||
&& crate::set_disk::put_rename_early_ack_enabled()
|
||||
&& write_context.acquires_namespace()
|
||||
&& write_context.allows_early_ack(opts.write_completion)
|
||||
&& publication_fence.is_none()
|
||||
&& opts.scanner_publication_commit_scope.is_none()
|
||||
&& opts.decommission_capacity_admission.is_none()
|
||||
&& !opts.data_movement
|
||||
&& !is_meta_bucketname(bucket)
|
||||
{
|
||||
if let Some(prepared) = reservations {
|
||||
crate::set_disk::align_rename_metadata_reservations(prepared, &shuffle_disks)
|
||||
} else if shuffle_disks.iter().flatten().all(|disk| disk.is_local())
|
||||
&& crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
|
||||
.await
|
||||
.is_none()
|
||||
&& let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil())
|
||||
{
|
||||
crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
|
||||
&shuffle_disks,
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
&self.ctx,
|
||||
false,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -5268,6 +5317,11 @@ impl SetDisks {
|
||||
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
|
||||
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
|
||||
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
|
||||
let commit_object_lock_snapshot = opts
|
||||
.object_lock_config_snapshot
|
||||
.as_ref()
|
||||
.filter(|snapshot| snapshot.owns_lifecycle_guard())
|
||||
.cloned();
|
||||
// A scanner publication scope owns the movement permit until the
|
||||
// complete rename fan-out drains. Keep this path synchronous so
|
||||
// its terminal state is known before the coordinator releases
|
||||
@@ -5281,6 +5335,26 @@ impl SetDisks {
|
||||
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& commit_owns_namespace_guard
|
||||
&& commit_scanner_publication_scope.is_none();
|
||||
let lifecycle_owned = commit_bucket_lifecycle_guard.is_some()
|
||||
|| commit_object_lock_snapshot
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| snapshot.owns_lifecycle_guard_for(bucket, source_bucket_incarnation_id));
|
||||
let reservation_ctx = (commit_allows_early_ack
|
||||
&& write_context.acquires_namespace()
|
||||
&& commit_object_lock_guard.is_some()
|
||||
&& commit_decommission_object_lock_guard.is_none()
|
||||
&& commit_decommission_capacity_guard.is_none()
|
||||
&& commit_publication_guard.is_none()
|
||||
&& !quota_mutation_fence
|
||||
&& !opts.data_movement
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& lifecycle_owned)
|
||||
.then(|| Arc::clone(&self.ctx));
|
||||
let prepared_metadata_reservations = if reservation_ctx.is_some() {
|
||||
prepared_metadata_reservations
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Full-tail callers also transfer owned guards to the coordinator:
|
||||
// cancelling their ACK waiter must not cancel an in-flight rename.
|
||||
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|
||||
@@ -5604,6 +5678,7 @@ impl SetDisks {
|
||||
)
|
||||
.with_publication_scope(commit_scanner_publication_scope.clone())
|
||||
.with_rollback_receipt(commit_rollback_receipt.clone())
|
||||
.with_reservation_context(reservation_ctx, prepared_metadata_reservations)
|
||||
.with_namespace_commit_guard(
|
||||
(!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
|
||||
),
|
||||
@@ -5627,6 +5702,7 @@ impl SetDisks {
|
||||
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
|
||||
}
|
||||
let mut rename_guard_release = None;
|
||||
let mut committed_tail_guard = None;
|
||||
let mut needs_immediate_heal = false;
|
||||
let mut tail_owns_tmp_cleanup = false;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -5658,7 +5734,14 @@ impl SetDisks {
|
||||
.or_else(|| commit_version_suspended.then(Uuid::nil))
|
||||
.map(|version_id| version_id.to_string());
|
||||
request.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let object_lock_guard = if rename_commit.tail_has_metadata_reservations {
|
||||
None
|
||||
} else {
|
||||
_object_lock_guard
|
||||
.take()
|
||||
.map(|guard| Arc::new(tokio::sync::Mutex::new(guard)))
|
||||
};
|
||||
committed_tail_guard = object_lock_guard.clone();
|
||||
let publication_guard = _publication_guard.take();
|
||||
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
|
||||
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
|
||||
@@ -5683,6 +5766,7 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
commit_object_lock_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5704,6 +5788,7 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
object_lock_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5717,6 +5802,7 @@ impl SetDisks {
|
||||
drop(write_context);
|
||||
drop(publication_guard);
|
||||
drop(bucket_lifecycle_guard);
|
||||
drop(object_lock_snapshot);
|
||||
cleanup_runtime::run(cleanup_reservation, async move {
|
||||
let zero_target_cleanup = targets.is_empty();
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
@@ -5972,6 +6058,13 @@ impl SetDisks {
|
||||
lock_held_post_rename_started,
|
||||
);
|
||||
let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Some(guard) = committed_tail_guard.take() {
|
||||
// The committed tail still owns every physical rename. Convert
|
||||
// only after metadata invalidation and before acknowledging PUT.
|
||||
// Unsupported peers keep their exclusive leases and the tail
|
||||
// retains ownership until its physical work has drained.
|
||||
let _ = guard.lock().await.downgrade_committed_tail().await;
|
||||
}
|
||||
if let Some(release) = rename_guard_release.take() {
|
||||
let _ = release.send(true);
|
||||
}
|
||||
@@ -22177,6 +22270,117 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_retains_owned_bucket_lifecycle_fence() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-tail-lifecycle";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
let lock = NamespaceLock::new("tail-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
Uuid::new_v4(),
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(snapshot),
|
||||
..Default::default()
|
||||
};
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer_set = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
writer_set
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("rename tail pauses");
|
||||
put.await.expect("PUT joins").expect("PUT acknowledged");
|
||||
let mut bucket_delete =
|
||||
Box::pin(lock.get_write_lock(ObjectKey::new(bucket, "lifecycle"), "delete", Duration::from_secs(10)));
|
||||
let early = tokio::time::timeout(Duration::from_secs(2), &mut bucket_delete).await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let guard = match early {
|
||||
Ok(result) => {
|
||||
drop(result.expect("early bucket lock result"));
|
||||
panic!("bucket lifecycle owner must outlive the acknowledged physical tail");
|
||||
}
|
||||
Err(_) => tokio::time::timeout(Duration::from_secs(15), bucket_delete)
|
||||
.await
|
||||
.expect("lifecycle lock released after tail")
|
||||
.expect("bucket delete acquires lock"),
|
||||
};
|
||||
drop(guard);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_drain_allows_read_after_acknowledged_put() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
|
||||
],
|
||||
async {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-early-ack-tail-read";
|
||||
let object = "early-ack-tail-read-object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let put_set = Arc::clone(&set_disks);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
put_set
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("tail disk should pause during rename");
|
||||
put.await
|
||||
.expect("PUT task should join")
|
||||
.expect("PUT should be acknowledged at quorum");
|
||||
let result = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await;
|
||||
let mut reader = result.expect("acknowledged object should remain readable while tail drains");
|
||||
let mut body = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut body))
|
||||
.await
|
||||
.expect("object body should not wait for the tail")
|
||||
.expect("object body should drain");
|
||||
assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
rename_barrier.release();
|
||||
drop(rename_barrier);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes() {
|
||||
@@ -22249,6 +22453,200 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_does_not_timeout_the_next_put() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-early-ack-next-writer";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
{
|
||||
let map = set.ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
let lifecycle_lock = NamespaceLock::new("next-put-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lifecycle_lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lifecycle_lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let incarnation = Uuid::new_v4();
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
incarnation,
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let mut opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
object_lock_config_snapshot: Some(snapshot.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let first_set = Arc::clone(&set);
|
||||
let first_opts = opts.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_set
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &first_opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first rename paused");
|
||||
first.await.expect("first PUT joins").expect("first PUT acknowledged");
|
||||
let second = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]), &opts),
|
||||
)
|
||||
.await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let second = second.expect("second PUT finishes").expect("second PUT must not time out");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("both rename tails must drain");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read newest object");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("read full body");
|
||||
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
let expected_etag = second.etag.as_ref().expect("second PUT ETag");
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("each disk must retain the latest version");
|
||||
assert_eq!(info.metadata.get("etag"), Some(expected_etag), "late rename must not restore an old copy");
|
||||
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
|
||||
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_cannot_restore_a_deleted_version() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-tail-version-delete";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
{
|
||||
let map = set.ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
let lock = NamespaceLock::new("version-delete-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let incarnation = Uuid::new_v4();
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
incarnation,
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: true,
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
object_lock_config_snapshot: Some(snapshot.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer = Arc::clone(&set);
|
||||
let put_opts = opts.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
writer
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &put_opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("rename pauses");
|
||||
let version = put
|
||||
.await
|
||||
.expect("PUT joins")
|
||||
.expect("PUT succeeds")
|
||||
.version_id
|
||||
.expect("version ID")
|
||||
.to_string();
|
||||
let delete_barrier = DeleteObjectCommitBarrier::install_before_namespace(bucket, object);
|
||||
let delete_set = Arc::clone(&set);
|
||||
let mut delete_opts = opts;
|
||||
delete_opts.version_id = Some(version.clone());
|
||||
let delete = tokio::spawn(async move { delete_set.delete_object(bucket, object, delete_opts).await });
|
||||
delete_barrier.wait_until_paused().await;
|
||||
delete_barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !delete_barrier.namespace_acquired() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("version delete acquires namespace while old tail is paused");
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
delete.await.expect("delete joins").expect("version delete succeeds");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("rename tails drain");
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.read_version("", bucket, object, &version, &ReadOptions::default()).await,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)
|
||||
),
|
||||
"late tail must not restore the deleted version"
|
||||
);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_successful_tail_reclaims_its_old_non_inline_data_dir() {
|
||||
|
||||
@@ -45,7 +45,8 @@
|
||||
use crate::error::{Result, StorageError};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
use std::time::Duration;
|
||||
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use rustfs_lock::distributed_lock::LockLostSignal;
|
||||
@@ -98,9 +99,74 @@ pub(super) struct FenceRegistration {
|
||||
#[derive(Default)]
|
||||
pub(crate) struct BucketFenceRegistry {
|
||||
entries: Mutex<HashMap<String, FenceEntry>>,
|
||||
reservation_slots: Mutex<HashMap<(String, Uuid), Weak<ReservationFenceSlot>>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ReservationFenceSlot {
|
||||
pub(super) owner: tokio::sync::Mutex<Weak<ReservationFenceGuard>>,
|
||||
}
|
||||
|
||||
/// Shared by short bursts of reservations, but never kept alive by the cache.
|
||||
pub(super) struct ReservationFenceGuard {
|
||||
publication: NamespaceLockGuard,
|
||||
lifecycle: BucketIncarnationFenceGuard,
|
||||
created_at: tokio::time::Instant,
|
||||
_slot: Arc<ReservationFenceSlot>,
|
||||
}
|
||||
|
||||
impl ReservationFenceGuard {
|
||||
pub(super) fn new(
|
||||
publication: NamespaceLockGuard,
|
||||
lifecycle: BucketIncarnationFenceGuard,
|
||||
slot: Arc<ReservationFenceSlot>,
|
||||
) -> Self {
|
||||
Self {
|
||||
publication,
|
||||
lifecycle,
|
||||
created_at: tokio::time::Instant::now(),
|
||||
_slot: slot,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_reusable(&self) -> bool {
|
||||
// Stop admitting new reservations through this shared reader so a
|
||||
// queued bucket writer can progress after the existing tails drain.
|
||||
self.created_at.elapsed() < Duration::from_secs(1) && self.is_held()
|
||||
}
|
||||
|
||||
fn is_held(&self) -> bool {
|
||||
!self.publication.is_lock_lost()
|
||||
&& !self.publication.is_released()
|
||||
&& !self.lifecycle.is_lock_lost()
|
||||
&& self
|
||||
.lifecycle
|
||||
.namespace_lock_guard()
|
||||
.is_some_and(|guard| !guard.is_released())
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::disk::os::MetadataReservationFence for ReservationFenceGuard {
|
||||
fn is_valid(&self) -> bool {
|
||||
self.is_held()
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketFenceRegistry {
|
||||
pub(super) fn reservation_slot(&self, bucket: &str, incarnation: Uuid) -> Arc<ReservationFenceSlot> {
|
||||
let mut slots = self.reservation_slots.lock().expect("reservation fence registry poisoned");
|
||||
let key = (bucket.to_owned(), incarnation);
|
||||
if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) {
|
||||
return slot;
|
||||
}
|
||||
if slots.len() >= 1024 {
|
||||
slots.retain(|_, slot| slot.strong_count() != 0);
|
||||
}
|
||||
let slot = Arc::new(ReservationFenceSlot::default());
|
||||
slots.insert(key, Arc::downgrade(&slot));
|
||||
slot
|
||||
}
|
||||
|
||||
/// Register a new live guard for `bucket` and return the memoized
|
||||
/// incarnation id if one is valid for the current coverage window.
|
||||
fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration {
|
||||
|
||||
@@ -456,6 +456,7 @@ mod rebalance;
|
||||
pub(crate) use rebalance::PoolSpaceForTest;
|
||||
pub(crate) mod utils;
|
||||
|
||||
pub(crate) use peer::admit_local_disk;
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
|
||||
|
||||
@@ -4814,7 +4814,44 @@ impl ECStore {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, opts)).await?;
|
||||
let mut early_opts = opts.clone();
|
||||
// Bucket publication precedes lifecycle and metadata transaction locks.
|
||||
let prepared = if self.single_pool()
|
||||
&& !self.ctx.lock_manager().is_disabled()
|
||||
&& crate::set_disk::put_rename_early_ack_enabled()
|
||||
&& early_opts.write_completion == crate::object_api::WriteCompletion::Quorum
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& !early_opts.no_lock
|
||||
&& !early_opts.data_movement
|
||||
&& early_opts.write_lock_context.is_none()
|
||||
&& early_opts.object_lock_config_snapshot.is_none()
|
||||
&& early_opts.bucket_lifecycle_lock_fence.is_none()
|
||||
&& early_opts.namespace_lock_fence.is_none()
|
||||
&& early_opts.scanner_publication_commit_scope.is_none()
|
||||
&& early_opts.decommission_capacity_admission.is_none()
|
||||
&& early_opts.decommission_capacity.is_none()
|
||||
&& early_opts.quota_admission.is_none()
|
||||
&& early_opts
|
||||
.put_object_cancellation
|
||||
.as_ref()
|
||||
.is_none_or(|token| !token.is_cancelled())
|
||||
&& rustfs_utils::path::object_name_disk_identity(object).as_ref()
|
||||
!= rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
{
|
||||
check_new_object_args(bucket, object, false)?;
|
||||
let incarnation = match early_opts.expected_bucket_incarnation_id {
|
||||
Some(incarnation) => incarnation,
|
||||
None => self.bucket_incarnation_id(bucket).await?,
|
||||
};
|
||||
early_opts.expected_bucket_incarnation_id = Some(incarnation);
|
||||
let encoded = encode_dir_object(object);
|
||||
let set = self.pools[0].get_disks_by_key(&encoded);
|
||||
let disks = set.disks.read().await.clone();
|
||||
crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, &encoded, incarnation, &self.ctx, true).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, &early_opts)).await?;
|
||||
let idx = self
|
||||
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
|
||||
.await?;
|
||||
@@ -4823,7 +4860,8 @@ impl ECStore {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
|
||||
let result = Box::pin(self.pools[idx].put_object_with_old_current_size(bucket, object.as_str(), data, &opts)).await;
|
||||
let result =
|
||||
Box::pin(self.pools[idx].put_object_with_prepared_reservations(bucket, object.as_str(), data, &opts, prepared)).await;
|
||||
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
|
||||
match result {
|
||||
Ok((object_info, old_current_size)) => {
|
||||
@@ -9938,6 +9976,305 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn ecstore_early_ack_prepares_before_snapshot_and_orders_the_next_put() {
|
||||
use crate::disk::ReadOptions;
|
||||
use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "ecstore-reservation-order";
|
||||
let object = "object";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let first_store = Arc::clone(&store);
|
||||
let first = tokio::spawn(async move {
|
||||
first_store
|
||||
.handle_put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'1'; 1 << 20]),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first tail pauses");
|
||||
first.await.expect("first task").expect("first PUT");
|
||||
let second = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
store.handle_put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'2'; 1 << 20]),
|
||||
&ObjectOptions::default(),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let (second, _) = second.expect("second PUT completes").expect("second PUT succeeds");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("rename tails drain");
|
||||
let expected_etag = second.etag.as_ref().expect("second ETag");
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read latest metadata");
|
||||
assert_eq!(info.metadata.get("etag"), Some(expected_etag));
|
||||
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
|
||||
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_prepared_reservations_hold_bucket_publication_fence() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.clone();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in disks.iter().flatten() {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "local-reservation-bucket-fence";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let reservations =
|
||||
crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, "object", incarnation, &ctx, true)
|
||||
.await
|
||||
.expect("prepare reservations")
|
||||
.expect("all disks prepared");
|
||||
let lock = store
|
||||
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("publication lock");
|
||||
let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(tokio::time::timeout(Duration::from_millis(50), &mut writer).await.is_err());
|
||||
drop(reservations);
|
||||
let guard = tokio::time::timeout(Duration::from_secs(5), writer)
|
||||
.await
|
||||
.expect("reservations release fence")
|
||||
.expect("writer lock");
|
||||
drop(guard);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bucket_reservations_do_not_exhaust_shared_lock_holders() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "reservation-reader-capacity";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let disk = disks[0].endpoint().to_string();
|
||||
let mut tokens = Vec::new();
|
||||
let mut failure = None;
|
||||
for index in 0..256 {
|
||||
match store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&disk,
|
||||
bucket,
|
||||
&format!("object-{index}"),
|
||||
incarnation,
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(token) => tokens.push(token),
|
||||
Err(error) => {
|
||||
failure = Some((index, error));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
for token in tokens {
|
||||
assert!(
|
||||
store
|
||||
.cancel_local_object_metadata_reservation(&disk, token)
|
||||
.await
|
||||
.expect("cancel token")
|
||||
);
|
||||
}
|
||||
assert!(failure.is_none(), "bucket reservations exhausted shared lock holders: {failure:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn expired_reservation_fence_does_not_bypass_a_waiting_bucket_writer() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disk = set.disks.read().await.iter().flatten().next().expect("local disk").clone();
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "reservation-writer-fairness";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let first = store
|
||||
.reserve_local_object_metadata_at_incarnation(&disk_ref, bucket, "first", incarnation, Duration::from_secs(5))
|
||||
.await
|
||||
.expect("first reservation");
|
||||
let lock = store
|
||||
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("publication lock");
|
||||
let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut writer).await.is_err(),
|
||||
"writer waits for first token"
|
||||
);
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
tokio::time::resume();
|
||||
let mut second = Box::pin(store.reserve_local_object_metadata_at_incarnation(
|
||||
&disk_ref,
|
||||
bucket,
|
||||
"second",
|
||||
incarnation,
|
||||
Duration::from_secs(5),
|
||||
));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(),
|
||||
"expired cached reader must not bypass writer"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, first)
|
||||
.await
|
||||
.expect("cancel first token")
|
||||
);
|
||||
let guard = tokio::time::timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("writer proceeds")
|
||||
.expect("writer lock");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(),
|
||||
"new reader waits while writer holds the lock"
|
||||
);
|
||||
drop(guard);
|
||||
let token = tokio::time::timeout(Duration::from_secs(2), second)
|
||||
.await
|
||||
.expect("reader proceeds")
|
||||
.expect("second reservation");
|
||||
assert!(
|
||||
store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, token)
|
||||
.await
|
||||
.expect("cancel second token")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lost_bucket_fence_cannot_activate_a_prepared_reservation() {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)));
|
||||
let client = Arc::new(RefreshLossLockClient::with_manager(manager));
|
||||
let lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
|
||||
"reservation-loss".to_string(),
|
||||
vec![client.clone() as Arc<dyn rustfs_lock::LockClient>],
|
||||
1,
|
||||
);
|
||||
let acquire = |object: &'static str| {
|
||||
rustfs_lock::LockRequest::new(
|
||||
rustfs_lock::ObjectKey::new("bucket", object),
|
||||
rustfs_lock::LockType::Shared,
|
||||
"reservation-loss",
|
||||
)
|
||||
.with_ttl(Duration::from_secs(2))
|
||||
.with_refresh_interval(Duration::from_millis(20))
|
||||
};
|
||||
let publication = lock
|
||||
.acquire_guard(&acquire("publication"))
|
||||
.await
|
||||
.expect("publication request")
|
||||
.expect("publication quorum");
|
||||
let signal = publication.lock_lost_signal().expect("distributed loss signal");
|
||||
let lifecycle = lock
|
||||
.acquire_guard(&acquire("lifecycle"))
|
||||
.await
|
||||
.expect("lifecycle request")
|
||||
.expect("lifecycle quorum");
|
||||
let bucket_registry = Arc::new(crate::store::bucket_fence::BucketFenceRegistry::default());
|
||||
let lifecycle =
|
||||
crate::store::bucket_fence::BucketIncarnationFenceGuard::new(bucket_registry.clone(), "bucket", lifecycle, true);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let slot = bucket_registry.reservation_slot("bucket", incarnation);
|
||||
let owner = Arc::new(crate::store::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot));
|
||||
let registry = crate::disk::os::MetadataReservationRegistry::new(1);
|
||||
let key = std::path::Path::new("bucket/object");
|
||||
let disk = Uuid::new_v4();
|
||||
let token = registry
|
||||
.prepare_with_owner(key, disk, Some(owner))
|
||||
.expect("prepare reservation");
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(Duration::from_secs(5), signal.notified())
|
||||
.await
|
||||
.expect("publication lock loss");
|
||||
assert!(
|
||||
registry.activate(token, key, disk).await.is_err(),
|
||||
"a lost bucket fence must not enter the mutation queue"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn borrowed_copy_tail_drained_waits_for_rename_tail() {
|
||||
|
||||
@@ -85,6 +85,171 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Prepare an inert user-object reservation on the currently admitted disk.
|
||||
/// An unclaimed token expires and does not hold the object's metadata lock.
|
||||
#[cfg(test)]
|
||||
pub async fn reserve_local_object_metadata(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Prepare before the caller acquires bucket lifecycle or object locks.
|
||||
/// The token holds the bucket publication fence through physical completion.
|
||||
pub async fn reserve_local_object_metadata_at_incarnation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
if self.ctx.lock_manager().is_disabled()
|
||||
|| crate::bucket::utils::check_valid_bucket_name(bucket).is_err()
|
||||
|| is_meta_bucketname(bucket)
|
||||
|| incarnation.is_nil()
|
||||
|| timeout.is_zero()
|
||||
|| rustfs_utils::path::object_name_disk_identity(object).as_ref()
|
||||
== rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
|
||||
let slot = self.bucket_fence_registry.reservation_slot(bucket, incarnation);
|
||||
let mut cached = slot.owner.lock().await;
|
||||
let owner = match cached.upgrade().filter(|owner| owner.is_reusable()) {
|
||||
Some(owner) => owner,
|
||||
None => {
|
||||
// Bucket deletion takes publication before lifecycle and metadata.
|
||||
let lock = self
|
||||
.handle_new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let publication = lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
// Older peers may acquire lifecycle without publication.
|
||||
let lifecycle = self
|
||||
.acquire_bucket_incarnation_fence(bucket, incarnation)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let owner = Arc::new(super::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot.clone()));
|
||||
if !owner.is_reusable() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
*cached = Arc::downgrade(&owner);
|
||||
owner
|
||||
}
|
||||
};
|
||||
drop(cached);
|
||||
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, Some(owner))
|
||||
.await
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn reserve_local_object_metadata_with_owner(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: std::time::Duration,
|
||||
owner: Option<Arc<dyn crate::disk::os::MetadataReservationFence>>,
|
||||
) -> DiskResult<Uuid> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
wrapper.prepare_object_metadata_with_owner(bucket, object, owner),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
/// Atomically enqueue a prepared token on its original disk and object.
|
||||
pub async fn activate_local_object_metadata_reservation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
) -> DiskResult<()> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
wrapper.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed token on the currently admitted disk.
|
||||
/// A token already claimed by a physical mutation cannot be cancelled.
|
||||
pub async fn cancel_local_object_metadata_reservation(&self, disk_ref: &str, token: Uuid) -> DiskResult<bool> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
Ok(wrapper.cancel_object_metadata_reservation(token))
|
||||
}
|
||||
|
||||
/// Publish an ordinary staged user-object write using a one-shot disk reservation.
|
||||
/// The token is bound to the admitted disk instance and destination. A claimed
|
||||
/// physical mutation continues to own its lease if the RPC waiter is cancelled.
|
||||
/// The caller must retain the bucket lifecycle fence until the physical tail
|
||||
/// finishes; the reservation does not identify a bucket incarnation.
|
||||
pub async fn rename_local_data_reserved(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
token: Uuid,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
if token.is_nil()
|
||||
|| source.0 != crate::disk::RUSTFS_META_TMP_BUCKET
|
||||
|| crate::bucket::utils::check_valid_bucket_name(destination.0).is_err()
|
||||
|| is_meta_bucketname(destination.0)
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&fi.metadata,
|
||||
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||
)
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
source,
|
||||
fi,
|
||||
destination,
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Resolve a quota lease against this request's captured storage instance.
|
||||
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
@@ -270,6 +435,7 @@ impl ECStore {
|
||||
scanner_publication_lease_token: scanner_token,
|
||||
external_guard,
|
||||
namespace_owner: None,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -351,7 +517,7 @@ fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admit_local_disk(
|
||||
pub(crate) async fn admit_local_disk(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk: &DiskStore,
|
||||
disk_id: Option<Uuid>,
|
||||
@@ -783,6 +949,116 @@ mod tests {
|
||||
.expect("seeded metadata bytes")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_target_rename_is_bound_to_the_admitted_disk() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let dir = tempfile::tempdir().expect("disk root");
|
||||
let id = Uuid::new_v4();
|
||||
let disk = target_disk(&ctx, dir.path(), id).await;
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let version = Uuid::new_v4();
|
||||
let fi = target_file_info("object", version, b"reserved");
|
||||
seed_target(&disk, crate::disk::RUSTFS_META_TMP_BUCKET, "staged", fi.clone()).await;
|
||||
tokio::fs::create_dir_all(dir.path().join("target-bucket"))
|
||||
.await
|
||||
.expect("user bucket");
|
||||
assert!(matches!(
|
||||
store
|
||||
.reserve_local_object_metadata(
|
||||
&disk_ref,
|
||||
"target-bucket/../.rustfs.sys",
|
||||
"object",
|
||||
std::time::Duration::from_secs(5),
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
assert!(matches!(
|
||||
store
|
||||
.reserve_local_object_metadata(&disk_ref, "", ".rustfs.sys/config", std::time::Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
let token = store
|
||||
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve current disk");
|
||||
assert!(matches!(
|
||||
store
|
||||
.rename_local_data_reserved(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "object"), token)
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
store
|
||||
.activate_local_object_metadata_reservation(&disk_ref, "target-bucket", "object", token)
|
||||
.await
|
||||
.expect("activate token");
|
||||
store
|
||||
.rename_local_data_reserved(
|
||||
&disk_ref,
|
||||
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
|
||||
&fi,
|
||||
("target-bucket", "object"),
|
||||
token,
|
||||
)
|
||||
.await
|
||||
.expect("reserved publication");
|
||||
assert!(
|
||||
!store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, token)
|
||||
.await
|
||||
.expect("consumed token")
|
||||
);
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"object",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read reserved publication");
|
||||
assert_eq!(read.data, fi.data);
|
||||
|
||||
let stale = store
|
||||
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve before disk replacement");
|
||||
let replacement = target_disk(&ctx, dir.path(), id).await;
|
||||
assert!(matches!(
|
||||
store
|
||||
.rename_local_data_reserved(
|
||||
&disk_ref,
|
||||
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
|
||||
&fi,
|
||||
("target-bucket", "object"),
|
||||
stale,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::DiskOngoingReq)
|
||||
));
|
||||
let after = replacement
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"object",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("replacement retains publication");
|
||||
assert_eq!(after.data, fi.data);
|
||||
assert!(store.cancel_local_object_metadata_reservation(&disk_ref, stale).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
|
||||
for warm in [false, true] {
|
||||
|
||||
@@ -382,6 +382,42 @@ impl LockClient for LocalClient {
|
||||
}
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let mut guards = shard.write().await;
|
||||
let Some(entry) = guards.get_mut(lock_id) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if entry.is_expired() {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(entry.guard.downgrade_committed_tail())
|
||||
}
|
||||
|
||||
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let mut guards = shard.write().await;
|
||||
let Some(entry) = guards.get_mut(lock_id) else { return Ok(false) };
|
||||
if entry.is_expired() {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(entry.guard.try_upgrade_committed_tail())
|
||||
}
|
||||
|
||||
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
|
||||
self.refresh(lock_id).await
|
||||
}
|
||||
|
||||
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let guards = shard.read().await;
|
||||
let Some(entry) = guards.get(lock_id) else { return Ok(None) };
|
||||
if entry.is_expired() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(entry.guard.waiting_writer())
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
self.release(lock_id).await
|
||||
}
|
||||
@@ -547,6 +583,32 @@ mod tests {
|
||||
assert!(!response.success);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn downgrade_tail_is_idempotent_and_release_preserves_readers() {
|
||||
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
|
||||
let resource = crate::ObjectKey::new("bucket", "tail-downgrade");
|
||||
let first = request(resource.clone(), "owner-a", Duration::from_secs(30));
|
||||
assert!(client.acquire_lock(&first).await.expect("initial lock").success);
|
||||
assert!(client.downgrade_committed_tail(&first.lock_id).await.expect("downgrade"));
|
||||
assert!(
|
||||
client
|
||||
.downgrade_committed_tail(&first.lock_id)
|
||||
.await
|
||||
.expect("retry downgrade")
|
||||
);
|
||||
let reader = crate::LockRequest::new(resource.clone(), LockType::Shared, "reader")
|
||||
.with_acquire_timeout(Duration::from_millis(500));
|
||||
assert!(client.acquire_lock(&reader).await.expect("reader").success);
|
||||
let writer = request(resource, "writer", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
|
||||
assert!(!client.acquire_lock(&writer).await.expect("writer blocked").success);
|
||||
assert!(client.release(&first.lock_id).await.expect("release tail"));
|
||||
assert!(!client.acquire_lock(&writer).await.expect("reader excludes writer").success);
|
||||
assert!(client.release(&reader.lock_id).await.expect("release reader"));
|
||||
assert!(client.acquire_lock(&writer).await.expect("writer after drain").success);
|
||||
assert!(client.release(&writer.lock_id).await.expect("release writer"));
|
||||
assert!(!client.downgrade_committed_tail(&first.lock_id).await.expect("released lease"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_rejects_a_lock_id_for_another_resource() {
|
||||
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
|
||||
|
||||
@@ -20,6 +20,12 @@ use async_trait::async_trait;
|
||||
use futures::future::join_all;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Marker exchanged in a refresh request and response for a committed-tail conversion.
|
||||
pub const COMMITTED_TAIL_CONVERSION_MARKER: &str = "rustfs-committed-tail-v1";
|
||||
pub const COMMITTED_TAIL_UPGRADE_MARKER: &str = "rustfs-committed-tail-upgrade-v1";
|
||||
pub const HEAL_LOCK_TRANSITION_PROBE_MARKER: &str = "rustfs-heal-lock-transition-probe-v1";
|
||||
pub const HEAL_WAITING_WRITER_MARKER: &str = "rustfs-heal-waiting-writer-v1";
|
||||
|
||||
/// Lock client trait
|
||||
#[async_trait]
|
||||
pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
@@ -48,6 +54,27 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
/// Refresh lock
|
||||
async fn refresh(&self, lock_id: &LockId) -> Result<bool>;
|
||||
|
||||
/// Convert an exclusive lease to a reader-compatible committed-tail lease.
|
||||
/// Unsupported peers retain their original lease and report `false`.
|
||||
async fn downgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Try to restore exclusive ownership, retaining the writer fence on failure.
|
||||
async fn try_upgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Check support without changing the lease mode.
|
||||
async fn supports_heal_lock_transition(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Observe foreground writer contention, or return `None` when unsupported.
|
||||
async fn waiting_writer(&self, _lock_id: &LockId) -> Result<Option<bool>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Force release lock
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool>;
|
||||
|
||||
|
||||
@@ -262,6 +262,8 @@ pub struct DistributedLockGuard {
|
||||
/// All underlying leases that should be refreshed and released with the guard.
|
||||
entries: Vec<HeldLockEntry>,
|
||||
lock_type: LockType,
|
||||
/// Shared and exclusive quorums must intersect before a tail may convert.
|
||||
committed_tail_quorums_intersect: bool,
|
||||
/// If true, Drop will not try to release (used if user manually released).
|
||||
disarmed: bool,
|
||||
/// Background lease task for renewable leases.
|
||||
@@ -277,7 +279,13 @@ impl DistributedLockGuard {
|
||||
/// - `entries` is the full list of underlying leases that should be refreshed and released
|
||||
/// with this guard.
|
||||
/// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics.
|
||||
fn new(lock_id: LockId, entries: Vec<HeldLockEntry>, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self {
|
||||
fn new(
|
||||
lock_id: LockId,
|
||||
entries: Vec<HeldLockEntry>,
|
||||
lock_type: LockType,
|
||||
heartbeat: LockHeartbeatConfig,
|
||||
committed_tail_quorums_intersect: bool,
|
||||
) -> Self {
|
||||
record_lock_held_acquire(lock_type);
|
||||
let lock_lost = Arc::new(LockLostSignal::default());
|
||||
match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) {
|
||||
@@ -296,6 +304,7 @@ impl DistributedLockGuard {
|
||||
lock_id,
|
||||
entries,
|
||||
lock_type,
|
||||
committed_tail_quorums_intersect,
|
||||
disarmed: false,
|
||||
refresh_task,
|
||||
lock_lost,
|
||||
@@ -412,6 +421,70 @@ impl DistributedLockGuard {
|
||||
&self.lock_id
|
||||
}
|
||||
|
||||
/// Convert retained peer leases without ever releasing writer exclusion.
|
||||
/// A peer without conversion support keeps its exclusive lease. Partial
|
||||
/// conversion is safe: both lease modes continue to exclude writers.
|
||||
pub async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.downgrade_committed_tail(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
/// Require every retained lease to confirm exclusivity before committing.
|
||||
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || self.is_lock_lost() {
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.try_upgrade_committed_tail(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
pub async fn supports_heal_lock_transition(&self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.supports_heal_lock_transition(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
/// Observe contention on every held lease. Unknown peers prevent a false idle result.
|
||||
pub async fn waiting_writer(&self) -> Option<bool> {
|
||||
if self.disarmed || self.is_lock_lost() {
|
||||
return None;
|
||||
}
|
||||
let results =
|
||||
futures::future::join_all(self.entries.iter().map(|entry| entry.client.waiting_writer(&entry.lock_id))).await;
|
||||
if self.is_lock_lost() {
|
||||
return None;
|
||||
}
|
||||
if results.iter().any(|result| matches!(result, Ok(Some(true)))) {
|
||||
return Some(true);
|
||||
}
|
||||
results
|
||||
.iter()
|
||||
.all(|result| matches!(result, Ok(Some(false))))
|
||||
.then_some(false)
|
||||
}
|
||||
|
||||
/// Actual granted peer identities, which may differ from the logical guard ID.
|
||||
pub fn granted_lock_ids(&self) -> impl Iterator<Item = &LockId> {
|
||||
self.entries.iter().map(|entry| &entry.lock_id)
|
||||
@@ -599,6 +672,7 @@ impl DistributedLock {
|
||||
individual_locks,
|
||||
request.lock_type,
|
||||
heartbeat,
|
||||
self.read_quorum().saturating_add(required_quorum) > self.clients.len(),
|
||||
)))
|
||||
} else {
|
||||
// Check if it's a timeout or quorum failure
|
||||
|
||||
@@ -39,6 +39,8 @@ pub struct FastLockGuard {
|
||||
shard: Option<Arc<LockShard>>, // None when locks are disabled
|
||||
released: bool,
|
||||
disabled: bool, // True when locks are disabled globally
|
||||
committed_tail: bool,
|
||||
sealed_for_commit: bool,
|
||||
/// Unique ID for this guard instance to prevent double-release
|
||||
guard_id: u64,
|
||||
}
|
||||
@@ -54,6 +56,8 @@ impl FastLockGuard {
|
||||
shard: Some(shard),
|
||||
released: false,
|
||||
disabled: false,
|
||||
committed_tail: false,
|
||||
sealed_for_commit: false,
|
||||
guard_id,
|
||||
}
|
||||
}
|
||||
@@ -68,6 +72,8 @@ impl FastLockGuard {
|
||||
shard: None,
|
||||
released: false,
|
||||
disabled: true,
|
||||
committed_tail: false,
|
||||
sealed_for_commit: false,
|
||||
guard_id,
|
||||
}
|
||||
}
|
||||
@@ -87,6 +93,83 @@ impl FastLockGuard {
|
||||
&self.owner
|
||||
}
|
||||
|
||||
/// The physical acquisition was force-released before registration.
|
||||
pub(crate) fn abandon_unregistered(&mut self) {
|
||||
if !self.released {
|
||||
self.released = true;
|
||||
record_lock_held_release(self.mode);
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep excluding writers while permitting reads of the committed version.
|
||||
pub fn downgrade_committed_tail(&mut self) -> bool {
|
||||
if self.released || self.sealed_for_commit {
|
||||
return false;
|
||||
}
|
||||
if self.committed_tail {
|
||||
return true;
|
||||
}
|
||||
if self.mode != LockMode::Exclusive {
|
||||
return false;
|
||||
}
|
||||
if self.disabled {
|
||||
self.mode = LockMode::Shared;
|
||||
self.committed_tail = true;
|
||||
return true;
|
||||
}
|
||||
let Some(shard) = &self.shard else { return false };
|
||||
if !shard.downgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
|
||||
return false;
|
||||
}
|
||||
record_lock_held_release(LockMode::Exclusive);
|
||||
record_lock_held_acquire(LockMode::Shared);
|
||||
self.mode = LockMode::Shared;
|
||||
self.committed_tail = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// Try to restore exclusive ownership without releasing the writer fence.
|
||||
pub fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
if self.released || (!self.committed_tail && self.mode != LockMode::Exclusive) {
|
||||
return false;
|
||||
}
|
||||
if !self.disabled {
|
||||
let Some(shard) = &self.shard else { return false };
|
||||
if !shard.try_upgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if self.committed_tail {
|
||||
record_lock_held_release(LockMode::Shared);
|
||||
record_lock_held_acquire(LockMode::Exclusive);
|
||||
}
|
||||
self.mode = LockMode::Exclusive;
|
||||
self.committed_tail = false;
|
||||
self.sealed_for_commit = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// Observe writers waiting on this guard's exact object key.
|
||||
pub fn waiting_writer(&self) -> Option<bool> {
|
||||
if self.released {
|
||||
return None;
|
||||
}
|
||||
if self.disabled {
|
||||
return Some(false);
|
||||
}
|
||||
self.shard
|
||||
.as_ref()?
|
||||
.waiting_writer_for_guard(&self.key, &self.owner, self.guard_id)
|
||||
}
|
||||
|
||||
fn release_physical(&self, shard: &LockShard) -> bool {
|
||||
if self.committed_tail {
|
||||
shard.release_committed_tail_with_guard(&self.key, &self.owner, self.guard_id)
|
||||
} else {
|
||||
shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Manually release the lock early
|
||||
///
|
||||
/// Returns true if the lock was successfully released, false if it was
|
||||
@@ -106,7 +189,7 @@ impl FastLockGuard {
|
||||
}
|
||||
|
||||
if let Some(shard) = &self.shard {
|
||||
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
|
||||
let success = self.release_physical(shard);
|
||||
if success {
|
||||
self.released = true;
|
||||
record_lock_held_release(self.mode);
|
||||
@@ -169,7 +252,7 @@ impl Drop for FastLockGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(shard) = &self.shard {
|
||||
if !self.released && !self.disabled {
|
||||
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
|
||||
let success = self.release_physical(shard);
|
||||
if !success {
|
||||
// For high-concurrency scenarios, this is likely due to:
|
||||
// 1. Lock was already released by another thread
|
||||
|
||||
@@ -96,11 +96,14 @@ impl FastObjectLockManager {
|
||||
/// Acquire object lock
|
||||
pub async fn acquire_lock(&self, request: ObjectLockRequest) -> Result<FastLockGuard, LockResult> {
|
||||
let shard = self.get_shard(&request.key);
|
||||
match shard.acquire_lock(&request).await {
|
||||
Ok(()) => {
|
||||
let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
|
||||
match shard.acquire_guarded(&request).await {
|
||||
Ok(acquired) => {
|
||||
let mut guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
|
||||
// Register guard to prevent premature cleanup
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
if !shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired) {
|
||||
guard.abandon_unregistered();
|
||||
return Err(LockResult::Timeout);
|
||||
}
|
||||
Ok(guard)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
@@ -201,31 +204,15 @@ impl FastObjectLockManager {
|
||||
let mut all_failed = Vec::new();
|
||||
let mut guards = Vec::new();
|
||||
|
||||
for (shard_id, requests) in shard_groups {
|
||||
let shard = self.shards[*shard_id].clone();
|
||||
|
||||
for (_, requests) in shard_groups {
|
||||
for request in requests {
|
||||
let key = request.key.clone();
|
||||
let owner = request.owner.clone();
|
||||
let mode = request.mode;
|
||||
|
||||
let acquired = if shard.try_fast_path_only(request) {
|
||||
true
|
||||
} else {
|
||||
match shard.acquire_lock(request).await {
|
||||
Ok(()) => true,
|
||||
Err(err) => {
|
||||
all_failed.push((key.clone(), err));
|
||||
false
|
||||
}
|
||||
match self.acquire_lock(request.clone()).await {
|
||||
Ok(guard) => {
|
||||
all_successful.push(key);
|
||||
guards.push(guard);
|
||||
}
|
||||
};
|
||||
|
||||
if acquired {
|
||||
let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
all_successful.push(key);
|
||||
guards.push(guard);
|
||||
Err(err) => all_failed.push((key, err)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -245,14 +232,10 @@ impl FastObjectLockManager {
|
||||
let mut acquired_guards = Vec::new();
|
||||
let mut failed_locks = Vec::new();
|
||||
|
||||
'outer: for (shard_id, requests) in shard_groups {
|
||||
let shard = self.shards[*shard_id].clone();
|
||||
|
||||
'outer: for (_, requests) in shard_groups {
|
||||
for request in requests {
|
||||
match shard.acquire_lock(request).await {
|
||||
Ok(()) => {
|
||||
let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
match self.acquire_lock(request.clone()).await {
|
||||
Ok(guard) => {
|
||||
acquired_guards.push(guard);
|
||||
}
|
||||
Err(err) => {
|
||||
|
||||
@@ -94,6 +94,7 @@ impl ObjectLockState {
|
||||
pub fn reset_for_reuse(&mut self) {
|
||||
// Reset atomic state
|
||||
self.atomic_state = crate::fast_lock::state::AtomicLockState::new();
|
||||
self.force_epoch.store(0, Ordering::Relaxed);
|
||||
|
||||
// Clear owners
|
||||
*self.current_owner.write() = None;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use tokio::time::timeout;
|
||||
|
||||
@@ -30,6 +31,12 @@ struct ActiveGuardInfo {
|
||||
key: ObjectKey,
|
||||
mode: LockMode,
|
||||
owner: Arc<str>,
|
||||
committed_tail: bool,
|
||||
}
|
||||
|
||||
pub(crate) struct AcquiredLock {
|
||||
state: Arc<ObjectLockState>,
|
||||
force_epoch: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Hash)]
|
||||
@@ -46,6 +53,8 @@ enum TestPausePoint {
|
||||
BeforeFastWrite,
|
||||
FastWriteExisting,
|
||||
SlowLookup,
|
||||
ForceAfterTailRelease,
|
||||
ReleaseAfterUnregister,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -69,7 +78,7 @@ fn test_pause(point: TestPausePoint, key: &ObjectKey) {
|
||||
|
||||
/// Lock shard to reduce global contention.
|
||||
///
|
||||
/// When both are needed, lock `objects` before a state's owner metadata.
|
||||
/// When combined, lock `active_guards`, then `objects`, then a state's owner metadata.
|
||||
#[derive(Debug)]
|
||||
pub struct LockShard {
|
||||
/// Object lock states - using parking_lot for better performance
|
||||
@@ -133,12 +142,16 @@ impl LockShard {
|
||||
|
||||
/// Acquire lock with fast path optimization
|
||||
pub async fn acquire_lock(&self, request: &ObjectLockRequest) -> Result<(), LockResult> {
|
||||
self.acquire_guarded(request).await.map(|_| ())
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_guarded(&self, request: &ObjectLockRequest) -> Result<AcquiredLock, LockResult> {
|
||||
let start_time = Instant::now();
|
||||
|
||||
// Try fast path first
|
||||
if let Some(_state) = self.try_fast_path(request) {
|
||||
if let Some(acquired) = self.try_fast_path(request) {
|
||||
self.metrics.record_fast_path_success();
|
||||
return Ok(());
|
||||
return Ok(acquired);
|
||||
}
|
||||
|
||||
// Slow path with waiting
|
||||
@@ -157,7 +170,7 @@ impl LockShard {
|
||||
}
|
||||
|
||||
/// Try fast path lock acquisition (lock-free when possible)
|
||||
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<Arc<ObjectLockState>> {
|
||||
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<AcquiredLock> {
|
||||
// First try to get existing state without write lock
|
||||
{
|
||||
let objects = self.objects.read();
|
||||
@@ -168,14 +181,16 @@ impl LockShard {
|
||||
test_pause(TestPausePoint::FastExisting, &request.key);
|
||||
|
||||
// Try atomic acquisition
|
||||
let success = match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let success = force_epoch != u64::MAX
|
||||
&& match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
drop(objects);
|
||||
|
||||
if success {
|
||||
return Some(state);
|
||||
return Some(AcquiredLock { state, force_epoch });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -195,10 +210,11 @@ impl LockShard {
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::FastWriteExisting, &request.key);
|
||||
|
||||
let acquired = state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let acquired = force_epoch != u64::MAX && state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
|
||||
drop(objects);
|
||||
if acquired {
|
||||
return Some(state);
|
||||
return Some(AcquiredLock { state, force_epoch });
|
||||
}
|
||||
} else {
|
||||
// Create new state from pool and acquire immediately
|
||||
@@ -206,7 +222,7 @@ impl LockShard {
|
||||
let state = Arc::new(*state_box);
|
||||
if state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout) {
|
||||
objects.insert(request.key.clone(), state.clone());
|
||||
return Some(state);
|
||||
return Some(AcquiredLock { state, force_epoch: 0 });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -215,7 +231,7 @@ impl LockShard {
|
||||
}
|
||||
|
||||
/// Slow path with async waiting
|
||||
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<(), LockResult> {
|
||||
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<AcquiredLock, LockResult> {
|
||||
// Use adaptive timeout based on current load and request priority
|
||||
let adaptive_timeout = self.calculate_adaptive_timeout(request);
|
||||
let deadline = start_time + adaptive_timeout;
|
||||
@@ -234,7 +250,7 @@ impl LockShard {
|
||||
|
||||
loop {
|
||||
// Keep the state in the map until its acquisition attempt completes.
|
||||
let (state, success) = {
|
||||
let (state, success, force_epoch) = {
|
||||
let mut objects = self.objects.write();
|
||||
let state = match objects.get(&request.key) {
|
||||
Some(state) => state.clone(),
|
||||
@@ -249,17 +265,19 @@ impl LockShard {
|
||||
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::SlowLookup, &request.key);
|
||||
let success = match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let success = force_epoch != u64::MAX
|
||||
&& match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
drop(objects);
|
||||
(state, success)
|
||||
(state, success, force_epoch)
|
||||
};
|
||||
|
||||
if success {
|
||||
self.metrics.record_slow_path_success();
|
||||
return Ok(());
|
||||
return Ok(AcquiredLock { state, force_epoch });
|
||||
}
|
||||
|
||||
// Check timeout
|
||||
@@ -381,14 +399,10 @@ impl LockShard {
|
||||
|
||||
/// Release lock with guard ID tracking for double-release prevention
|
||||
pub fn release_lock_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, mode: LockMode, guard_id: u64) -> bool {
|
||||
// First, try to remove the guard from active set
|
||||
let guard_was_active = {
|
||||
let mut guards = self.active_guards.lock();
|
||||
guards.remove(&guard_id).is_some()
|
||||
};
|
||||
|
||||
// If guard was not active, this is a double-release attempt
|
||||
if !guard_was_active {
|
||||
// Keep force-unlock from releasing this holder and admitting a new
|
||||
// same-owner holder between guard removal and physical release.
|
||||
let mut guards = self.active_guards.lock();
|
||||
if guards.remove(&guard_id).is_none() {
|
||||
tracing::debug!(
|
||||
"Double-release attempt blocked: key={}, owner={}, mode={:?}, guard_id={}",
|
||||
key,
|
||||
@@ -399,6 +413,9 @@ impl LockShard {
|
||||
return false;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::ReleaseAfterUnregister, key);
|
||||
|
||||
// Proceed with normal release
|
||||
let should_cleanup;
|
||||
let result;
|
||||
@@ -423,6 +440,8 @@ impl LockShard {
|
||||
}
|
||||
}
|
||||
|
||||
drop(guards);
|
||||
|
||||
if should_cleanup {
|
||||
self.schedule_cleanup(key.clone());
|
||||
}
|
||||
@@ -430,21 +449,110 @@ impl LockShard {
|
||||
result
|
||||
}
|
||||
|
||||
/// Convert an active exclusive guard without opening a writer gap.
|
||||
pub(crate) fn downgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Exclusive {
|
||||
return false;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
if !state.downgrade_committed_tail(owner) {
|
||||
return false;
|
||||
}
|
||||
info.mode = LockMode::Shared;
|
||||
info.committed_tail = true;
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn try_upgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
|
||||
return false;
|
||||
}
|
||||
if info.mode == LockMode::Exclusive && !info.committed_tail {
|
||||
return true;
|
||||
}
|
||||
if info.mode != LockMode::Shared || !info.committed_tail {
|
||||
return false;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
if !state.try_upgrade_committed_tail(owner) {
|
||||
return false;
|
||||
}
|
||||
info.mode = LockMode::Exclusive;
|
||||
info.committed_tail = false;
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn waiting_writer_for_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> Option<bool> {
|
||||
let guards = self.active_guards.lock();
|
||||
let info = guards.get(&guard_id)?.as_ref()?;
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
|
||||
return None;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let state = objects.get(key)?;
|
||||
Some(state.atomic_state.writer_observed_during_tail())
|
||||
}
|
||||
|
||||
/// Release a converted guard and its special reader-admission state.
|
||||
pub(crate) fn release_committed_tail_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Shared || !info.committed_tail {
|
||||
return false;
|
||||
}
|
||||
let (success, should_cleanup) = {
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
let success = state.release_committed_tail(owner);
|
||||
(success, success && !state.is_locked() && !state.atomic_state.has_waiters())
|
||||
};
|
||||
if success {
|
||||
guards.remove(&guard_id);
|
||||
self.metrics.record_release();
|
||||
}
|
||||
drop(guards);
|
||||
if should_cleanup {
|
||||
self.schedule_cleanup(key.clone());
|
||||
}
|
||||
success
|
||||
}
|
||||
|
||||
/// Register a guard to prevent premature cleanup
|
||||
pub fn register_guard(&self, guard_id: u64) {
|
||||
self.active_guards.lock().insert(guard_id, None);
|
||||
}
|
||||
|
||||
pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc<str>) {
|
||||
pub(crate) fn register_guard_with_info(
|
||||
&self,
|
||||
guard_id: u64,
|
||||
key: &ObjectKey,
|
||||
mode: LockMode,
|
||||
owner: &Arc<str>,
|
||||
acquired: &AcquiredLock,
|
||||
) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let objects = self.objects.read();
|
||||
if !objects.get(key).is_some_and(|state| {
|
||||
Arc::ptr_eq(state, &acquired.state) && state.force_epoch.load(Ordering::Acquire) == acquired.force_epoch
|
||||
}) {
|
||||
return false;
|
||||
}
|
||||
guards.insert(
|
||||
guard_id,
|
||||
Some(ActiveGuardInfo {
|
||||
key: key.clone(),
|
||||
mode,
|
||||
owner: owner.clone(),
|
||||
committed_tail: false,
|
||||
}),
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
/// Unregister a guard (called when guard is dropped)
|
||||
@@ -719,27 +827,57 @@ impl LockShard {
|
||||
/// Returns the number of owners that were released. Used by the admin
|
||||
/// force-unlock path to clear a stuck resource.
|
||||
pub fn force_release_all(&self, key: &ObjectKey) -> usize {
|
||||
let owners_modes: Vec<(Arc<str>, LockMode)> = {
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else {
|
||||
return 0;
|
||||
};
|
||||
let mut pairs = Vec::new();
|
||||
if let Some(info) = state.current_owner.read().clone() {
|
||||
pairs.push((info.owner, LockMode::Exclusive));
|
||||
}
|
||||
for entry in state.shared_owners.read().iter() {
|
||||
pairs.push((entry.owner.clone(), LockMode::Shared));
|
||||
}
|
||||
pairs
|
||||
};
|
||||
|
||||
// Serialize tail conversion and prevent new acquisitions while taking
|
||||
// and releasing the owner snapshot. A new tail must never be mistaken
|
||||
// for an ordinary shared owner by force unlock.
|
||||
let mut guards = self.active_guards.lock();
|
||||
let objects = self.objects.write();
|
||||
let Some(state) = objects.get(key) else { return 0 };
|
||||
// Acquisitions completed before this point but not yet registered
|
||||
// must not later create guards for the released generation.
|
||||
let _ = state
|
||||
.force_epoch
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |epoch| epoch.checked_add(1));
|
||||
let tail = guards.iter().find_map(|(&id, info)| {
|
||||
info.as_ref()
|
||||
.filter(|info| info.committed_tail && info.key == *key)
|
||||
.map(|info| (id, info.owner.clone()))
|
||||
});
|
||||
let mut released = 0;
|
||||
for (owner, mode) in owners_modes {
|
||||
if self.release_lock(key, &owner, mode) {
|
||||
if let Some((id, owner)) = tail
|
||||
&& state.release_committed_tail(&owner)
|
||||
{
|
||||
guards.remove(&id);
|
||||
self.metrics.record_release();
|
||||
released += 1;
|
||||
}
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::ForceAfterTailRelease, key);
|
||||
let mut owners_modes = Vec::new();
|
||||
if let Some(info) = state.current_owner.read().clone() {
|
||||
owners_modes.push((info.owner, LockMode::Exclusive, 1));
|
||||
}
|
||||
for entry in state.shared_owners.read().iter() {
|
||||
owners_modes.push((entry.owner.clone(), LockMode::Shared, entry.count));
|
||||
}
|
||||
for (owner, mode, count) in owners_modes {
|
||||
let mut owner_released = false;
|
||||
for _ in 0..count {
|
||||
let success = match mode {
|
||||
LockMode::Shared => state.release_shared(&owner),
|
||||
LockMode::Exclusive => state.release_exclusive(&owner),
|
||||
};
|
||||
if success {
|
||||
self.metrics.record_release();
|
||||
owner_released = true;
|
||||
}
|
||||
}
|
||||
if owner_released {
|
||||
released += 1;
|
||||
}
|
||||
}
|
||||
// Old guard drops must not release a later holder with the same owner.
|
||||
guards.retain(|_, info| info.as_ref().is_none_or(|info| info.key != *key));
|
||||
released
|
||||
}
|
||||
|
||||
@@ -979,6 +1117,315 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn force_unlock_cannot_interleave_an_old_release_with_a_new_holder() {
|
||||
for reader in [false, true] {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-release-gap");
|
||||
let old = if reader {
|
||||
manager.acquire_read_lock(key.clone(), "same-owner").await
|
||||
} else {
|
||||
manager.acquire_write_lock(key.clone(), "same-owner").await
|
||||
}
|
||||
.expect("old holder");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let (entered, resume) = install_pause(TestPausePoint::ReleaseAfterUnregister, &key);
|
||||
let old_drop = tokio::task::spawn_blocking(move || drop(old));
|
||||
entered.recv_timeout(Duration::from_secs(5)).expect("old release paused");
|
||||
|
||||
let can_force = shard.active_guards.try_lock().is_some();
|
||||
let mut force = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
|
||||
};
|
||||
let mut new_guard = if can_force {
|
||||
assert_eq!(force.await.expect("force join"), 1);
|
||||
let mut guard = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer");
|
||||
if reader {
|
||||
assert!(guard.downgrade_committed_tail());
|
||||
}
|
||||
resume.send(()).expect("resume old release");
|
||||
old_drop.await.expect("old release join");
|
||||
guard
|
||||
} else {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut force).await.is_err());
|
||||
resume.send(()).expect("resume old release");
|
||||
old_drop.await.expect("old release join");
|
||||
assert_eq!(force.await.expect("force join"), 0);
|
||||
manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer")
|
||||
};
|
||||
if reader && !can_force {
|
||||
assert!(new_guard.downgrade_committed_tail());
|
||||
}
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"an old release must not unlock the new writer"
|
||||
);
|
||||
drop(new_guard);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn force_unlock_does_not_capture_a_new_committed_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-tail-race");
|
||||
let mut old = manager.acquire_write_lock(key.clone(), "old").await.expect("old writer");
|
||||
assert!(old.downgrade_committed_tail());
|
||||
let (entered, resume) = install_pause(TestPausePoint::ForceAfterTailRelease, &key);
|
||||
let force = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
|
||||
};
|
||||
entered.recv_timeout(Duration::from_secs(5)).expect("force unlock paused");
|
||||
let mut next = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_write_lock(key, "new").await })
|
||||
};
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut next).await.is_err());
|
||||
resume.send(()).expect("resume force unlock");
|
||||
assert_eq!(force.await.expect("force unlock task"), 1);
|
||||
let mut new = next.await.expect("new writer task").expect("new writer");
|
||||
assert!(new.downgrade_committed_tail());
|
||||
drop(old);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"the new committed tail must still exclude writers"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_tail_upgrade_waits_for_readers_and_keeps_writer_fenced() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgrade-tail");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = manager
|
||||
.acquire_read_lock(key.clone(), "reader")
|
||||
.await
|
||||
.expect("reader during tail");
|
||||
assert!(!tail.try_upgrade_committed_tail(), "an active reader must prevent exclusive commit");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "late-reader"))
|
||||
.await
|
||||
.is_err(),
|
||||
"new readers must not starve a pending upgrade"
|
||||
);
|
||||
drop(reader);
|
||||
assert!(tail.try_upgrade_committed_tail());
|
||||
assert!(tail.try_upgrade_committed_tail(), "a repeated request must be idempotent");
|
||||
assert!(
|
||||
!tail.downgrade_committed_tail(),
|
||||
"a delayed downgrade must not reopen reads after upgrade"
|
||||
);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "reader"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
drop(tail);
|
||||
let _writer = manager.acquire_write_lock(key, "writer").await.expect("writer after release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tail_remembers_a_writer_that_stopped_waiting_between_samples() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "brief-writer-observation");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(tail.waiting_writer(), Some(true), "a short wait must not be missed by polling");
|
||||
assert!(!tail.try_upgrade_committed_tail(), "observed contention must prevent a later commit");
|
||||
drop(tail);
|
||||
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
|
||||
assert_eq!(next.waiting_writer(), Some(false), "contention must not leak to the next owner");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upgraded_tail_remembers_a_writer_that_stopped_waiting() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgraded-tail-brief-writer");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert!(tail.try_upgrade_committed_tail());
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(tail.waiting_writer(), Some(true), "upgraded leases must retain observed contention");
|
||||
drop(tail);
|
||||
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
|
||||
assert_eq!(next.waiting_writer(), Some(false));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guard_observes_waiting_writer_and_rejects_stale_identity() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "waiting-writer-observation");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert_eq!(tail.waiting_writer(), Some(false));
|
||||
let writer = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_write_lock(key, "foreground").await })
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while tail.waiting_writer() != Some(true) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("writer becomes observable");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
assert_eq!(tail.waiting_writer(), None, "force-released guard must not inspect a replacement owner");
|
||||
let _writer = writer.await.expect("writer task").expect("writer succeeds after release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dropping_a_pending_upgrade_allows_readers_again() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "cancel-upgrade");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = manager.acquire_read_lock(key.clone(), "reader").await.expect("first reader");
|
||||
assert!(!tail.try_upgrade_committed_tail());
|
||||
let mut waiting = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_read_lock(key, "waiting-reader").await })
|
||||
};
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut waiting).await.is_err());
|
||||
drop(tail);
|
||||
let second = tokio::time::timeout(Duration::from_secs(2), waiting)
|
||||
.await
|
||||
.expect("reader wakes after cancellation")
|
||||
.expect("reader task joins")
|
||||
.expect("reader obtains lock");
|
||||
drop((reader, second));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_released_tail_cannot_claim_an_exclusive_upgrade() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgrade-stale");
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let _new = manager.acquire_write_lock(key, "same-owner").await.expect("new owner");
|
||||
assert!(!tail.try_upgrade_committed_tail());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exclusive_upgrade_ack_fences_a_late_downgrade() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "late-downgrade");
|
||||
let mut guard = manager.acquire_write_lock(key, "heal").await.expect("write lock");
|
||||
assert!(guard.try_upgrade_committed_tail());
|
||||
assert!(!guard.downgrade_committed_tail());
|
||||
assert!(guard.try_upgrade_committed_tail());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_old_readers_cannot_release_a_new_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-old-readers");
|
||||
let old1 = manager
|
||||
.acquire_read_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("first reader");
|
||||
let old2 = manager
|
||||
.acquire_read_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("second reader");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
drop((old1, old2));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"old reader drops must not release the new tail"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forced_unregistered_reader_cannot_release_a_new_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-unregistered-reader");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let owner: Arc<str> = Arc::from("same-owner");
|
||||
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
|
||||
let acquired = shard
|
||||
.acquire_guarded(&request)
|
||||
.await
|
||||
.expect("old reader acquired before registration");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let mut old = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
|
||||
assert!(!shard.register_guard_with_info(old.guard_id(), old.key(), old.mode(), old.owner(), &acquired));
|
||||
old.abandon_unregistered();
|
||||
drop(old);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"an old reader must not release a new committed tail"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_on_another_key_keeps_unregistered_acquisition_valid() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "unrelated-acquisition");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let owner: Arc<str> = Arc::from("owner");
|
||||
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
|
||||
let acquired = shard.acquire_guarded(&request).await.expect("reader acquired");
|
||||
let other = (0..1000)
|
||||
.map(|index| ObjectKey::new("bucket", format!("other-{index}")))
|
||||
.find(|other| Arc::ptr_eq(&shard, manager.get_shard(other)))
|
||||
.expect("another key on the same shard");
|
||||
let other_guard = manager
|
||||
.acquire_write_lock(other.clone(), "other")
|
||||
.await
|
||||
.expect("other writer");
|
||||
assert_eq!(manager.force_unlock(&other), 1);
|
||||
drop(other_guard);
|
||||
let guard = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
|
||||
assert!(shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired));
|
||||
drop(guard);
|
||||
assert!(manager.acquire_write_lock(key, "next").await.is_ok());
|
||||
}
|
||||
|
||||
fn start_acquisition(
|
||||
manager: &Arc<FastObjectLockManager>,
|
||||
request: ObjectLockRequest,
|
||||
|
||||
@@ -23,6 +23,11 @@ use crate::fast_lock::types::{LockMode, LockPriority};
|
||||
/// Optimized atomic lock state encoding in u64
|
||||
/// Bits: [63:48] reserved | [47:32] writers_waiting | [31:16] readers_waiting | [15:8] readers_count | [7:1] flags | [0] writer_flag
|
||||
const WRITER_FLAG_MASK: u64 = 0x1;
|
||||
// A committed write tail is a reader-compatible owner that still excludes writers.
|
||||
const COMMITTED_TAIL_FLAG: u64 = 0x2;
|
||||
const TAIL_UPGRADE_PENDING_FLAG: u64 = 0x4;
|
||||
const TAIL_WRITER_OBSERVED_FLAG: u64 = 0x8;
|
||||
const TAIL_TRACK_WRITERS_FLAG: u64 = 0x10;
|
||||
const READERS_SHIFT: u8 = 8;
|
||||
const READERS_MASK: u64 = 0xFF << READERS_SHIFT; // Support up to 255 concurrent readers
|
||||
const READERS_WAITING_SHIFT: u8 = 16;
|
||||
@@ -31,7 +36,6 @@ const WRITERS_WAITING_SHIFT: u8 = 32;
|
||||
const WRITERS_WAITING_MASK: u64 = 0xFFFF << WRITERS_WAITING_SHIFT;
|
||||
|
||||
// Fast path check masks
|
||||
const NO_WRITER_AND_NO_WAITING_WRITERS: u64 = WRITER_FLAG_MASK | WRITERS_WAITING_MASK;
|
||||
const COMPLETELY_UNLOCKED: u64 = 0;
|
||||
|
||||
/// Fast atomic lock state for single version
|
||||
@@ -66,8 +70,8 @@ impl AtomicLockState {
|
||||
let state = self.state.load(Ordering::Relaxed); // Use Relaxed for better performance
|
||||
match mode {
|
||||
LockMode::Shared => {
|
||||
// No writer and no waiting writers
|
||||
(state & NO_WRITER_AND_NO_WAITING_WRITERS) == 0
|
||||
(state & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) == 0
|
||||
&& ((state & WRITERS_WAITING_MASK) == 0 || (state & COMMITTED_TAIL_FLAG) != 0)
|
||||
}
|
||||
LockMode::Exclusive => {
|
||||
// Completely unlocked
|
||||
@@ -83,8 +87,11 @@ impl AtomicLockState {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
|
||||
// Fast path check - cannot acquire if there's a writer or writers waiting
|
||||
if (current & NO_WRITER_AND_NO_WAITING_WRITERS) != 0 {
|
||||
// While a committed tail owns a shared slot, let reads through
|
||||
// even if a subsequent writer is waiting for that tail to drain.
|
||||
if (current & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) != 0
|
||||
|| ((current & WRITERS_WAITING_MASK) != 0 && (current & COMMITTED_TAIL_FLAG) == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -137,6 +144,78 @@ impl AtomicLockState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomically retain writer exclusion while allowing reads of a committed tail.
|
||||
fn downgrade_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & WRITER_FLAG_MASK) == 0 || (current & (READERS_MASK | COMMITTED_TAIL_FLAG)) != 0 {
|
||||
return false;
|
||||
}
|
||||
let next = (current & !WRITER_FLAG_MASK)
|
||||
| COMMITTED_TAIL_FLAG
|
||||
| (1 << READERS_SHIFT)
|
||||
| if current & WRITERS_WAITING_MASK != 0 {
|
||||
TAIL_WRITER_OBSERVED_FLAG
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Restore exclusive ownership only after every other reader has left.
|
||||
fn try_upgrade_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & COMMITTED_TAIL_FLAG) == 0 || (current & TAIL_WRITER_OBSERVED_FLAG) != 0 {
|
||||
return false;
|
||||
}
|
||||
let readers_drained = (current & READERS_MASK) == (1 << READERS_SHIFT);
|
||||
let next = if readers_drained {
|
||||
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG | READERS_MASK))
|
||||
| WRITER_FLAG_MASK
|
||||
| TAIL_TRACK_WRITERS_FLAG
|
||||
} else {
|
||||
current | TAIL_UPGRADE_PENDING_FLAG
|
||||
};
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return readers_drained;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Release the committed tail's own shared slot and restore ordinary fairness.
|
||||
fn release_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & COMMITTED_TAIL_FLAG) == 0 || self.readers_count(current) == 0 {
|
||||
return false;
|
||||
}
|
||||
let next =
|
||||
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG)) - (1 << READERS_SHIFT);
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Release shared lock
|
||||
pub fn release_shared(&self) -> bool {
|
||||
loop {
|
||||
@@ -169,7 +248,7 @@ impl AtomicLockState {
|
||||
return false; // No exclusive lock to release
|
||||
}
|
||||
|
||||
let new_state = current & !WRITER_FLAG_MASK;
|
||||
let new_state = current & !(WRITER_FLAG_MASK | TAIL_TRACK_WRITERS_FLAG | TAIL_WRITER_OBSERVED_FLAG);
|
||||
|
||||
if self
|
||||
.state
|
||||
@@ -236,7 +315,10 @@ impl AtomicLockState {
|
||||
return false; // Max waiting writers
|
||||
}
|
||||
|
||||
let new_state = current + (1 << WRITERS_WAITING_SHIFT);
|
||||
let mut new_state = current + (1 << WRITERS_WAITING_SHIFT);
|
||||
if current & (COMMITTED_TAIL_FLAG | TAIL_TRACK_WRITERS_FLAG) != 0 {
|
||||
new_state |= TAIL_WRITER_OBSERVED_FLAG;
|
||||
}
|
||||
|
||||
if self
|
||||
.state
|
||||
@@ -321,10 +403,15 @@ impl AtomicLockState {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn writers_waiting_count(&self) -> u16 {
|
||||
pub(crate) fn writers_waiting_count(&self) -> u16 {
|
||||
let state = self.state.load(Ordering::Acquire);
|
||||
self.writers_waiting(state)
|
||||
}
|
||||
|
||||
pub(crate) fn writer_observed_during_tail(&self) -> bool {
|
||||
let state = self.state.load(Ordering::Acquire);
|
||||
self.writers_waiting(state) > 0 || state & TAIL_WRITER_OBSERVED_FLAG != 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Object lock state with version support - optimized memory layout
|
||||
@@ -334,6 +421,8 @@ pub struct ObjectLockState {
|
||||
// First cache line: Most frequently accessed data
|
||||
/// Atomic state for fast operations
|
||||
pub atomic_state: AtomicLockState,
|
||||
/// Changes whenever an administrator force-releases this state.
|
||||
pub(crate) force_epoch: AtomicU64,
|
||||
|
||||
// Second cache line: Notification mechanisms
|
||||
/// Notification for readers (traditional)
|
||||
@@ -377,6 +466,7 @@ impl ObjectLockState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
atomic_state: AtomicLockState::new(),
|
||||
force_epoch: AtomicU64::new(0),
|
||||
read_notify: Notify::new(),
|
||||
write_notify: Notify::new(),
|
||||
optimized_notify: OptimizedNotify::new(),
|
||||
@@ -425,6 +515,70 @@ impl ObjectLockState {
|
||||
true
|
||||
}
|
||||
|
||||
/// Convert the current exclusive owner into the committed-tail owner.
|
||||
pub(crate) fn downgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut current = self.current_owner.write();
|
||||
let Some(info) = current.as_ref().filter(|info| info.owner.as_ref() == owner.as_ref()).cloned() else {
|
||||
return false;
|
||||
};
|
||||
let mut shared = self.shared_owners.write();
|
||||
if !shared.is_empty() || !self.atomic_state.downgrade_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
shared.push(SharedOwnerEntry {
|
||||
owner: info.owner,
|
||||
count: 1,
|
||||
acquired_at: info.acquired_at,
|
||||
lock_timeout: info.lock_timeout,
|
||||
});
|
||||
*current = None;
|
||||
drop(shared);
|
||||
drop(current);
|
||||
self.optimized_notify.notify_readers();
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn try_upgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut current = self.current_owner.write();
|
||||
let mut shared = self.shared_owners.write();
|
||||
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
|
||||
return false;
|
||||
};
|
||||
if current.is_some() || !self.atomic_state.try_upgrade_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
let info = shared.remove(pos);
|
||||
*current = Some(ExclusiveOwnerInfo {
|
||||
owner: info.owner,
|
||||
acquired_at: info.acquired_at,
|
||||
lock_timeout: info.lock_timeout,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
/// Release the committed-tail owner's shared slot without releasing readers.
|
||||
pub(crate) fn release_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut shared = self.shared_owners.write();
|
||||
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
|
||||
return false;
|
||||
};
|
||||
if !self.atomic_state.release_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
if shared[pos].count > 1 {
|
||||
shared[pos].count -= 1;
|
||||
} else {
|
||||
shared.remove(pos);
|
||||
}
|
||||
let no_readers = shared.is_empty();
|
||||
drop(shared);
|
||||
if no_readers {
|
||||
self.optimized_notify.notify_writer();
|
||||
}
|
||||
self.optimized_notify.notify_readers();
|
||||
true
|
||||
}
|
||||
|
||||
/// Release shared lock
|
||||
pub fn release_shared(&self, owner: &Arc<str>) -> bool {
|
||||
let mut shared = self.shared_owners.write();
|
||||
@@ -555,6 +709,24 @@ mod tests {
|
||||
assert!(!state.release_exclusive());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn committed_tail_allows_reads_but_preserves_writer_exclusion() {
|
||||
let state = AtomicLockState::new();
|
||||
assert!(state.try_acquire_exclusive());
|
||||
assert!(state.inc_writers_waiting());
|
||||
assert!(state.downgrade_committed_tail());
|
||||
assert!(!state.try_acquire_exclusive(), "the tail must still exclude writers");
|
||||
assert!(state.try_acquire_shared(), "a queued writer must not block reads of the committed tail");
|
||||
assert!(state.release_committed_tail());
|
||||
assert!(!state.try_acquire_shared(), "writer preference resumes when the tail ends");
|
||||
assert!(!state.try_acquire_exclusive(), "an outstanding reader must still exclude writers");
|
||||
assert!(state.release_shared());
|
||||
assert!(state.try_acquire_exclusive());
|
||||
assert!(state.release_exclusive());
|
||||
state.dec_writers_waiting();
|
||||
assert!(state.is_free());
|
||||
}
|
||||
|
||||
// Regression for the waiter-preserving exclusive CAS.
|
||||
//
|
||||
// The acquisition CAS used to demand a fully-zero state word, which
|
||||
|
||||
@@ -111,6 +111,67 @@ mod fast_lock_tests {
|
||||
assert!(guard.is_released(), "Guard should be marked as released");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_old_tail_cannot_release_new_tail() {
|
||||
let manager = create_test_manager();
|
||||
let key = ObjectKey::new("test-bucket", "force-tail-generation");
|
||||
let owner: Arc<str> = Arc::from("same-owner");
|
||||
let mut old = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("old writer");
|
||||
assert!(old.downgrade_committed_tail());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut new = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(new.downgrade_committed_tail());
|
||||
drop(old);
|
||||
let attempt =
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, Arc::from("competing-writer")))
|
||||
.await;
|
||||
assert!(attempt.is_err(), "dropping the old guard must not release the new tail");
|
||||
drop(new);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_tail_allows_reader_while_writer_waits() {
|
||||
let manager = Arc::new(create_test_manager());
|
||||
let key = ObjectKey::new("test-bucket", "tail-handoff");
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), Arc::from("first-writer"))
|
||||
.await
|
||||
.expect("initial writer");
|
||||
let writer = tokio::spawn({
|
||||
let manager = Arc::clone(&manager);
|
||||
let key = key.clone();
|
||||
async move { manager.acquire_write_lock(key, Arc::from("second-writer")).await }
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while manager.current_lock_counts().1 == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second writer should be queued");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = tokio::time::timeout(Duration::from_secs(2), manager.acquire_read_lock(key, Arc::from("reader")))
|
||||
.await
|
||||
.expect("reader should not wait behind committed tail")
|
||||
.expect("reader should acquire");
|
||||
assert!(!writer.is_finished(), "tail and reader must exclude the next writer");
|
||||
drop(tail);
|
||||
assert!(!writer.is_finished(), "reader still excludes the next writer");
|
||||
drop(reader);
|
||||
let writer = tokio::time::timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("writer should resume")
|
||||
.expect("writer task should join")
|
||||
.expect("writer should acquire");
|
||||
drop(writer);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_lock_counts_report_live_holders_and_waiters() {
|
||||
let manager = Arc::new(create_test_manager());
|
||||
|
||||
@@ -107,6 +107,38 @@ impl NamespaceLockGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Permit reads while continuing to exclude competing writers.
|
||||
pub async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.downgrade_committed_tail().await,
|
||||
Self::Fast(guard) => guard.downgrade_committed_tail(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to restore exclusive access while retaining the writer fence.
|
||||
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.try_upgrade_committed_tail().await,
|
||||
Self::Fast(guard) => guard.try_upgrade_committed_tail(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether the held leases can safely use heal lock transitions.
|
||||
pub async fn supports_heal_lock_transition(&self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.supports_heal_lock_transition().await,
|
||||
Self::Fast(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe whether a writer is waiting on this object's held leases.
|
||||
pub async fn waiting_writer(&self) -> Option<bool> {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.waiting_writer().await,
|
||||
Self::Fast(guard) => guard.waiting_writer(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Manually release the lock early
|
||||
pub fn release(&mut self) -> bool {
|
||||
match self {
|
||||
|
||||
@@ -154,6 +154,43 @@ impl crate::client::LockClient for DelayedClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PendingTailClient(LocalClient);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::client::LockClient for PendingTailClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
|
||||
self.0.acquire_lock(request).await
|
||||
}
|
||||
async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.release(lock_id).await
|
||||
}
|
||||
async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.refresh(lock_id).await
|
||||
}
|
||||
async fn downgrade_committed_tail(&self, _: &LockId) -> crate::Result<bool> {
|
||||
std::future::pending().await
|
||||
}
|
||||
async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.force_release(lock_id).await
|
||||
}
|
||||
async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
|
||||
self.0.check_status(lock_id).await
|
||||
}
|
||||
async fn get_stats(&self) -> crate::Result<LockStats> {
|
||||
self.0.get_stats().await
|
||||
}
|
||||
async fn close(&self) -> crate::Result<()> {
|
||||
self.0.close().await
|
||||
}
|
||||
async fn is_online(&self) -> bool {
|
||||
self.0.is_online().await
|
||||
}
|
||||
async fn is_local(&self) -> bool {
|
||||
self.0.is_local().await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FlakyAcquireClient {
|
||||
inner: LocalClient,
|
||||
@@ -579,6 +616,286 @@ async fn test_namespace_lock_guard_lock_id() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tail_conversion_rejects_non_intersecting_quorums() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..4)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients_and_quorum("non-intersecting".to_owned(), clients, 2);
|
||||
let resource = ObjectKey::new("bucket", "non-intersecting-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("write lease");
|
||||
assert!(
|
||||
!tail.downgrade_committed_tail().await,
|
||||
"conversion requires intersecting read and write quorums"
|
||||
);
|
||||
assert!(tail.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_tail_conversion_keeps_writers_excluded() {
|
||||
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = Vec::new();
|
||||
for index in 0..4 {
|
||||
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
|
||||
if index < 2 {
|
||||
clients.push(inner);
|
||||
} else {
|
||||
clients.push(Arc::new(DelayedClient {
|
||||
inner,
|
||||
delay: Duration::ZERO,
|
||||
}));
|
||||
}
|
||||
}
|
||||
let lock = NamespaceLock::with_clients("mixed-peers".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "partial-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(!tail.downgrade_committed_tail().await, "mixed peers cannot all confirm conversion");
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"mixed leases still exclude writers"
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_tail_conversion_keeps_writers_excluded() {
|
||||
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
clients.push(Arc::new(PendingTailClient(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))));
|
||||
let lock = NamespaceLock::with_clients_and_quorum("cancel-tail".to_owned(), clients, 4);
|
||||
let resource = ObjectKey::new("bucket", "cancel-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), tail.downgrade_committed_tail())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("other peers converted before cancellation");
|
||||
assert!(reader.release());
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"cancelled conversion must still exclude writers"
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsupported_peer_keeps_exclusive_tail_lease() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| {
|
||||
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
|
||||
Arc::new(DelayedClient {
|
||||
inner,
|
||||
delay: Duration::ZERO,
|
||||
}) as Arc<dyn crate::client::LockClient>
|
||||
})
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("old-peers".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "old-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(!tail.supports_heal_lock_transition().await, "old peers must not claim support");
|
||||
assert_eq!(tail.waiting_writer().await, None, "old peers cannot report contention");
|
||||
assert!(!tail.downgrade_committed_tail().await, "old peers must not claim conversion");
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "old peers must not claim upgrade support");
|
||||
assert!(
|
||||
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "writer", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after tail");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_committed_tail_allows_reader_and_excludes_writer() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("tail-test".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "committed-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(tail.downgrade_committed_tail().await, "all held peer leases should convert");
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("committed reader");
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"tail and reader must exclude writer"
|
||||
);
|
||||
assert!(tail.release());
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"reader still excludes writer"
|
||||
);
|
||||
assert!(reader.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after drain");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_tail_upgrade_requires_all_leases_and_drained_readers() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("tail-upgrade".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "tail-upgrade");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("write lock");
|
||||
assert!(tail.supports_heal_lock_transition().await);
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("reader");
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "live readers prevent full upgrade");
|
||||
assert!(reader.release());
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while !tail.try_upgrade_committed_tail().await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retry must finish partial upgrades after asynchronous reader release");
|
||||
assert!(tail.try_upgrade_committed_tail().await, "upgrade retries must be idempotent");
|
||||
assert!(
|
||||
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_tail_observes_a_waiting_writer() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = Arc::new(NamespaceLock::with_clients("writer-observation".to_owned(), clients));
|
||||
let resource = ObjectKey::new("bucket", "writer-observation");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("heal lock");
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
assert_eq!(tail.waiting_writer().await, Some(false));
|
||||
let writer = {
|
||||
let lock = Arc::clone(&lock);
|
||||
let resource = resource.clone();
|
||||
tokio::spawn(async move { lock.get_write_lock(resource, "writer", Duration::from_secs(3)).await })
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while tail.waiting_writer().await != Some(true) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("waiting writer becomes visible on a held peer");
|
||||
assert!(tail.release());
|
||||
let mut guard = writer.await.expect("writer task").expect("writer succeeds after release");
|
||||
assert!(guard.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_partial_upgrade_remembers_contention_on_upgraded_peer() {
|
||||
use crate::client::LockClient;
|
||||
|
||||
let clients: Vec<Arc<LocalClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))))
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients(
|
||||
"partial-upgrade-writer".to_owned(),
|
||||
clients.iter().cloned().map(|client| client as Arc<dyn LockClient>).collect(),
|
||||
);
|
||||
let resource = ObjectKey::new("bucket", "partial-upgrade-writer");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("heal lock");
|
||||
let ids = match &tail {
|
||||
NamespaceLockGuard::Standard(guard) => guard.granted_lock_ids().cloned().collect::<Vec<_>>(),
|
||||
NamespaceLockGuard::Fast(_) => panic!("distributed guard expected"),
|
||||
};
|
||||
let mut holders = Vec::new();
|
||||
for client in &clients {
|
||||
for id in &ids {
|
||||
if client.check_status(id).await.expect("peer status").is_some() {
|
||||
holders.push(Arc::clone(client));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(holders.len() >= 2, "write quorum must hold multiple peers");
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
let reader = LockRequest::new(resource.clone(), LockType::Shared, "reader");
|
||||
assert!(holders[0].acquire_lock(&reader).await.expect("reader request").success);
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "reader prevents a full upgrade");
|
||||
let writer = LockRequest::new(resource, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_millis(100));
|
||||
assert!(!holders[1].acquire_lock(&writer).await.expect("writer request").success);
|
||||
assert_eq!(
|
||||
tail.waiting_writer().await,
|
||||
Some(true),
|
||||
"partial upgrade must retain the earlier contention"
|
||||
);
|
||||
assert!(holders[0].release(&reader.lock_id).await.expect("release reader"));
|
||||
assert!(tail.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_distributed_multi_node_simulation() {
|
||||
// Simulate a 3-node distributed environment where each node has its own lock backend
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1092,6 +1092,51 @@ impl_canonical_mutation_body!(
|
||||
// its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
|
||||
// the JSON fallback decode path, breaks the signature-bound digest.
|
||||
|
||||
pub fn canonical_metadata_reservation_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_str(&request.object)?;
|
||||
body.push_u64(request.timeout_ms);
|
||||
body.push_bytes(&request.bucket_incarnation_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_metadata_reservation_cancel_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationCancelRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-cancel-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_metadata_reservation_activate_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationActivateRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_str(&request.object)?;
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_reserved_rename_data_request_body(
|
||||
request: &proto_gen::node_service::ReservedRenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-reserved-rename-data-v1\0");
|
||||
if let Some(rename) = request.rename.as_ref() {
|
||||
body.push_bytes(&canonical_rename_data_request_body(rename)?)?;
|
||||
} else {
|
||||
body.push_bytes(&[])?;
|
||||
}
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_data_request_body(
|
||||
request: &proto_gen::node_service::RenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
@@ -1722,6 +1767,84 @@ mod disk_mutation_canonical_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_reservation_canonical_bodies_bind_scope_and_token() {
|
||||
use proto_gen::node_service::{
|
||||
MetadataReservationActivateRequest, MetadataReservationCancelRequest, MetadataReservationRequest,
|
||||
ReservedRenameDataRequest,
|
||||
};
|
||||
|
||||
let request = MetadataReservationRequest {
|
||||
disk: "disk".into(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
timeout_ms: 5000,
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_metadata_reservation_request_body(&request).expect("reservation body")];
|
||||
for mutate in [
|
||||
|r: &mut MetadataReservationRequest| r.disk = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.bucket = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.object = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.timeout_ms = 1,
|
||||
|r: &mut MetadataReservationRequest| r.bucket_incarnation_id = vec![2; 16].into(),
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_metadata_reservation_request_body(&changed).expect("changed body"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let activation = MetadataReservationActivateRequest {
|
||||
disk: "disk".into(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_metadata_reservation_activate_request_body(&activation).expect("activation body")];
|
||||
for mutate in [
|
||||
|r: &mut MetadataReservationActivateRequest| r.disk = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.bucket = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.object = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.token = vec![2; 16].into(),
|
||||
] {
|
||||
let mut changed = activation.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_metadata_reservation_activate_request_body(&changed).expect("changed activation"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let cancellation = MetadataReservationCancelRequest {
|
||||
disk: "disk".into(),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut changed = cancellation.clone();
|
||||
changed.token = vec![2; 16].into();
|
||||
assert_ne!(
|
||||
canonical_metadata_reservation_cancel_request_body(&cancellation).expect("cancel body"),
|
||||
canonical_metadata_reservation_cancel_request_body(&changed).expect("changed cancel body")
|
||||
);
|
||||
let rename = ReservedRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
dst_path: "object".into(),
|
||||
..Default::default()
|
||||
}),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut changed_token = rename.clone();
|
||||
changed_token.token = vec![2; 16].into();
|
||||
let mut changed_object = rename.clone();
|
||||
changed_object.rename.as_mut().expect("rename").dst_path = "other".into();
|
||||
assert_all_distinct(&[
|
||||
canonical_reserved_rename_data_request_body(&rename).expect("rename body"),
|
||||
canonical_reserved_rename_data_request_body(&changed_token).expect("changed token"),
|
||||
canonical_reserved_rename_data_request_body(&changed_object).expect("changed object"),
|
||||
canonical_reserved_rename_data_request_body(&ReservedRenameDataRequest {
|
||||
rename: None,
|
||||
token: rename.token,
|
||||
})
|
||||
.expect("missing rename"),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_canonical_body_binds_every_field() {
|
||||
let baseline = RenameDataRequest {
|
||||
|
||||
@@ -339,6 +339,31 @@ message RenameDataResponse {
|
||||
bytes rename_data_resp_bin = 4;
|
||||
}
|
||||
|
||||
message MetadataReservationRequest {
|
||||
string disk = 1;
|
||||
string bucket = 2;
|
||||
string object = 3;
|
||||
uint64 timeout_ms = 4;
|
||||
bytes bucket_incarnation_id = 5;
|
||||
}
|
||||
|
||||
message MetadataReservationCancelRequest {
|
||||
string disk = 1;
|
||||
bytes token = 2;
|
||||
}
|
||||
|
||||
message MetadataReservationActivateRequest {
|
||||
string disk = 1;
|
||||
string bucket = 2;
|
||||
string object = 3;
|
||||
bytes token = 4;
|
||||
}
|
||||
|
||||
message ReservedRenameDataRequest {
|
||||
RenameDataRequest rename = 1;
|
||||
bytes token = 2;
|
||||
}
|
||||
|
||||
// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
|
||||
// accepts an arbitrary volume or object path.
|
||||
message PoolMetaNativeRecoveryRequest {
|
||||
@@ -1314,6 +1339,10 @@ service NodeService {
|
||||
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
|
||||
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
|
||||
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc ReserveObjectMetadata(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
|
||||
rpc ActivateObjectMetadataReservation(MetadataReservationActivateRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
|
||||
rpc CancelObjectMetadataReservation(MetadataReservationCancelRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameReservedData(ReservedRenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
// The method itself requires source-preserving multipart publication on every peer.
|
||||
rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
|
||||
@@ -1826,6 +1826,34 @@ impl Node for NodeService {
|
||||
self.handle_rename_data(request, false, true, false).await
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata(
|
||||
&self,
|
||||
request: Request<MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
self.handle_reserve_object_metadata(request).await
|
||||
}
|
||||
|
||||
async fn cancel_object_metadata_reservation(
|
||||
&self,
|
||||
request: Request<MetadataReservationCancelRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
self.handle_cancel_object_metadata_reservation(request).await
|
||||
}
|
||||
|
||||
async fn activate_object_metadata_reservation(
|
||||
&self,
|
||||
request: Request<MetadataReservationActivateRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
self.handle_activate_object_metadata_reservation(request).await
|
||||
}
|
||||
|
||||
async fn rename_reserved_data(
|
||||
&self,
|
||||
request: Request<ReservedRenameDataRequest>,
|
||||
) -> Result<Response<RenameDataResponse>, Status> {
|
||||
self.handle_rename_reserved_data(request).await
|
||||
}
|
||||
|
||||
async fn undo_pool_meta_write(
|
||||
&self,
|
||||
request: Request<DeleteVersionRequest>,
|
||||
@@ -3245,15 +3273,17 @@ mod tests {
|
||||
HealControlRequest, HealControlResponse, ListBucketRequest, ListDirRequest, ListVolumesRequest,
|
||||
LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest,
|
||||
LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest, PreparePartTransactionRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest, MetadataReservationCancelRequest,
|
||||
MetadataReservationRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest, PreparePartTransactionRequest,
|
||||
ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
|
||||
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest,
|
||||
SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest,
|
||||
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest,
|
||||
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest,
|
||||
RenameFileRequest, RenamePartRequest, ReservedRenameDataRequest, ScannerActivityRequest,
|
||||
ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
|
||||
ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest,
|
||||
StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, TierMutationPeerState,
|
||||
TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, WriteRequest,
|
||||
heal_control_service_client::HealControlServiceClient,
|
||||
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
|
||||
node_service_client::NodeServiceClient,
|
||||
@@ -3271,8 +3301,12 @@ mod tests {
|
||||
use tonic::{Request, Response, Status};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DISK_MUTATION_RPC_METHODS: [&str; 34] = [
|
||||
const DISK_MUTATION_RPC_METHODS: [&str; 38] = [
|
||||
"renamedata",
|
||||
"reserveobjectmetadata",
|
||||
"activateobjectmetadatareservation",
|
||||
"cancelobjectmetadatareservation",
|
||||
"renamereserveddata",
|
||||
"renamemultipartdata",
|
||||
"recovermultipartsource",
|
||||
"renamedataatincarnation",
|
||||
@@ -4970,6 +5004,43 @@ mod tests {
|
||||
},
|
||||
rustfs_protos::canonical_rename_data_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
reserve_object_metadata,
|
||||
MetadataReservationRequest {
|
||||
disk: disk.clone(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
timeout_ms: 5000,
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
},
|
||||
rustfs_protos::canonical_metadata_reservation_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
activate_object_metadata_reservation,
|
||||
MetadataReservationActivateRequest {
|
||||
disk: disk.clone(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
token: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
},
|
||||
rustfs_protos::canonical_metadata_reservation_activate_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
cancel_object_metadata_reservation,
|
||||
MetadataReservationCancelRequest {
|
||||
disk: disk.clone(),
|
||||
token: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
},
|
||||
rustfs_protos::canonical_metadata_reservation_cancel_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
rename_reserved_data,
|
||||
ReservedRenameDataRequest {
|
||||
rename: Some(RenameDataRequest::default()),
|
||||
token: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
},
|
||||
rustfs_protos::canonical_reserved_rename_data_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
recover_multipart_source,
|
||||
RecoverMultipartSourceRequest::default(),
|
||||
@@ -7475,11 +7546,15 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn target_rpc_fixture() -> TargetRpcFixture {
|
||||
target_rpc_fixture_with_bucket_metadata(false).await
|
||||
}
|
||||
|
||||
async fn target_rpc_fixture_with_bucket_metadata(initialize_bucket_metadata: bool) -> TargetRpcFixture {
|
||||
super::timeout(Duration::from_secs(90), async {
|
||||
let root = tempfile::tempdir().expect("target RPC root");
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(root.path())
|
||||
.init_bucket_metadata(false)
|
||||
.init_bucket_metadata(initialize_bucket_metadata)
|
||||
.build()
|
||||
.await;
|
||||
ObjectStore::new(env.ecstore.clone())
|
||||
@@ -8146,6 +8221,13 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn stage_target_rpc(fixture: &TargetRpcFixture) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec<u8>) {
|
||||
stage_target_rpc_inner(fixture, true).await
|
||||
}
|
||||
|
||||
async fn stage_target_rpc_inner(
|
||||
fixture: &TargetRpcFixture,
|
||||
create_volume: bool,
|
||||
) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec<u8>) {
|
||||
use crate::storage::storage_api::ecstore_disk::{DiskAPI, ReadOptions};
|
||||
let set = fixture
|
||||
.env
|
||||
@@ -8168,7 +8250,9 @@ mod tests {
|
||||
}];
|
||||
fi.data = Some(Bytes::from_static(b"target-rpc-inline"));
|
||||
fi.set_inline_data();
|
||||
disk.make_volume("target-rpc").await.expect("target volume");
|
||||
if create_volume {
|
||||
disk.make_volume("target-rpc").await.expect("target volume");
|
||||
}
|
||||
disk.write_metadata("target-rpc", "target-rpc", "staged", fi.clone())
|
||||
.await
|
||||
.expect("stage real inline body");
|
||||
@@ -8220,6 +8304,303 @@ mod tests {
|
||||
request
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_reservation_handlers_bind_and_publish_on_the_target_disk() {
|
||||
use crate::storage::storage_api::ecstore_disk::{DiskAPI, RUSTFS_META_TMP_BUCKET, ReadOptions};
|
||||
|
||||
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
|
||||
fixture.env.make_bucket("target-rpc", false).await;
|
||||
let incarnation = fixture
|
||||
.env
|
||||
.ecstore
|
||||
.bucket_incarnation_id("target-rpc")
|
||||
.await
|
||||
.expect("bucket incarnation");
|
||||
let (disk, fi, _) = stage_target_rpc_inner(&fixture, false).await;
|
||||
disk.write_metadata(RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET, "reserved-stage", fi.clone())
|
||||
.await
|
||||
.expect("stage reserved body");
|
||||
let service = make_server_for_context(Some(fixture.context.clone()));
|
||||
let mut reserve = Request::new(MetadataReservationRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
bucket: "target-rpc".into(),
|
||||
object: "destination".into(),
|
||||
timeout_ms: 5000,
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let mut stale = Request::new(reserve.get_ref().clone());
|
||||
stale.get_mut().bucket_incarnation_id = Uuid::new_v4().as_bytes().to_vec().into();
|
||||
let stale_body = rustfs_protos::canonical_metadata_reservation_request_body(stale.get_ref()).expect("stale body");
|
||||
set_tonic_canonical_body_digest(&mut stale, &stale_body).expect("stale digest");
|
||||
mark_v2_authenticated(&mut stale);
|
||||
assert!(
|
||||
!service
|
||||
.reserve_object_metadata(stale)
|
||||
.await
|
||||
.expect("stale handler")
|
||||
.into_inner()
|
||||
.success,
|
||||
"a token must not be issued for a different bucket incarnation"
|
||||
);
|
||||
let mut self_lock = Request::new(reserve.get_ref().clone());
|
||||
self_lock.get_mut().object = rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH.to_owned();
|
||||
let self_lock_body =
|
||||
rustfs_protos::canonical_metadata_reservation_request_body(self_lock.get_ref()).expect("self-lock body");
|
||||
set_tonic_canonical_body_digest(&mut self_lock, &self_lock_body).expect("self-lock digest");
|
||||
mark_v2_authenticated(&mut self_lock);
|
||||
assert!(
|
||||
!service
|
||||
.reserve_object_metadata(self_lock)
|
||||
.await
|
||||
.expect("self-lock handler")
|
||||
.into_inner()
|
||||
.success,
|
||||
"a token cannot hold the same lock that its object commit must acquire"
|
||||
);
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(reserve.get_ref()).expect("reserve body");
|
||||
set_tonic_canonical_body_digest(&mut reserve, &body).expect("reserve digest");
|
||||
mark_v2_authenticated(&mut reserve);
|
||||
let response = service
|
||||
.reserve_object_metadata(reserve)
|
||||
.await
|
||||
.expect("reserve RPC")
|
||||
.into_inner();
|
||||
assert!(response.success, "reservation must succeed: {:?}", response.error);
|
||||
assert_eq!(response.protocol_version, 2);
|
||||
let token = response.token;
|
||||
let mut activate = Request::new(MetadataReservationActivateRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
bucket: "target-rpc".into(),
|
||||
object: "destination".into(),
|
||||
token: token.clone(),
|
||||
});
|
||||
let body =
|
||||
rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activate body");
|
||||
set_tonic_canonical_body_digest(&mut activate, &body).expect("activate digest");
|
||||
mark_v2_authenticated(&mut activate);
|
||||
let activated = service
|
||||
.activate_object_metadata_reservation(activate)
|
||||
.await
|
||||
.expect("activate RPC")
|
||||
.into_inner();
|
||||
assert!(activated.success, "activation must succeed: {:?}", activated.error);
|
||||
let message = ReservedRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
src_volume: RUSTFS_META_TMP_BUCKET.into(),
|
||||
src_path: "reserved-stage".into(),
|
||||
file_info: serde_json::to_string(&fi).expect("encode metadata"),
|
||||
dst_volume: "target-rpc".into(),
|
||||
dst_path: "destination".into(),
|
||||
..Default::default()
|
||||
}),
|
||||
token,
|
||||
};
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(&message).expect("rename body");
|
||||
let mut tampered = Request::new(message.clone());
|
||||
set_tonic_canonical_body_digest(&mut tampered, &body).expect("rename digest");
|
||||
mark_v2_authenticated(&mut tampered);
|
||||
tampered.get_mut().rename.as_mut().expect("rename").dst_path = "other".into();
|
||||
assert_eq!(
|
||||
service
|
||||
.rename_reserved_data(tampered)
|
||||
.await
|
||||
.expect_err("tampered object")
|
||||
.code(),
|
||||
tonic::Code::PermissionDenied
|
||||
);
|
||||
let mut rename = Request::new(message);
|
||||
set_tonic_canonical_body_digest(&mut rename, &body).expect("rename digest");
|
||||
mark_v2_authenticated(&mut rename);
|
||||
let response = service.rename_reserved_data(rename).await.expect("rename RPC").into_inner();
|
||||
assert!(response.success, "reserved rename must succeed: {:?}", response.error);
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-rpc",
|
||||
"target-rpc",
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read committed object");
|
||||
assert_eq!(read.data, fi.data);
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn reserved_rename_rpc_cancellation_does_not_stop_physical_publication() {
|
||||
use crate::storage::storage_api::{
|
||||
LocalPublicationPause, LocalPublicationStage,
|
||||
contract::namespace::NamespaceLocking as _,
|
||||
ecstore_disk::{DiskAPI, RUSTFS_META_TMP_BUCKET, ReadOptions},
|
||||
};
|
||||
|
||||
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
|
||||
fixture.env.make_bucket("target-rpc", false).await;
|
||||
let incarnation = fixture
|
||||
.env
|
||||
.ecstore
|
||||
.bucket_incarnation_id("target-rpc")
|
||||
.await
|
||||
.expect("bucket incarnation");
|
||||
let (disk, fi, _) = stage_target_rpc_inner(&fixture, false).await;
|
||||
disk.write_metadata(RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET, "reserved-cancel", fi.clone())
|
||||
.await
|
||||
.expect("stage reserved body");
|
||||
let service = make_server_for_context(Some(fixture.context.clone()));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind RPC listener");
|
||||
let addr = listener.local_addr().expect("RPC listener address");
|
||||
let (shutdown, stopped) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
tonic::transport::Server::builder()
|
||||
.add_service(NodeServiceServer::new(service))
|
||||
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
|
||||
let _ = stopped.await;
|
||||
})
|
||||
.await
|
||||
.expect("serve RPC target");
|
||||
});
|
||||
let mut client = NodeServiceClient::connect(format!("http://{addr}"))
|
||||
.await
|
||||
.expect("connect RPC client");
|
||||
let mut prepare = Request::new(MetadataReservationRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
bucket: "target-rpc".into(),
|
||||
object: "destination".into(),
|
||||
timeout_ms: 5000,
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(prepare.get_ref()).expect("prepare body");
|
||||
set_tonic_canonical_body_digest(&mut prepare, &body).expect("prepare digest");
|
||||
mark_v2_authenticated(&mut prepare);
|
||||
let prepared = client
|
||||
.reserve_object_metadata(prepare)
|
||||
.await
|
||||
.expect("prepare handler")
|
||||
.into_inner();
|
||||
assert!(prepared.success, "prepare: {:?}", prepared.error);
|
||||
let mut activate = Request::new(MetadataReservationActivateRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
bucket: "target-rpc".into(),
|
||||
object: "destination".into(),
|
||||
token: prepared.token.clone(),
|
||||
});
|
||||
let body =
|
||||
rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activate body");
|
||||
set_tonic_canonical_body_digest(&mut activate, &body).expect("activate digest");
|
||||
mark_v2_authenticated(&mut activate);
|
||||
let activated = client
|
||||
.activate_object_metadata_reservation(activate)
|
||||
.await
|
||||
.expect("activate handler")
|
||||
.into_inner();
|
||||
assert!(activated.success, "activate: {:?}", activated.error);
|
||||
let mut rename = Request::new(ReservedRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
src_volume: RUSTFS_META_TMP_BUCKET.into(),
|
||||
src_path: "reserved-cancel".into(),
|
||||
dst_volume: "target-rpc".into(),
|
||||
dst_path: "destination".into(),
|
||||
file_info: serde_json::to_string(&fi).expect("encode metadata"),
|
||||
..Default::default()
|
||||
}),
|
||||
token: prepared.token,
|
||||
});
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(rename.get_ref()).expect("rename body");
|
||||
set_tonic_canonical_body_digest(&mut rename, &body).expect("rename digest");
|
||||
mark_v2_authenticated(&mut rename);
|
||||
let mut pause =
|
||||
LocalPublicationPause::install(&disk, "target-rpc", "destination/xl.meta", LocalPublicationStage::PreparedRename)
|
||||
.expect("install physical pause");
|
||||
let mut handler = Box::pin(client.rename_reserved_data(rename));
|
||||
super::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
result = &mut handler => panic!("handler completed before physical entry: {result:?}"),
|
||||
entered = pause.entered() => entered.expect("physical executor entered"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded physical entry");
|
||||
drop(handler);
|
||||
assert!(
|
||||
fixture.env.ecstore.namespace_commits_pending_for_test(),
|
||||
"physical publication remains active"
|
||||
);
|
||||
let lifecycle = fixture
|
||||
.env
|
||||
.ecstore
|
||||
.new_ns_lock("target-rpc", rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("bucket publication lock");
|
||||
let mut bucket_writer = Box::pin(lifecycle.get_write_lock(Duration::from_secs(10)));
|
||||
let early = super::timeout(Duration::from_millis(500), &mut bucket_writer).await;
|
||||
let old_lifecycle = fixture
|
||||
.env
|
||||
.ecstore
|
||||
.new_ns_lock("target-rpc", "")
|
||||
.await
|
||||
.expect("bucket lifecycle lock");
|
||||
let mut old_writer = Box::pin(old_lifecycle.get_write_lock(Duration::from_secs(10)));
|
||||
let old_early = super::timeout(Duration::from_millis(500), &mut old_writer).await;
|
||||
drop(pause);
|
||||
match early {
|
||||
Ok(result) => {
|
||||
drop(result.expect("early lifecycle result"));
|
||||
panic!("bucket publication writer acquired while the remote physical write was still paused");
|
||||
}
|
||||
Err(_) => drop(
|
||||
super::timeout(Duration::from_secs(10), bucket_writer)
|
||||
.await
|
||||
.expect("lifecycle owner drains")
|
||||
.expect("bucket writer acquires"),
|
||||
),
|
||||
}
|
||||
match old_early {
|
||||
Ok(result) => {
|
||||
drop(result.expect("early lifecycle result"));
|
||||
panic!("bucket lifecycle writer acquired while the remote physical write was still paused");
|
||||
}
|
||||
Err(_) => drop(
|
||||
super::timeout(Duration::from_secs(10), old_writer)
|
||||
.await
|
||||
.expect("lifecycle owner drains")
|
||||
.expect("bucket lifecycle writer acquires"),
|
||||
),
|
||||
}
|
||||
super::timeout(Duration::from_secs(10), async {
|
||||
while fixture.env.ecstore.namespace_commits_pending_for_test() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical publication drains");
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-rpc",
|
||||
"target-rpc",
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read published object");
|
||||
assert_eq!(read.data, fi.data);
|
||||
drop(client);
|
||||
let _ = shutdown.send(());
|
||||
super::timeout(Duration::from_secs(10), server)
|
||||
.await
|
||||
.expect("RPC server shuts down")
|
||||
.expect("RPC server task");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn quota_fence_handlers_commit_real_data_and_drain_before_revocation() {
|
||||
@@ -8974,28 +9355,47 @@ mod tests {
|
||||
))
|
||||
}
|
||||
|
||||
async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>> {
|
||||
async fn internal_snapshot_once(
|
||||
root: &std::path::Path,
|
||||
) -> std::io::Result<std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>>> {
|
||||
let mut snapshot = std::collections::BTreeMap::new();
|
||||
let mut directories = (0..4)
|
||||
.map(|index| std::path::PathBuf::from(format!("disk{index}/{RUSTFS_META_BUCKET}")))
|
||||
.collect::<Vec<_>>();
|
||||
while let Some(relative) = directories.pop() {
|
||||
let mut entries = tokio::fs::read_dir(root.join(&relative))
|
||||
.await
|
||||
.expect("read internal snapshot directory");
|
||||
let mut entries = tokio::fs::read_dir(root.join(&relative)).await?;
|
||||
snapshot.insert(relative.clone(), None);
|
||||
while let Some(entry) = entries.next_entry().await.expect("read internal snapshot entry") {
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = relative.join(entry.file_name());
|
||||
let file_type = entry.file_type().await.expect("read internal snapshot entry type");
|
||||
let file_type = entry.file_type().await?;
|
||||
if file_type.is_dir() {
|
||||
directories.push(path);
|
||||
} else {
|
||||
assert!(file_type.is_file(), "fixture snapshot must contain only directories and regular files");
|
||||
snapshot.insert(path, Some(tokio::fs::read(entry.path()).await.expect("read snapshot file bytes")));
|
||||
snapshot.insert(path, Some(tokio::fs::read(entry.path()).await?));
|
||||
}
|
||||
}
|
||||
}
|
||||
snapshot
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>> {
|
||||
// Startup cleanup can remove temporary paths after the namespace
|
||||
// commit drains. Capture the complete tree only once it is stable.
|
||||
super::timeout(Duration::from_secs(10), async {
|
||||
let mut previous = None;
|
||||
loop {
|
||||
match internal_snapshot_once(root).await {
|
||||
Ok(snapshot) if previous.as_ref() == Some(&snapshot) => return snapshot,
|
||||
Ok(snapshot) => previous = Some(snapshot),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => previous = None,
|
||||
Err(error) => panic!("read internal snapshot: {error}"),
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("internal snapshot must stabilize within ten seconds")
|
||||
}
|
||||
|
||||
fn file_info(object: &str, version: Uuid, body: Bytes) -> FileInfo {
|
||||
|
||||
@@ -507,6 +507,161 @@ impl NodeService {
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
pub(super) async fn handle_reserve_object_metadata(
|
||||
&self,
|
||||
request: Request<MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("invalid reservation body"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &body)
|
||||
.map_err(|_| Status::permission_denied("metadata reservation requires body-bound authentication"))?;
|
||||
let request = request.into_inner();
|
||||
if request.timeout_ms == 0 {
|
||||
return Err(Status::invalid_argument("reservation timeout must be positive"));
|
||||
}
|
||||
let incarnation = Uuid::from_slice(&request.bucket_incarnation_id)
|
||||
.ok()
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Status::invalid_argument("invalid bucket incarnation"))?;
|
||||
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
|
||||
return Err(Status::failed_precondition("storage instance is unavailable"));
|
||||
};
|
||||
let result = store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&request.disk,
|
||||
&request.bucket,
|
||||
&request.object,
|
||||
incarnation,
|
||||
std::time::Duration::from_millis(request.timeout_ms.min(5000)),
|
||||
)
|
||||
.await;
|
||||
Ok(Response::new(match result {
|
||||
Ok(token) => SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
protocol_version: 2,
|
||||
error: None,
|
||||
},
|
||||
Err(error) => SnapshotLeaseResponse {
|
||||
success: false,
|
||||
token: Bytes::new(),
|
||||
protocol_version: 2,
|
||||
error: Some(error.into()),
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) async fn handle_activate_object_metadata_reservation(
|
||||
&self,
|
||||
request: Request<MetadataReservationActivateRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("invalid reservation activation body"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &body)
|
||||
.map_err(|_| Status::permission_denied("reservation activation requires body-bound authentication"))?;
|
||||
let request = request.into_inner();
|
||||
let token = Uuid::from_slice(&request.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
|
||||
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
|
||||
return Err(Status::failed_precondition("storage instance is unavailable"));
|
||||
};
|
||||
Ok(Response::new(
|
||||
match store
|
||||
.activate_local_object_metadata_reservation(&request.disk, &request.bucket, &request.object, token)
|
||||
.await
|
||||
{
|
||||
Ok(()) => SnapshotLeaseMutationResponse {
|
||||
success: true,
|
||||
error: None,
|
||||
},
|
||||
Err(error) => SnapshotLeaseMutationResponse {
|
||||
success: false,
|
||||
error: Some(error.into()),
|
||||
},
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn handle_cancel_object_metadata_reservation(
|
||||
&self,
|
||||
request: Request<MetadataReservationCancelRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("invalid reservation cancellation body"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &body)
|
||||
.map_err(|_| Status::permission_denied("reservation cancellation requires body-bound authentication"))?;
|
||||
let request = request.into_inner();
|
||||
let token = Uuid::from_slice(&request.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
|
||||
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
|
||||
return Err(Status::failed_precondition("storage instance is unavailable"));
|
||||
};
|
||||
Ok(Response::new(
|
||||
match store.cancel_local_object_metadata_reservation(&request.disk, token).await {
|
||||
Ok(cancelled) => SnapshotLeaseMutationResponse {
|
||||
success: cancelled,
|
||||
error: None,
|
||||
},
|
||||
Err(error) => SnapshotLeaseMutationResponse {
|
||||
success: false,
|
||||
error: Some(error.into()),
|
||||
},
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn handle_rename_reserved_data(
|
||||
&self,
|
||||
request: Request<ReservedRenameDataRequest>,
|
||||
) -> Result<Response<RenameDataResponse>, Status> {
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("invalid reserved rename body"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &body)
|
||||
.map_err(|_| Status::permission_denied("reserved rename requires body-bound authentication"))?;
|
||||
let request = request.into_inner();
|
||||
let token = Uuid::from_slice(&request.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
|
||||
let rename = request.rename.ok_or_else(|| Status::invalid_argument("missing rename"))?;
|
||||
if !rename.scanner_publication_lease_token.is_empty() || !rename.bucket_incarnation_id.is_empty() {
|
||||
return Err(Status::invalid_argument("reserved rename does not support scanner or heal scope"));
|
||||
}
|
||||
let decoded = decode_rename_data_request_file_info(&rename.file_info_bin, &rename.file_info)
|
||||
.map_err(|_| Status::invalid_argument("invalid reserved rename FileInfo"))?;
|
||||
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
|
||||
return Err(Status::failed_precondition("storage instance is unavailable"));
|
||||
};
|
||||
let result = store
|
||||
.rename_local_data_reserved(
|
||||
&rename.disk,
|
||||
(&rename.src_volume, &rename.src_path),
|
||||
&decoded.value,
|
||||
(&rename.dst_volume, &rename.dst_path),
|
||||
token,
|
||||
)
|
||||
.await;
|
||||
let encoded = result.and_then(|value| encode_rename_data_response_payloads(&value, decoded.from_msgpack));
|
||||
Ok(Response::new(match encoded {
|
||||
Ok((rename_data_resp, rename_data_resp_bin)) => RenameDataResponse {
|
||||
success: true,
|
||||
rename_data_resp,
|
||||
rename_data_resp_bin: rename_data_resp_bin.into(),
|
||||
error: None,
|
||||
},
|
||||
Err(error) => RenameDataResponse {
|
||||
success: false,
|
||||
rename_data_resp: String::new(),
|
||||
rename_data_resp_bin: Bytes::new(),
|
||||
error: Some(error.into()),
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) async fn handle_acquire_snapshot_lease(
|
||||
&self,
|
||||
request: Request<SnapshotLeaseRequest>,
|
||||
|
||||
@@ -49,6 +49,91 @@ async fn refresh_lock(
|
||||
lock_client: &std::sync::Arc<dyn rustfs_lock::client::LockClient>,
|
||||
args: &LockRequest,
|
||||
) -> GenerallyLockResponse {
|
||||
let writer_marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
|
||||
if args
|
||||
.metadata
|
||||
.tags
|
||||
.get(writer_marker)
|
||||
.is_some_and(|value| value == writer_marker)
|
||||
{
|
||||
return match lock_client.waiting_writer(&args.lock_id).await {
|
||||
Ok(Some(waiting)) => GenerallyLockResponse {
|
||||
success: true,
|
||||
error_info: None,
|
||||
lock_info: Some(format!("{writer_marker}:{}", u8::from(waiting))),
|
||||
},
|
||||
Ok(None) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
},
|
||||
Err(error) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(error.to_string()),
|
||||
lock_info: None,
|
||||
},
|
||||
};
|
||||
}
|
||||
let probe_marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
|
||||
if args
|
||||
.metadata
|
||||
.tags
|
||||
.get(probe_marker)
|
||||
.is_some_and(|value| value == probe_marker)
|
||||
{
|
||||
return match lock_client.supports_heal_lock_transition(&args.lock_id).await {
|
||||
Ok(success) => GenerallyLockResponse {
|
||||
success,
|
||||
error_info: None,
|
||||
lock_info: success.then(|| probe_marker.to_owned()),
|
||||
},
|
||||
Err(error) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(error.to_string()),
|
||||
lock_info: None,
|
||||
},
|
||||
};
|
||||
}
|
||||
let upgrade_marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
|
||||
if args
|
||||
.metadata
|
||||
.tags
|
||||
.get(upgrade_marker)
|
||||
.is_some_and(|value| value == upgrade_marker)
|
||||
{
|
||||
return match lock_client.try_upgrade_committed_tail(&args.lock_id).await {
|
||||
Ok(success) => GenerallyLockResponse {
|
||||
success,
|
||||
error_info: None,
|
||||
lock_info: success.then(|| upgrade_marker.to_owned()),
|
||||
},
|
||||
Err(error) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(error.to_string()),
|
||||
lock_info: None,
|
||||
},
|
||||
};
|
||||
}
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
|
||||
if args.metadata.tags.get(marker).is_some_and(|value| value == marker) {
|
||||
return match lock_client.downgrade_committed_tail(&args.lock_id).await {
|
||||
Ok(true) => GenerallyLockResponse {
|
||||
success: true,
|
||||
error_info: None,
|
||||
lock_info: Some(marker.to_owned()),
|
||||
},
|
||||
Ok(false) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
},
|
||||
Err(error) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(error.to_string()),
|
||||
lock_info: None,
|
||||
},
|
||||
};
|
||||
}
|
||||
match lock_client.refresh(&args.lock_id).await {
|
||||
Ok(true) => GenerallyLockResponse {
|
||||
success: true,
|
||||
@@ -347,6 +432,115 @@ mod tests {
|
||||
assert!(result.lock_info.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_reports_waiting_writer_for_held_lease() {
|
||||
use super::refresh_lock;
|
||||
use rustfs_lock::client::{HEAL_WAITING_WRITER_MARKER, LockClient};
|
||||
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
|
||||
let key = ObjectKey::new("bucket", "writer-observation-rpc");
|
||||
let mut held = LockRequest::new(key.clone(), LockType::Exclusive, "heal");
|
||||
assert!(client.acquire_lock(&held).await.expect("acquire heal lease").success);
|
||||
held.metadata
|
||||
.tags
|
||||
.insert(HEAL_WAITING_WRITER_MARKER.into(), HEAL_WAITING_WRITER_MARKER.into());
|
||||
let writer = {
|
||||
let client = Arc::clone(&client);
|
||||
tokio::spawn(async move {
|
||||
let request = LockRequest::new(key, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_secs(3));
|
||||
let response = client.acquire_lock(&request).await.expect("writer request");
|
||||
(request, response)
|
||||
})
|
||||
};
|
||||
let expected = format!("{HEAL_WAITING_WRITER_MARKER}:1");
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
let observed = refresh_lock(&client, &held).await;
|
||||
if observed.lock_info.as_deref() == Some(expected.as_str()) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("waiting writer reported");
|
||||
assert!(client.release(&held.lock_id).await.expect("release heal lease"));
|
||||
let (request, response) = writer.await.expect("writer task");
|
||||
assert!(response.success);
|
||||
assert!(client.release(&request.lock_id).await.expect("release writer"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_upgrade_seals_lease_against_late_conversion() {
|
||||
use super::refresh_lock;
|
||||
use rustfs_lock::client::{
|
||||
COMMITTED_TAIL_CONVERSION_MARKER, COMMITTED_TAIL_UPGRADE_MARKER, HEAL_LOCK_TRANSITION_PROBE_MARKER,
|
||||
HEAL_WAITING_WRITER_MARKER, LockClient,
|
||||
};
|
||||
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
|
||||
use std::sync::Arc;
|
||||
|
||||
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
|
||||
let mut request = LockRequest::new(ObjectKey::new("bucket", "upgrade-rpc"), LockType::Exclusive, "owner");
|
||||
assert!(client.acquire_lock(&request).await.expect("acquire lease").success);
|
||||
request
|
||||
.metadata
|
||||
.tags
|
||||
.insert(HEAL_LOCK_TRANSITION_PROBE_MARKER.into(), HEAL_LOCK_TRANSITION_PROBE_MARKER.into());
|
||||
let probe = refresh_lock(&client, &request).await;
|
||||
assert!(probe.success);
|
||||
assert_eq!(probe.lock_info.as_deref(), Some(HEAL_LOCK_TRANSITION_PROBE_MARKER));
|
||||
request.metadata.tags.clear();
|
||||
request
|
||||
.metadata
|
||||
.tags
|
||||
.insert(HEAL_WAITING_WRITER_MARKER.into(), HEAL_WAITING_WRITER_MARKER.into());
|
||||
let observation = refresh_lock(&client, &request).await;
|
||||
assert!(observation.success);
|
||||
assert_eq!(observation.lock_info.as_deref(), Some(format!("{HEAL_WAITING_WRITER_MARKER}:0").as_str()));
|
||||
request.metadata.tags.clear();
|
||||
request
|
||||
.metadata
|
||||
.tags
|
||||
.insert(COMMITTED_TAIL_UPGRADE_MARKER.into(), COMMITTED_TAIL_UPGRADE_MARKER.into());
|
||||
let response = refresh_lock(&client, &request).await;
|
||||
assert!(response.success);
|
||||
assert_eq!(response.lock_info.as_deref(), Some(COMMITTED_TAIL_UPGRADE_MARKER));
|
||||
request.metadata.tags.clear();
|
||||
request
|
||||
.metadata
|
||||
.tags
|
||||
.insert(COMMITTED_TAIL_CONVERSION_MARKER.into(), COMMITTED_TAIL_CONVERSION_MARKER.into());
|
||||
assert!(!refresh_lock(&client, &request).await.success, "late conversion must be rejected");
|
||||
assert!(client.release(&request.lock_id).await.expect("release lease"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_can_convert_committed_tail_and_acknowledges_protocol_version() {
|
||||
use super::refresh_lock;
|
||||
use rustfs_lock::client::{COMMITTED_TAIL_CONVERSION_MARKER, LockClient};
|
||||
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
|
||||
let resource = ObjectKey::new("bucket", "tail-conversion-rpc");
|
||||
let mut request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner");
|
||||
assert!(client.acquire_lock(&request).await.expect("acquire write lease").success);
|
||||
request
|
||||
.metadata
|
||||
.tags
|
||||
.insert(COMMITTED_TAIL_CONVERSION_MARKER.into(), COMMITTED_TAIL_CONVERSION_MARKER.into());
|
||||
let response = refresh_lock(&client, &request).await;
|
||||
assert!(response.success);
|
||||
assert_eq!(response.lock_info.as_deref(), Some(COMMITTED_TAIL_CONVERSION_MARKER));
|
||||
let read = LockRequest::new(resource, LockType::Shared, "reader").with_acquire_timeout(Duration::from_secs(1));
|
||||
assert!(client.acquire_lock(&read).await.expect("read committed tail").success);
|
||||
assert!(client.release(&read.lock_id).await.expect("release reader"));
|
||||
assert!(client.release(&request.lock_id).await.expect("release tail"));
|
||||
}
|
||||
|
||||
// B1 -- refresh_lock delegates to LocalClient::refresh for a held lock, returns
|
||||
// success=true, and actually extends expires_at (inverts the "parse then drop" stub).
|
||||
#[tokio::test]
|
||||
|
||||
Reference in New Issue
Block a user