mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-01 08:40:23 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Hauser Zhi
|
647846956f | ||
|
GatewayJ
|
04690f7da5 | ||
|
Ramakrishna Chilaka
|
3c0829049d |
@@ -970,10 +970,10 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# Keep storage-heavy cases isolated in the full suite too. The large pagination
|
||||
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
|
||||
# Keep the storage-heavy smoke cases isolated in the full suite too. They
|
||||
# start multiple server fixtures and issue many sequential S3 operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Fixed, small contract for every executable server change. These existing
|
||||
|
||||
@@ -576,34 +576,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn activate_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationActivateRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn cancel_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationCancelRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_reserved_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ReservedRenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_data_with_pool_meta_cas(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
@@ -690,7 +691,7 @@ where
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
@@ -5209,6 +5210,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "catalog-cleanup-incarnation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation");
|
||||
let called = std::sync::atomic::AtomicBool::new(false);
|
||||
let result = store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
Uuid::new_v4(),
|
||||
|| {
|
||||
called.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
|
||||
assert!(
|
||||
!called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"incarnation must be checked before the external fence"
|
||||
);
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read unchanged bucket metadata");
|
||||
assert!(
|
||||
metadata.quota_config_json.is_empty(),
|
||||
"a stale cleanup must not modify the replacement bucket"
|
||||
);
|
||||
store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
incarnation,
|
||||
|| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("the current incarnation permits the validated update");
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read updated bucket metadata");
|
||||
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
|
||||
@@ -67,15 +67,14 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest,
|
||||
MetadataReservationCancelRequest, MetadataReservationRequest,
|
||||
PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest,
|
||||
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest,
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -2453,194 +2452,6 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
/// Prepare a remote token without retrying the RPC. It owns the bucket
|
||||
/// publication fence but does not enter the object metadata queue yet.
|
||||
pub async fn reserve_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: Duration,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"reserve_object_metadata",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?,
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.reserve_object_metadata(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
if response.protocol_version != 2 {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
Uuid::from_slice(&response.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or(DiskError::Unexpected)
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token. The caller must cancel it on an indeterminate
|
||||
/// result before allowing any subsequent unprotected mutation.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"activate_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationActivateRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation activation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.activate_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed remote reservation. False cannot prove that an
|
||||
/// earlier claim did not start a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"cancel_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationCancelRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation cancellation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.cancel_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error {
|
||||
return Err(error.into());
|
||||
}
|
||||
Ok(response.success)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Publish a staged user-object write using a remote one-shot reservation.
|
||||
/// The caller must retain the bucket lifecycle fence until the tail drains.
|
||||
pub async fn rename_data_with_metadata_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
token: Uuid,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_reserved_data",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let rename = RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
scanner_publication_lease_token: Bytes::new(),
|
||||
bucket_incarnation_id: Bytes::new(),
|
||||
};
|
||||
let mut request = Request::new(ReservedRenameDataRequest {
|
||||
rename: Some(rename),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reserved rename request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.rename_reserved_data(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
@@ -5298,56 +5109,6 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let token = Uuid::new_v4();
|
||||
assert!(matches!(
|
||||
remote
|
||||
.reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.activate_object_metadata("bucket", "object", token, Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote.cancel_object_metadata_reservation(token).await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.rename_data_with_metadata_reservation(
|
||||
crate::disk::RUSTFS_META_TMP_BUCKET,
|
||||
"staged",
|
||||
&sample_rename_data_file_info(),
|
||||
"bucket",
|
||||
"object",
|
||||
token,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
|
||||
vec![
|
||||
"/node_service.NodeService/ReserveObjectMetadata",
|
||||
"/node_service.NodeService/ActivateObjectMetadataReservation",
|
||||
"/node_service.NodeService/CancelObjectMetadataReservation",
|
||||
"/node_service.NodeService/RenameReservedData",
|
||||
],
|
||||
"reservation operations must never downgrade to legacy methods",
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
|
||||
@@ -1315,122 +1315,6 @@ impl LockClient for RemoteClient {
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
// Older nodes refresh normally but return no conversion marker.
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
|
||||
let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
if !response.success {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) {
|
||||
Some(":0") => Some(false),
|
||||
Some(":1") => Some(true),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote force_release for {}", lock_id);
|
||||
let force_request = Self::create_unlock_request(lock_id);
|
||||
|
||||
@@ -655,19 +655,6 @@ impl Sets {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_prepared_reservations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
self.get_disks_by_key(object)
|
||||
.put_object_with_prepared_reservations(bucket, object, data, opts, reservations)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -604,69 +604,8 @@ impl LocalDiskWrapper {
|
||||
let super::RenameDataGuards {
|
||||
external_guard,
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
metadata_reservation_token,
|
||||
..
|
||||
} = guards;
|
||||
let metadata_reservation = match (metadata_reservation, metadata_reservation_token) {
|
||||
(Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)),
|
||||
(Some(reservation), None) => Some(reservation),
|
||||
(None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) {
|
||||
Ok(reservation) => Some(reservation),
|
||||
// A missing token may already have been claimed by an earlier
|
||||
// request that is still publishing; do not claim it was safe.
|
||||
Err(DiskError::DiskOngoingReq) => {
|
||||
return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq));
|
||||
}
|
||||
Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
},
|
||||
(None, None) => None,
|
||||
};
|
||||
// A queued reservation must acquire its metadata lease before disk
|
||||
// admission. Otherwise a waiting background mutation could hold the
|
||||
// disk permit needed by the earlier lease owner.
|
||||
let metadata_reservation = match metadata_reservation {
|
||||
Some(reservation) => Some(reservation.wait().await),
|
||||
None => None,
|
||||
};
|
||||
// The claimed reservation stays in this cancellable future while disk
|
||||
// admission can wait. Only physical work moves into an owned task.
|
||||
let prechecked_permit = if metadata_reservation.is_some() {
|
||||
// Background admission is exclusive. Waiting for it while holding
|
||||
// this object's lease can deadlock an admitted mutation waiting for
|
||||
// the same lease. An already owned permit does not need to wait.
|
||||
if current_disk_io_context().is_some_and(|context| {
|
||||
context.class == DiskIoClass::Background
|
||||
&& !context
|
||||
.permit
|
||||
.as_ref()
|
||||
.is_some_and(|permit| permit.is_for(&self.io_admission))
|
||||
}) {
|
||||
return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let preflight = self
|
||||
.with_native_resources(async {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.acquire_disk_io_permit().await
|
||||
})
|
||||
.await;
|
||||
match preflight {
|
||||
Ok(Ok(Some(permit))) => Some(permit),
|
||||
Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// The reserved metadata lease must outlive a cancelled RPC waiter.
|
||||
let external_guard: Option<Arc<dyn Send + Sync>> = if metadata_reservation.is_some() {
|
||||
Some(Arc::new(external_guard))
|
||||
} else {
|
||||
external_guard
|
||||
};
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -685,54 +624,27 @@ impl LocalDiskWrapper {
|
||||
};
|
||||
let observed = run_owned_mutation(external_guard, move || async move {
|
||||
let mut preflight_rejection = None;
|
||||
let rename = || async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = if let Some(reservation) = metadata_reservation {
|
||||
Box::pin(operation.disk.rename_data_observed_with_reservation(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
(namespace_owner, Some(reservation)),
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await
|
||||
};
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
};
|
||||
let result = if let Some(permit) = prechecked_permit {
|
||||
with_disk_io_permit(Some(permit), async {
|
||||
operation
|
||||
.with_native_resources(operation.track_admitted_disk_operation(
|
||||
"rename_data",
|
||||
move |io_permit| async move {
|
||||
let _io_permit = io_permit;
|
||||
rename().await
|
||||
},
|
||||
Duration::ZERO,
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::Write,
|
||||
true,
|
||||
let result = operation
|
||||
.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await?
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
operation
|
||||
.track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration)
|
||||
.await
|
||||
};
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
},
|
||||
timeout_duration,
|
||||
)
|
||||
.await;
|
||||
// Health tracking must observe the real disk error, not an Ok tuple.
|
||||
Ok(super::RenameDataObservation {
|
||||
result,
|
||||
@@ -1686,58 +1598,6 @@ pub struct LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
/// Reserve the metadata mutation lock for one object on this disk instance.
|
||||
/// Unclaimed reservations expire after 60 seconds.
|
||||
pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.reserve_object_metadata(bucket, object, timeout).await
|
||||
}
|
||||
|
||||
/// Register an object metadata reservation in FIFO order on this disk.
|
||||
/// The returned token may be claimed before its physical lease is available.
|
||||
pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.enqueue_object_metadata(bucket, object).await
|
||||
}
|
||||
|
||||
/// Prepare an inert reservation token bound to this disk and object.
|
||||
pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
self.prepare_object_metadata_with_owner(bucket, object, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_object_metadata_with_owner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
|
||||
) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.prepare_object_metadata_with_owner(bucket, object, owner)
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token; a cancelled token cannot be reactivated.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. A claimed physical operation retains its own lock.
|
||||
pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool {
|
||||
self.disk.cancel_object_metadata_reservation(token)
|
||||
}
|
||||
|
||||
/// Create a new LocalDiskWrapper
|
||||
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
|
||||
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
|
||||
@@ -2538,6 +2398,10 @@ impl LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2717,7 +2581,6 @@ impl LocalDiskWrapper {
|
||||
timeout_duration,
|
||||
timeout_health_action,
|
||||
mutation,
|
||||
false,
|
||||
))
|
||||
.await?
|
||||
}
|
||||
@@ -2729,7 +2592,6 @@ impl LocalDiskWrapper {
|
||||
timeout_duration: Duration,
|
||||
timeout_health_action: TimeoutHealthAction,
|
||||
mutation: DiskMetricMutation,
|
||||
identity_prechecked: bool,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
@@ -2751,7 +2613,7 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
|
||||
// Check if disk is stale
|
||||
if !identity_prechecked && let Err(err) = self.check_disk_stale().await {
|
||||
if let Err(err) = self.check_disk_stale().await {
|
||||
self.metrics.record_availability_error();
|
||||
return Err(err);
|
||||
}
|
||||
@@ -5719,115 +5581,6 @@ mod tests {
|
||||
drop(metadata_lease);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancelled_reserved_rename_releases_lease_before_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let meta = dir.path().join(RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
|
||||
.await
|
||||
.expect("write format");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
|
||||
disk.make_volume("bucket").await.expect("bucket volume");
|
||||
let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false));
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
wrapper.set_disk_id_state(Some(format.erasure.this)).await;
|
||||
disk.format_info.write().await.last_check = None;
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
disk.install_format_probe_hook(false, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv_timeout(Duration::from_secs(30));
|
||||
});
|
||||
let publisher = Arc::clone(&wrapper);
|
||||
let rename = tokio::spawn(async move {
|
||||
publisher
|
||||
.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&FileInfo::default(),
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), entered_rx)
|
||||
.await
|
||||
.expect("disk identity preflight starts")
|
||||
.expect("preflight hook");
|
||||
rename.abort();
|
||||
assert!(matches!(rename.await, Err(error) if error.is_cancelled()));
|
||||
let key = disk.get_object_path("bucket", "object").expect("object path");
|
||||
let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None));
|
||||
let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok();
|
||||
release_tx.send(()).expect("release disk identity preflight");
|
||||
if !released_before_io {
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), successor)
|
||||
.await
|
||||
.expect("preflight finishes"),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
released_before_io,
|
||||
"a cancelled request must not retain the reservation before physical mutation starts"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_background_rename_does_not_wait_for_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
let held = wrapper
|
||||
.io_admission
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("background permit");
|
||||
let fi = FileInfo::default();
|
||||
let rename = crate::disk::with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
wrapper.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&fi,
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
);
|
||||
let result = tokio::time::timeout(Duration::from_secs(1), rename).await;
|
||||
drop(held);
|
||||
let observation = result.expect("reserved mutation must not wait while holding the object lease");
|
||||
assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq)));
|
||||
assert!(observation.rejected_before_publication());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_versions_counts_faulty_drive_availability_rejection() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
+459
-1245
File diff suppressed because it is too large
Load Diff
@@ -184,6 +184,8 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
remove_file_if_exists(&backup_path)?;
|
||||
if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
|
||||
let write_res = (|| -> std::io::Result<()> {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, old_metadata)?;
|
||||
if sync && !old_metadata.is_empty() {
|
||||
@@ -367,9 +369,12 @@ fn prepare_rename_commit_and_destination_metadata_std(
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum FusedCommitAttempt {
|
||||
Committed(RenameDataResp),
|
||||
Committed(Box<RenameDataResp>),
|
||||
CommitErr(DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
|
||||
#[cfg(test)]
|
||||
ResumePanic(Box<dyn std::any::Any + Send + 'static>),
|
||||
}
|
||||
|
||||
async fn prepare_rename_commit_and_destination_metadata(
|
||||
@@ -439,29 +444,12 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
#[derive(Debug)]
|
||||
pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
|
||||
impl LocalRenamePreflightRejection {
|
||||
pub(in crate::disk) fn not_dispatched() -> Self {
|
||||
Self(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl RenameDataState {
|
||||
pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self {
|
||||
Self {
|
||||
metadata_reservation: Some(reservation),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keep optional native MPU state machines out of ordinary rename frames.
|
||||
// Construct the future here as well as pinning it, so its temporary stays off
|
||||
// the caller's poll stack. Call only from the existing native MPU branches.
|
||||
@@ -503,15 +491,6 @@ impl LocalDisk {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let mut fi = fi;
|
||||
// Pre-acquired metadata must never reverse quota or multipart-source
|
||||
// lock order. These paths need their own reservation protocol.
|
||||
if state.metadata_reservation.is_some()
|
||||
&& (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX))
|
||||
{
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) {
|
||||
Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
@@ -577,7 +556,9 @@ impl LocalDisk {
|
||||
None
|
||||
};
|
||||
let multipart_publication = if let Some(staging) = multipart_staging.as_ref() {
|
||||
Some(pin_multipart_future(|| self.prepare_native_multipart_publication(src_path, &mut fi, staging)).await?)
|
||||
Some(Arc::new(
|
||||
pin_multipart_future(|| self.prepare_native_multipart_publication_deferred(src_path, &mut fi, staging)).await?,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -605,27 +586,7 @@ impl LocalDisk {
|
||||
Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))),
|
||||
None => state.namespace_owner.take(),
|
||||
};
|
||||
// A reserved lease is single-use and bound to the exact disk path.
|
||||
// Keep the independent namespace/source owner alongside it; a pre-held
|
||||
// mutex cannot be acquired again without deadlocking.
|
||||
let reservation_fence = state
|
||||
.metadata_reservation
|
||||
.as_ref()
|
||||
.and_then(|reservation| reservation.fence());
|
||||
let metadata_owner: Arc<dyn Send + Sync> = if let Some(reservation) = state.metadata_reservation.take() {
|
||||
if multipart_staging.is_some() || quota_fence_claim.is_some() {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let lease = reservation
|
||||
.into_lease_for(&root_destination_object_path, self.reservation_instance)
|
||||
.inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
Arc::new((lease, metadata_owner))
|
||||
} else {
|
||||
os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await
|
||||
};
|
||||
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await;
|
||||
drop(metadata_wait);
|
||||
#[cfg(unix)]
|
||||
let parent_receipt = if native_timing {
|
||||
@@ -639,23 +600,17 @@ impl LocalDisk {
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let mut receipt_mutation = native_timing
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone()));
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone()));
|
||||
let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait");
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&root_destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_owner),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await;
|
||||
drop(namespace_wait);
|
||||
// The volume read lease prevents deletion of this disk's bucket until
|
||||
// publication finishes. Recheck after its wait, before any mutation.
|
||||
if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -804,11 +759,6 @@ impl LocalDisk {
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !super::has_rename_data_test_failpoints(dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
|
||||
&& !super::has_delete_version_test_failpoints()
|
||||
{
|
||||
#[cfg(test)]
|
||||
if !no_inline {
|
||||
@@ -852,7 +802,7 @@ impl LocalDisk {
|
||||
// requires a rollback directory; appending a new version keeps
|
||||
// an inline rollback backup for post-rename directory sync errors.
|
||||
if is_cancelled() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
@@ -875,7 +825,19 @@ impl LocalDisk {
|
||||
}
|
||||
});
|
||||
if rollback_data_dir.is_some() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
if no_inline
|
||||
&& sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
|
||||
&& os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf.is_some(),
|
||||
@@ -894,20 +856,31 @@ impl LocalDisk {
|
||||
skip_parent = dst_parent_owned.clone();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path_owned);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, false)?;
|
||||
if !no_inline {
|
||||
run_inline_preparation_before_backup(&dst_path_str);
|
||||
}
|
||||
|
||||
let mut staged_rollback_path = None;
|
||||
if sync_commit_metadata && let Some(ref old_metadata) = has_dst_buf {
|
||||
let backup_path =
|
||||
create_local_inline_rollback_backup(&dst_file_path_owned, &src_file_path_owned, old_metadata, true)?;
|
||||
if let Some(ref old_metadata) = has_dst_buf
|
||||
&& (sync_commit_metadata || cfg!(test))
|
||||
{
|
||||
let backup_path = create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
&src_file_path_owned,
|
||||
old_metadata,
|
||||
sync_commit_metadata,
|
||||
)?;
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
|
||||
let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
let sync_result = if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
// The payload was synced, but its part.1 directory entry
|
||||
// must also be persisted before moving the data directory.
|
||||
os::fsync_dir_std(src_data_path)
|
||||
@@ -923,37 +896,124 @@ impl LocalDisk {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if is_cancelled() {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(std::io::Error::new(ErrorKind::Interrupted, "commit cancelled"));
|
||||
}
|
||||
|
||||
let guard = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
guard,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
std::fs::remove_file(&src_file_path_owned)?;
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::FileNotFound));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
|
||||
&& let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
|
||||
{
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if has_data_dir_path_owned.is_some() {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if has_data_dir_path_owned.is_some()
|
||||
&& let Err(panic_payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
}))
|
||||
{
|
||||
return Ok(FusedCommitAttempt::ResumePanic(panic_payload));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(err) = os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
) {
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
if no_inline {
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
|
||||
.err()
|
||||
.unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) = fsync_inline_commit_std(
|
||||
&dst_file_path_owned,
|
||||
@@ -963,8 +1023,11 @@ impl LocalDisk {
|
||||
&guard,
|
||||
)
|
||||
{
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
if no_inline {
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
@@ -977,7 +1040,7 @@ impl LocalDisk {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
Ok(FusedCommitAttempt::Committed(RenameDataResp {
|
||||
Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: None,
|
||||
old_data_dir: None,
|
||||
rollback_data_dir: None,
|
||||
@@ -986,7 +1049,7 @@ impl LocalDisk {
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))
|
||||
})))
|
||||
};
|
||||
|
||||
let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
|
||||
@@ -1006,7 +1069,23 @@ impl LocalDisk {
|
||||
if let Some(receipt_mutation) = receipt_mutation.as_mut() {
|
||||
receipt_mutation.complete();
|
||||
}
|
||||
return Ok(resp);
|
||||
return Ok(*resp);
|
||||
}
|
||||
Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
@@ -1014,11 +1093,269 @@ impl LocalDisk {
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
Ok(FusedCommitAttempt::ResumePanic(panic_payload)) => {
|
||||
std::panic::resume_unwind(panic_payload);
|
||||
}
|
||||
Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
|
||||
fi = restored_fi;
|
||||
fi = *restored_fi;
|
||||
(guard, dst_buf)
|
||||
}
|
||||
}
|
||||
} else if let Some(publication) = multipart_publication.as_ref()
|
||||
&& no_inline
|
||||
&& !preserve_upload_metadata
|
||||
&& !publication.legacy
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
{
|
||||
enum MultipartFusedCommitOutcome {
|
||||
Committed(Box<RenameDataResp>),
|
||||
AbortBeforeDataRename(DiskError),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
RestoreTargetAfterError(DiskError, bool),
|
||||
}
|
||||
let publication_owned = Arc::clone(publication);
|
||||
let src_parent_owned = src_file_parent.to_path_buf();
|
||||
let dst_parent_owned = dst_file_parent.to_path_buf();
|
||||
let dst_file_path_owned = dst_file_path.clone();
|
||||
let src_file_path_owned = src_file_path.clone();
|
||||
let dst_volume_dir_owned = dst_volume_dir.clone();
|
||||
let publication_root_owned = self.publication_root.clone();
|
||||
let has_data_dir_path_owned = has_data_dir_path.clone();
|
||||
let sync_data_shards = durability.syncs_data_shards();
|
||||
let sync_commit_metadata = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
let dst_volume_str = dst_volume.to_string();
|
||||
let dst_path_str = dst_path.to_string();
|
||||
let fi_moved = fi;
|
||||
#[cfg(test)]
|
||||
let root_owned = self.root.clone();
|
||||
|
||||
let outcome = os::run_blocking_namespace_operation_with_sync(
|
||||
mutation_lease.clone(),
|
||||
true,
|
||||
move || -> std::io::Result<MultipartFusedCommitOutcome> {
|
||||
let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
|
||||
false,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(res) => res,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
if let Some(dst_buf) = has_dst_buf_owned.as_ref() {
|
||||
if FileMeta::is_xl2_v1_format(dst_buf)
|
||||
&& let Ok(nmeta) = FileMeta::load(dst_buf)
|
||||
{
|
||||
xlmeta = nmeta;
|
||||
} else {
|
||||
dst_meta_unparsable = true;
|
||||
}
|
||||
}
|
||||
let skip_parent_owned = if has_dst_buf_owned.is_some() {
|
||||
dst_parent_owned.clone()
|
||||
} else {
|
||||
dst_volume_dir_owned.clone()
|
||||
};
|
||||
let version_id = fi_moved.version_id.unwrap_or_default();
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf_owned.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_str,
|
||||
&dst_path_str,
|
||||
version_id,
|
||||
);
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
if let Some(old_data_dir) = has_old_data_dir.as_ref() {
|
||||
let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
|
||||
}
|
||||
if let Err(err) = xlmeta.add_version(fi_moved) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into()));
|
||||
}
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf_owned = match xlmeta.marshal_msg() {
|
||||
Ok(buf) => buf,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into())),
|
||||
};
|
||||
|
||||
let prepared_metadata_source = match publication_owned.prepare_and_install_head_on_thread(
|
||||
&new_dst_buf_owned,
|
||||
has_dst_buf_owned.as_deref(),
|
||||
&dst_file_path_owned,
|
||||
&directory_guard_owned,
|
||||
) {
|
||||
Ok(source) => source,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
// When presynced, shard payloads were synced during UploadPart and
|
||||
// prepare_and_install_head_on_thread just fsynced src_data_path after
|
||||
// writing RETIREMENT_ANCHOR.
|
||||
if sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::take_presynced_tmp_data_dir(src_data_path))
|
||||
&& let Err(err) = os::sync_dir_files_std(src_data_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
let directory_guard_owned = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
directory_guard_owned,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Err(err) = std::fs::remove_file(&src_file_path_owned) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::FileNotFound));
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let Some(src_parent) = src_data_path.parent() else {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::InvalidPath));
|
||||
};
|
||||
let payload_guard = match os::multipart_payload_commit_guard_std(&directory_guard_owned, src_parent) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
if let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &payload_guard)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(
|
||||
directory_guard_owned,
|
||||
DiskError::Unexpected,
|
||||
));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent_owned,
|
||||
&publication_root_owned,
|
||||
&directory_guard_owned,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(DiskError::Unexpected, false));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if sync_commit_metadata {
|
||||
if let Some(parent) = dst_file_path_owned.parent()
|
||||
&& let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(to_file_error(err).into(), true));
|
||||
}
|
||||
if has_dst_buf_owned.is_none() {
|
||||
let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
|
||||
while let Some(dir) = ancestor {
|
||||
if !dir.starts_with(&dst_volume_dir_owned) {
|
||||
break;
|
||||
}
|
||||
if let Err(err) = os::fsync_dir_limited_std(dir) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
|
||||
to_file_error(err).into(),
|
||||
true,
|
||||
));
|
||||
}
|
||||
if dir == dst_volume_dir_owned.as_path() {
|
||||
break;
|
||||
}
|
||||
ancestor = dir.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
|
||||
Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: Some(Box::new(result)),
|
||||
old_data_dir: has_old_data_dir,
|
||||
rollback_data_dir: None,
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))),
|
||||
Err(err) => Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(err, false)),
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?;
|
||||
|
||||
match outcome {
|
||||
MultipartFusedCommitOutcome::Committed(resp) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Ok(*resp);
|
||||
}
|
||||
MultipartFusedCommitOutcome::AbortBeforeDataRename(err) => return Err(err),
|
||||
MultipartFusedCommitOutcome::RestoreDataSource(directory_guard, err) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
MultipartFusedCommitOutcome::RestoreTargetAfterError(err, invalidate_before_return) => {
|
||||
if !invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
pin_multipart_future(|| publication.restore_target_after_error(mutation_lease.clone())).await?;
|
||||
if invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
prepare_rename_commit_and_destination_metadata(
|
||||
!no_inline && !preserve_upload_metadata,
|
||||
@@ -1142,13 +1479,29 @@ impl LocalDisk {
|
||||
// after the commit rename, and a crash before the rename means the
|
||||
// PUT was never acknowledged. A metadata commit: relaxed tiers
|
||||
// leave it to the page cache.
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(all(not(test), target_os = "linux"))]
|
||||
let probe_active = false;
|
||||
let shard_sync = async {
|
||||
if durability.syncs_data_shards()
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
#[cfg(target_os = "linux")]
|
||||
if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
if multipart_publication.is_none()
|
||||
&& let Err(err) = os::fsync_dir_with_owner(src_data_path, Some(commit.mutation_lease.clone())).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
@@ -1540,13 +1893,22 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let multipart_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => match pin_multipart_future(|| publication.finish(commit.mutation_lease.clone())).await {
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone())).await?;
|
||||
return Err(error);
|
||||
Some(publication) => {
|
||||
let target_already_synced = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
match pin_multipart_future(|| {
|
||||
publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone()))
|
||||
.await?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
Ok(RenameDataResp {
|
||||
@@ -2187,23 +2549,8 @@ impl LocalDisk {
|
||||
dst_path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::super::RenameDataObservation {
|
||||
self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed_with_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
owners: (Option<Arc<dyn Send + Sync>>, Option<os::MetadataMutationReservation>),
|
||||
) -> super::super::RenameDataObservation {
|
||||
let (namespace_owner, metadata_reservation) = owners;
|
||||
let mut state = RenameDataState {
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
..Default::default()
|
||||
};
|
||||
let result = self
|
||||
|
||||
@@ -1000,6 +1000,25 @@ pub(super) fn inspection_with_markers(
|
||||
Ok((inspection, has_physical_data_dirs_or_markers))
|
||||
}
|
||||
|
||||
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
|
||||
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
|
||||
|
||||
fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
|
||||
let mut map = PREPARED_DELETE_RECEIPTS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(rollback, entry);
|
||||
}
|
||||
|
||||
fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
|
||||
PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
// Native MPU source closure has its own conditional protocol. Generic
|
||||
// object recovery must never restore its head or acquire SourceRW under
|
||||
@@ -1523,6 +1542,9 @@ impl LocalDisk {
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if let Ok(root) = self.io_get_object_path(volume, object) {
|
||||
let _ = take_prepared_delete_receipt(&root.join(transaction.to_string()));
|
||||
}
|
||||
|
||||
let (mut observed, mut has_data_dir_work) = self
|
||||
.delete_recovery_inspect_owned_with_markers(volume, object, transaction, owner.clone())
|
||||
@@ -1911,8 +1933,14 @@ impl LocalDisk {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let bucket_path = self.io_get_bucket_path(volume)?;
|
||||
let mut file_leases = Vec::with_capacity(3);
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
let mut file_leases = Vec::with_capacity(5);
|
||||
for name in [
|
||||
STORAGE_FORMAT_FILE_BACKUP,
|
||||
DELETE_MARKER_ROLLBACK_FILE,
|
||||
TARGET,
|
||||
PREPARED,
|
||||
FINISHED,
|
||||
] {
|
||||
let path = rollback.join(name);
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, owner.clone()).await;
|
||||
file_leases.push((path, lease));
|
||||
@@ -2132,114 +2160,128 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>);
|
||||
let (old_dir, target, journal_raw) = match (|| -> Result<Option<PreparedDeleteTuple>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let mut deleted_headers: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
|
||||
.collect();
|
||||
let original_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
|
||||
.collect::<Result<_>>()?;
|
||||
let old_dir = meta.delete_version(&fi_clone)?;
|
||||
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
if let Some(uuid) = old_dir {
|
||||
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
|
||||
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
|
||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||
if old_path.join(STORAGE_FORMAT_FILE).exists() {
|
||||
return Ok(None);
|
||||
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool);
|
||||
let (old_dir, target, journal_raw, journal, sole_transaction) =
|
||||
match (|| -> Result<Option<PreparedDeleteTuple>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let mut deleted_headers: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
|
||||
.collect();
|
||||
let original_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
|
||||
.collect::<Result<_>>()?;
|
||||
let old_dir = meta.delete_version(&fi_clone)?;
|
||||
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
}
|
||||
let surviving: HashSet<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
|
||||
.collect();
|
||||
deleted_headers.retain(|header| !surviving.contains(header));
|
||||
let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect();
|
||||
let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
|
||||
let mut predecessors = BTreeSet::new();
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
if let Some(uuid) = old_dir {
|
||||
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
|
||||
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
|
||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||
if old_path.join(STORAGE_FORMAT_FILE).exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let Some(id) = entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.and_then(|name| Uuid::parse_str(name).ok())
|
||||
.filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let prepared = regular_bytes(&entry.path().join(PREPARED))?;
|
||||
let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
let mut records = Vec::new();
|
||||
if let Some(bytes) = prepared {
|
||||
records.push(
|
||||
serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?,
|
||||
);
|
||||
}
|
||||
if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
|
||||
records.push(
|
||||
serde_json::from_slice::<CommitReceipt>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?
|
||||
.journal,
|
||||
);
|
||||
}
|
||||
for record in records {
|
||||
if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
|| record.volume != volume_str
|
||||
|| record.object != object_str
|
||||
|| record.transaction != id
|
||||
|| incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
|
||||
|| record.predecessors.iter().any(|before| before.is_nil() || *before == id)
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let surviving: HashSet<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
|
||||
.collect();
|
||||
deleted_headers.retain(|header| !surviving.contains(header));
|
||||
let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect();
|
||||
let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut sole_transaction = true;
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let file_name = entry.file_name();
|
||||
let Some(name_str) = file_name.to_str() else {
|
||||
sole_transaction = false;
|
||||
continue;
|
||||
};
|
||||
let Some(id) = Uuid::parse_str(name_str).ok().filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let dir_path = entry.path();
|
||||
let prepared = regular_bytes(&dir_path.join(PREPARED))?;
|
||||
let receipt = regular_bytes(&dir_path.join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
if sole_transaction && name_str == id.hyphenated().to_string() && old_dir != Some(id) {
|
||||
let is_surviving_data_dir = original_versions.iter().any(|v| v.data_dir == Some(id))
|
||||
&& prepared.is_none()
|
||||
&& receipt.is_none()
|
||||
&& !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
|
||||
if !is_surviving_data_dir {
|
||||
sole_transaction = false;
|
||||
}
|
||||
}
|
||||
let mut records = Vec::new();
|
||||
if let Some(bytes) = prepared {
|
||||
records.push(
|
||||
serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?,
|
||||
);
|
||||
}
|
||||
if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
|
||||
records.push(
|
||||
serde_json::from_slice::<CommitReceipt>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?
|
||||
.journal,
|
||||
);
|
||||
}
|
||||
for record in records {
|
||||
if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
|| record.volume != volume_str
|
||||
|| record.object != object_str
|
||||
|| record.transaction != id
|
||||
|| incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
|
||||
|| record.predecessors.iter().any(|before| before.is_nil() || *before == id)
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
predecessors.insert(id);
|
||||
}
|
||||
predecessors.insert(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut sparse = fi_clone;
|
||||
sparse.data = None;
|
||||
let journal = DeleteRecoveryJournal {
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
volume: volume_str,
|
||||
object: object_str,
|
||||
transaction,
|
||||
bucket_incarnation: incarnation,
|
||||
original_digest: Some(digest(&buf)),
|
||||
target_digest: target.as_deref().map(digest),
|
||||
data_dirs: old_dir.into_iter().collect(),
|
||||
deleted_versions,
|
||||
requests: vec![sparse],
|
||||
original_versions,
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
sole_transaction &= predecessors.is_empty();
|
||||
let mut sparse = fi_clone;
|
||||
sparse.data = None;
|
||||
let journal = DeleteRecoveryJournal {
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
volume: volume_str,
|
||||
object: object_str,
|
||||
transaction,
|
||||
bucket_incarnation: incarnation,
|
||||
original_digest: Some(digest(&buf)),
|
||||
target_digest: target.as_deref().map(digest),
|
||||
data_dirs: old_dir.into_iter().collect(),
|
||||
deleted_versions,
|
||||
requests: vec![sparse],
|
||||
original_versions,
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
};
|
||||
let raw = journal_bytes(&journal)?;
|
||||
Ok(Some((old_dir, target, raw, journal, sole_transaction)))
|
||||
})() {
|
||||
Ok(Some(tuple)) => tuple,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(err) => return Ok(Err(err)),
|
||||
};
|
||||
Ok(Some((old_dir, target, journal_bytes(&journal)?)))
|
||||
})() {
|
||||
Ok(Some(tuple)) => tuple,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(err) => return Ok(Err(err)),
|
||||
};
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let staged = tempfile::Builder::new()
|
||||
.prefix("delete-preparation-")
|
||||
.tempdir_in(&staging_root)?;
|
||||
let backup_path = staged.path().join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
let staged = tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root)?;
|
||||
let staged_dir = staged.path().to_path_buf();
|
||||
let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if std::fs::hard_link(&xl_path, &backup_path).is_err() {
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, &buf)?;
|
||||
@@ -2247,18 +2289,22 @@ impl LocalDisk {
|
||||
file.sync_data()?;
|
||||
}
|
||||
}
|
||||
let mut written_files = Vec::with_capacity(2);
|
||||
for (name, bytes) in [(TARGET, target.as_deref()), (PREPARED, Some(journal_raw.as_slice()))] {
|
||||
let Some(bytes) = bytes else {
|
||||
continue;
|
||||
};
|
||||
let mut file = std::fs::File::create(staged.path().join(name))?;
|
||||
let mut file = std::fs::File::create(staged_dir.join(name))?;
|
||||
std::io::Write::write_all(&mut file, bytes)?;
|
||||
if sync && !bytes.is_empty() {
|
||||
file.sync_data()?;
|
||||
written_files.push(file);
|
||||
}
|
||||
}
|
||||
for file in written_files {
|
||||
file.sync_data()?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(staged.path())?;
|
||||
os::fsync_dir_std(&staged_dir)?;
|
||||
}
|
||||
match std::fs::symlink_metadata(&destination) {
|
||||
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
|
||||
@@ -2271,53 +2317,54 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
if let Some(parent) = destination.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
let staged_dir = staged.keep();
|
||||
if let Err(err) = std::fs::rename(&staged_dir, &destination) {
|
||||
let _ = std::fs::remove_dir_all(&staged_dir);
|
||||
return Err(err);
|
||||
}
|
||||
std::fs::rename(staged.path(), &destination)?;
|
||||
let old_data_path = old_dir.map(|uuid| file_path.join(uuid.to_string())).filter(|p| p.is_dir());
|
||||
if sync && let Err(err) = os::fsync_dir_std(&file_path) {
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
let mut reserved_data_path = None;
|
||||
if let Some(uuid) = old_dir {
|
||||
let old_path = file_path.join(uuid.to_string());
|
||||
if old_path.is_dir() {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
let reserve_res = (|| -> std::io::Result<()> {
|
||||
let marker = std::fs::File::create(&reserved_marker)?;
|
||||
if sync {
|
||||
marker.sync_all()?;
|
||||
os::fsync_dir_std(&old_path)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = reserve_res {
|
||||
let _ = std::fs::remove_file(&reserved_marker);
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
if let Some(old_path) = old_data_path {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
let reserve_res = (|| -> std::io::Result<()> {
|
||||
let marker = std::fs::File::create(&reserved_marker)?;
|
||||
if sync {
|
||||
marker.sync_all()?;
|
||||
os::fsync_dir_std(&old_path)?;
|
||||
}
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = reserve_res {
|
||||
let _ = std::fs::remove_file(&reserved_marker);
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
}
|
||||
if let Some(ref target_bytes) = target {
|
||||
let mut staged_xl = match tempfile::NamedTempFile::new_in(&staging_root) {
|
||||
Ok(staged_xl) => staged_xl,
|
||||
Err(err) => {
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
let _ = std::fs::remove_file(reserved_marker);
|
||||
let target_in_rollback = destination.join(TARGET);
|
||||
let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
let write_res = if std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok() {
|
||||
let rename_res = std::fs::rename(&linked_xl, &xl_path);
|
||||
if rename_res.is_err() {
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
}
|
||||
rename_res
|
||||
} else {
|
||||
(|| -> std::io::Result<()> {
|
||||
let mut staged_xl = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged_xl.as_file_mut(), target_bytes)?;
|
||||
if sync && !target_bytes.is_empty() {
|
||||
staged_xl.as_file().sync_data()?;
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
std::fs::rename(staged_xl.path(), &xl_path)
|
||||
})()
|
||||
};
|
||||
let write_res = (|| -> std::io::Result<()> {
|
||||
std::io::Write::write_all(staged_xl.as_file_mut(), target_bytes)?;
|
||||
if sync && !target_bytes.is_empty() {
|
||||
staged_xl.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged_xl.path(), &xl_path)
|
||||
})();
|
||||
if let Err(err) = write_res {
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
let _ = std::fs::remove_file(reserved_marker);
|
||||
@@ -2346,6 +2393,7 @@ impl LocalDisk {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
put_prepared_delete_receipt(destination, (journal, sole_transaction));
|
||||
Ok(Ok(Some(old_dir)))
|
||||
},
|
||||
)
|
||||
@@ -2391,6 +2439,26 @@ impl LocalDisk {
|
||||
lease,
|
||||
sync,
|
||||
move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
|
||||
if let Some((journal, sole_transaction)) = take_prepared_delete_receipt(&rollback)
|
||||
&& rollback.join(PREPARED).is_file()
|
||||
{
|
||||
let receipt = CommitReceipt {
|
||||
journal: journal.clone(),
|
||||
finished: false,
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged.path(), &receipt_path)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
return Ok(Ok(Some((journal, sole_transaction, true))));
|
||||
}
|
||||
let prepared = match regular_bytes(&rollback.join(PREPARED)) {
|
||||
Ok(Some(bytes)) => bytes,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
@@ -2526,7 +2594,7 @@ impl LocalDisk {
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let data_dirs = journal.data_dirs.clone();
|
||||
let has_target = journal.target_digest.is_some();
|
||||
let journal_for_receipt = has_target.then(|| journal.clone());
|
||||
let journal_for_receipt = journal.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let prune_paths: Vec<_> = rollback
|
||||
.ancestors()
|
||||
@@ -2538,11 +2606,13 @@ impl LocalDisk {
|
||||
let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let mut had_physical_data_dirs = false;
|
||||
for data_dir in &data_dirs {
|
||||
for base in [&root, &rollback] {
|
||||
let dir_path = base.join(data_dir.to_string());
|
||||
match std::fs::read_dir(&dir_path) {
|
||||
Ok(entries) => {
|
||||
had_physical_data_dirs = true;
|
||||
let mut removable = Vec::new();
|
||||
let mut preserve_dir = false;
|
||||
for entry in entries {
|
||||
@@ -2583,14 +2653,17 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
if !data_dirs.is_empty() && sync && root.is_dir() {
|
||||
if had_physical_data_dirs && sync && root.is_dir() {
|
||||
os::fsync_dir_std(&root)?;
|
||||
}
|
||||
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
|
||||
ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
|
||||
}
|
||||
if let Some(journal) = journal_for_receipt {
|
||||
let finished_receipt = CommitReceipt { journal, finished: true };
|
||||
if has_target || had_physical_data_dirs {
|
||||
let finished_receipt = CommitReceipt {
|
||||
journal: journal_for_receipt,
|
||||
finished: true,
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&finished_receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
@@ -2605,13 +2678,20 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
|
||||
let path = rollback.join(name);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
ignore_not_found(std::fs::remove_file(path))?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
|
||||
ignore_not_found(std::fs::remove_file(&receipt_path))?;
|
||||
for dir_path in &prune_paths {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, dir_path);
|
||||
if let Err(err) = std::fs::remove_dir(dir_path) {
|
||||
if is_dir_not_empty_error(&err) {
|
||||
break;
|
||||
@@ -6509,6 +6589,63 @@ mod tests {
|
||||
"surviving v1 data_dir must remain intact"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_unversioned_physical_delete_retains_receipt_until_finalize() {
|
||||
let (_temp, disk, mut info, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
|
||||
info.data = None;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut info.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
|
||||
);
|
||||
let data_dir = Uuid::new_v4();
|
||||
info.data_dir = Some(data_dir);
|
||||
fs::create_dir_all(root.join(data_dir.to_string()))
|
||||
.await
|
||||
.expect("create data_dir");
|
||||
fs::write(root.join(data_dir.to_string()).join("part.1"), b"physical-payload")
|
||||
.await
|
||||
.expect("write part.1");
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(info.clone()).expect("add version");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
|
||||
.await
|
||||
.expect("write xl.meta");
|
||||
|
||||
let transaction = Uuid::new_v4();
|
||||
let opts = DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let fused = disk
|
||||
.try_fused_recoverable_delete_version("recovery", "object", &info, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused unversioned physical delete must succeed");
|
||||
assert_eq!(fused, Some(()));
|
||||
|
||||
let (journal, sole_transaction, target_satisfied) = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish must succeed")
|
||||
.expect("absent target must satisfy local publish");
|
||||
assert!(sole_transaction);
|
||||
assert!(target_satisfied);
|
||||
assert!(journal.target_digest.is_none());
|
||||
|
||||
let rollback = root.join(transaction.to_string());
|
||||
assert!(rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file());
|
||||
|
||||
let finalized = disk
|
||||
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
|
||||
.await
|
||||
.expect("local finalize must succeed");
|
||||
assert!(finalized);
|
||||
assert!(!rollback.exists(), "rollback directory must be pruned");
|
||||
assert!(!root.join(data_dir.to_string()).exists(), "physical data_dir must be removed");
|
||||
assert!(!root.exists(), "empty object directory must be pruned");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -421,6 +421,7 @@ impl LocalDisk {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let permit = AbortDeletePermit::new(&store, &expected, certificate.clone())?;
|
||||
super::multipart_source_state::invalidate_active_multipart_source_head(&store.source);
|
||||
let destination = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, &Uuid::new_v4().to_string())?;
|
||||
let source = store.source.clone();
|
||||
let base = store.base.clone();
|
||||
@@ -618,6 +619,9 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
|
||||
root: os::PublicationRoot,
|
||||
}
|
||||
impl OrdinaryMultipartDeleteCheck {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn validate(&self) -> Result<()> {
|
||||
let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
|
||||
Ok(pinned) => pinned,
|
||||
|
||||
@@ -238,9 +238,192 @@ pub(super) fn read_optional_in_pinned(
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
struct FileStamp {
|
||||
len: u64,
|
||||
modified: Option<std::time::SystemTime>,
|
||||
#[cfg(unix)]
|
||||
ino: u64,
|
||||
}
|
||||
|
||||
impl FileStamp {
|
||||
fn from_path(path: &Path) -> Option<Self> {
|
||||
let meta = std::fs::symlink_metadata(path).ok()?;
|
||||
Some(Self {
|
||||
len: meta.len(),
|
||||
modified: meta.modified().ok(),
|
||||
#[cfg(unix)]
|
||||
ino: {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
meta.ino()
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ActiveMultipartSourceEntry {
|
||||
raw: Bytes,
|
||||
fi: FileInfo,
|
||||
head_stamp: FileStamp,
|
||||
unsettled_parts: std::collections::HashSet<usize>,
|
||||
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
|
||||
}
|
||||
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
|
||||
parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
|
||||
> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
|
||||
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
|
||||
return;
|
||||
};
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(
|
||||
source,
|
||||
ActiveMultipartSourceEntry {
|
||||
raw,
|
||||
fi,
|
||||
head_stamp,
|
||||
unsettled_parts: std::collections::HashSet::new(),
|
||||
parts: std::collections::HashMap::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
|
||||
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
|
||||
&& !source.join(STATE).exists()
|
||||
&& !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
&& !source.join(".mpu-completions").exists()
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
let (raw, fi, head_stamp) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
(entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
Some((raw, fi))
|
||||
}
|
||||
|
||||
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
|
||||
let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
clean
|
||||
}
|
||||
|
||||
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.unsettled_parts.insert(part_number);
|
||||
entry.parts.remove(&part_number);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
|
||||
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.and_then(|entry| entry.fi.data_dir)
|
||||
.filter(|dir| !dir.is_nil())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{}.meta", part.number));
|
||||
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
|
||||
return;
|
||||
};
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.parts.insert(part.number, (part_stamp, part));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn finish_active_multipart_part_transaction(
|
||||
source: &Path,
|
||||
part_number: Option<usize>,
|
||||
committed: bool,
|
||||
retired_existing_txn: bool,
|
||||
) {
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let Some(part_number) = part_number.filter(|_| committed) else {
|
||||
map.remove(source);
|
||||
return;
|
||||
};
|
||||
if !retired_existing_txn {
|
||||
return;
|
||||
}
|
||||
if let Some(entry) = map.get_mut(source)
|
||||
&& !entry.unsettled_parts.remove(&part_number)
|
||||
{
|
||||
map.remove(source);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
(entry.head_stamp, data_dir, *expected_stamp, part.clone())
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
|
||||
if FileStamp::from_path(&part_meta_path) != Some(expected_stamp) {
|
||||
return None;
|
||||
}
|
||||
Some(part)
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
|
||||
}
|
||||
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
/// not on the earlier async caller that may have timed out.
|
||||
pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
|
||||
if !initial && has_active_multipart_source_head(source) {
|
||||
return Ok(());
|
||||
}
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
@@ -440,6 +623,7 @@ impl SourceStore {
|
||||
terminal: Option<&Path>,
|
||||
sync_source_before: bool,
|
||||
) -> std::io::Result<()> {
|
||||
invalidate_active_multipart_source_head(source);
|
||||
#[cfg(unix)]
|
||||
let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
#[cfg(not(unix))]
|
||||
@@ -470,7 +654,7 @@ impl SourceStore {
|
||||
let terminal = terminal_record_path(&source, state);
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation_with_sync(self.lease.clone(), true, move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)?;
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, false)?;
|
||||
Self::persist_state_std(&source, &base, &root, &raw, terminal.as_deref(), false)
|
||||
})
|
||||
.await
|
||||
@@ -539,7 +723,7 @@ impl SourceStore {
|
||||
let root = self.publication_root.clone();
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, true)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
@@ -551,6 +735,7 @@ impl SourceStore {
|
||||
owner: Uuid,
|
||||
target: bool,
|
||||
bytes: &[u8],
|
||||
sync_source_root: bool,
|
||||
) -> std::io::Result<()> {
|
||||
let path = if target {
|
||||
target_backup_path(source, owner)
|
||||
@@ -582,9 +767,10 @@ impl SourceStore {
|
||||
if !existed {
|
||||
os::replace_multipart_source_metadata(&path, bytes, base, root)?;
|
||||
}
|
||||
os::sync_multipart_owned_record_parents(source, parent, existed, base, root)
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
let path = self.source.join(STORAGE_FORMAT_FILE);
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
@@ -1097,6 +1283,82 @@ pub(super) fn load_source(
|
||||
})
|
||||
}
|
||||
|
||||
fn load_uninitialized_source(
|
||||
raw: Bytes,
|
||||
path: &str,
|
||||
disk_id: Uuid,
|
||||
physical_slot: usize,
|
||||
disk_ids: &[Uuid],
|
||||
) -> Result<LoadedSource> {
|
||||
let sha = raw_sha(&raw);
|
||||
let length = raw.len() as u64;
|
||||
let current = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
native_source_protocol(¤t)?;
|
||||
if source_status(¤t)?.is_some()
|
||||
|| publication_identity(¤t)?.is_some()
|
||||
|| SourceConditionV1::take(&mut current.clone())?.is_some()
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let bucket = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-bucket")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let object = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-object")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let condition = capture_source_condition(bucket, object, path, ¤t, current.data_dir.ok_or(DiskError::FileCorrupt)?)?;
|
||||
if condition.generation.epoch != 0 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let slot = condition
|
||||
.lineage
|
||||
.geometry
|
||||
.distribution
|
||||
.iter()
|
||||
.position(|index| *index == current.erasure.index)
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
if slot != physical_slot || condition.lineage.geometry.disks() != disk_ids.len() {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let inventory = raw_sha(&encode(&(None::<[u8; 32]>, Some(sha), sha, None::<[u8; 32]>, None::<[u8; 32]>))?);
|
||||
let snapshot = Some(MultipartSourceSnapshot {
|
||||
disk_id,
|
||||
physical_slot: slot,
|
||||
lineage: condition.lineage,
|
||||
generation: condition.generation,
|
||||
revision: 0,
|
||||
phase: MultipartSourcePhase::Active,
|
||||
state_sha256: [0; 32],
|
||||
head_sha256: Some(sha),
|
||||
inventory_sha256: inventory,
|
||||
capability_sha256: None,
|
||||
publication: None,
|
||||
});
|
||||
Ok(LoadedSource {
|
||||
root_missing: false,
|
||||
state: None,
|
||||
head: Some(SourceHead {
|
||||
sha256: sha,
|
||||
length,
|
||||
raw: Some(raw.clone()),
|
||||
}),
|
||||
original: Some(raw),
|
||||
info: Some(current),
|
||||
snapshot,
|
||||
head_matches_state: true,
|
||||
head_matches_activation: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) const RETIREMENT_ANCHOR: &str = ".mpu-source-retirement";
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
@@ -1421,6 +1683,8 @@ struct ValidatedPreparedPublication {
|
||||
length: u64,
|
||||
}
|
||||
|
||||
type DeferredInitialPublication = (Vec<u8>, PathBuf, Vec<u8>);
|
||||
|
||||
pub(super) struct NativeMultipartPublication {
|
||||
pub store: Arc<SourceStore>,
|
||||
pub publication: PublicationIdentity,
|
||||
@@ -1428,22 +1692,49 @@ pub(super) struct NativeMultipartPublication {
|
||||
state: parking_lot::Mutex<SourceState>,
|
||||
capability: parking_lot::Mutex<Option<MultipartUndoCapability>>,
|
||||
validated_prepared: parking_lot::Mutex<Option<ValidatedPreparedPublication>>,
|
||||
deferred_initial: parking_lot::Mutex<Option<DeferredInitialPublication>>,
|
||||
pub legacy: bool,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[cfg(test)]
|
||||
pub(super) async fn prepare_native_multipart_publication(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_native_multipart_publication_deferred(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_native_multipart_publication_inner(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
defer_persist: bool,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
// Real Complete is parts-backed, including a zero-byte last part.
|
||||
if incoming.data.is_some() || incoming.inline_data() {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
let store = Arc::new(self.multipart_store_from_staging(path, staging).await?);
|
||||
let loaded = store.load().await?;
|
||||
let loaded = if defer_persist && let Some(raw) = staging.fresh_source_head() {
|
||||
load_uninitialized_source(raw, &store.path, store.disk_id, store.physical_slot, &store.disk_ids)?
|
||||
} else {
|
||||
store.load().await?
|
||||
};
|
||||
let snapshot = loaded.snapshot.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
let original = loaded
|
||||
.head
|
||||
@@ -1545,20 +1836,33 @@ impl LocalDisk {
|
||||
};
|
||||
let anchor_bytes = encode(&anchor_record)?;
|
||||
state.validate(store.disk_id, &store.path)?;
|
||||
let state_raw = encode(&state)?;
|
||||
let source_dir = store.source.clone();
|
||||
let base_dir = store.base.clone();
|
||||
let pub_root = store.publication_root.clone();
|
||||
let terminal = terminal_record_path(&source_dir, &state);
|
||||
let original_bytes = original.to_vec();
|
||||
os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
|
||||
SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes)?;
|
||||
SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
let deferred_initial = if defer_persist {
|
||||
Some((original_bytes, directory, anchor_bytes))
|
||||
} else {
|
||||
let state_raw = encode(&state)?;
|
||||
let source_dir = store.source.clone();
|
||||
let base_dir = store.base.clone();
|
||||
let pub_root = store.publication_root.clone();
|
||||
let terminal = terminal_record_path(&source_dir, &state);
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source_dir)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
|
||||
SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
|
||||
SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
None
|
||||
};
|
||||
let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
|
||||
let context = NativeMultipartPublication {
|
||||
store,
|
||||
@@ -1567,6 +1871,7 @@ impl LocalDisk {
|
||||
state: parking_lot::Mutex::new(state),
|
||||
capability: parking_lot::Mutex::new(Some(capability)),
|
||||
validated_prepared: parking_lot::Mutex::new(None),
|
||||
deferred_initial: parking_lot::Mutex::new(deferred_initial),
|
||||
legacy,
|
||||
};
|
||||
Ok(context)
|
||||
@@ -1621,7 +1926,7 @@ impl NativeMultipartPublication {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous)?;
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
|
||||
})
|
||||
@@ -1631,6 +1936,132 @@ impl NativeMultipartPublication {
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn prepare_and_install_head_on_thread(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let mut state = self.state.lock().clone();
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
length: new_metadata.len() as u64,
|
||||
};
|
||||
if let Some(previous) = old_metadata {
|
||||
state.previous_target_sha256 = Some(raw_sha(previous));
|
||||
state.previous_target_length = Some(previous.len() as u64);
|
||||
}
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
let prepared_path = &self.prepared_path;
|
||||
let canonical = self.canonical_head();
|
||||
let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
|
||||
if let Some((ref original_bytes, _, _)) = deferred_initial {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(prepared_path, dst_file_path, directory_guard)?;
|
||||
if let Err(err) = prepared_metadata_source.write_all(new_metadata, false) {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
let setup_res: std::io::Result<()> = (|| {
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = setup_res {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
if let Err(err) = prepared_metadata_source.sync_data() {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub async fn prepare_and_install_head(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
@@ -1643,7 +2074,11 @@ impl NativeMultipartPublication {
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
@@ -1666,6 +2101,9 @@ impl NativeMultipartPublication {
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
@@ -1675,18 +2113,51 @@ impl NativeMultipartPublication {
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(&prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
@@ -1700,6 +2171,28 @@ impl NativeMultipartPublication {
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub async fn prepare_and_install_head(
|
||||
self: &Arc<Self>,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let this = Arc::clone(self);
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let dst_file_path = dst_file_path.to_path_buf();
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
this.prepare_and_install_head_on_thread(&new_bytes, old_bytes.as_deref(), &dst_file_path, &owned_directory_guard)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
|
||||
{
|
||||
@@ -1771,7 +2264,11 @@ impl NativeMultipartPublication {
|
||||
// return an error after consuming the last claimable source heads.
|
||||
self.store.restore_exact_target_with_lease(&state, lease).await
|
||||
}
|
||||
pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
|
||||
pub(super) fn finish_on_thread(
|
||||
&self,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let (expected_sha256, expected_length) = {
|
||||
@@ -1790,16 +2287,16 @@ impl NativeMultipartPublication {
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
|
||||
let source = self.store.source.clone();
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
let path = self.store.path.clone();
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
(|| -> std::io::Result<()> {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
@@ -1808,28 +2305,64 @@ impl NativeMultipartPublication {
|
||||
if current.head.is_some() || current_raw != expected_state_raw {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
&base,
|
||||
&root,
|
||||
|target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
if target_already_synced {
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
})?;
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), true)
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
|target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
|
||||
if let Some((io_path, cleaned)) = staging_cleanup {
|
||||
match std::fs::remove_dir(&io_path) {
|
||||
Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
|
||||
cleaned.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
let proof = self
|
||||
@@ -1842,6 +2375,21 @@ impl NativeMultipartPublication {
|
||||
undo_capability: capability,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn finish(
|
||||
self: &Arc<Self>,
|
||||
target_lease: Arc<os::NamespaceMutationLease>,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let this = Arc::clone(self);
|
||||
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
|
||||
this.finish_on_thread(target_already_synced, staging_cleanup)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
|
||||
@@ -207,6 +207,12 @@ pub(super) struct MultipartSourceAdmission {
|
||||
initial: bool,
|
||||
}
|
||||
impl MultipartSourceAdmission {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn is_initial(&self) -> bool {
|
||||
self.initial
|
||||
}
|
||||
pub(super) fn precondition(&self) -> Arc<dyn Fn() -> std::io::Result<()> + Send + Sync> {
|
||||
let admission = self.clone();
|
||||
Arc::new(move || admission.validate())
|
||||
@@ -250,6 +256,8 @@ pub(super) struct MultipartCommitStaging {
|
||||
logical_path: PathBuf,
|
||||
io_path: PathBuf,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
cleaned: Arc<std::sync::atomic::AtomicBool>,
|
||||
fresh_source_head: Option<bytes::Bytes>,
|
||||
}
|
||||
|
||||
impl MultipartCommitStaging {
|
||||
@@ -259,10 +267,19 @@ impl MultipartCommitStaging {
|
||||
pub(super) fn private_directory(&self, directory: Uuid) -> PathBuf {
|
||||
self.io_path.join(directory.to_string())
|
||||
}
|
||||
pub(super) fn cleanup_handle(&self) -> (PathBuf, Arc<std::sync::atomic::AtomicBool>) {
|
||||
(self.io_path.clone(), Arc::clone(&self.cleaned))
|
||||
}
|
||||
pub(super) fn fresh_source_head(&self) -> Option<bytes::Bytes> {
|
||||
self.fresh_source_head.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MultipartCommitStaging {
|
||||
fn drop(&mut self) {
|
||||
if self.cleaned.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let logical_path = self.logical_path.clone();
|
||||
let io_path = self.io_path.clone();
|
||||
let owner = self.owner.clone();
|
||||
@@ -285,6 +302,14 @@ impl Drop for MultipartCommitStaging {
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
let root = upload_scope_root(upload_or_part_path, part_path);
|
||||
let Ok(source) = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, root) else {
|
||||
return false;
|
||||
};
|
||||
super::multipart_source_state::has_clean_active_multipart_source(&source)
|
||||
}
|
||||
|
||||
pub(super) fn multipart_source_admission(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -356,12 +381,14 @@ impl LocalDisk {
|
||||
let io_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &attempt)?;
|
||||
// The attempt root is exactly the namespace used by the tmp sweeper.
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical_path, owner.clone()).await;
|
||||
let staging = MultipartCommitStaging {
|
||||
let mut staging = MultipartCommitStaging {
|
||||
path: attempt,
|
||||
lease,
|
||||
logical_path,
|
||||
io_path,
|
||||
owner,
|
||||
cleaned: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
fresh_source_head: None,
|
||||
};
|
||||
let canonical_metadata = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &format!("{source_path}/xl.meta"))?;
|
||||
let source_object = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, source_path)?;
|
||||
@@ -371,31 +398,37 @@ impl LocalDisk {
|
||||
let source_path = source_path.to_owned();
|
||||
let incoming = incoming.clone();
|
||||
let sync_directory = effective_durability(destination_volume).syncs_data_shards();
|
||||
os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
let cached_head = super::multipart_source_state::get_active_multipart_source_head(&source_object);
|
||||
let (raw, source, from_active_cache) = if let Some((raw, source)) = cached_head {
|
||||
(raw, source, true)
|
||||
} else {
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
(bytes::Bytes::from(raw), source, false)
|
||||
};
|
||||
let source_dir = source
|
||||
.data_dir
|
||||
.filter(|dir| !dir.is_nil())
|
||||
@@ -412,23 +445,31 @@ impl LocalDisk {
|
||||
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
|
||||
if incoming.data.is_none() && !incoming.is_remote() {
|
||||
let source_data_dir = source_object.join(source_dir.to_string());
|
||||
let pinned_source_data = (!incoming.parts.is_empty())
|
||||
.then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
|
||||
.transpose()?;
|
||||
let mut pinned_source_data = None;
|
||||
let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
|
||||
for part in &incoming.parts {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
|
||||
let actual = if let Some(cached_part) =
|
||||
super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
|
||||
{
|
||||
cached_part
|
||||
} else {
|
||||
if pinned_source_data.is_none() {
|
||||
pinned_source_data =
|
||||
Some(os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root)?);
|
||||
}
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?
|
||||
};
|
||||
let normalized = i64::try_from(actual.size)
|
||||
.map_err(std::io::Error::other)?
|
||||
.max(actual.actual_size);
|
||||
@@ -464,10 +505,15 @@ impl LocalDisk {
|
||||
cancelled,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
let has_state = !from_active_cache
|
||||
&& (source_object.join(".mpu-source-state").exists()
|
||||
|| source_object.join(".mpu-recovery").exists()
|
||||
|| source_object.join(".mpu-completions").exists());
|
||||
Ok((!has_state).then_some(raw))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
staging.fresh_source_head = fresh_head;
|
||||
Ok(staging)
|
||||
}
|
||||
}
|
||||
@@ -1219,4 +1265,211 @@ mod tests {
|
||||
.expect("non-part empty hash cleanup must remain supported");
|
||||
assert!(!directory.exists(), "a per-upload safety guard must not strand empty hash directories");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_active_head_cache_populates_and_invalidates_across_part_and_delete_paths() {
|
||||
use crate::disk::{DiskAPI as _, PartTransactionAction, ReadOptions};
|
||||
let (_root, disk, completed, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let upload = format!("{}/{}", "c".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
disk.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &upload, init_fi.clone())
|
||||
.await
|
||||
.expect("fresh multipart upload metadata");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
let read_fi = disk
|
||||
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("cached read_version");
|
||||
assert_eq!(read_fi.data_dir, Some(data_dir));
|
||||
|
||||
let destination = format!("{upload}/{data_dir}/part.1");
|
||||
let part_meta = bytes::Bytes::from(completed.parts[0].marshal_msg().expect("part meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part transaction");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"in-flight part transaction must mark active source as having unsettled part txn"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part transaction");
|
||||
assert!(
|
||||
disk.has_clean_active_multipart_source(&upload, false),
|
||||
"settled commit must restore clean active multipart source status"
|
||||
);
|
||||
let read_parts = disk
|
||||
.read_parts(RUSTFS_META_MULTIPART_BUCKET, &[format!("{destination}.meta")])
|
||||
.await
|
||||
.expect("cached read_parts");
|
||||
assert_eq!(read_parts.len(), 1);
|
||||
assert_eq!(read_parts[0].number, 1);
|
||||
|
||||
// Prepare part 2, then replay Commit for already-settled part 1: part 2's
|
||||
// pending transaction must keep the upload marked as having unsettled part txns.
|
||||
let destination2 = format!("{upload}/{data_dir}/part.2");
|
||||
let mut part2_info = completed.parts[0].clone();
|
||||
part2_info.number = 2;
|
||||
let part2_meta = bytes::Bytes::from(part2_info.marshal_msg().expect("part 2 meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part 2");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part 2 transaction");
|
||||
assert!(!disk.has_clean_active_multipart_source(&upload, false));
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("replayed part 1 commit settlement");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"replayed settlement for part 1 must not clear part 2's pending transaction"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part 2");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination2, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part 2 transaction");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
// Deleting {upload}/xl.meta through delete_paths must invalidate the canonical {upload} cache key.
|
||||
disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[format!("{upload}/xl.meta")])
|
||||
.await
|
||||
.expect("delete xl.meta via delete_paths");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"per-file head deletion must invalidate the canonical upload cache key"
|
||||
);
|
||||
assert!(matches!(
|
||||
disk.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_source_cache_publication_holds_source_lock_against_concurrent_retirement() {
|
||||
use crate::disk::{DiskAPI as _, disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks};
|
||||
|
||||
let (_root, disk, _, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let disk = Arc::new(disk);
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let upload = format!("{}/{}", "d".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
let source_dir = disk
|
||||
.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &upload)
|
||||
.expect("source dir");
|
||||
let base_dir = disk
|
||||
.io_get_bucket_path(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.expect("multipart bucket path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = hooks::install_at(hooks::Stage::DirFsync, &source_dir, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
|
||||
let init_wrapper = wrapper.clone();
|
||||
let init_upload = upload.clone();
|
||||
let init_task = tokio::spawn(async move {
|
||||
init_wrapper
|
||||
.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &init_upload, init_fi)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("initial write_metadata reaches parent dir fsync")
|
||||
.expect("entered channel");
|
||||
|
||||
let retire_wrapper = wrapper.clone();
|
||||
let retire_path = format!("{upload}/xl.meta");
|
||||
let mut retire_fut = Box::pin(async move {
|
||||
retire_wrapper
|
||||
.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[retire_path])
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
futures::poll!(tokio::task::unconstrained(retire_fut.as_mut())).is_pending(),
|
||||
"concurrent retirement must block on SourceRW while initial head publication is in flight"
|
||||
);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
init_task.await.expect("init join").expect("initial write_metadata succeeds");
|
||||
retire_fut
|
||||
.await
|
||||
.expect("concurrent retirement completes after SourceRW release");
|
||||
|
||||
assert!(
|
||||
!super::super::multipart_source_state::has_active_multipart_source_head(&source_dir),
|
||||
"retired source must not remain active in cache after interleaved initial publication"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
super::super::multipart_source_state::validate_source_mutation(
|
||||
&source_dir,
|
||||
&base_dir,
|
||||
&disk.publication_root,
|
||||
false
|
||||
),
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"validate_source_mutation must reject non-initial mutation on retired source"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,8 +106,6 @@ pub(crate) struct RenameDataGuards {
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
pub(crate) metadata_reservation_token: Option<Uuid>,
|
||||
}
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
@@ -124,13 +122,6 @@ impl RenameDataObservation {
|
||||
}
|
||||
}
|
||||
|
||||
fn not_dispatched(error: DiskError) -> Self {
|
||||
Self {
|
||||
result: Err(error),
|
||||
preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rejected_before_publication(&self) -> bool {
|
||||
self.result.is_err() && self.preflight_rejection.is_some()
|
||||
}
|
||||
@@ -320,60 +311,6 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// Prepare an inert single-use reservation on this disk.
|
||||
/// A lost remote response holds its bucket publication fence until expiration, but does not lock the object.
|
||||
pub async fn prepare_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.prepare_object_metadata(bucket, object),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token in object mutation order.
|
||||
pub async fn activate_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.activate_object_metadata(bucket, object, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. False does not prove that a claim has
|
||||
/// not already started a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
match self {
|
||||
Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)),
|
||||
Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Physical ownership belongs to the node hosting the local attachment.
|
||||
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
|
||||
match self {
|
||||
@@ -464,6 +401,13 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
match self {
|
||||
Self::Local(disk) => disk.has_clean_active_multipart_source(upload_or_part_path, part_path),
|
||||
Self::Remote(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1307,33 +1251,18 @@ impl Disk {
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => {
|
||||
if guards.metadata_reservation.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
if let Some(token) = guards.metadata_reservation_token {
|
||||
if guards.scanner_publication_lease_token.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
return RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+228
-388
@@ -22,10 +22,8 @@ use parking_lot::Mutex;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
future::Future,
|
||||
io,
|
||||
path::{Component, Path, PathBuf},
|
||||
pin::Pin,
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
@@ -1670,7 +1668,7 @@ fn clear_file_fdatasync_group_commit_for_test() {
|
||||
|
||||
// Small object directories are cheaper to flush in one blocking task. Multipart
|
||||
// directories fan out only once enough files can amortize per-task scheduling.
|
||||
const PARALLEL_FILE_SYNC_THRESHOLD: usize = 16;
|
||||
pub(crate) const PARALLEL_FILE_SYNC_THRESHOLD: usize = 16;
|
||||
pub(crate) const MAX_PARALLEL_FILE_SYNCS: usize = 16;
|
||||
// Scale aggregate fan-out for wider nodes while reserving at least half of the
|
||||
// configured Tokio blocking pool for unrelated filesystem work.
|
||||
@@ -1784,10 +1782,13 @@ pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
|
||||
limiter
|
||||
}
|
||||
|
||||
// Keep one warm draining worker per disk/lane and spin briefly in user space
|
||||
// Keep warm draining workers per disk/lane and spin briefly in user space
|
||||
// before exiting `spawn_blocking` so back-to-back stages of `PUT` and `DELETE`
|
||||
// on the same drive do not thrash Tokio's `BlockingPool` (`futex_wake`/`futex_wait`).
|
||||
// on the same drive do not thrash Tokio's `BlockingPool` (`futex_wake`/`futex_wait`),
|
||||
// while allowing concurrent fsync callers to enter the kernel simultaneously and
|
||||
// coalesce in the filesystem journal (`jbd2`/`xfs` group commit).
|
||||
const MAX_NAMESPACE_WORKERS_PER_DISK: usize = 1;
|
||||
pub(crate) const MAX_FSYNC_WORKERS_PER_DISK: usize = 8;
|
||||
// Return busy workers to the shared pool so other disks can acquire a slot.
|
||||
const MAX_BLOCKING_JOBS_PER_WORKER: usize = 64;
|
||||
const WORKER_IDLE_SPIN_ITERS: usize = 256;
|
||||
@@ -1815,6 +1816,7 @@ impl DiskBlockingLane {
|
||||
struct DiskBlockingQueueState {
|
||||
jobs: VecDeque<Box<dyn FnOnce() + Send>>,
|
||||
active_workers: usize,
|
||||
idle_workers: usize,
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
paused_workers: usize,
|
||||
}
|
||||
@@ -1858,6 +1860,7 @@ impl Drop for DiskBlockingWorkerGuard {
|
||||
let cancelled = {
|
||||
let mut state = self.queue.lane(self.lane).state.lock();
|
||||
state.active_workers = state.active_workers.saturating_sub(1);
|
||||
state.idle_workers = state.idle_workers.saturating_sub(1);
|
||||
if state.active_workers == 0 {
|
||||
self.queue
|
||||
.lane(self.lane)
|
||||
@@ -1876,9 +1879,10 @@ impl Drop for DiskBlockingWorkerGuard {
|
||||
let should_respawn = {
|
||||
let mut state = self.queue.lane(self.lane).state.lock();
|
||||
state.active_workers = state.active_workers.saturating_sub(1);
|
||||
let max_workers = self.queue.effective_max_workers(&state);
|
||||
if !state.jobs.is_empty() && state.active_workers < max_workers {
|
||||
let max_workers = self.queue.effective_max_workers(self.lane, &state);
|
||||
if state.jobs.len() > state.idle_workers && state.active_workers < max_workers {
|
||||
state.active_workers += 1;
|
||||
state.idle_workers += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -1903,15 +1907,23 @@ impl DiskBlockingQueue {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn effective_max_workers(&self, state: &DiskBlockingQueueState) -> usize {
|
||||
fn lane_max_workers(&self, lane: DiskBlockingLane) -> usize {
|
||||
match lane {
|
||||
DiskBlockingLane::Foreground | DiskBlockingLane::Isolated => MAX_NAMESPACE_WORKERS_PER_DISK,
|
||||
DiskBlockingLane::Fsync => MAX_FSYNC_WORKERS_PER_DISK,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn effective_max_workers(&self, lane: DiskBlockingLane, state: &DiskBlockingQueueState) -> usize {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
{
|
||||
MAX_NAMESPACE_WORKERS_PER_DISK.saturating_add(state.paused_workers)
|
||||
self.lane_max_workers(lane).saturating_add(state.paused_workers)
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
{
|
||||
let _ = state;
|
||||
MAX_NAMESPACE_WORKERS_PER_DISK
|
||||
self.lane_max_workers(lane)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1920,8 +1932,9 @@ impl DiskBlockingQueue {
|
||||
let mut state = self.lane(lane).state.lock();
|
||||
state.jobs.push_back(job);
|
||||
self.lane(lane).ready.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
if state.active_workers < self.effective_max_workers(&state) {
|
||||
if state.jobs.len() > state.idle_workers && state.active_workers < self.effective_max_workers(lane, &state) {
|
||||
state.active_workers += 1;
|
||||
state.idle_workers += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -1964,13 +1977,19 @@ impl DiskBlockingQueue {
|
||||
CURRENT_DISK_BLOCKING_QUEUE.with(|slot| {
|
||||
*slot.borrow_mut() = Some((Arc::clone(&queue), lane));
|
||||
});
|
||||
guard.state = DiskBlockingWorkerState::Running;
|
||||
let mut remaining_jobs = MAX_BLOCKING_JOBS_PER_WORKER;
|
||||
let mut idle_polled = false;
|
||||
let mut counted_idle = true;
|
||||
loop {
|
||||
let job = {
|
||||
let mut state = queue.lane(lane).state.lock();
|
||||
if state.active_workers > queue.effective_max_workers(&state) || (idle_polled && state.jobs.is_empty()) {
|
||||
if counted_idle {
|
||||
state.idle_workers = state.idle_workers.saturating_sub(1);
|
||||
counted_idle = false;
|
||||
}
|
||||
guard.state = DiskBlockingWorkerState::Running;
|
||||
if state.active_workers > queue.effective_max_workers(lane, &state) || (idle_polled && state.jobs.is_empty())
|
||||
{
|
||||
state.active_workers = state.active_workers.saturating_sub(1);
|
||||
guard.state = DiskBlockingWorkerState::Retired;
|
||||
return;
|
||||
@@ -1980,6 +1999,15 @@ impl DiskBlockingQueue {
|
||||
.lane(lane)
|
||||
.ready
|
||||
.store(!state.jobs.is_empty(), std::sync::atomic::Ordering::Relaxed);
|
||||
if job.is_none() {
|
||||
if state.idle_workers > 0 {
|
||||
state.active_workers = state.active_workers.saturating_sub(1);
|
||||
guard.state = DiskBlockingWorkerState::Retired;
|
||||
return;
|
||||
}
|
||||
state.idle_workers += 1;
|
||||
counted_idle = true;
|
||||
}
|
||||
job
|
||||
};
|
||||
if let Some(job) = job {
|
||||
@@ -1990,15 +2018,11 @@ impl DiskBlockingQueue {
|
||||
}
|
||||
idle_polled = false;
|
||||
} else {
|
||||
for i in 0..WORKER_IDLE_SPIN_ITERS {
|
||||
for _ in 0..WORKER_IDLE_SPIN_ITERS {
|
||||
if queue.lane(lane).ready.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
if i & 15 == 15 {
|
||||
std::thread::yield_now();
|
||||
} else {
|
||||
std::hint::spin_loop();
|
||||
}
|
||||
std::hint::spin_loop();
|
||||
}
|
||||
// Recheck and retire under the same mutex used by enqueue.
|
||||
idle_polled = true;
|
||||
@@ -2108,8 +2132,9 @@ pub(crate) fn enter_blocking_test_hook() -> BlockingTestHookGuard {
|
||||
let should_spawn = {
|
||||
let mut state = queue.lane(lane).state.lock();
|
||||
state.paused_workers = state.paused_workers.saturating_add(1);
|
||||
if !state.jobs.is_empty() && state.active_workers < queue.effective_max_workers(&state) {
|
||||
if state.jobs.len() > state.idle_workers && state.active_workers < queue.effective_max_workers(lane, &state) {
|
||||
state.active_workers += 1;
|
||||
state.idle_workers += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -2204,326 +2229,6 @@ pub(crate) struct MetadataMutationLease {
|
||||
_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
pub(crate) trait MetadataReservationFence: Send + Sync {
|
||||
fn is_valid(&self) -> bool;
|
||||
}
|
||||
|
||||
/// Single-use ownership of the metadata lock for an exact physical object.
|
||||
pub(crate) struct MetadataMutationReservation {
|
||||
key: PathBuf,
|
||||
disk_instance: uuid::Uuid,
|
||||
lease: MetadataReservationLease,
|
||||
capacity: Option<OwnedSemaphorePermit>,
|
||||
fence: Option<Arc<dyn MetadataReservationFence>>,
|
||||
}
|
||||
|
||||
enum MetadataReservationLease {
|
||||
Held(Arc<MetadataMutationLease>),
|
||||
Queued(Pin<Box<dyn Future<Output = Arc<MetadataMutationLease>> + Send>>),
|
||||
}
|
||||
|
||||
impl MetadataMutationReservation {
|
||||
pub(crate) fn fence(&self) -> Option<Arc<dyn MetadataReservationFence>> {
|
||||
self.fence.clone()
|
||||
}
|
||||
|
||||
/// Wait for a previously enqueued lease without holding an I/O permit.
|
||||
/// Dropping this future removes an unclaimed waiter from the FIFO queue.
|
||||
pub(crate) async fn wait(self) -> Self {
|
||||
let Self {
|
||||
key,
|
||||
disk_instance,
|
||||
lease,
|
||||
capacity,
|
||||
fence,
|
||||
} = self;
|
||||
let lease = match lease {
|
||||
MetadataReservationLease::Held(lease) => lease,
|
||||
MetadataReservationLease::Queued(waiter) => waiter.await,
|
||||
};
|
||||
Self {
|
||||
key,
|
||||
disk_instance,
|
||||
lease: MetadataReservationLease::Held(lease),
|
||||
capacity,
|
||||
fence,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn into_lease_for(self, key: &Path, disk_instance: uuid::Uuid) -> Result<Arc<dyn Send + Sync>> {
|
||||
if self.key != key || self.disk_instance != disk_instance {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if self.fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let MetadataReservationLease::Held(lease) = self.lease else {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
};
|
||||
Ok(match self.capacity {
|
||||
Some(capacity) => Arc::new((lease, capacity)),
|
||||
None => lease,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reserve_metadata_mutation_lease(
|
||||
object: &Path,
|
||||
disk_instance: uuid::Uuid,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> MetadataMutationReservation {
|
||||
MetadataMutationReservation {
|
||||
key: object.to_path_buf(),
|
||||
disk_instance,
|
||||
lease: MetadataReservationLease::Held(acquire_metadata_mutation_lease(object, owner).await),
|
||||
capacity: None,
|
||||
fence: None,
|
||||
}
|
||||
}
|
||||
|
||||
struct MetadataReservationEntry {
|
||||
state: MetadataReservationState,
|
||||
expires_at: tokio::time::Instant,
|
||||
}
|
||||
|
||||
enum MetadataReservationState {
|
||||
Prepared {
|
||||
key: PathBuf,
|
||||
disk_instance: uuid::Uuid,
|
||||
capacity: Option<OwnedSemaphorePermit>,
|
||||
owner: Option<Arc<dyn MetadataReservationFence>>,
|
||||
},
|
||||
Queued(MetadataMutationReservation),
|
||||
}
|
||||
|
||||
/// Process-local, bounded one-shot reservations for one disk instance.
|
||||
pub(crate) struct MetadataReservationRegistry {
|
||||
entries: Mutex<HashMap<uuid::Uuid, MetadataReservationEntry>>,
|
||||
capacity: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl MetadataReservationRegistry {
|
||||
pub(crate) fn new(limit: usize) -> Arc<Self> {
|
||||
let registry = Arc::new(Self {
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
capacity: Arc::new(Semaphore::new(limit)),
|
||||
});
|
||||
let weak = Arc::downgrade(®istry);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
let Some(registry) = weak.upgrade() else { break };
|
||||
registry.remove_expired();
|
||||
}
|
||||
});
|
||||
registry
|
||||
}
|
||||
|
||||
pub(crate) async fn reserve(
|
||||
&self,
|
||||
key: &Path,
|
||||
disk_instance: uuid::Uuid,
|
||||
acquire_timeout: Duration,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<uuid::Uuid> {
|
||||
let capacity = match Arc::clone(&self.capacity).try_acquire_owned() {
|
||||
Ok(capacity) => capacity,
|
||||
Err(_) => {
|
||||
self.remove_expired();
|
||||
Arc::clone(&self.capacity)
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| DiskError::DiskOngoingReq)?
|
||||
}
|
||||
};
|
||||
let mut reservation = tokio::time::timeout(acquire_timeout, reserve_metadata_mutation_lease(key, disk_instance, owner))
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?;
|
||||
reservation.capacity = Some(capacity);
|
||||
let token = uuid::Uuid::new_v4();
|
||||
let mut entries = self.entries.lock();
|
||||
match entries.entry(token) {
|
||||
std::collections::hash_map::Entry::Vacant(entry) => entry.insert(MetadataReservationEntry {
|
||||
state: MetadataReservationState::Queued(reservation),
|
||||
expires_at: tokio::time::Instant::now() + Duration::from_secs(60),
|
||||
}),
|
||||
std::collections::hash_map::Entry::Occupied(_) => return Err(DiskError::Unexpected),
|
||||
};
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// Register this mutation in FIFO order before returning its token.
|
||||
/// The mutex may still be held by an earlier operation when this returns.
|
||||
pub(crate) async fn enqueue(&self, key: &Path, disk_instance: uuid::Uuid) -> Result<uuid::Uuid> {
|
||||
let capacity = match Arc::clone(&self.capacity).try_acquire_owned() {
|
||||
Ok(capacity) => capacity,
|
||||
Err(_) => {
|
||||
self.remove_expired();
|
||||
Arc::clone(&self.capacity)
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| DiskError::DiskOngoingReq)?
|
||||
}
|
||||
};
|
||||
let object = key.to_path_buf();
|
||||
let mut waiter: Pin<Box<dyn Future<Output = Arc<MetadataMutationLease>> + Send>> =
|
||||
Box::pin(async move { acquire_metadata_mutation_lease(&object, None).await });
|
||||
// Poll once so Tokio registers this waiter before the token can be
|
||||
// acknowledged; retaining the future retains its position in the queue.
|
||||
let lease = match futures::poll!(waiter.as_mut()) {
|
||||
std::task::Poll::Ready(lease) => MetadataReservationLease::Held(lease),
|
||||
std::task::Poll::Pending => MetadataReservationLease::Queued(waiter),
|
||||
};
|
||||
let token = uuid::Uuid::new_v4();
|
||||
let reservation = MetadataMutationReservation {
|
||||
key: key.to_path_buf(),
|
||||
disk_instance,
|
||||
lease,
|
||||
capacity: Some(capacity),
|
||||
fence: None,
|
||||
};
|
||||
let mut entries = self.entries.lock();
|
||||
match entries.entry(token) {
|
||||
std::collections::hash_map::Entry::Vacant(entry) => entry.insert(MetadataReservationEntry {
|
||||
state: MetadataReservationState::Queued(reservation),
|
||||
expires_at: tokio::time::Instant::now() + Duration::from_secs(60),
|
||||
}),
|
||||
std::collections::hash_map::Entry::Occupied(_) => return Err(DiskError::Unexpected),
|
||||
};
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// Allocate an inert token before a client asks to enter the FIFO queue.
|
||||
/// A lost preparation response cannot hold an object metadata lock.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn prepare(&self, key: &Path, disk_instance: uuid::Uuid) -> Result<uuid::Uuid> {
|
||||
self.prepare_with_owner(key, disk_instance, None)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_with_owner(
|
||||
&self,
|
||||
key: &Path,
|
||||
disk_instance: uuid::Uuid,
|
||||
owner: Option<Arc<dyn MetadataReservationFence>>,
|
||||
) -> Result<uuid::Uuid> {
|
||||
let capacity = match Arc::clone(&self.capacity).try_acquire_owned() {
|
||||
Ok(capacity) => capacity,
|
||||
Err(_) => {
|
||||
self.remove_expired();
|
||||
Arc::clone(&self.capacity)
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| DiskError::DiskOngoingReq)?
|
||||
}
|
||||
};
|
||||
let token = uuid::Uuid::new_v4();
|
||||
let entry = MetadataReservationEntry {
|
||||
state: MetadataReservationState::Prepared {
|
||||
key: key.to_path_buf(),
|
||||
disk_instance,
|
||||
capacity: Some(capacity),
|
||||
owner,
|
||||
},
|
||||
expires_at: tokio::time::Instant::now() + Duration::from_secs(60),
|
||||
};
|
||||
match self.entries.lock().entry(token) {
|
||||
std::collections::hash_map::Entry::Vacant(slot) => {
|
||||
slot.insert(entry);
|
||||
Ok(token)
|
||||
}
|
||||
std::collections::hash_map::Entry::Occupied(_) => Err(DiskError::Unexpected),
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomically enqueue a prepared token. Cancellation and enqueue serialize
|
||||
/// on the registry, so a delayed enqueue cannot revive a cancelled token.
|
||||
pub(crate) async fn activate(&self, token: uuid::Uuid, key: &Path, disk_instance: uuid::Uuid) -> Result<()> {
|
||||
std::future::poll_fn(|cx| {
|
||||
let mut entries = self.entries.lock();
|
||||
let Some(entry) = entries.get_mut(&token) else {
|
||||
return std::task::Poll::Ready(Err(DiskError::DiskOngoingReq));
|
||||
};
|
||||
if entry.expires_at <= tokio::time::Instant::now() {
|
||||
return std::task::Poll::Ready(Err(DiskError::Timeout));
|
||||
}
|
||||
let (stored_key, stored_instance, capacity, owner) = match &mut entry.state {
|
||||
MetadataReservationState::Prepared {
|
||||
key,
|
||||
disk_instance,
|
||||
capacity,
|
||||
owner,
|
||||
} => (key, disk_instance, capacity, owner),
|
||||
MetadataReservationState::Queued(reservation) => {
|
||||
return std::task::Poll::Ready(if reservation.key != key || reservation.disk_instance != disk_instance {
|
||||
Err(DiskError::InvalidPath)
|
||||
} else if reservation.fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
Err(DiskError::DiskOngoingReq)
|
||||
} else {
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
};
|
||||
if stored_key != key || *stored_instance != disk_instance {
|
||||
return std::task::Poll::Ready(Err(DiskError::InvalidPath));
|
||||
}
|
||||
if owner.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
return std::task::Poll::Ready(Err(DiskError::DiskOngoingReq));
|
||||
}
|
||||
let Some(capacity) = capacity.take() else {
|
||||
return std::task::Poll::Ready(Err(DiskError::Unexpected));
|
||||
};
|
||||
let object = stored_key.clone();
|
||||
let fence = owner.take();
|
||||
let owner: Option<Arc<dyn Send + Sync>> = fence.clone().map(|fence| fence as Arc<dyn Send + Sync>);
|
||||
let mut waiter: Pin<Box<dyn Future<Output = Arc<MetadataMutationLease>> + Send>> =
|
||||
Box::pin(async move { acquire_metadata_mutation_lease(&object, owner).await });
|
||||
let lease = match waiter.as_mut().poll(cx) {
|
||||
std::task::Poll::Ready(lease) => MetadataReservationLease::Held(lease),
|
||||
std::task::Poll::Pending => MetadataReservationLease::Queued(waiter),
|
||||
};
|
||||
entry.state = MetadataReservationState::Queued(MetadataMutationReservation {
|
||||
key: stored_key.clone(),
|
||||
disk_instance,
|
||||
lease,
|
||||
capacity: Some(capacity),
|
||||
fence,
|
||||
});
|
||||
std::task::Poll::Ready(Ok(()))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn claim(&self, token: uuid::Uuid, key: &Path, disk_instance: uuid::Uuid) -> Result<MetadataMutationReservation> {
|
||||
let entry = self.entries.lock().remove(&token).ok_or(DiskError::DiskOngoingReq)?;
|
||||
if entry.expires_at <= tokio::time::Instant::now() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
let MetadataReservationState::Queued(reservation) = entry.state else {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
};
|
||||
if reservation.key != key || reservation.disk_instance != disk_instance {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if reservation.fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
Ok(reservation)
|
||||
}
|
||||
|
||||
pub(crate) fn cancel(&self, token: uuid::Uuid) -> bool {
|
||||
let removed = self.entries.lock().remove(&token);
|
||||
removed.is_some()
|
||||
}
|
||||
|
||||
fn remove_expired(&self) {
|
||||
let now = tokio::time::Instant::now();
|
||||
let expired: Vec<_> = self
|
||||
.entries
|
||||
.lock()
|
||||
.extract_if(|_, entry| entry.expires_at <= now)
|
||||
.map(|(_, entry)| entry)
|
||||
.collect();
|
||||
drop(expired);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_metadata_mutation_lease(
|
||||
object: &Path,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
@@ -2891,7 +2596,7 @@ fn sync_files(paths: &[PathBuf]) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
|
||||
pub(crate) fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
|
||||
let mut files = Vec::with_capacity(PARALLEL_FILE_SYNC_THRESHOLD);
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
@@ -4064,16 +3769,21 @@ impl PreparedRenameSource {
|
||||
#[cfg(not(windows))]
|
||||
std::io::Write::write_all(&mut self.source, data)?;
|
||||
if sync {
|
||||
#[cfg(test)]
|
||||
multipart_source_syscall_failures::check(multipart_source_syscall_failures::Stage::FileSync, &self.test_destination)?;
|
||||
let _sync = StageTimer::new(true, "disk_prepared_metadata_fdatasync");
|
||||
#[cfg(windows)]
|
||||
self.source.as_file().sync_data()?;
|
||||
#[cfg(not(windows))]
|
||||
self.source.sync_data()?;
|
||||
self.sync_data()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn sync_data(&self) -> io::Result<()> {
|
||||
#[cfg(test)]
|
||||
multipart_source_syscall_failures::check(multipart_source_syscall_failures::Stage::FileSync, &self.test_destination)?;
|
||||
let _sync = StageTimer::new(true, "disk_prepared_metadata_fdatasync");
|
||||
#[cfg(windows)]
|
||||
self.source.as_file().sync_data()?;
|
||||
#[cfg(not(windows))]
|
||||
self.source.sync_data()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn create_prepared_rename_source_with_commit_guard(
|
||||
@@ -5735,6 +5445,34 @@ impl RenameCommitGuard {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) fn multipart_payload_commit_guard_std(
|
||||
metadata_guard: &RenameCommitGuard,
|
||||
source_parent: &Path,
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
let original = &metadata_guard.directories;
|
||||
original.validate()?;
|
||||
let source = LinuxCommitDirectory::new(original.root.open_directory(source_parent)?)?;
|
||||
let destination = original.prepare_destination()?;
|
||||
// Clone the pinned file identity, never recapture the target pathname
|
||||
// between payload and canonical-head publication.
|
||||
let destination = LinuxCommitDirectory {
|
||||
file: Arc::clone(&destination.file),
|
||||
device: destination.device,
|
||||
inode: destination.inode,
|
||||
};
|
||||
Ok(RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
root: original.root.clone(),
|
||||
source_path: source_parent.to_path_buf(),
|
||||
destination_path: original.destination_path.clone(),
|
||||
source,
|
||||
destination: std::sync::OnceLock::from(destination),
|
||||
destination_base: original.destination_base.clone(),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// Native MPU metadata and payload have distinct source parents. Reuse the
|
||||
/// exact target identity already held by the metadata transaction, while
|
||||
/// independently pinning the private payload source under the same root.
|
||||
@@ -5746,30 +5484,7 @@ pub(crate) async fn multipart_payload_commit_guard(
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
let metadata_guard = metadata_guard.clone();
|
||||
let source_parent = source_parent.to_path_buf();
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
let original = &metadata_guard.directories;
|
||||
original.validate()?;
|
||||
let source = LinuxCommitDirectory::new(original.root.open_directory(&source_parent)?)?;
|
||||
let destination = original.prepare_destination()?;
|
||||
// Clone the pinned file identity, never recapture the target pathname
|
||||
// between payload and canonical-head publication.
|
||||
let destination = LinuxCommitDirectory {
|
||||
file: Arc::clone(&destination.file),
|
||||
device: destination.device,
|
||||
inode: destination.inode,
|
||||
};
|
||||
Ok(RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
root: original.root.clone(),
|
||||
source_path: source_parent,
|
||||
destination_path: original.destination_path.clone(),
|
||||
source,
|
||||
destination: std::sync::OnceLock::from(destination),
|
||||
destination_base: original.destination_base.clone(),
|
||||
}),
|
||||
})
|
||||
})
|
||||
.await
|
||||
run_blocking_namespace_operation(lease, move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent)).await
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_rename_commit_guard(
|
||||
@@ -6951,6 +6666,17 @@ pub(crate) fn sync_multipart_owned_record_parents(
|
||||
include_leaf: bool,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<()> {
|
||||
sync_multipart_owned_record_parents_options(source, record_parent, include_leaf, true, base, publication_root)
|
||||
}
|
||||
|
||||
pub(crate) fn sync_multipart_owned_record_parents_options(
|
||||
source: &Path,
|
||||
record_parent: &Path,
|
||||
include_leaf: bool,
|
||||
sync_source_root: bool,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<()> {
|
||||
let relative = record_parent
|
||||
.strip_prefix(source)
|
||||
@@ -6974,6 +6700,9 @@ pub(crate) fn sync_multipart_owned_record_parents(
|
||||
if !directory.starts_with(source) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "native record barrier escaped source"));
|
||||
}
|
||||
if directory == source && !sync_source_root {
|
||||
return Ok(());
|
||||
}
|
||||
#[cfg_attr(not(unix), allow(unused_mut))]
|
||||
let mut pinned = pin_multipart_source_directory(directory, base, publication_root)?;
|
||||
#[cfg(unix)]
|
||||
@@ -11714,19 +11443,24 @@ mod tests {
|
||||
let queue = register_disk_blocking_queue(root.path(), root.path());
|
||||
std::fs::write(root.path().join("intermediate"), b"backup").expect("backup");
|
||||
std::fs::write(root.path().join("receipt"), b"committed").expect("receipt");
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let blocker_queue = Arc::clone(&queue);
|
||||
let blocker = tokio::spawn(async move {
|
||||
blocker_queue
|
||||
.run_fsync_cancellable(move |_| {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.blocking_recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
started_rx.await.expect("fsync lane blocked");
|
||||
let mut release_txs = Vec::new();
|
||||
let mut blockers = Vec::new();
|
||||
for _ in 0..MAX_FSYNC_WORKERS_PER_DISK {
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let blocker_queue = Arc::clone(&queue);
|
||||
blockers.push(tokio::spawn(async move {
|
||||
blocker_queue
|
||||
.run_fsync_cancellable(move |_| {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.blocking_recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}));
|
||||
started_rx.await.expect("fsync worker blocked");
|
||||
release_txs.push(release_tx);
|
||||
}
|
||||
let mut cleanup = std::pin::pin!(remove_files_and_fsync_with_owner(
|
||||
root.path(),
|
||||
&["intermediate"],
|
||||
@@ -11737,8 +11471,12 @@ mod tests {
|
||||
assert!(futures::FutureExt::now_or_never(&mut cleanup).is_none());
|
||||
let foreground = tokio::time::timeout(Duration::from_secs(5), queue.run(|| Ok(()))).await;
|
||||
let retained = root.path().join("receipt").exists();
|
||||
release_tx.send(()).expect("release fsync lane");
|
||||
blocker.await.expect("blocker joined").expect("blocker completed");
|
||||
for release_tx in release_txs {
|
||||
release_tx.send(()).expect("release fsync worker");
|
||||
}
|
||||
for blocker in blockers {
|
||||
blocker.await.expect("blocker joined").expect("blocker completed");
|
||||
}
|
||||
cleanup.await.expect("cleanup completed");
|
||||
foreground.expect("foreground must progress").expect("foreground operation");
|
||||
assert_eq!(retained, sync, "only durable cleanup must wait for the fsync lane");
|
||||
@@ -11960,6 +11698,25 @@ mod tests {
|
||||
|
||||
for lane in [DiskBlockingLane::Foreground, DiskBlockingLane::Fsync] {
|
||||
let queue = DiskBlockingQueue::new();
|
||||
let max_workers = queue.lane_max_workers(lane);
|
||||
let mut hold_txs = Vec::new();
|
||||
let mut hold_tasks = Vec::new();
|
||||
for _ in 1..max_workers {
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let hold_queue = Arc::clone(&queue);
|
||||
hold_tasks.push(tokio::spawn(async move {
|
||||
hold_queue
|
||||
.run_on_lane(lane, "hold operation failed", DiskBlockingQueue::test_stages_for_lane(lane), move |_| {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.blocking_recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}));
|
||||
started_rx.await.expect("hold worker started");
|
||||
hold_txs.push(release_tx);
|
||||
}
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
|
||||
let (first_started_tx, first_started_rx) = oneshot::channel();
|
||||
let (first_release_tx, first_release_rx) = oneshot::channel();
|
||||
@@ -11993,10 +11750,91 @@ mod tests {
|
||||
let retained = gate.snapshot().active;
|
||||
second_release_tx.send(()).expect("release second producer");
|
||||
second.await.expect("second producer completed");
|
||||
for tx in hold_txs {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
for task in hold_tasks {
|
||||
task.await.expect("hold task joined").expect("hold task completed");
|
||||
}
|
||||
assert_eq!(retained, 0, "shared worker must not retain a completed producer's unit");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn disk_blocking_queue_fsync_lane_scales_workers_and_retires_idle() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let queue = DiskBlockingQueue::new();
|
||||
let active = Arc::new(AtomicUsize::new(0));
|
||||
let max_seen = Arc::new(AtomicUsize::new(0));
|
||||
let entered = Arc::new(AtomicUsize::new(0));
|
||||
let (all_entered_tx, all_entered_rx) = oneshot::channel();
|
||||
let all_entered_tx = Arc::new(Mutex::new(Some(all_entered_tx)));
|
||||
let (release_tx, _) = tokio::sync::broadcast::channel::<()>(1);
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for i in 0..(MAX_FSYNC_WORKERS_PER_DISK + 4) {
|
||||
let q = Arc::clone(&queue);
|
||||
let active = Arc::clone(&active);
|
||||
let max_seen = Arc::clone(&max_seen);
|
||||
let entered = Arc::clone(&entered);
|
||||
let all_entered_tx = Arc::clone(&all_entered_tx);
|
||||
let mut release_rx = release_tx.subscribe();
|
||||
handles.push(tokio::spawn(async move {
|
||||
use_current_runtime_for_fsync();
|
||||
q.run_on_lane(
|
||||
DiskBlockingLane::Fsync,
|
||||
"fsync scale test failed",
|
||||
DiskBlockingQueue::test_stages_for_lane(DiskBlockingLane::Fsync),
|
||||
move |_| {
|
||||
let cur = active.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
max_seen.fetch_max(cur, Ordering::SeqCst);
|
||||
let ord = entered.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if ord == MAX_FSYNC_WORKERS_PER_DISK
|
||||
&& let Some(tx) = all_entered_tx.lock().expect("lock completion sender").take()
|
||||
{
|
||||
let _ = tx.send(());
|
||||
}
|
||||
if ord <= MAX_FSYNC_WORKERS_PER_DISK {
|
||||
let _ = release_rx.blocking_recv();
|
||||
}
|
||||
active.fetch_sub(1, Ordering::SeqCst);
|
||||
Ok(i)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}));
|
||||
}
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), all_entered_rx)
|
||||
.await
|
||||
.expect("all fsync workers should enter within timeout")
|
||||
.expect("all_entered channel");
|
||||
let _ = release_tx.send(());
|
||||
for (i, handle) in handles.into_iter().enumerate() {
|
||||
assert_eq!(handle.await.expect("task join").expect("fsync job"), i);
|
||||
}
|
||||
assert_eq!(
|
||||
max_seen.load(Ordering::SeqCst),
|
||||
MAX_FSYNC_WORKERS_PER_DISK,
|
||||
"fsync lane must run MAX_FSYNC_WORKERS_PER_DISK concurrent workers up to its configured bound"
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
loop {
|
||||
{
|
||||
let state = queue.lane(DiskBlockingLane::Fsync).state.lock();
|
||||
if state.active_workers == 0 && state.idle_workers == 0 {
|
||||
assert!(state.jobs.is_empty());
|
||||
break;
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all fsync workers must retire with zero idle count");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn disk_blocking_queue_bounds_concurrency_and_recovers_from_panic() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
@@ -12004,6 +11842,7 @@ mod tests {
|
||||
let queue = DiskBlockingQueue::new();
|
||||
let active = Arc::new(AtomicUsize::new(0));
|
||||
let max_seen = Arc::new(AtomicUsize::new(0));
|
||||
let entered = Arc::new(AtomicUsize::new(0));
|
||||
let barrier = Arc::new(std::sync::Barrier::new(MAX_NAMESPACE_WORKERS_PER_DISK + 1));
|
||||
|
||||
let mut handles = Vec::new();
|
||||
@@ -12011,12 +11850,13 @@ mod tests {
|
||||
let q = Arc::clone(&queue);
|
||||
let active = Arc::clone(&active);
|
||||
let max_seen = Arc::clone(&max_seen);
|
||||
let entered = Arc::clone(&entered);
|
||||
let barrier = Arc::clone(&barrier);
|
||||
handles.push(tokio::spawn(async move {
|
||||
q.run(move || {
|
||||
let cur = active.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
max_seen.fetch_max(cur, Ordering::SeqCst);
|
||||
if i < MAX_NAMESPACE_WORKERS_PER_DISK {
|
||||
if entered.fetch_add(1, Ordering::SeqCst) < MAX_NAMESPACE_WORKERS_PER_DISK {
|
||||
barrier.wait();
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
|
||||
@@ -333,6 +333,34 @@ struct ProofCache {
|
||||
preferred: usize,
|
||||
}
|
||||
|
||||
type VerifiedStripeKey = (uuid::Uuid, [u8; 32], u32);
|
||||
|
||||
static VERIFIED_STRIPE_RECORDS: std::sync::LazyLock<parking_lot::Mutex<std::collections::HashMap<VerifiedStripeKey, Bytes>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
fn get_verified_stripe_record(part: &PartIntegrity, stripe: u32) -> Option<Bytes> {
|
||||
VERIFIED_STRIPE_RECORDS
|
||||
.lock()
|
||||
.get(&(part.generation, part.root, stripe))
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn put_verified_stripe_record(part: &PartIntegrity, stripe: u32, record: Bytes) {
|
||||
let mut map = VERIFIED_STRIPE_RECORDS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().copied()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert((part.generation, part.root, stripe), record);
|
||||
}
|
||||
|
||||
pub(crate) fn invalidate_verified_stripe_records(generation: uuid::Uuid) {
|
||||
VERIFIED_STRIPE_RECORDS
|
||||
.lock()
|
||||
.retain(|(stored_gen, _, _), _| *stored_gen != generation);
|
||||
}
|
||||
|
||||
impl PartProofReader {
|
||||
pub(crate) async fn reconstruction_proof(
|
||||
&self,
|
||||
@@ -410,6 +438,9 @@ impl PartProofReader {
|
||||
|
||||
pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
|
||||
let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
|
||||
if let Some(bytes) = get_verified_stripe_record(&self.part, stripe) {
|
||||
return Ok(bytes);
|
||||
}
|
||||
let mut cached = self.cached.lock().await;
|
||||
if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
|
||||
return Ok(bytes.clone());
|
||||
@@ -465,6 +496,9 @@ impl PartProofReader {
|
||||
bytes
|
||||
};
|
||||
self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
|
||||
if self.part.stripes <= 4 {
|
||||
put_verified_stripe_record(&self.part, stripe, record.clone());
|
||||
}
|
||||
if cached.records.len() == 2 {
|
||||
cached.records.pop_front();
|
||||
}
|
||||
@@ -863,4 +897,41 @@ mod tests {
|
||||
prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verified_stripe_record_cache_serves_verified_record_and_still_verifies_payload() {
|
||||
let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
|
||||
let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
|
||||
let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
|
||||
builder.push(shards.into_iter()).await.expect("stripe");
|
||||
let prepared = builder.finish(8).await.expect("index");
|
||||
let inline_bytes = prepared.inline_bytes().expect("inline bytes");
|
||||
let part = prepared.part;
|
||||
|
||||
let warm_reader = PartProofReader {
|
||||
part: part.clone(),
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: Some(inline_bytes),
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
};
|
||||
warm_reader.verify(0, 0, shards[0]).await.expect("warm verify");
|
||||
assert!(
|
||||
get_verified_stripe_record(&part, 0).is_some(),
|
||||
"small verified stripe record should populate the verified stripe cache"
|
||||
);
|
||||
|
||||
let cached_reader = Arc::new(PartProofReader {
|
||||
part,
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: None,
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
});
|
||||
cached_reader.verify(0, 0, shards[0]).await.expect("cached verify");
|
||||
assert!(
|
||||
cached_reader.verify(0, 0, shards[1]).await.is_err(),
|
||||
"cached record must still reject corrupt or mismatched shard payloads"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,20 +160,6 @@ pub struct ObjectLockConfigSnapshot {
|
||||
}
|
||||
|
||||
impl ObjectLockConfigSnapshot {
|
||||
pub(crate) fn owns_lifecycle_guard(&self) -> bool {
|
||||
self._lifecycle_guard.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option<Uuid>) -> bool {
|
||||
self.bucket.as_deref() == Some(bucket)
|
||||
&& incarnation.is_some()
|
||||
&& self.bucket_incarnation_id == incarnation
|
||||
&& self
|
||||
._lifecycle_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released())
|
||||
}
|
||||
|
||||
pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self {
|
||||
Self {
|
||||
store_id: None,
|
||||
|
||||
@@ -2538,7 +2538,6 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
||||
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
|
||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
|
||||
pub(in crate::set_disk) tail_has_metadata_reservations: bool,
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) struct RenameTailCleanup {
|
||||
@@ -3469,8 +3468,6 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: Option<RenameRollbackReceipt>,
|
||||
multipart_proof_authority: Option<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
multipart_commit_owner: Option<Arc<MultipartLogicalGuards>>,
|
||||
reservation_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
prepared_reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
}
|
||||
|
||||
impl<'a> RenameDataFenceOptions<'a> {
|
||||
@@ -3486,8 +3483,6 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: None,
|
||||
multipart_proof_authority: None,
|
||||
multipart_commit_owner: None,
|
||||
reservation_ctx: None,
|
||||
prepared_reservations: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3524,165 +3519,6 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
self.namespace_commit_guard = namespace_commit_guard;
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_reservation_context(
|
||||
mut self,
|
||||
ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
) -> Self {
|
||||
self.reservation_ctx = ctx;
|
||||
self.prepared_reservations = reservations;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameMetadataReservation {
|
||||
disk: DiskStore,
|
||||
token: Option<Uuid>,
|
||||
prepared_at: tokio::time::Instant,
|
||||
}
|
||||
|
||||
impl RenameMetadataReservation {
|
||||
async fn cancel(&mut self) -> disk::error::Result<()> {
|
||||
let Some(token) = self.token else { return Ok(()) };
|
||||
match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await {
|
||||
Ok(Ok(true)) => {
|
||||
self.token = None;
|
||||
Ok(())
|
||||
}
|
||||
Ok(Ok(false)) => Err(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) => Err(error),
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameMetadataReservation {
|
||||
fn drop(&mut self) {
|
||||
let Some(token) = self.token.take() else { return };
|
||||
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
|
||||
let disk = self.disk.clone();
|
||||
runtime.spawn(async move {
|
||||
let _ = disk.cancel_object_metadata_reservation(token).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_rename_metadata_reservations(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
owns_bucket_fence: bool,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
let prepared: Vec<disk::error::Result<Option<RenameMetadataReservation>>> = join_all(disks.iter().map(|disk| async move {
|
||||
let Some(disk) = disk else { return Ok(None) };
|
||||
if disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, disk, None, false).await?;
|
||||
}
|
||||
let prepared_at = tokio::time::Instant::now();
|
||||
let token = if disk.is_local() && owns_bucket_fence {
|
||||
let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx)
|
||||
.await
|
||||
.ok_or(DiskError::DiskOngoingReq)?;
|
||||
store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&disk.endpoint().to_string(),
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5))
|
||||
.await?
|
||||
};
|
||||
// No await separates receipt from installing the cancellation owner.
|
||||
Ok(Some(RenameMetadataReservation {
|
||||
disk: disk.clone(),
|
||||
token: Some(token),
|
||||
prepared_at,
|
||||
}))
|
||||
}))
|
||||
.await;
|
||||
let mut reservations = Vec::with_capacity(prepared.len());
|
||||
let mut failed = false;
|
||||
for result in prepared {
|
||||
match result {
|
||||
Ok(reservation) => reservations.push(reservation),
|
||||
Err(_) => {
|
||||
failed = true;
|
||||
reservations.push(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
// Prepared tokens are inert; dropping them cannot leave a queued writer.
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
pub(crate) fn align_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> Option<Vec<Option<RenameMetadataReservation>>> {
|
||||
if reservations.len() != disks.len() {
|
||||
return None;
|
||||
}
|
||||
let mut aligned = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
let Some(disk) = disk else {
|
||||
aligned.push(None);
|
||||
continue;
|
||||
};
|
||||
let index = reservations
|
||||
.iter()
|
||||
.position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?;
|
||||
aligned.push(reservations[index].take());
|
||||
}
|
||||
// Disks lost while staging need no activation; dropping their still-inert
|
||||
// reservations schedules cancellation without blocking the commit.
|
||||
Some(aligned)
|
||||
}
|
||||
|
||||
async fn activate_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
// Tokens expire after 60 seconds. Before sending any activation, fall
|
||||
// back if a long upload consumed the safety margin for its RPC deadline.
|
||||
if reservations
|
||||
.iter()
|
||||
.flatten()
|
||||
.any(|reservation| reservation.prepared_at.elapsed() >= Duration::from_secs(50))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let activated = join_all(reservations.iter().flatten().map(|reservation| async move {
|
||||
if reservation.disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?;
|
||||
}
|
||||
let token = reservation.token.ok_or(DiskError::Unexpected)?;
|
||||
reservation
|
||||
.disk
|
||||
.activate_object_metadata(bucket, object, token, Duration::from_secs(5))
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
if activated.into_iter().any(|result| result.is_err()) {
|
||||
let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await;
|
||||
if cancelled.into_iter().any(|result| result.is_err()) {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
@@ -3694,7 +3530,7 @@ type RenameDataLegacyTuple = (
|
||||
Option<OldCurrentSize>,
|
||||
);
|
||||
|
||||
pub(crate) fn put_rename_early_ack_enabled() -> bool {
|
||||
fn put_rename_early_ack_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
|
||||
}
|
||||
|
||||
@@ -4060,7 +3896,6 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4157,8 +3992,6 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority: _,
|
||||
multipart_commit_owner: _,
|
||||
reservation_ctx,
|
||||
prepared_reservations,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -4174,11 +4007,6 @@ impl SetDisks {
|
||||
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) {
|
||||
(Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?,
|
||||
(_, _) => None,
|
||||
};
|
||||
let tail_has_metadata_reservations = reservations.is_some();
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
|
||||
let coordinator_disks = fanout_disks.clone();
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
@@ -4211,11 +4039,6 @@ impl SetDisks {
|
||||
let file_info = file_info.clone();
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let reservation = reservations
|
||||
.as_mut()
|
||||
.and_then(|entries| entries.get_mut(i))
|
||||
.and_then(Option::take);
|
||||
let reservation_ctx = reservation_ctx.clone();
|
||||
tasks.spawn(async move {
|
||||
let _namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
@@ -4226,12 +4049,6 @@ impl SetDisks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let disk_admission = if reservation.is_some() && disk.is_local() {
|
||||
let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?;
|
||||
crate::store::admit_local_disk(ctx, &disk, None, true).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let disk_wait_location_stage = if disk.is_local() {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
|
||||
} else {
|
||||
@@ -4257,11 +4074,6 @@ impl SetDisks {
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
dispatch_state = RenameDispatchState::MayHavePublished;
|
||||
let _foreground_active_guard = rename_foreground_active_guard(i);
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> = if disk_admission.is_some() {
|
||||
Some(Arc::new((namespace_commit_guard.clone(), disk_admission)))
|
||||
} else {
|
||||
namespace_commit_guard.clone().map(|owner| owner as Arc<dyn Send + Sync>)
|
||||
};
|
||||
let observed = disk
|
||||
.rename_data_borrowed_with_fence_observed(
|
||||
&src_bucket,
|
||||
@@ -4271,8 +4083,9 @@ impl SetDisks {
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token),
|
||||
namespace_owner,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -4550,7 +4363,6 @@ impl SetDisks {
|
||||
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
|
||||
commit.map(|mut commit| {
|
||||
commit.tail_drain = Some(tail_drain);
|
||||
commit.tail_has_metadata_reservations = tail_has_metadata_reservations;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
|
||||
attach_tail_owner_started,
|
||||
@@ -4620,8 +4432,6 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority,
|
||||
multipart_commit_owner,
|
||||
reservation_ctx: _,
|
||||
prepared_reservations: _,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -5091,7 +4901,6 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5531,6 +5340,14 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref()
|
||||
.is_some_and(|disk| disk.has_clean_active_multipart_source(dst_object, true))
|
||||
})
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let transaction_path = part_transaction_path(dst_object);
|
||||
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
|
||||
|
||||
@@ -5730,6 +5547,14 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref()
|
||||
.is_some_and(|disk| disk.has_clean_active_multipart_source(part_path, false))
|
||||
})
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let listings = join_all(disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
async move {
|
||||
@@ -5937,7 +5762,7 @@ impl SetDisks {
|
||||
let all_prepared_succeeded = errs
|
||||
.iter()
|
||||
.zip(prepared_disks.iter())
|
||||
.all(|(err, disk)| disk.is_none() || err.is_none());
|
||||
.all(|(err, disk)| err.is_none() && disk.as_ref().is_some_and(|d| d.is_local()));
|
||||
let committed = if all_prepared_succeeded {
|
||||
let settles = prepared_disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
|
||||
@@ -161,7 +161,7 @@ impl SetDisks {
|
||||
)
|
||||
});
|
||||
let mut metadata = FileMeta::new();
|
||||
for parts in versions.into_values() {
|
||||
for (version_id, parts) in versions {
|
||||
let errors = parts
|
||||
.iter()
|
||||
.zip(&missing)
|
||||
@@ -173,15 +173,25 @@ impl SetDisks {
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let read_quorum = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((quorum, _)) => usize::try_from(quorum).map_err(|_| DiskError::FileCorrupt)?,
|
||||
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((read, write)) => (
|
||||
usize::try_from(read).map_err(|_| DiskError::FileCorrupt)?,
|
||||
usize::try_from(write).map_err(|_| DiskError::FileCorrupt)?,
|
||||
),
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) if complete => continue,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => return Err(DiskError::ErasureReadQuorum),
|
||||
Err(DiskError::ErasureReadQuorum) if complete => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
let selected = Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?;
|
||||
let selected = if crate::bucket::utils::is_meta_bucketname(bucket) && version_id.is_none() {
|
||||
// A null-version system record must carry the same commit proof
|
||||
// as GET. Listing only needs the selected metadata, so no disk
|
||||
// handles are retained from the shared read selector.
|
||||
Self::select_committed_fileinfo(&vec![None; disk_count], &parts, &errors, "", read_quorum, write_quorum)?.1
|
||||
} else {
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?
|
||||
};
|
||||
metadata.add_version(selected)?;
|
||||
}
|
||||
if metadata.versions.is_empty() {
|
||||
|
||||
@@ -410,10 +410,6 @@ impl ObjectLockDiagGuard {
|
||||
self
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
self.guard.downgrade_committed_tail().await
|
||||
}
|
||||
|
||||
/// Whether the underlying namespace lock's heartbeat has observed a
|
||||
/// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
|
||||
/// point on this so a stale lock holder does not race a double-write.
|
||||
@@ -906,10 +902,6 @@ mod core;
|
||||
pub(crate) use core::io_primitives::disk_call_counters;
|
||||
#[cfg(test)]
|
||||
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
pub(crate) use core::io_primitives::{
|
||||
RenameMetadataReservation, align_rename_metadata_reservations, prepare_rename_metadata_reservations,
|
||||
put_rename_early_ack_enabled,
|
||||
};
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
|
||||
@@ -34,7 +34,6 @@ mod heal_lock_wait_tests;
|
||||
#[derive(Clone, Copy)]
|
||||
struct ExplicitVersionHeal<'a> {
|
||||
opts: &'a HealOpts,
|
||||
staging_guard: Option<&'a Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>>,
|
||||
allow_regeneration: bool,
|
||||
preserve_unrepairable: bool,
|
||||
commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>,
|
||||
@@ -42,58 +41,6 @@ struct ExplicitVersionHeal<'a> {
|
||||
retirement: Option<&'a MarkerRetirementContext<'a>>,
|
||||
}
|
||||
|
||||
async fn restore_heal_exclusive_guard(
|
||||
guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>,
|
||||
) -> disk::error::Result<()> {
|
||||
tokio::time::timeout(get_lock_acquire_timeout(), async {
|
||||
loop {
|
||||
{
|
||||
let mut guard = guard.lock().await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail())
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
// A writer may have waited on a peer that upgraded while
|
||||
// another peer was still waiting for its readers to drain.
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
return if matches!(waiting, Ok(Some(false))) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::Timeout)
|
||||
};
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn wait_for_heal_writer(guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>) {
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
let waiting =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) {
|
||||
drop(tokio::spawn(async move {
|
||||
let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}));
|
||||
}
|
||||
|
||||
// Keep the large heal state machine and its construction temporary out of
|
||||
// each dispatch caller's poll frame. Ordinary object writes do not use this
|
||||
// heal-only allocation boundary.
|
||||
@@ -897,7 +844,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: None,
|
||||
@@ -1048,7 +994,6 @@ impl SetDisks {
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
let ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard,
|
||||
allow_regeneration: allow_explicit_version_regen,
|
||||
preserve_unrepairable,
|
||||
commit_fence,
|
||||
@@ -1756,29 +1701,7 @@ impl SetDisks {
|
||||
let dst_data_dir = data_dir;
|
||||
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
let mut shared_staging = false;
|
||||
if let Some(guard) = staging_guard {
|
||||
let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.supports_heal_lock_transition().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if supported {
|
||||
let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.downgrade_committed_tail().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if converted {
|
||||
shared_staging = true;
|
||||
} else {
|
||||
// Some peers may have converted. Abort without committing;
|
||||
// dropping the guard releases each retained lease.
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
let stage = async {
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = async {
|
||||
let erasure_info = latest_meta.erasure.clone();
|
||||
let mut writer_failures = HealWriterFailures::new(disks_to_heal_count);
|
||||
|
||||
@@ -2003,43 +1926,17 @@ impl SetDisks {
|
||||
|
||||
writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable");
|
||||
Ok(None)
|
||||
};
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = if shared_staging {
|
||||
if let Some(guard) = staging_guard {
|
||||
tokio::select! {
|
||||
result = stage => result,
|
||||
_ = wait_for_heal_writer(guard) => {
|
||||
// The UUID prefix is private to this attempt. Cleanup must not
|
||||
// retain the namespace lock while a foreground writer waits;
|
||||
// the stale-tmp sweeper also covers a cancelled disk operation.
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
};
|
||||
}
|
||||
.await;
|
||||
match stage_result {
|
||||
Ok(None) => {}
|
||||
failure => {
|
||||
if shared_staging {
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
} else {
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}
|
||||
// Release readers and writers before removing all parts
|
||||
// staged under this heal's exclusively owned prefix.
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
return failure.map(|error| (result, error));
|
||||
}
|
||||
}
|
||||
if shared_staging
|
||||
&& let Some(guard) = staging_guard
|
||||
&& let Err(error) = restore_heal_exclusive_guard(guard).await
|
||||
{
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
|
||||
if _read_repair_read_lock_guard
|
||||
@@ -3719,7 +3616,6 @@ impl SetDisks {
|
||||
&version,
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(can_commit),
|
||||
@@ -3780,7 +3676,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &read_opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -3807,7 +3702,7 @@ impl SetDisks {
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
|
||||
Some(Arc::new(tokio::sync::Mutex::new(
|
||||
Some(Arc::new(
|
||||
measure_heal_object_lock(
|
||||
"heal_object",
|
||||
HealObjectLockKind::Write,
|
||||
@@ -3819,7 +3714,7 @@ impl SetDisks {
|
||||
.narrow_to_disk()
|
||||
.unwrap_or_else(DiskError::other)
|
||||
})?,
|
||||
)))
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -3837,18 +3732,10 @@ impl SetDisks {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let write_lock_loss = if let Some(guard) = &_write_lock_guard {
|
||||
guard.lock().await.lock_lost_signal()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let locks_lost = || {
|
||||
write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost())
|
||||
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
};
|
||||
// Recovery cleanup can outlive its future. Never permit reads until
|
||||
// every physical operation holding its recovery owner has finished.
|
||||
let recovery_tail = Arc::new(());
|
||||
let mut recovery_removed_version = false;
|
||||
if !opts.read_repair
|
||||
&& _write_lock_guard.is_some()
|
||||
@@ -3875,7 +3762,6 @@ impl SetDisks {
|
||||
Arc::new((
|
||||
scope.clone(),
|
||||
self.ctx.begin_namespace_commit(),
|
||||
Arc::clone(&recovery_tail),
|
||||
_write_lock_guard.clone(),
|
||||
transaction_guard.clone(),
|
||||
)) as Arc<dyn Send + Sync>
|
||||
@@ -3999,9 +3885,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -4026,9 +3909,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -6167,7 +6047,6 @@ mod heal_result_report_tests {
|
||||
"",
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -10758,369 +10637,6 @@ mod heal_result_report_tests {
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-storage-entry";
|
||||
let object = "object.bin";
|
||||
let expected = vec![0x37; 1024 * 1024];
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external object data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(&missing).await.expect("remove shard");
|
||||
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
let result = heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
read_repair: false,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
(result, absence)
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("read-repair reached shard reconstruction");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_millis(200),
|
||||
set.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("background heal reconstruction should allow concurrent readers")
|
||||
.expect("open object while reconstruction is paused");
|
||||
let mut during_heal = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal))
|
||||
.await
|
||||
.expect("read completes while reconstruction is paused")
|
||||
.expect("read object during reconstruction");
|
||||
assert_eq!(during_heal, expected);
|
||||
drop(reader);
|
||||
release.notify_one();
|
||||
let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("repair finishes")
|
||||
.expect("repair task joins");
|
||||
let (_, error) = result.expect("repair operation succeeds");
|
||||
assert!(error.is_none(), "repair failed: {error:?}");
|
||||
assert!(absence.is_none());
|
||||
assert!(missing.exists(), "missing shard was not restored");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read repaired object");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-long-put";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let put = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes after releasing reconstruction")
|
||||
.expect("heal task joins");
|
||||
put.expect("foreground PUT finishes during reconstruction")
|
||||
.expect("foreground PUT must not time out");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
drop(_pause);
|
||||
let mut absence = None;
|
||||
let (_, retry_error) = set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a later heal must retry the current object");
|
||||
assert!(retry_error.is_none(), "retry must complete: {retry_error:?}");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read foreground write after heal");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_yields_to_version_delete_without_resurrecting_it() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-version-delete";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create versioned bucket");
|
||||
let original = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original version");
|
||||
let version = original.version_id.expect("version id").to_string();
|
||||
let survivor = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x52; 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write surviving version")
|
||||
.version_id
|
||||
.expect("surviving version id");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, &version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("read original metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal_version = version.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
&heal_version,
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut None,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let delete = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes")
|
||||
.expect("heal task joins");
|
||||
delete
|
||||
.expect("foreground delete finishes")
|
||||
.expect("foreground delete succeeds");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
let result = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version),
|
||||
"stale heal must not resurrect the deleted version: {result:?}"
|
||||
);
|
||||
let mut survivor = set
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(survivor.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("surviving version remains readable");
|
||||
let mut body = Vec::new();
|
||||
survivor.stream.read_to_end(&mut body).await.expect("read surviving version");
|
||||
assert_eq!(body, vec![0x52; 1024]);
|
||||
for disk in &disks {
|
||||
let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await;
|
||||
assert!(
|
||||
matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
|
||||
"deleted version must remain absent on each disk: {physical:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() {
|
||||
let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
|
||||
@@ -41,8 +41,8 @@ use super::super::{
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError,
|
||||
Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
|
||||
complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, contains_key_str,
|
||||
create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
|
||||
get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
|
||||
create_bitrot_writer_with_orig_volume, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5,
|
||||
get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
|
||||
log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
|
||||
prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
|
||||
reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
|
||||
@@ -124,9 +124,10 @@ async fn create_part_writers(
|
||||
let Some(disk) = disk else {
|
||||
return (None, Some(DiskError::DiskNotFound));
|
||||
};
|
||||
match create_bitrot_writer(
|
||||
match create_bitrot_writer_with_orig_volume(
|
||||
false,
|
||||
Some(disk),
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
path,
|
||||
length,
|
||||
@@ -977,7 +978,7 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
|
||||
if !root_prefix.is_empty() {
|
||||
if !root_prefix.is_empty() && root_prefix != DATA_MOVEMENT_MULTIPART_PREFIX {
|
||||
let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
@@ -1002,6 +1003,13 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
.into_iter()
|
||||
.map(|sha_dir| sha_dir.trim_end_matches('/').to_owned())
|
||||
.filter(|sha_dir| sha_dir.len() == 64 && sha_dir.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
.map(|sha_dir| {
|
||||
if root_prefix.is_empty() {
|
||||
sha_dir
|
||||
} else {
|
||||
format!("{root_prefix}/{sha_dir}")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let listed_upload_dirs = stream::iter(sha_dirs)
|
||||
.map(|sha_dir| {
|
||||
@@ -1124,6 +1132,7 @@ impl SetDisks {
|
||||
orig_bucket: &str,
|
||||
error_path: &str,
|
||||
root_prefix: &str,
|
||||
require_all_disks: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.is_empty() {
|
||||
@@ -1165,7 +1174,8 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
|
||||
let required_discoveries = if require_all_disks { disks.len() } else { discovery_quorum };
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, required_discoveries) {
|
||||
return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
|
||||
}
|
||||
|
||||
@@ -1187,7 +1197,7 @@ impl SetDisks {
|
||||
|
||||
pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
|
||||
let (_, paths, _, _) = self
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "")
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "", false)
|
||||
.await?;
|
||||
Ok(paths.into_iter().next())
|
||||
}
|
||||
@@ -1200,8 +1210,9 @@ impl SetDisks {
|
||||
upload_identity: &str,
|
||||
) -> Result<Vec<String>> {
|
||||
let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object));
|
||||
let (_, candidate_paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
|
||||
let (_, candidate_paths, _, has_minority_candidate) = self
|
||||
.discover_multipart_upload_paths(bucket, object, &expected_parent, false)
|
||||
.await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::DecommissionCapacityBlocked {
|
||||
message: "data movement multipart cleanup found an upload path on fewer than the discovery quorum of disks"
|
||||
@@ -1750,6 +1761,54 @@ impl SetDisks {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
// Every disk must confirm absence in both namespaces before cleanup can
|
||||
// remove catalog protection from the bucket.
|
||||
for root_prefix in ["", DATA_MOVEMENT_MULTIPART_PREFIX] {
|
||||
let (disks, paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, "", root_prefix, true).await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
for upload_path in paths {
|
||||
let (sha_dir, _) = upload_path
|
||||
.rsplit_once('/')
|
||||
.filter(|(directory, upload_id)| !directory.is_empty() && !upload_id.is_empty())
|
||||
.ok_or(DiskError::CorruptedFormat)?;
|
||||
let (metadata, errors) =
|
||||
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, "", false, false, false)
|
||||
.await?;
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
if let Some(error) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(to_object_err(error.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
let (_, modified, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum);
|
||||
let file = Self::pick_valid_fileinfo(&metadata, modified, etag, read_quorum)?;
|
||||
let (Some(owner_bucket), Some(owner_object)) = (
|
||||
file.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
|
||||
file.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
|
||||
) else {
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
};
|
||||
if owner_bucket.is_empty()
|
||||
|| owner_object.is_empty()
|
||||
|| Self::get_multipart_sha_dir(owner_bucket, owner_object)
|
||||
!= sha_dir
|
||||
.strip_prefix(DATA_MOVEMENT_MULTIPART_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.unwrap_or(sha_dir)
|
||||
{
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
if owner_bucket == bucket {
|
||||
return Err(Error::BucketNotEmpty(bucket.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
||||
&self,
|
||||
@@ -1761,7 +1820,8 @@ impl SetDisks {
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
let (disks, candidate_paths, discovery_quorum, _) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
|
||||
let (disks, candidate_paths, discovery_quorum, _) =
|
||||
self.discover_multipart_upload_paths(bucket, prefix, "", false).await?;
|
||||
let listed_uploads = stream::iter(candidate_paths)
|
||||
.map(|upload_path| {
|
||||
let disks = &disks;
|
||||
@@ -2436,7 +2496,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
|
||||
let part_integrity = if let Some(integrity) = integrity {
|
||||
Some(integrity.write(&mut shuffle_disks, bucket, RUSTFS_META_TMP_BUCKET, &tmp_part).await?)
|
||||
Some(
|
||||
integrity
|
||||
.write(
|
||||
&mut shuffle_disks,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_part,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else { None };
|
||||
if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
@@ -4458,7 +4527,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// parts; deleting them before the commit would strand the upload
|
||||
// permanently. This mirrors the "clean up only after commit" pattern
|
||||
// already used for the old data-dir GC and the upload-dir delete_all below.
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
let can_skip_part_cleanup = !needs_upgrade
|
||||
&& commit_disks
|
||||
.iter()
|
||||
.all(|disk| disk.as_ref().is_some_and(|disk| disk.is_local()));
|
||||
if !can_skip_part_cleanup {
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
}
|
||||
|
||||
if let Some(old_dir) = op_old_dir {
|
||||
// backlog#898: best-effort reclaim of the dereferenced old data dir.
|
||||
@@ -11197,6 +11272,163 @@ mod tests {
|
||||
assert_eq!(upload_uuid_suffix(&recovered[0]), upload_uuid_suffix(&upload.upload_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_distinguishes_bucket_ownership() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-target";
|
||||
let other = "multipart-cleanup-other";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create target bucket");
|
||||
disk.make_volume(other).await.expect("create other bucket");
|
||||
}
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an empty upload namespace permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(other, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in other bucket");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an upload owned by another bucket permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in target bucket");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_requires_every_disk_online() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-offline";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks.disks.write().await[3] = None;
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing preserves its discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("an offline disk must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the offline disk must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
set_disks.disks.write().await[3] = Some(disk_stores[3].clone());
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_a_failed_disk_enumeration() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-list-error";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let sha_dir = SetDisks::get_multipart_sha_dir(bucket, "object.bin");
|
||||
let multipart_dir = temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET);
|
||||
tokio::fs::create_dir_all(&multipart_dir)
|
||||
.await
|
||||
.expect("create multipart staging directory");
|
||||
tokio::fs::write(multipart_dir.join(&sha_dir), b"not a directory")
|
||||
.await
|
||||
.expect("replace one upload parent directory with a file");
|
||||
multipart_upload_paths_on_disk(disk_stores[0].clone(), bucket, "")
|
||||
.await
|
||||
.expect_err("the affected disk must fail directory enumeration");
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing still accepts the remaining discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("a failed disk enumeration must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the failed enumeration must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
tokio::fs::remove_file(temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET).join(sha_dir))
|
||||
.await
|
||||
.expect("remove the invalid directory entry");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after repair");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_data_movement_uploads() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-data-movement";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos()),
|
||||
);
|
||||
set_disks
|
||||
.new_multipart_upload(
|
||||
bucket,
|
||||
"object.bin",
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create data movement upload");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_unreadable_ownership() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-corrupt";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload");
|
||||
let mut corrupted = 0;
|
||||
for temp_dir in &temp_dirs {
|
||||
for path in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
|
||||
tokio::fs::write(path, b"invalid upload metadata")
|
||||
.await
|
||||
.expect("corrupt upload metadata");
|
||||
corrupted += 1;
|
||||
}
|
||||
}
|
||||
assert!(corrupted > 0, "upload metadata must exist before corruption");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("unreadable upload ownership must block cleanup");
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): a single upload directory whose `xl.meta` was
|
||||
/// destroyed (crash mid-write, torn disk state) must degrade to that upload
|
||||
/// alone. Failing the whole ListMultipartUploads turns one piece of stale
|
||||
|
||||
@@ -4184,19 +4184,7 @@ impl SetDisks {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_prepared_reservations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, reservations)
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -4215,7 +4203,7 @@ impl SetDisks {
|
||||
// The owned publication capability supplies the exact target write
|
||||
// domain at commit time; the generic PUT path must not reacquire it.
|
||||
opts.no_lock = true;
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence), None)
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -4226,9 +4214,8 @@ impl SetDisks {
|
||||
data: &'a mut PutObjReader,
|
||||
opts: &'a ObjectOptions,
|
||||
publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> impl Future<Output = Result<(ObjectInfo, Option<OldCurrentSize>)>> + Send + 'a {
|
||||
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence, reservations))
|
||||
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence))
|
||||
}
|
||||
|
||||
async fn put_object_with_old_current_size_inner(
|
||||
@@ -4238,7 +4225,6 @@ impl SetDisks {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
mut publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
let write_context = WriteCommitContext::from_options(opts, bucket, object, publication_fence.is_some())?;
|
||||
let protect_write = opts.shard_integrity_write_enabled();
|
||||
@@ -4803,41 +4789,6 @@ impl SetDisks {
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Remote publication owners were prepared before bucket metadata
|
||||
// locks. Local-only fixtures can prepare before the object lock.
|
||||
let prepared_metadata_reservations = if !self.ctx.lock_manager().is_disabled()
|
||||
&& crate::set_disk::put_rename_early_ack_enabled()
|
||||
&& write_context.acquires_namespace()
|
||||
&& write_context.allows_early_ack(opts.write_completion)
|
||||
&& publication_fence.is_none()
|
||||
&& opts.scanner_publication_commit_scope.is_none()
|
||||
&& opts.decommission_capacity_admission.is_none()
|
||||
&& !opts.data_movement
|
||||
&& !is_meta_bucketname(bucket)
|
||||
{
|
||||
if let Some(prepared) = reservations {
|
||||
crate::set_disk::align_rename_metadata_reservations(prepared, &shuffle_disks)
|
||||
} else if shuffle_disks.iter().flatten().all(|disk| disk.is_local())
|
||||
&& crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
|
||||
.await
|
||||
.is_none()
|
||||
&& let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil())
|
||||
{
|
||||
crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
|
||||
&shuffle_disks,
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
&self.ctx,
|
||||
false,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -5317,11 +5268,6 @@ impl SetDisks {
|
||||
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
|
||||
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
|
||||
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
|
||||
let commit_object_lock_snapshot = opts
|
||||
.object_lock_config_snapshot
|
||||
.as_ref()
|
||||
.filter(|snapshot| snapshot.owns_lifecycle_guard())
|
||||
.cloned();
|
||||
// A scanner publication scope owns the movement permit until the
|
||||
// complete rename fan-out drains. Keep this path synchronous so
|
||||
// its terminal state is known before the coordinator releases
|
||||
@@ -5335,26 +5281,6 @@ impl SetDisks {
|
||||
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& commit_owns_namespace_guard
|
||||
&& commit_scanner_publication_scope.is_none();
|
||||
let lifecycle_owned = commit_bucket_lifecycle_guard.is_some()
|
||||
|| commit_object_lock_snapshot
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| snapshot.owns_lifecycle_guard_for(bucket, source_bucket_incarnation_id));
|
||||
let reservation_ctx = (commit_allows_early_ack
|
||||
&& write_context.acquires_namespace()
|
||||
&& commit_object_lock_guard.is_some()
|
||||
&& commit_decommission_object_lock_guard.is_none()
|
||||
&& commit_decommission_capacity_guard.is_none()
|
||||
&& commit_publication_guard.is_none()
|
||||
&& !quota_mutation_fence
|
||||
&& !opts.data_movement
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& lifecycle_owned)
|
||||
.then(|| Arc::clone(&self.ctx));
|
||||
let prepared_metadata_reservations = if reservation_ctx.is_some() {
|
||||
prepared_metadata_reservations
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Full-tail callers also transfer owned guards to the coordinator:
|
||||
// cancelling their ACK waiter must not cancel an in-flight rename.
|
||||
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|
||||
@@ -5678,7 +5604,6 @@ impl SetDisks {
|
||||
)
|
||||
.with_publication_scope(commit_scanner_publication_scope.clone())
|
||||
.with_rollback_receipt(commit_rollback_receipt.clone())
|
||||
.with_reservation_context(reservation_ctx, prepared_metadata_reservations)
|
||||
.with_namespace_commit_guard(
|
||||
(!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
|
||||
),
|
||||
@@ -5702,7 +5627,6 @@ impl SetDisks {
|
||||
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
|
||||
}
|
||||
let mut rename_guard_release = None;
|
||||
let mut committed_tail_guard = None;
|
||||
let mut needs_immediate_heal = false;
|
||||
let mut tail_owns_tmp_cleanup = false;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -5734,14 +5658,7 @@ impl SetDisks {
|
||||
.or_else(|| commit_version_suspended.then(Uuid::nil))
|
||||
.map(|version_id| version_id.to_string());
|
||||
request.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
let object_lock_guard = if rename_commit.tail_has_metadata_reservations {
|
||||
None
|
||||
} else {
|
||||
_object_lock_guard
|
||||
.take()
|
||||
.map(|guard| Arc::new(tokio::sync::Mutex::new(guard)))
|
||||
};
|
||||
committed_tail_guard = object_lock_guard.clone();
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let publication_guard = _publication_guard.take();
|
||||
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
|
||||
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
|
||||
@@ -5766,7 +5683,6 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
commit_object_lock_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5788,7 +5704,6 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
object_lock_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5802,7 +5717,6 @@ impl SetDisks {
|
||||
drop(write_context);
|
||||
drop(publication_guard);
|
||||
drop(bucket_lifecycle_guard);
|
||||
drop(object_lock_snapshot);
|
||||
cleanup_runtime::run(cleanup_reservation, async move {
|
||||
let zero_target_cleanup = targets.is_empty();
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
@@ -6058,13 +5972,6 @@ impl SetDisks {
|
||||
lock_held_post_rename_started,
|
||||
);
|
||||
let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Some(guard) = committed_tail_guard.take() {
|
||||
// The committed tail still owns every physical rename. Convert
|
||||
// only after metadata invalidation and before acknowledging PUT.
|
||||
// Unsupported peers keep their exclusive leases and the tail
|
||||
// retains ownership until its physical work has drained.
|
||||
let _ = guard.lock().await.downgrade_committed_tail().await;
|
||||
}
|
||||
if let Some(release) = rename_guard_release.take() {
|
||||
let _ = release.send(true);
|
||||
}
|
||||
@@ -6222,7 +6129,11 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
if !tail_owns_tmp_cleanup {
|
||||
let all_local_disks_cleaned_tmp = all_commit_disks_local
|
||||
&& online_disks.len() == commit_set.set_drive_count
|
||||
&& online_disks.iter().all(Option::is_some)
|
||||
&& !issue3031_diag_enabled();
|
||||
if !tail_owns_tmp_cleanup && !all_local_disks_cleaned_tmp {
|
||||
let cleanup_set = commit_set.clone();
|
||||
let cleanup_tmp_dir = commit_tmp_dir.clone();
|
||||
cleanup_runtime::spawn(commit_cleanup_reservation.clone(), async move {
|
||||
@@ -22270,117 +22181,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_retains_owned_bucket_lifecycle_fence() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-tail-lifecycle";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
let lock = NamespaceLock::new("tail-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
Uuid::new_v4(),
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(snapshot),
|
||||
..Default::default()
|
||||
};
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer_set = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
writer_set
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("rename tail pauses");
|
||||
put.await.expect("PUT joins").expect("PUT acknowledged");
|
||||
let mut bucket_delete =
|
||||
Box::pin(lock.get_write_lock(ObjectKey::new(bucket, "lifecycle"), "delete", Duration::from_secs(10)));
|
||||
let early = tokio::time::timeout(Duration::from_secs(2), &mut bucket_delete).await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let guard = match early {
|
||||
Ok(result) => {
|
||||
drop(result.expect("early bucket lock result"));
|
||||
panic!("bucket lifecycle owner must outlive the acknowledged physical tail");
|
||||
}
|
||||
Err(_) => tokio::time::timeout(Duration::from_secs(15), bucket_delete)
|
||||
.await
|
||||
.expect("lifecycle lock released after tail")
|
||||
.expect("bucket delete acquires lock"),
|
||||
};
|
||||
drop(guard);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_drain_allows_read_after_acknowledged_put() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
|
||||
],
|
||||
async {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-early-ack-tail-read";
|
||||
let object = "early-ack-tail-read-object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let put_set = Arc::clone(&set_disks);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
put_set
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("tail disk should pause during rename");
|
||||
put.await
|
||||
.expect("PUT task should join")
|
||||
.expect("PUT should be acknowledged at quorum");
|
||||
let result = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await;
|
||||
let mut reader = result.expect("acknowledged object should remain readable while tail drains");
|
||||
let mut body = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut body))
|
||||
.await
|
||||
.expect("object body should not wait for the tail")
|
||||
.expect("object body should drain");
|
||||
assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
rename_barrier.release();
|
||||
drop(rename_barrier);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes() {
|
||||
@@ -22453,200 +22253,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_does_not_timeout_the_next_put() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-early-ack-next-writer";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
{
|
||||
let map = set.ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
let lifecycle_lock = NamespaceLock::new("next-put-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lifecycle_lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lifecycle_lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let incarnation = Uuid::new_v4();
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
incarnation,
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let mut opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
object_lock_config_snapshot: Some(snapshot.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let first_set = Arc::clone(&set);
|
||||
let first_opts = opts.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_set
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &first_opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first rename paused");
|
||||
first.await.expect("first PUT joins").expect("first PUT acknowledged");
|
||||
let second = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]), &opts),
|
||||
)
|
||||
.await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let second = second.expect("second PUT finishes").expect("second PUT must not time out");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("both rename tails must drain");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read newest object");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("read full body");
|
||||
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
let expected_etag = second.etag.as_ref().expect("second PUT ETag");
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("each disk must retain the latest version");
|
||||
assert_eq!(info.metadata.get("etag"), Some(expected_etag), "late rename must not restore an old copy");
|
||||
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
|
||||
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_cannot_restore_a_deleted_version() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-tail-version-delete";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
{
|
||||
let map = set.ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
let lock = NamespaceLock::new("version-delete-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let incarnation = Uuid::new_v4();
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
incarnation,
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: true,
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
object_lock_config_snapshot: Some(snapshot.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer = Arc::clone(&set);
|
||||
let put_opts = opts.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
writer
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &put_opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("rename pauses");
|
||||
let version = put
|
||||
.await
|
||||
.expect("PUT joins")
|
||||
.expect("PUT succeeds")
|
||||
.version_id
|
||||
.expect("version ID")
|
||||
.to_string();
|
||||
let delete_barrier = DeleteObjectCommitBarrier::install_before_namespace(bucket, object);
|
||||
let delete_set = Arc::clone(&set);
|
||||
let mut delete_opts = opts;
|
||||
delete_opts.version_id = Some(version.clone());
|
||||
let delete = tokio::spawn(async move { delete_set.delete_object(bucket, object, delete_opts).await });
|
||||
delete_barrier.wait_until_paused().await;
|
||||
delete_barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !delete_barrier.namespace_acquired() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("version delete acquires namespace while old tail is paused");
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
delete.await.expect("delete joins").expect("version delete succeeds");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("rename tails drain");
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.read_version("", bucket, object, &version, &ReadOptions::default()).await,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)
|
||||
),
|
||||
"late tail must not restore the deleted version"
|
||||
);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_successful_tail_reclaims_its_old_non_inline_data_dir() {
|
||||
|
||||
@@ -44,15 +44,16 @@ pub(crate) mod lifecycle {
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker, WalkOptions as StorageWalkOptions,
|
||||
WalkVersionsSortOrder,
|
||||
AuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker,
|
||||
WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadResult, PartInfo,
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
|
||||
MultipartUploadResult, PartInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -523,7 +523,22 @@ impl ECStore {
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, validate).await
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, None, validate).await
|
||||
}
|
||||
|
||||
/// Validate an external fence and the bucket incarnation in one metadata transaction.
|
||||
pub async fn update_bucket_metadata_config_for_incarnation_validated<F>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, Some(expected_incarnation_id), validate).await
|
||||
}
|
||||
|
||||
pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
|
||||
@@ -45,8 +45,7 @@
|
||||
use crate::error::{Result, StorageError};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
use std::time::Duration;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use rustfs_lock::distributed_lock::LockLostSignal;
|
||||
@@ -99,74 +98,9 @@ pub(super) struct FenceRegistration {
|
||||
#[derive(Default)]
|
||||
pub(crate) struct BucketFenceRegistry {
|
||||
entries: Mutex<HashMap<String, FenceEntry>>,
|
||||
reservation_slots: Mutex<HashMap<(String, Uuid), Weak<ReservationFenceSlot>>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ReservationFenceSlot {
|
||||
pub(super) owner: tokio::sync::Mutex<Weak<ReservationFenceGuard>>,
|
||||
}
|
||||
|
||||
/// Shared by short bursts of reservations, but never kept alive by the cache.
|
||||
pub(super) struct ReservationFenceGuard {
|
||||
publication: NamespaceLockGuard,
|
||||
lifecycle: BucketIncarnationFenceGuard,
|
||||
created_at: tokio::time::Instant,
|
||||
_slot: Arc<ReservationFenceSlot>,
|
||||
}
|
||||
|
||||
impl ReservationFenceGuard {
|
||||
pub(super) fn new(
|
||||
publication: NamespaceLockGuard,
|
||||
lifecycle: BucketIncarnationFenceGuard,
|
||||
slot: Arc<ReservationFenceSlot>,
|
||||
) -> Self {
|
||||
Self {
|
||||
publication,
|
||||
lifecycle,
|
||||
created_at: tokio::time::Instant::now(),
|
||||
_slot: slot,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_reusable(&self) -> bool {
|
||||
// Stop admitting new reservations through this shared reader so a
|
||||
// queued bucket writer can progress after the existing tails drain.
|
||||
self.created_at.elapsed() < Duration::from_secs(1) && self.is_held()
|
||||
}
|
||||
|
||||
fn is_held(&self) -> bool {
|
||||
!self.publication.is_lock_lost()
|
||||
&& !self.publication.is_released()
|
||||
&& !self.lifecycle.is_lock_lost()
|
||||
&& self
|
||||
.lifecycle
|
||||
.namespace_lock_guard()
|
||||
.is_some_and(|guard| !guard.is_released())
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::disk::os::MetadataReservationFence for ReservationFenceGuard {
|
||||
fn is_valid(&self) -> bool {
|
||||
self.is_held()
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketFenceRegistry {
|
||||
pub(super) fn reservation_slot(&self, bucket: &str, incarnation: Uuid) -> Arc<ReservationFenceSlot> {
|
||||
let mut slots = self.reservation_slots.lock().expect("reservation fence registry poisoned");
|
||||
let key = (bucket.to_owned(), incarnation);
|
||||
if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) {
|
||||
return slot;
|
||||
}
|
||||
if slots.len() >= 1024 {
|
||||
slots.retain(|_, slot| slot.strong_count() != 0);
|
||||
}
|
||||
let slot = Arc::new(ReservationFenceSlot::default());
|
||||
slots.insert(key, Arc::downgrade(&slot));
|
||||
slot
|
||||
}
|
||||
|
||||
/// Register a new live guard for `bucket` and return the memoized
|
||||
/// incarnation id if one is valid for the current coverage window.
|
||||
fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration {
|
||||
|
||||
@@ -37,8 +37,8 @@ use crate::error::{
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::list::{
|
||||
ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr,
|
||||
StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
AuthoritativeListOperations, ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo,
|
||||
StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
@@ -80,6 +80,21 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl AuthoritativeListOperations for ECStore {
|
||||
async fn walk_authoritative(
|
||||
self: Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
check_list_objs_args(bucket, prefix, &None)?;
|
||||
self.walk_authoritatively(rx, bucket, prefix, result, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the versioned listing options used by recursive object walks.
|
||||
///
|
||||
/// `list_path_result` normally derives these scan bounds before dispatching to
|
||||
@@ -1550,6 +1565,57 @@ impl ECStore {
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn walk_authoritatively(
|
||||
self: Arc<Self>,
|
||||
cancellation: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
let cancel = cancellation.child_token();
|
||||
let _cancel_on_drop = cancel.clone().drop_guard();
|
||||
let mut scans = Vec::new();
|
||||
let mut inputs = Vec::new();
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
inputs.push(receiver);
|
||||
let mut scan = recursive_walk_list_path_options(bucket.to_owned(), prefix, &opts);
|
||||
// Internal absence checks must include reserved object prefixes.
|
||||
scan.skip_hidden_prefix_check = true;
|
||||
scans.push(set.list_versions_authoritatively(cancel.clone(), scan, sender));
|
||||
}
|
||||
}
|
||||
if scans.is_empty() {
|
||||
return Err(StorageError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let (sender, receiver) = mpsc::channel(100);
|
||||
let forwarder = spawn_walk_entry_forwarder(cancel.clone(), receiver, result, opts, bucket.to_owned());
|
||||
let merge = spawn_listing_merge(cancel.clone(), inputs, sender);
|
||||
let scan_results = join_all(scans).await;
|
||||
let merge_result = merge.await.map_err(Error::from)?;
|
||||
let forward_result = forwarder.await.map_err(Error::from);
|
||||
|
||||
// An unavailable set cannot establish absence, even when another set
|
||||
// completed. Only an explicitly missing directory or volume is empty.
|
||||
for scan in scan_results {
|
||||
match scan {
|
||||
Ok(outcome) if !outcome.limit_reached => {}
|
||||
Ok(_) => return Err(StorageError::ErasureReadQuorum),
|
||||
Err(StorageError::FileNotFound | StorageError::VolumeNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
merge_result?;
|
||||
forward_result?;
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(Error::other("authoritative object walk was cancelled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Forwards merged raw entries to a walk consumer as `ObjectInfoOrErr`,
|
||||
@@ -2520,7 +2586,8 @@ impl SetDisks {
|
||||
/// Version scan for set-scoped callers that act on absence: set walks
|
||||
/// (tier reference proofs, tier-delete journals, pool maintenance) and the
|
||||
/// recursive prefix-delete scan. Every online drive is read and each
|
||||
/// version is resolved at the read quorum of its own erasure layout; a
|
||||
/// version uses the read quorum of its own erasure layout. Null versions
|
||||
/// in the system bucket also retain GET's commit-proof selection. A
|
||||
/// version that cannot be ruled out fails the scan instead of being
|
||||
/// omitted. Merged `ListObjectVersions` pages go through `list_path`.
|
||||
async fn list_versions_authoritatively(
|
||||
@@ -2859,6 +2926,7 @@ mod test {
|
||||
use crate::error::{Result, StorageError};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::{SetDisks, hermetic_set_disks_isolated};
|
||||
use crate::storage_api_contracts::list::AuthoritativeListOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use bytes::Bytes;
|
||||
@@ -5719,6 +5787,126 @@ mod test {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_rejects_uncommitted_null_version_split() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
for new_etag in ["new", "old"] {
|
||||
let new =
|
||||
test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), new_etag, Uuid::from_u128(2));
|
||||
let entries = MetaCacheEntries(vec![Some(old.clone()), Some(old.clone()), Some(new.clone()), Some(new)]);
|
||||
|
||||
assert!(matches!(
|
||||
SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2),
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_selects_committed_null_version() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (new_copies, expected) in [(1, "old"), (3, "new"), (4, "new")] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| Some(if slot < new_copies { new.clone() } else { old.clone() }))
|
||||
.collect(),
|
||||
);
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2)
|
||||
.expect("committed generation should resolve")
|
||||
.expect("committed record should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_matches_degraded_null_version_reads() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (old_copies, new_copies, expected) in [
|
||||
(2, 1, Some("old")),
|
||||
(1, 2, None),
|
||||
(0, 3, Some("new")),
|
||||
(0, 2, Some("new")),
|
||||
(0, 1, None),
|
||||
] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| {
|
||||
if slot < old_copies {
|
||||
Some(old.clone())
|
||||
} else if slot < old_copies + new_copies {
|
||||
Some(new.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let errors = (0..4)
|
||||
.map(|slot| (slot >= old_copies + new_copies).then_some(DiskError::DiskNotFound))
|
||||
.collect::<Vec<_>>();
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2);
|
||||
|
||||
if let Some(expected) = expected {
|
||||
let info = resolved
|
||||
.expect("degraded committed record should resolve")
|
||||
.expect("degraded committed record should be present")
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
} else {
|
||||
assert!(matches!(resolved, Err(DiskError::ErasureReadQuorum)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_user_bucket_scan_keeps_null_version_read_quorum_selection() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for old_copies in [1, 2] {
|
||||
let entries = MetaCacheEntries(vec![
|
||||
Some(new.clone()),
|
||||
Some(new.clone()),
|
||||
Some(old.clone()),
|
||||
(old_copies == 2).then(|| old.clone()),
|
||||
]);
|
||||
let errors = [None, None, None, (old_copies == 1).then_some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions("bucket", entries, &errors, 4, 2)
|
||||
.expect("user bucket read quorum should resolve")
|
||||
.expect("user bucket record should be present");
|
||||
let info = resolved.to_fileinfo("bucket").expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_keeps_explicit_version_read_quorum() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let entry = test_object_meta_entry_with_erasure_versions("entry.json", &[(time, "version", 2, 2)]);
|
||||
let entries = MetaCacheEntries(vec![Some(entry.clone()), Some(entry), None, None]);
|
||||
let errors = [None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2)
|
||||
.expect("explicit version read quorum should resolve")
|
||||
.expect("explicit version should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("version"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_version_scan_tolerates_corrupt_replicas_only_with_version_quorum() {
|
||||
let entry = test_object_meta_entry_with_erasure_versions(
|
||||
@@ -6060,6 +6248,49 @@ mod test {
|
||||
Bytes::from(meta.marshal_msg().expect("encode fixture xl.meta"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_store_walk_includes_reserved_prefix_versions() {
|
||||
let bucket = "catalog-cleanup-versions";
|
||||
let name = ".rustfs-table/namespaces/ns/tables/table/metadata/metadata.json";
|
||||
let (_dirs, store) = single_set_store(16).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
for (slot, disk) in set.disks.read().await.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("fixture drive is online");
|
||||
disk.make_volume(bucket).await.expect("create bucket volume");
|
||||
disk.write_all(
|
||||
bucket,
|
||||
&format!("{name}/{STORAGE_FORMAT_FILE}"),
|
||||
versioned_object_metadata(bucket, name, &[(2, 1_700_000_002), (1, 1_700_000_001)], slot),
|
||||
)
|
||||
.await
|
||||
.expect("persist reserved metadata versions");
|
||||
}
|
||||
let (sender, mut receiver) = mpsc::channel(1);
|
||||
let walk = store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let collect = async {
|
||||
let mut versions = Vec::new();
|
||||
while let Some(item) = receiver.recv().await {
|
||||
assert!(item.err.is_none(), "reserved metadata must decode: {:?}", item.err);
|
||||
let item = item.item.expect("walk must return an object");
|
||||
versions.push((item.name, item.version_id));
|
||||
}
|
||||
versions
|
||||
};
|
||||
let (walk, versions) = timeout(Duration::from_secs(10), async { tokio::join!(walk, collect) })
|
||||
.await
|
||||
.expect("authoritative reserved-prefix walk must terminate");
|
||||
walk.expect("authoritative walk must succeed");
|
||||
assert_eq!(
|
||||
versions,
|
||||
[
|
||||
(name.to_string(), Some(Uuid::from_u128(1))),
|
||||
(name.to_string(), Some(Uuid::from_u128(2)))
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
// A versioned DeleteObject(versionId) can reach write quorum while three
|
||||
// drives are offline; they keep the deleted version after they return. A
|
||||
// different drive going offline later must not turn that sub-quorum
|
||||
@@ -6130,6 +6361,16 @@ mod test {
|
||||
.await
|
||||
.expect("set walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let (sender, mut receiver) = mpsc::channel(16);
|
||||
let walk =
|
||||
store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let drain = async { while receiver.recv().await.is_some() {} };
|
||||
let (walk, ()) = timeout(Duration::from_secs(10), async { tokio::join!(walk, drain) })
|
||||
.await
|
||||
.expect("authoritative store walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let recursive_delete_scan = set
|
||||
.clone()
|
||||
.inner_list_object_versions_for_recursive_delete(bucket, "", None, None, 1000)
|
||||
|
||||
@@ -456,7 +456,6 @@ mod rebalance;
|
||||
pub(crate) use rebalance::PoolSpaceForTest;
|
||||
pub(crate) mod utils;
|
||||
|
||||
pub(crate) use peer::admit_local_disk;
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
|
||||
|
||||
@@ -23,6 +23,23 @@ use std::collections::HashSet;
|
||||
|
||||
const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::multipart::MultipartUploadInspection for ECStore {
|
||||
type Error = Error;
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
let (_, lifecycle_guard) = self
|
||||
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
|
||||
.await?;
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
set.ensure_no_multipart_uploads(bucket).await?;
|
||||
}
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_guard_held(lifecycle_guard.as_ref(), bucket, "")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, usize>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
@@ -4814,44 +4814,7 @@ impl ECStore {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
let mut early_opts = opts.clone();
|
||||
// Bucket publication precedes lifecycle and metadata transaction locks.
|
||||
let prepared = if self.single_pool()
|
||||
&& !self.ctx.lock_manager().is_disabled()
|
||||
&& crate::set_disk::put_rename_early_ack_enabled()
|
||||
&& early_opts.write_completion == crate::object_api::WriteCompletion::Quorum
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& !early_opts.no_lock
|
||||
&& !early_opts.data_movement
|
||||
&& early_opts.write_lock_context.is_none()
|
||||
&& early_opts.object_lock_config_snapshot.is_none()
|
||||
&& early_opts.bucket_lifecycle_lock_fence.is_none()
|
||||
&& early_opts.namespace_lock_fence.is_none()
|
||||
&& early_opts.scanner_publication_commit_scope.is_none()
|
||||
&& early_opts.decommission_capacity_admission.is_none()
|
||||
&& early_opts.decommission_capacity.is_none()
|
||||
&& early_opts.quota_admission.is_none()
|
||||
&& early_opts
|
||||
.put_object_cancellation
|
||||
.as_ref()
|
||||
.is_none_or(|token| !token.is_cancelled())
|
||||
&& rustfs_utils::path::object_name_disk_identity(object).as_ref()
|
||||
!= rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
{
|
||||
check_new_object_args(bucket, object, false)?;
|
||||
let incarnation = match early_opts.expected_bucket_incarnation_id {
|
||||
Some(incarnation) => incarnation,
|
||||
None => self.bucket_incarnation_id(bucket).await?,
|
||||
};
|
||||
early_opts.expected_bucket_incarnation_id = Some(incarnation);
|
||||
let encoded = encode_dir_object(object);
|
||||
let set = self.pools[0].get_disks_by_key(&encoded);
|
||||
let disks = set.disks.read().await.clone();
|
||||
crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, &encoded, incarnation, &self.ctx, true).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, &early_opts)).await?;
|
||||
let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, opts)).await?;
|
||||
let idx = self
|
||||
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
|
||||
.await?;
|
||||
@@ -4860,8 +4823,7 @@ impl ECStore {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
|
||||
let result =
|
||||
Box::pin(self.pools[idx].put_object_with_prepared_reservations(bucket, object.as_str(), data, &opts, prepared)).await;
|
||||
let result = Box::pin(self.pools[idx].put_object_with_old_current_size(bucket, object.as_str(), data, &opts)).await;
|
||||
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
|
||||
match result {
|
||||
Ok((object_info, old_current_size)) => {
|
||||
@@ -9976,305 +9938,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn ecstore_early_ack_prepares_before_snapshot_and_orders_the_next_put() {
|
||||
use crate::disk::ReadOptions;
|
||||
use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "ecstore-reservation-order";
|
||||
let object = "object";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let first_store = Arc::clone(&store);
|
||||
let first = tokio::spawn(async move {
|
||||
first_store
|
||||
.handle_put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'1'; 1 << 20]),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first tail pauses");
|
||||
first.await.expect("first task").expect("first PUT");
|
||||
let second = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
store.handle_put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'2'; 1 << 20]),
|
||||
&ObjectOptions::default(),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let (second, _) = second.expect("second PUT completes").expect("second PUT succeeds");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("rename tails drain");
|
||||
let expected_etag = second.etag.as_ref().expect("second ETag");
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read latest metadata");
|
||||
assert_eq!(info.metadata.get("etag"), Some(expected_etag));
|
||||
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
|
||||
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_prepared_reservations_hold_bucket_publication_fence() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.clone();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in disks.iter().flatten() {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "local-reservation-bucket-fence";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let reservations =
|
||||
crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, "object", incarnation, &ctx, true)
|
||||
.await
|
||||
.expect("prepare reservations")
|
||||
.expect("all disks prepared");
|
||||
let lock = store
|
||||
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("publication lock");
|
||||
let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(tokio::time::timeout(Duration::from_millis(50), &mut writer).await.is_err());
|
||||
drop(reservations);
|
||||
let guard = tokio::time::timeout(Duration::from_secs(5), writer)
|
||||
.await
|
||||
.expect("reservations release fence")
|
||||
.expect("writer lock");
|
||||
drop(guard);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bucket_reservations_do_not_exhaust_shared_lock_holders() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "reservation-reader-capacity";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let disk = disks[0].endpoint().to_string();
|
||||
let mut tokens = Vec::new();
|
||||
let mut failure = None;
|
||||
for index in 0..256 {
|
||||
match store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&disk,
|
||||
bucket,
|
||||
&format!("object-{index}"),
|
||||
incarnation,
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(token) => tokens.push(token),
|
||||
Err(error) => {
|
||||
failure = Some((index, error));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
for token in tokens {
|
||||
assert!(
|
||||
store
|
||||
.cancel_local_object_metadata_reservation(&disk, token)
|
||||
.await
|
||||
.expect("cancel token")
|
||||
);
|
||||
}
|
||||
assert!(failure.is_none(), "bucket reservations exhausted shared lock holders: {failure:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn expired_reservation_fence_does_not_bypass_a_waiting_bucket_writer() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disk = set.disks.read().await.iter().flatten().next().expect("local disk").clone();
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "reservation-writer-fairness";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let first = store
|
||||
.reserve_local_object_metadata_at_incarnation(&disk_ref, bucket, "first", incarnation, Duration::from_secs(5))
|
||||
.await
|
||||
.expect("first reservation");
|
||||
let lock = store
|
||||
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("publication lock");
|
||||
let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut writer).await.is_err(),
|
||||
"writer waits for first token"
|
||||
);
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
tokio::time::resume();
|
||||
let mut second = Box::pin(store.reserve_local_object_metadata_at_incarnation(
|
||||
&disk_ref,
|
||||
bucket,
|
||||
"second",
|
||||
incarnation,
|
||||
Duration::from_secs(5),
|
||||
));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(),
|
||||
"expired cached reader must not bypass writer"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, first)
|
||||
.await
|
||||
.expect("cancel first token")
|
||||
);
|
||||
let guard = tokio::time::timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("writer proceeds")
|
||||
.expect("writer lock");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(),
|
||||
"new reader waits while writer holds the lock"
|
||||
);
|
||||
drop(guard);
|
||||
let token = tokio::time::timeout(Duration::from_secs(2), second)
|
||||
.await
|
||||
.expect("reader proceeds")
|
||||
.expect("second reservation");
|
||||
assert!(
|
||||
store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, token)
|
||||
.await
|
||||
.expect("cancel second token")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lost_bucket_fence_cannot_activate_a_prepared_reservation() {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)));
|
||||
let client = Arc::new(RefreshLossLockClient::with_manager(manager));
|
||||
let lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
|
||||
"reservation-loss".to_string(),
|
||||
vec![client.clone() as Arc<dyn rustfs_lock::LockClient>],
|
||||
1,
|
||||
);
|
||||
let acquire = |object: &'static str| {
|
||||
rustfs_lock::LockRequest::new(
|
||||
rustfs_lock::ObjectKey::new("bucket", object),
|
||||
rustfs_lock::LockType::Shared,
|
||||
"reservation-loss",
|
||||
)
|
||||
.with_ttl(Duration::from_secs(2))
|
||||
.with_refresh_interval(Duration::from_millis(20))
|
||||
};
|
||||
let publication = lock
|
||||
.acquire_guard(&acquire("publication"))
|
||||
.await
|
||||
.expect("publication request")
|
||||
.expect("publication quorum");
|
||||
let signal = publication.lock_lost_signal().expect("distributed loss signal");
|
||||
let lifecycle = lock
|
||||
.acquire_guard(&acquire("lifecycle"))
|
||||
.await
|
||||
.expect("lifecycle request")
|
||||
.expect("lifecycle quorum");
|
||||
let bucket_registry = Arc::new(crate::store::bucket_fence::BucketFenceRegistry::default());
|
||||
let lifecycle =
|
||||
crate::store::bucket_fence::BucketIncarnationFenceGuard::new(bucket_registry.clone(), "bucket", lifecycle, true);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let slot = bucket_registry.reservation_slot("bucket", incarnation);
|
||||
let owner = Arc::new(crate::store::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot));
|
||||
let registry = crate::disk::os::MetadataReservationRegistry::new(1);
|
||||
let key = std::path::Path::new("bucket/object");
|
||||
let disk = Uuid::new_v4();
|
||||
let token = registry
|
||||
.prepare_with_owner(key, disk, Some(owner))
|
||||
.expect("prepare reservation");
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(Duration::from_secs(5), signal.notified())
|
||||
.await
|
||||
.expect("publication lock loss");
|
||||
assert!(
|
||||
registry.activate(token, key, disk).await.is_err(),
|
||||
"a lost bucket fence must not enter the mutation queue"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn borrowed_copy_tail_drained_waits_for_rename_tail() {
|
||||
|
||||
@@ -85,171 +85,6 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Prepare an inert user-object reservation on the currently admitted disk.
|
||||
/// An unclaimed token expires and does not hold the object's metadata lock.
|
||||
#[cfg(test)]
|
||||
pub async fn reserve_local_object_metadata(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Prepare before the caller acquires bucket lifecycle or object locks.
|
||||
/// The token holds the bucket publication fence through physical completion.
|
||||
pub async fn reserve_local_object_metadata_at_incarnation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
if self.ctx.lock_manager().is_disabled()
|
||||
|| crate::bucket::utils::check_valid_bucket_name(bucket).is_err()
|
||||
|| is_meta_bucketname(bucket)
|
||||
|| incarnation.is_nil()
|
||||
|| timeout.is_zero()
|
||||
|| rustfs_utils::path::object_name_disk_identity(object).as_ref()
|
||||
== rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
|
||||
let slot = self.bucket_fence_registry.reservation_slot(bucket, incarnation);
|
||||
let mut cached = slot.owner.lock().await;
|
||||
let owner = match cached.upgrade().filter(|owner| owner.is_reusable()) {
|
||||
Some(owner) => owner,
|
||||
None => {
|
||||
// Bucket deletion takes publication before lifecycle and metadata.
|
||||
let lock = self
|
||||
.handle_new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let publication = lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
// Older peers may acquire lifecycle without publication.
|
||||
let lifecycle = self
|
||||
.acquire_bucket_incarnation_fence(bucket, incarnation)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let owner = Arc::new(super::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot.clone()));
|
||||
if !owner.is_reusable() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
*cached = Arc::downgrade(&owner);
|
||||
owner
|
||||
}
|
||||
};
|
||||
drop(cached);
|
||||
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, Some(owner))
|
||||
.await
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn reserve_local_object_metadata_with_owner(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: std::time::Duration,
|
||||
owner: Option<Arc<dyn crate::disk::os::MetadataReservationFence>>,
|
||||
) -> DiskResult<Uuid> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
wrapper.prepare_object_metadata_with_owner(bucket, object, owner),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
/// Atomically enqueue a prepared token on its original disk and object.
|
||||
pub async fn activate_local_object_metadata_reservation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
) -> DiskResult<()> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
wrapper.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed token on the currently admitted disk.
|
||||
/// A token already claimed by a physical mutation cannot be cancelled.
|
||||
pub async fn cancel_local_object_metadata_reservation(&self, disk_ref: &str, token: Uuid) -> DiskResult<bool> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
Ok(wrapper.cancel_object_metadata_reservation(token))
|
||||
}
|
||||
|
||||
/// Publish an ordinary staged user-object write using a one-shot disk reservation.
|
||||
/// The token is bound to the admitted disk instance and destination. A claimed
|
||||
/// physical mutation continues to own its lease if the RPC waiter is cancelled.
|
||||
/// The caller must retain the bucket lifecycle fence until the physical tail
|
||||
/// finishes; the reservation does not identify a bucket incarnation.
|
||||
pub async fn rename_local_data_reserved(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
token: Uuid,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
if token.is_nil()
|
||||
|| source.0 != crate::disk::RUSTFS_META_TMP_BUCKET
|
||||
|| crate::bucket::utils::check_valid_bucket_name(destination.0).is_err()
|
||||
|| is_meta_bucketname(destination.0)
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&fi.metadata,
|
||||
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||
)
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
source,
|
||||
fi,
|
||||
destination,
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Resolve a quota lease against this request's captured storage instance.
|
||||
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
@@ -435,7 +270,6 @@ impl ECStore {
|
||||
scanner_publication_lease_token: scanner_token,
|
||||
external_guard,
|
||||
namespace_owner: None,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -517,7 +351,7 @@ fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn admit_local_disk(
|
||||
async fn admit_local_disk(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk: &DiskStore,
|
||||
disk_id: Option<Uuid>,
|
||||
@@ -949,116 +783,6 @@ mod tests {
|
||||
.expect("seeded metadata bytes")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_target_rename_is_bound_to_the_admitted_disk() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let dir = tempfile::tempdir().expect("disk root");
|
||||
let id = Uuid::new_v4();
|
||||
let disk = target_disk(&ctx, dir.path(), id).await;
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let version = Uuid::new_v4();
|
||||
let fi = target_file_info("object", version, b"reserved");
|
||||
seed_target(&disk, crate::disk::RUSTFS_META_TMP_BUCKET, "staged", fi.clone()).await;
|
||||
tokio::fs::create_dir_all(dir.path().join("target-bucket"))
|
||||
.await
|
||||
.expect("user bucket");
|
||||
assert!(matches!(
|
||||
store
|
||||
.reserve_local_object_metadata(
|
||||
&disk_ref,
|
||||
"target-bucket/../.rustfs.sys",
|
||||
"object",
|
||||
std::time::Duration::from_secs(5),
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
assert!(matches!(
|
||||
store
|
||||
.reserve_local_object_metadata(&disk_ref, "", ".rustfs.sys/config", std::time::Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
let token = store
|
||||
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve current disk");
|
||||
assert!(matches!(
|
||||
store
|
||||
.rename_local_data_reserved(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "object"), token)
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
store
|
||||
.activate_local_object_metadata_reservation(&disk_ref, "target-bucket", "object", token)
|
||||
.await
|
||||
.expect("activate token");
|
||||
store
|
||||
.rename_local_data_reserved(
|
||||
&disk_ref,
|
||||
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
|
||||
&fi,
|
||||
("target-bucket", "object"),
|
||||
token,
|
||||
)
|
||||
.await
|
||||
.expect("reserved publication");
|
||||
assert!(
|
||||
!store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, token)
|
||||
.await
|
||||
.expect("consumed token")
|
||||
);
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"object",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read reserved publication");
|
||||
assert_eq!(read.data, fi.data);
|
||||
|
||||
let stale = store
|
||||
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve before disk replacement");
|
||||
let replacement = target_disk(&ctx, dir.path(), id).await;
|
||||
assert!(matches!(
|
||||
store
|
||||
.rename_local_data_reserved(
|
||||
&disk_ref,
|
||||
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
|
||||
&fi,
|
||||
("target-bucket", "object"),
|
||||
stale,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::DiskOngoingReq)
|
||||
));
|
||||
let after = replacement
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"object",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("replacement retains publication");
|
||||
assert_eq!(after.data, fi.data);
|
||||
assert!(store.cancel_local_object_metadata_reservation(&disk_ref, stale).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
|
||||
for warm in [false, true] {
|
||||
|
||||
@@ -382,42 +382,6 @@ impl LockClient for LocalClient {
|
||||
}
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let mut guards = shard.write().await;
|
||||
let Some(entry) = guards.get_mut(lock_id) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if entry.is_expired() {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(entry.guard.downgrade_committed_tail())
|
||||
}
|
||||
|
||||
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let mut guards = shard.write().await;
|
||||
let Some(entry) = guards.get_mut(lock_id) else { return Ok(false) };
|
||||
if entry.is_expired() {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(entry.guard.try_upgrade_committed_tail())
|
||||
}
|
||||
|
||||
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
|
||||
self.refresh(lock_id).await
|
||||
}
|
||||
|
||||
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let guards = shard.read().await;
|
||||
let Some(entry) = guards.get(lock_id) else { return Ok(None) };
|
||||
if entry.is_expired() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(entry.guard.waiting_writer())
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
self.release(lock_id).await
|
||||
}
|
||||
@@ -583,32 +547,6 @@ mod tests {
|
||||
assert!(!response.success);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn downgrade_tail_is_idempotent_and_release_preserves_readers() {
|
||||
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
|
||||
let resource = crate::ObjectKey::new("bucket", "tail-downgrade");
|
||||
let first = request(resource.clone(), "owner-a", Duration::from_secs(30));
|
||||
assert!(client.acquire_lock(&first).await.expect("initial lock").success);
|
||||
assert!(client.downgrade_committed_tail(&first.lock_id).await.expect("downgrade"));
|
||||
assert!(
|
||||
client
|
||||
.downgrade_committed_tail(&first.lock_id)
|
||||
.await
|
||||
.expect("retry downgrade")
|
||||
);
|
||||
let reader = crate::LockRequest::new(resource.clone(), LockType::Shared, "reader")
|
||||
.with_acquire_timeout(Duration::from_millis(500));
|
||||
assert!(client.acquire_lock(&reader).await.expect("reader").success);
|
||||
let writer = request(resource, "writer", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
|
||||
assert!(!client.acquire_lock(&writer).await.expect("writer blocked").success);
|
||||
assert!(client.release(&first.lock_id).await.expect("release tail"));
|
||||
assert!(!client.acquire_lock(&writer).await.expect("reader excludes writer").success);
|
||||
assert!(client.release(&reader.lock_id).await.expect("release reader"));
|
||||
assert!(client.acquire_lock(&writer).await.expect("writer after drain").success);
|
||||
assert!(client.release(&writer.lock_id).await.expect("release writer"));
|
||||
assert!(!client.downgrade_committed_tail(&first.lock_id).await.expect("released lease"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_rejects_a_lock_id_for_another_resource() {
|
||||
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
|
||||
|
||||
@@ -20,12 +20,6 @@ use async_trait::async_trait;
|
||||
use futures::future::join_all;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Marker exchanged in a refresh request and response for a committed-tail conversion.
|
||||
pub const COMMITTED_TAIL_CONVERSION_MARKER: &str = "rustfs-committed-tail-v1";
|
||||
pub const COMMITTED_TAIL_UPGRADE_MARKER: &str = "rustfs-committed-tail-upgrade-v1";
|
||||
pub const HEAL_LOCK_TRANSITION_PROBE_MARKER: &str = "rustfs-heal-lock-transition-probe-v1";
|
||||
pub const HEAL_WAITING_WRITER_MARKER: &str = "rustfs-heal-waiting-writer-v1";
|
||||
|
||||
/// Lock client trait
|
||||
#[async_trait]
|
||||
pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
@@ -54,27 +48,6 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
/// Refresh lock
|
||||
async fn refresh(&self, lock_id: &LockId) -> Result<bool>;
|
||||
|
||||
/// Convert an exclusive lease to a reader-compatible committed-tail lease.
|
||||
/// Unsupported peers retain their original lease and report `false`.
|
||||
async fn downgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Try to restore exclusive ownership, retaining the writer fence on failure.
|
||||
async fn try_upgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Check support without changing the lease mode.
|
||||
async fn supports_heal_lock_transition(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Observe foreground writer contention, or return `None` when unsupported.
|
||||
async fn waiting_writer(&self, _lock_id: &LockId) -> Result<Option<bool>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Force release lock
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool>;
|
||||
|
||||
|
||||
@@ -262,8 +262,6 @@ pub struct DistributedLockGuard {
|
||||
/// All underlying leases that should be refreshed and released with the guard.
|
||||
entries: Vec<HeldLockEntry>,
|
||||
lock_type: LockType,
|
||||
/// Shared and exclusive quorums must intersect before a tail may convert.
|
||||
committed_tail_quorums_intersect: bool,
|
||||
/// If true, Drop will not try to release (used if user manually released).
|
||||
disarmed: bool,
|
||||
/// Background lease task for renewable leases.
|
||||
@@ -279,13 +277,7 @@ impl DistributedLockGuard {
|
||||
/// - `entries` is the full list of underlying leases that should be refreshed and released
|
||||
/// with this guard.
|
||||
/// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics.
|
||||
fn new(
|
||||
lock_id: LockId,
|
||||
entries: Vec<HeldLockEntry>,
|
||||
lock_type: LockType,
|
||||
heartbeat: LockHeartbeatConfig,
|
||||
committed_tail_quorums_intersect: bool,
|
||||
) -> Self {
|
||||
fn new(lock_id: LockId, entries: Vec<HeldLockEntry>, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self {
|
||||
record_lock_held_acquire(lock_type);
|
||||
let lock_lost = Arc::new(LockLostSignal::default());
|
||||
match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) {
|
||||
@@ -304,7 +296,6 @@ impl DistributedLockGuard {
|
||||
lock_id,
|
||||
entries,
|
||||
lock_type,
|
||||
committed_tail_quorums_intersect,
|
||||
disarmed: false,
|
||||
refresh_task,
|
||||
lock_lost,
|
||||
@@ -421,70 +412,6 @@ impl DistributedLockGuard {
|
||||
&self.lock_id
|
||||
}
|
||||
|
||||
/// Convert retained peer leases without ever releasing writer exclusion.
|
||||
/// A peer without conversion support keeps its exclusive lease. Partial
|
||||
/// conversion is safe: both lease modes continue to exclude writers.
|
||||
pub async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.downgrade_committed_tail(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
/// Require every retained lease to confirm exclusivity before committing.
|
||||
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || self.is_lock_lost() {
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.try_upgrade_committed_tail(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
pub async fn supports_heal_lock_transition(&self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.supports_heal_lock_transition(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
/// Observe contention on every held lease. Unknown peers prevent a false idle result.
|
||||
pub async fn waiting_writer(&self) -> Option<bool> {
|
||||
if self.disarmed || self.is_lock_lost() {
|
||||
return None;
|
||||
}
|
||||
let results =
|
||||
futures::future::join_all(self.entries.iter().map(|entry| entry.client.waiting_writer(&entry.lock_id))).await;
|
||||
if self.is_lock_lost() {
|
||||
return None;
|
||||
}
|
||||
if results.iter().any(|result| matches!(result, Ok(Some(true)))) {
|
||||
return Some(true);
|
||||
}
|
||||
results
|
||||
.iter()
|
||||
.all(|result| matches!(result, Ok(Some(false))))
|
||||
.then_some(false)
|
||||
}
|
||||
|
||||
/// Actual granted peer identities, which may differ from the logical guard ID.
|
||||
pub fn granted_lock_ids(&self) -> impl Iterator<Item = &LockId> {
|
||||
self.entries.iter().map(|entry| &entry.lock_id)
|
||||
@@ -672,7 +599,6 @@ impl DistributedLock {
|
||||
individual_locks,
|
||||
request.lock_type,
|
||||
heartbeat,
|
||||
self.read_quorum().saturating_add(required_quorum) > self.clients.len(),
|
||||
)))
|
||||
} else {
|
||||
// Check if it's a timeout or quorum failure
|
||||
|
||||
@@ -39,8 +39,6 @@ pub struct FastLockGuard {
|
||||
shard: Option<Arc<LockShard>>, // None when locks are disabled
|
||||
released: bool,
|
||||
disabled: bool, // True when locks are disabled globally
|
||||
committed_tail: bool,
|
||||
sealed_for_commit: bool,
|
||||
/// Unique ID for this guard instance to prevent double-release
|
||||
guard_id: u64,
|
||||
}
|
||||
@@ -56,8 +54,6 @@ impl FastLockGuard {
|
||||
shard: Some(shard),
|
||||
released: false,
|
||||
disabled: false,
|
||||
committed_tail: false,
|
||||
sealed_for_commit: false,
|
||||
guard_id,
|
||||
}
|
||||
}
|
||||
@@ -72,8 +68,6 @@ impl FastLockGuard {
|
||||
shard: None,
|
||||
released: false,
|
||||
disabled: true,
|
||||
committed_tail: false,
|
||||
sealed_for_commit: false,
|
||||
guard_id,
|
||||
}
|
||||
}
|
||||
@@ -93,83 +87,6 @@ impl FastLockGuard {
|
||||
&self.owner
|
||||
}
|
||||
|
||||
/// The physical acquisition was force-released before registration.
|
||||
pub(crate) fn abandon_unregistered(&mut self) {
|
||||
if !self.released {
|
||||
self.released = true;
|
||||
record_lock_held_release(self.mode);
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep excluding writers while permitting reads of the committed version.
|
||||
pub fn downgrade_committed_tail(&mut self) -> bool {
|
||||
if self.released || self.sealed_for_commit {
|
||||
return false;
|
||||
}
|
||||
if self.committed_tail {
|
||||
return true;
|
||||
}
|
||||
if self.mode != LockMode::Exclusive {
|
||||
return false;
|
||||
}
|
||||
if self.disabled {
|
||||
self.mode = LockMode::Shared;
|
||||
self.committed_tail = true;
|
||||
return true;
|
||||
}
|
||||
let Some(shard) = &self.shard else { return false };
|
||||
if !shard.downgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
|
||||
return false;
|
||||
}
|
||||
record_lock_held_release(LockMode::Exclusive);
|
||||
record_lock_held_acquire(LockMode::Shared);
|
||||
self.mode = LockMode::Shared;
|
||||
self.committed_tail = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// Try to restore exclusive ownership without releasing the writer fence.
|
||||
pub fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
if self.released || (!self.committed_tail && self.mode != LockMode::Exclusive) {
|
||||
return false;
|
||||
}
|
||||
if !self.disabled {
|
||||
let Some(shard) = &self.shard else { return false };
|
||||
if !shard.try_upgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if self.committed_tail {
|
||||
record_lock_held_release(LockMode::Shared);
|
||||
record_lock_held_acquire(LockMode::Exclusive);
|
||||
}
|
||||
self.mode = LockMode::Exclusive;
|
||||
self.committed_tail = false;
|
||||
self.sealed_for_commit = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// Observe writers waiting on this guard's exact object key.
|
||||
pub fn waiting_writer(&self) -> Option<bool> {
|
||||
if self.released {
|
||||
return None;
|
||||
}
|
||||
if self.disabled {
|
||||
return Some(false);
|
||||
}
|
||||
self.shard
|
||||
.as_ref()?
|
||||
.waiting_writer_for_guard(&self.key, &self.owner, self.guard_id)
|
||||
}
|
||||
|
||||
fn release_physical(&self, shard: &LockShard) -> bool {
|
||||
if self.committed_tail {
|
||||
shard.release_committed_tail_with_guard(&self.key, &self.owner, self.guard_id)
|
||||
} else {
|
||||
shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Manually release the lock early
|
||||
///
|
||||
/// Returns true if the lock was successfully released, false if it was
|
||||
@@ -189,7 +106,7 @@ impl FastLockGuard {
|
||||
}
|
||||
|
||||
if let Some(shard) = &self.shard {
|
||||
let success = self.release_physical(shard);
|
||||
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
|
||||
if success {
|
||||
self.released = true;
|
||||
record_lock_held_release(self.mode);
|
||||
@@ -252,7 +169,7 @@ impl Drop for FastLockGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(shard) = &self.shard {
|
||||
if !self.released && !self.disabled {
|
||||
let success = self.release_physical(shard);
|
||||
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
|
||||
if !success {
|
||||
// For high-concurrency scenarios, this is likely due to:
|
||||
// 1. Lock was already released by another thread
|
||||
|
||||
@@ -96,14 +96,11 @@ impl FastObjectLockManager {
|
||||
/// Acquire object lock
|
||||
pub async fn acquire_lock(&self, request: ObjectLockRequest) -> Result<FastLockGuard, LockResult> {
|
||||
let shard = self.get_shard(&request.key);
|
||||
match shard.acquire_guarded(&request).await {
|
||||
Ok(acquired) => {
|
||||
let mut guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
|
||||
match shard.acquire_lock(&request).await {
|
||||
Ok(()) => {
|
||||
let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
|
||||
// Register guard to prevent premature cleanup
|
||||
if !shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired) {
|
||||
guard.abandon_unregistered();
|
||||
return Err(LockResult::Timeout);
|
||||
}
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
Ok(guard)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
@@ -204,15 +201,31 @@ impl FastObjectLockManager {
|
||||
let mut all_failed = Vec::new();
|
||||
let mut guards = Vec::new();
|
||||
|
||||
for (_, requests) in shard_groups {
|
||||
for (shard_id, requests) in shard_groups {
|
||||
let shard = self.shards[*shard_id].clone();
|
||||
|
||||
for request in requests {
|
||||
let key = request.key.clone();
|
||||
match self.acquire_lock(request.clone()).await {
|
||||
Ok(guard) => {
|
||||
all_successful.push(key);
|
||||
guards.push(guard);
|
||||
let owner = request.owner.clone();
|
||||
let mode = request.mode;
|
||||
|
||||
let acquired = if shard.try_fast_path_only(request) {
|
||||
true
|
||||
} else {
|
||||
match shard.acquire_lock(request).await {
|
||||
Ok(()) => true,
|
||||
Err(err) => {
|
||||
all_failed.push((key.clone(), err));
|
||||
false
|
||||
}
|
||||
}
|
||||
Err(err) => all_failed.push((key, err)),
|
||||
};
|
||||
|
||||
if acquired {
|
||||
let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
all_successful.push(key);
|
||||
guards.push(guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -232,10 +245,14 @@ impl FastObjectLockManager {
|
||||
let mut acquired_guards = Vec::new();
|
||||
let mut failed_locks = Vec::new();
|
||||
|
||||
'outer: for (_, requests) in shard_groups {
|
||||
'outer: for (shard_id, requests) in shard_groups {
|
||||
let shard = self.shards[*shard_id].clone();
|
||||
|
||||
for request in requests {
|
||||
match self.acquire_lock(request.clone()).await {
|
||||
Ok(guard) => {
|
||||
match shard.acquire_lock(request).await {
|
||||
Ok(()) => {
|
||||
let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
acquired_guards.push(guard);
|
||||
}
|
||||
Err(err) => {
|
||||
|
||||
@@ -94,7 +94,6 @@ impl ObjectLockState {
|
||||
pub fn reset_for_reuse(&mut self) {
|
||||
// Reset atomic state
|
||||
self.atomic_state = crate::fast_lock::state::AtomicLockState::new();
|
||||
self.force_epoch.store(0, Ordering::Relaxed);
|
||||
|
||||
// Clear owners
|
||||
*self.current_owner.write() = None;
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use tokio::time::timeout;
|
||||
|
||||
@@ -31,12 +30,6 @@ struct ActiveGuardInfo {
|
||||
key: ObjectKey,
|
||||
mode: LockMode,
|
||||
owner: Arc<str>,
|
||||
committed_tail: bool,
|
||||
}
|
||||
|
||||
pub(crate) struct AcquiredLock {
|
||||
state: Arc<ObjectLockState>,
|
||||
force_epoch: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Hash)]
|
||||
@@ -53,8 +46,6 @@ enum TestPausePoint {
|
||||
BeforeFastWrite,
|
||||
FastWriteExisting,
|
||||
SlowLookup,
|
||||
ForceAfterTailRelease,
|
||||
ReleaseAfterUnregister,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -78,7 +69,7 @@ fn test_pause(point: TestPausePoint, key: &ObjectKey) {
|
||||
|
||||
/// Lock shard to reduce global contention.
|
||||
///
|
||||
/// When combined, lock `active_guards`, then `objects`, then a state's owner metadata.
|
||||
/// When both are needed, lock `objects` before a state's owner metadata.
|
||||
#[derive(Debug)]
|
||||
pub struct LockShard {
|
||||
/// Object lock states - using parking_lot for better performance
|
||||
@@ -142,16 +133,12 @@ impl LockShard {
|
||||
|
||||
/// Acquire lock with fast path optimization
|
||||
pub async fn acquire_lock(&self, request: &ObjectLockRequest) -> Result<(), LockResult> {
|
||||
self.acquire_guarded(request).await.map(|_| ())
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_guarded(&self, request: &ObjectLockRequest) -> Result<AcquiredLock, LockResult> {
|
||||
let start_time = Instant::now();
|
||||
|
||||
// Try fast path first
|
||||
if let Some(acquired) = self.try_fast_path(request) {
|
||||
if let Some(_state) = self.try_fast_path(request) {
|
||||
self.metrics.record_fast_path_success();
|
||||
return Ok(acquired);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Slow path with waiting
|
||||
@@ -170,7 +157,7 @@ impl LockShard {
|
||||
}
|
||||
|
||||
/// Try fast path lock acquisition (lock-free when possible)
|
||||
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<AcquiredLock> {
|
||||
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<Arc<ObjectLockState>> {
|
||||
// First try to get existing state without write lock
|
||||
{
|
||||
let objects = self.objects.read();
|
||||
@@ -181,16 +168,14 @@ impl LockShard {
|
||||
test_pause(TestPausePoint::FastExisting, &request.key);
|
||||
|
||||
// Try atomic acquisition
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let success = force_epoch != u64::MAX
|
||||
&& match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
let success = match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
drop(objects);
|
||||
|
||||
if success {
|
||||
return Some(AcquiredLock { state, force_epoch });
|
||||
return Some(state);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -210,11 +195,10 @@ impl LockShard {
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::FastWriteExisting, &request.key);
|
||||
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let acquired = force_epoch != u64::MAX && state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
|
||||
let acquired = state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
|
||||
drop(objects);
|
||||
if acquired {
|
||||
return Some(AcquiredLock { state, force_epoch });
|
||||
return Some(state);
|
||||
}
|
||||
} else {
|
||||
// Create new state from pool and acquire immediately
|
||||
@@ -222,7 +206,7 @@ impl LockShard {
|
||||
let state = Arc::new(*state_box);
|
||||
if state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout) {
|
||||
objects.insert(request.key.clone(), state.clone());
|
||||
return Some(AcquiredLock { state, force_epoch: 0 });
|
||||
return Some(state);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -231,7 +215,7 @@ impl LockShard {
|
||||
}
|
||||
|
||||
/// Slow path with async waiting
|
||||
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<AcquiredLock, LockResult> {
|
||||
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<(), LockResult> {
|
||||
// Use adaptive timeout based on current load and request priority
|
||||
let adaptive_timeout = self.calculate_adaptive_timeout(request);
|
||||
let deadline = start_time + adaptive_timeout;
|
||||
@@ -250,7 +234,7 @@ impl LockShard {
|
||||
|
||||
loop {
|
||||
// Keep the state in the map until its acquisition attempt completes.
|
||||
let (state, success, force_epoch) = {
|
||||
let (state, success) = {
|
||||
let mut objects = self.objects.write();
|
||||
let state = match objects.get(&request.key) {
|
||||
Some(state) => state.clone(),
|
||||
@@ -265,19 +249,17 @@ impl LockShard {
|
||||
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::SlowLookup, &request.key);
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let success = force_epoch != u64::MAX
|
||||
&& match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
let success = match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
drop(objects);
|
||||
(state, success, force_epoch)
|
||||
(state, success)
|
||||
};
|
||||
|
||||
if success {
|
||||
self.metrics.record_slow_path_success();
|
||||
return Ok(AcquiredLock { state, force_epoch });
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Check timeout
|
||||
@@ -399,10 +381,14 @@ impl LockShard {
|
||||
|
||||
/// Release lock with guard ID tracking for double-release prevention
|
||||
pub fn release_lock_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, mode: LockMode, guard_id: u64) -> bool {
|
||||
// Keep force-unlock from releasing this holder and admitting a new
|
||||
// same-owner holder between guard removal and physical release.
|
||||
let mut guards = self.active_guards.lock();
|
||||
if guards.remove(&guard_id).is_none() {
|
||||
// First, try to remove the guard from active set
|
||||
let guard_was_active = {
|
||||
let mut guards = self.active_guards.lock();
|
||||
guards.remove(&guard_id).is_some()
|
||||
};
|
||||
|
||||
// If guard was not active, this is a double-release attempt
|
||||
if !guard_was_active {
|
||||
tracing::debug!(
|
||||
"Double-release attempt blocked: key={}, owner={}, mode={:?}, guard_id={}",
|
||||
key,
|
||||
@@ -413,9 +399,6 @@ impl LockShard {
|
||||
return false;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::ReleaseAfterUnregister, key);
|
||||
|
||||
// Proceed with normal release
|
||||
let should_cleanup;
|
||||
let result;
|
||||
@@ -440,8 +423,6 @@ impl LockShard {
|
||||
}
|
||||
}
|
||||
|
||||
drop(guards);
|
||||
|
||||
if should_cleanup {
|
||||
self.schedule_cleanup(key.clone());
|
||||
}
|
||||
@@ -449,110 +430,21 @@ impl LockShard {
|
||||
result
|
||||
}
|
||||
|
||||
/// Convert an active exclusive guard without opening a writer gap.
|
||||
pub(crate) fn downgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Exclusive {
|
||||
return false;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
if !state.downgrade_committed_tail(owner) {
|
||||
return false;
|
||||
}
|
||||
info.mode = LockMode::Shared;
|
||||
info.committed_tail = true;
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn try_upgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
|
||||
return false;
|
||||
}
|
||||
if info.mode == LockMode::Exclusive && !info.committed_tail {
|
||||
return true;
|
||||
}
|
||||
if info.mode != LockMode::Shared || !info.committed_tail {
|
||||
return false;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
if !state.try_upgrade_committed_tail(owner) {
|
||||
return false;
|
||||
}
|
||||
info.mode = LockMode::Exclusive;
|
||||
info.committed_tail = false;
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn waiting_writer_for_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> Option<bool> {
|
||||
let guards = self.active_guards.lock();
|
||||
let info = guards.get(&guard_id)?.as_ref()?;
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
|
||||
return None;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let state = objects.get(key)?;
|
||||
Some(state.atomic_state.writer_observed_during_tail())
|
||||
}
|
||||
|
||||
/// Release a converted guard and its special reader-admission state.
|
||||
pub(crate) fn release_committed_tail_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Shared || !info.committed_tail {
|
||||
return false;
|
||||
}
|
||||
let (success, should_cleanup) = {
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
let success = state.release_committed_tail(owner);
|
||||
(success, success && !state.is_locked() && !state.atomic_state.has_waiters())
|
||||
};
|
||||
if success {
|
||||
guards.remove(&guard_id);
|
||||
self.metrics.record_release();
|
||||
}
|
||||
drop(guards);
|
||||
if should_cleanup {
|
||||
self.schedule_cleanup(key.clone());
|
||||
}
|
||||
success
|
||||
}
|
||||
|
||||
/// Register a guard to prevent premature cleanup
|
||||
pub fn register_guard(&self, guard_id: u64) {
|
||||
self.active_guards.lock().insert(guard_id, None);
|
||||
}
|
||||
|
||||
pub(crate) fn register_guard_with_info(
|
||||
&self,
|
||||
guard_id: u64,
|
||||
key: &ObjectKey,
|
||||
mode: LockMode,
|
||||
owner: &Arc<str>,
|
||||
acquired: &AcquiredLock,
|
||||
) -> bool {
|
||||
pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc<str>) {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let objects = self.objects.read();
|
||||
if !objects.get(key).is_some_and(|state| {
|
||||
Arc::ptr_eq(state, &acquired.state) && state.force_epoch.load(Ordering::Acquire) == acquired.force_epoch
|
||||
}) {
|
||||
return false;
|
||||
}
|
||||
guards.insert(
|
||||
guard_id,
|
||||
Some(ActiveGuardInfo {
|
||||
key: key.clone(),
|
||||
mode,
|
||||
owner: owner.clone(),
|
||||
committed_tail: false,
|
||||
}),
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
/// Unregister a guard (called when guard is dropped)
|
||||
@@ -827,57 +719,27 @@ impl LockShard {
|
||||
/// Returns the number of owners that were released. Used by the admin
|
||||
/// force-unlock path to clear a stuck resource.
|
||||
pub fn force_release_all(&self, key: &ObjectKey) -> usize {
|
||||
// Serialize tail conversion and prevent new acquisitions while taking
|
||||
// and releasing the owner snapshot. A new tail must never be mistaken
|
||||
// for an ordinary shared owner by force unlock.
|
||||
let mut guards = self.active_guards.lock();
|
||||
let objects = self.objects.write();
|
||||
let Some(state) = objects.get(key) else { return 0 };
|
||||
// Acquisitions completed before this point but not yet registered
|
||||
// must not later create guards for the released generation.
|
||||
let _ = state
|
||||
.force_epoch
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |epoch| epoch.checked_add(1));
|
||||
let tail = guards.iter().find_map(|(&id, info)| {
|
||||
info.as_ref()
|
||||
.filter(|info| info.committed_tail && info.key == *key)
|
||||
.map(|info| (id, info.owner.clone()))
|
||||
});
|
||||
let mut released = 0;
|
||||
if let Some((id, owner)) = tail
|
||||
&& state.release_committed_tail(&owner)
|
||||
{
|
||||
guards.remove(&id);
|
||||
self.metrics.record_release();
|
||||
released += 1;
|
||||
}
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::ForceAfterTailRelease, key);
|
||||
let mut owners_modes = Vec::new();
|
||||
if let Some(info) = state.current_owner.read().clone() {
|
||||
owners_modes.push((info.owner, LockMode::Exclusive, 1));
|
||||
}
|
||||
for entry in state.shared_owners.read().iter() {
|
||||
owners_modes.push((entry.owner.clone(), LockMode::Shared, entry.count));
|
||||
}
|
||||
for (owner, mode, count) in owners_modes {
|
||||
let mut owner_released = false;
|
||||
for _ in 0..count {
|
||||
let success = match mode {
|
||||
LockMode::Shared => state.release_shared(&owner),
|
||||
LockMode::Exclusive => state.release_exclusive(&owner),
|
||||
};
|
||||
if success {
|
||||
self.metrics.record_release();
|
||||
owner_released = true;
|
||||
}
|
||||
let owners_modes: Vec<(Arc<str>, LockMode)> = {
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else {
|
||||
return 0;
|
||||
};
|
||||
let mut pairs = Vec::new();
|
||||
if let Some(info) = state.current_owner.read().clone() {
|
||||
pairs.push((info.owner, LockMode::Exclusive));
|
||||
}
|
||||
if owner_released {
|
||||
for entry in state.shared_owners.read().iter() {
|
||||
pairs.push((entry.owner.clone(), LockMode::Shared));
|
||||
}
|
||||
pairs
|
||||
};
|
||||
|
||||
let mut released = 0;
|
||||
for (owner, mode) in owners_modes {
|
||||
if self.release_lock(key, &owner, mode) {
|
||||
released += 1;
|
||||
}
|
||||
}
|
||||
// Old guard drops must not release a later holder with the same owner.
|
||||
guards.retain(|_, info| info.as_ref().is_none_or(|info| info.key != *key));
|
||||
released
|
||||
}
|
||||
|
||||
@@ -1117,315 +979,6 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn force_unlock_cannot_interleave_an_old_release_with_a_new_holder() {
|
||||
for reader in [false, true] {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-release-gap");
|
||||
let old = if reader {
|
||||
manager.acquire_read_lock(key.clone(), "same-owner").await
|
||||
} else {
|
||||
manager.acquire_write_lock(key.clone(), "same-owner").await
|
||||
}
|
||||
.expect("old holder");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let (entered, resume) = install_pause(TestPausePoint::ReleaseAfterUnregister, &key);
|
||||
let old_drop = tokio::task::spawn_blocking(move || drop(old));
|
||||
entered.recv_timeout(Duration::from_secs(5)).expect("old release paused");
|
||||
|
||||
let can_force = shard.active_guards.try_lock().is_some();
|
||||
let mut force = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
|
||||
};
|
||||
let mut new_guard = if can_force {
|
||||
assert_eq!(force.await.expect("force join"), 1);
|
||||
let mut guard = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer");
|
||||
if reader {
|
||||
assert!(guard.downgrade_committed_tail());
|
||||
}
|
||||
resume.send(()).expect("resume old release");
|
||||
old_drop.await.expect("old release join");
|
||||
guard
|
||||
} else {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut force).await.is_err());
|
||||
resume.send(()).expect("resume old release");
|
||||
old_drop.await.expect("old release join");
|
||||
assert_eq!(force.await.expect("force join"), 0);
|
||||
manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer")
|
||||
};
|
||||
if reader && !can_force {
|
||||
assert!(new_guard.downgrade_committed_tail());
|
||||
}
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"an old release must not unlock the new writer"
|
||||
);
|
||||
drop(new_guard);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn force_unlock_does_not_capture_a_new_committed_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-tail-race");
|
||||
let mut old = manager.acquire_write_lock(key.clone(), "old").await.expect("old writer");
|
||||
assert!(old.downgrade_committed_tail());
|
||||
let (entered, resume) = install_pause(TestPausePoint::ForceAfterTailRelease, &key);
|
||||
let force = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
|
||||
};
|
||||
entered.recv_timeout(Duration::from_secs(5)).expect("force unlock paused");
|
||||
let mut next = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_write_lock(key, "new").await })
|
||||
};
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut next).await.is_err());
|
||||
resume.send(()).expect("resume force unlock");
|
||||
assert_eq!(force.await.expect("force unlock task"), 1);
|
||||
let mut new = next.await.expect("new writer task").expect("new writer");
|
||||
assert!(new.downgrade_committed_tail());
|
||||
drop(old);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"the new committed tail must still exclude writers"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_tail_upgrade_waits_for_readers_and_keeps_writer_fenced() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgrade-tail");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = manager
|
||||
.acquire_read_lock(key.clone(), "reader")
|
||||
.await
|
||||
.expect("reader during tail");
|
||||
assert!(!tail.try_upgrade_committed_tail(), "an active reader must prevent exclusive commit");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "late-reader"))
|
||||
.await
|
||||
.is_err(),
|
||||
"new readers must not starve a pending upgrade"
|
||||
);
|
||||
drop(reader);
|
||||
assert!(tail.try_upgrade_committed_tail());
|
||||
assert!(tail.try_upgrade_committed_tail(), "a repeated request must be idempotent");
|
||||
assert!(
|
||||
!tail.downgrade_committed_tail(),
|
||||
"a delayed downgrade must not reopen reads after upgrade"
|
||||
);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "reader"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
drop(tail);
|
||||
let _writer = manager.acquire_write_lock(key, "writer").await.expect("writer after release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tail_remembers_a_writer_that_stopped_waiting_between_samples() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "brief-writer-observation");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(tail.waiting_writer(), Some(true), "a short wait must not be missed by polling");
|
||||
assert!(!tail.try_upgrade_committed_tail(), "observed contention must prevent a later commit");
|
||||
drop(tail);
|
||||
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
|
||||
assert_eq!(next.waiting_writer(), Some(false), "contention must not leak to the next owner");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upgraded_tail_remembers_a_writer_that_stopped_waiting() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgraded-tail-brief-writer");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert!(tail.try_upgrade_committed_tail());
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(tail.waiting_writer(), Some(true), "upgraded leases must retain observed contention");
|
||||
drop(tail);
|
||||
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
|
||||
assert_eq!(next.waiting_writer(), Some(false));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guard_observes_waiting_writer_and_rejects_stale_identity() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "waiting-writer-observation");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert_eq!(tail.waiting_writer(), Some(false));
|
||||
let writer = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_write_lock(key, "foreground").await })
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while tail.waiting_writer() != Some(true) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("writer becomes observable");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
assert_eq!(tail.waiting_writer(), None, "force-released guard must not inspect a replacement owner");
|
||||
let _writer = writer.await.expect("writer task").expect("writer succeeds after release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dropping_a_pending_upgrade_allows_readers_again() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "cancel-upgrade");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = manager.acquire_read_lock(key.clone(), "reader").await.expect("first reader");
|
||||
assert!(!tail.try_upgrade_committed_tail());
|
||||
let mut waiting = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_read_lock(key, "waiting-reader").await })
|
||||
};
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut waiting).await.is_err());
|
||||
drop(tail);
|
||||
let second = tokio::time::timeout(Duration::from_secs(2), waiting)
|
||||
.await
|
||||
.expect("reader wakes after cancellation")
|
||||
.expect("reader task joins")
|
||||
.expect("reader obtains lock");
|
||||
drop((reader, second));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_released_tail_cannot_claim_an_exclusive_upgrade() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgrade-stale");
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let _new = manager.acquire_write_lock(key, "same-owner").await.expect("new owner");
|
||||
assert!(!tail.try_upgrade_committed_tail());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exclusive_upgrade_ack_fences_a_late_downgrade() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "late-downgrade");
|
||||
let mut guard = manager.acquire_write_lock(key, "heal").await.expect("write lock");
|
||||
assert!(guard.try_upgrade_committed_tail());
|
||||
assert!(!guard.downgrade_committed_tail());
|
||||
assert!(guard.try_upgrade_committed_tail());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_old_readers_cannot_release_a_new_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-old-readers");
|
||||
let old1 = manager
|
||||
.acquire_read_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("first reader");
|
||||
let old2 = manager
|
||||
.acquire_read_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("second reader");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
drop((old1, old2));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"old reader drops must not release the new tail"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forced_unregistered_reader_cannot_release_a_new_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-unregistered-reader");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let owner: Arc<str> = Arc::from("same-owner");
|
||||
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
|
||||
let acquired = shard
|
||||
.acquire_guarded(&request)
|
||||
.await
|
||||
.expect("old reader acquired before registration");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let mut old = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
|
||||
assert!(!shard.register_guard_with_info(old.guard_id(), old.key(), old.mode(), old.owner(), &acquired));
|
||||
old.abandon_unregistered();
|
||||
drop(old);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"an old reader must not release a new committed tail"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_on_another_key_keeps_unregistered_acquisition_valid() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "unrelated-acquisition");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let owner: Arc<str> = Arc::from("owner");
|
||||
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
|
||||
let acquired = shard.acquire_guarded(&request).await.expect("reader acquired");
|
||||
let other = (0..1000)
|
||||
.map(|index| ObjectKey::new("bucket", format!("other-{index}")))
|
||||
.find(|other| Arc::ptr_eq(&shard, manager.get_shard(other)))
|
||||
.expect("another key on the same shard");
|
||||
let other_guard = manager
|
||||
.acquire_write_lock(other.clone(), "other")
|
||||
.await
|
||||
.expect("other writer");
|
||||
assert_eq!(manager.force_unlock(&other), 1);
|
||||
drop(other_guard);
|
||||
let guard = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
|
||||
assert!(shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired));
|
||||
drop(guard);
|
||||
assert!(manager.acquire_write_lock(key, "next").await.is_ok());
|
||||
}
|
||||
|
||||
fn start_acquisition(
|
||||
manager: &Arc<FastObjectLockManager>,
|
||||
request: ObjectLockRequest,
|
||||
|
||||
@@ -23,11 +23,6 @@ use crate::fast_lock::types::{LockMode, LockPriority};
|
||||
/// Optimized atomic lock state encoding in u64
|
||||
/// Bits: [63:48] reserved | [47:32] writers_waiting | [31:16] readers_waiting | [15:8] readers_count | [7:1] flags | [0] writer_flag
|
||||
const WRITER_FLAG_MASK: u64 = 0x1;
|
||||
// A committed write tail is a reader-compatible owner that still excludes writers.
|
||||
const COMMITTED_TAIL_FLAG: u64 = 0x2;
|
||||
const TAIL_UPGRADE_PENDING_FLAG: u64 = 0x4;
|
||||
const TAIL_WRITER_OBSERVED_FLAG: u64 = 0x8;
|
||||
const TAIL_TRACK_WRITERS_FLAG: u64 = 0x10;
|
||||
const READERS_SHIFT: u8 = 8;
|
||||
const READERS_MASK: u64 = 0xFF << READERS_SHIFT; // Support up to 255 concurrent readers
|
||||
const READERS_WAITING_SHIFT: u8 = 16;
|
||||
@@ -36,6 +31,7 @@ const WRITERS_WAITING_SHIFT: u8 = 32;
|
||||
const WRITERS_WAITING_MASK: u64 = 0xFFFF << WRITERS_WAITING_SHIFT;
|
||||
|
||||
// Fast path check masks
|
||||
const NO_WRITER_AND_NO_WAITING_WRITERS: u64 = WRITER_FLAG_MASK | WRITERS_WAITING_MASK;
|
||||
const COMPLETELY_UNLOCKED: u64 = 0;
|
||||
|
||||
/// Fast atomic lock state for single version
|
||||
@@ -70,8 +66,8 @@ impl AtomicLockState {
|
||||
let state = self.state.load(Ordering::Relaxed); // Use Relaxed for better performance
|
||||
match mode {
|
||||
LockMode::Shared => {
|
||||
(state & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) == 0
|
||||
&& ((state & WRITERS_WAITING_MASK) == 0 || (state & COMMITTED_TAIL_FLAG) != 0)
|
||||
// No writer and no waiting writers
|
||||
(state & NO_WRITER_AND_NO_WAITING_WRITERS) == 0
|
||||
}
|
||||
LockMode::Exclusive => {
|
||||
// Completely unlocked
|
||||
@@ -87,11 +83,8 @@ impl AtomicLockState {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
|
||||
// While a committed tail owns a shared slot, let reads through
|
||||
// even if a subsequent writer is waiting for that tail to drain.
|
||||
if (current & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) != 0
|
||||
|| ((current & WRITERS_WAITING_MASK) != 0 && (current & COMMITTED_TAIL_FLAG) == 0)
|
||||
{
|
||||
// Fast path check - cannot acquire if there's a writer or writers waiting
|
||||
if (current & NO_WRITER_AND_NO_WAITING_WRITERS) != 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -144,78 +137,6 @@ impl AtomicLockState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomically retain writer exclusion while allowing reads of a committed tail.
|
||||
fn downgrade_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & WRITER_FLAG_MASK) == 0 || (current & (READERS_MASK | COMMITTED_TAIL_FLAG)) != 0 {
|
||||
return false;
|
||||
}
|
||||
let next = (current & !WRITER_FLAG_MASK)
|
||||
| COMMITTED_TAIL_FLAG
|
||||
| (1 << READERS_SHIFT)
|
||||
| if current & WRITERS_WAITING_MASK != 0 {
|
||||
TAIL_WRITER_OBSERVED_FLAG
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Restore exclusive ownership only after every other reader has left.
|
||||
fn try_upgrade_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & COMMITTED_TAIL_FLAG) == 0 || (current & TAIL_WRITER_OBSERVED_FLAG) != 0 {
|
||||
return false;
|
||||
}
|
||||
let readers_drained = (current & READERS_MASK) == (1 << READERS_SHIFT);
|
||||
let next = if readers_drained {
|
||||
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG | READERS_MASK))
|
||||
| WRITER_FLAG_MASK
|
||||
| TAIL_TRACK_WRITERS_FLAG
|
||||
} else {
|
||||
current | TAIL_UPGRADE_PENDING_FLAG
|
||||
};
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return readers_drained;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Release the committed tail's own shared slot and restore ordinary fairness.
|
||||
fn release_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & COMMITTED_TAIL_FLAG) == 0 || self.readers_count(current) == 0 {
|
||||
return false;
|
||||
}
|
||||
let next =
|
||||
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG)) - (1 << READERS_SHIFT);
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Release shared lock
|
||||
pub fn release_shared(&self) -> bool {
|
||||
loop {
|
||||
@@ -248,7 +169,7 @@ impl AtomicLockState {
|
||||
return false; // No exclusive lock to release
|
||||
}
|
||||
|
||||
let new_state = current & !(WRITER_FLAG_MASK | TAIL_TRACK_WRITERS_FLAG | TAIL_WRITER_OBSERVED_FLAG);
|
||||
let new_state = current & !WRITER_FLAG_MASK;
|
||||
|
||||
if self
|
||||
.state
|
||||
@@ -315,10 +236,7 @@ impl AtomicLockState {
|
||||
return false; // Max waiting writers
|
||||
}
|
||||
|
||||
let mut new_state = current + (1 << WRITERS_WAITING_SHIFT);
|
||||
if current & (COMMITTED_TAIL_FLAG | TAIL_TRACK_WRITERS_FLAG) != 0 {
|
||||
new_state |= TAIL_WRITER_OBSERVED_FLAG;
|
||||
}
|
||||
let new_state = current + (1 << WRITERS_WAITING_SHIFT);
|
||||
|
||||
if self
|
||||
.state
|
||||
@@ -403,15 +321,10 @@ impl AtomicLockState {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn writers_waiting_count(&self) -> u16 {
|
||||
pub fn writers_waiting_count(&self) -> u16 {
|
||||
let state = self.state.load(Ordering::Acquire);
|
||||
self.writers_waiting(state)
|
||||
}
|
||||
|
||||
pub(crate) fn writer_observed_during_tail(&self) -> bool {
|
||||
let state = self.state.load(Ordering::Acquire);
|
||||
self.writers_waiting(state) > 0 || state & TAIL_WRITER_OBSERVED_FLAG != 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Object lock state with version support - optimized memory layout
|
||||
@@ -421,8 +334,6 @@ pub struct ObjectLockState {
|
||||
// First cache line: Most frequently accessed data
|
||||
/// Atomic state for fast operations
|
||||
pub atomic_state: AtomicLockState,
|
||||
/// Changes whenever an administrator force-releases this state.
|
||||
pub(crate) force_epoch: AtomicU64,
|
||||
|
||||
// Second cache line: Notification mechanisms
|
||||
/// Notification for readers (traditional)
|
||||
@@ -466,7 +377,6 @@ impl ObjectLockState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
atomic_state: AtomicLockState::new(),
|
||||
force_epoch: AtomicU64::new(0),
|
||||
read_notify: Notify::new(),
|
||||
write_notify: Notify::new(),
|
||||
optimized_notify: OptimizedNotify::new(),
|
||||
@@ -515,70 +425,6 @@ impl ObjectLockState {
|
||||
true
|
||||
}
|
||||
|
||||
/// Convert the current exclusive owner into the committed-tail owner.
|
||||
pub(crate) fn downgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut current = self.current_owner.write();
|
||||
let Some(info) = current.as_ref().filter(|info| info.owner.as_ref() == owner.as_ref()).cloned() else {
|
||||
return false;
|
||||
};
|
||||
let mut shared = self.shared_owners.write();
|
||||
if !shared.is_empty() || !self.atomic_state.downgrade_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
shared.push(SharedOwnerEntry {
|
||||
owner: info.owner,
|
||||
count: 1,
|
||||
acquired_at: info.acquired_at,
|
||||
lock_timeout: info.lock_timeout,
|
||||
});
|
||||
*current = None;
|
||||
drop(shared);
|
||||
drop(current);
|
||||
self.optimized_notify.notify_readers();
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn try_upgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut current = self.current_owner.write();
|
||||
let mut shared = self.shared_owners.write();
|
||||
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
|
||||
return false;
|
||||
};
|
||||
if current.is_some() || !self.atomic_state.try_upgrade_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
let info = shared.remove(pos);
|
||||
*current = Some(ExclusiveOwnerInfo {
|
||||
owner: info.owner,
|
||||
acquired_at: info.acquired_at,
|
||||
lock_timeout: info.lock_timeout,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
/// Release the committed-tail owner's shared slot without releasing readers.
|
||||
pub(crate) fn release_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut shared = self.shared_owners.write();
|
||||
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
|
||||
return false;
|
||||
};
|
||||
if !self.atomic_state.release_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
if shared[pos].count > 1 {
|
||||
shared[pos].count -= 1;
|
||||
} else {
|
||||
shared.remove(pos);
|
||||
}
|
||||
let no_readers = shared.is_empty();
|
||||
drop(shared);
|
||||
if no_readers {
|
||||
self.optimized_notify.notify_writer();
|
||||
}
|
||||
self.optimized_notify.notify_readers();
|
||||
true
|
||||
}
|
||||
|
||||
/// Release shared lock
|
||||
pub fn release_shared(&self, owner: &Arc<str>) -> bool {
|
||||
let mut shared = self.shared_owners.write();
|
||||
@@ -709,24 +555,6 @@ mod tests {
|
||||
assert!(!state.release_exclusive());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn committed_tail_allows_reads_but_preserves_writer_exclusion() {
|
||||
let state = AtomicLockState::new();
|
||||
assert!(state.try_acquire_exclusive());
|
||||
assert!(state.inc_writers_waiting());
|
||||
assert!(state.downgrade_committed_tail());
|
||||
assert!(!state.try_acquire_exclusive(), "the tail must still exclude writers");
|
||||
assert!(state.try_acquire_shared(), "a queued writer must not block reads of the committed tail");
|
||||
assert!(state.release_committed_tail());
|
||||
assert!(!state.try_acquire_shared(), "writer preference resumes when the tail ends");
|
||||
assert!(!state.try_acquire_exclusive(), "an outstanding reader must still exclude writers");
|
||||
assert!(state.release_shared());
|
||||
assert!(state.try_acquire_exclusive());
|
||||
assert!(state.release_exclusive());
|
||||
state.dec_writers_waiting();
|
||||
assert!(state.is_free());
|
||||
}
|
||||
|
||||
// Regression for the waiter-preserving exclusive CAS.
|
||||
//
|
||||
// The acquisition CAS used to demand a fully-zero state word, which
|
||||
|
||||
@@ -111,67 +111,6 @@ mod fast_lock_tests {
|
||||
assert!(guard.is_released(), "Guard should be marked as released");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_old_tail_cannot_release_new_tail() {
|
||||
let manager = create_test_manager();
|
||||
let key = ObjectKey::new("test-bucket", "force-tail-generation");
|
||||
let owner: Arc<str> = Arc::from("same-owner");
|
||||
let mut old = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("old writer");
|
||||
assert!(old.downgrade_committed_tail());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut new = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(new.downgrade_committed_tail());
|
||||
drop(old);
|
||||
let attempt =
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, Arc::from("competing-writer")))
|
||||
.await;
|
||||
assert!(attempt.is_err(), "dropping the old guard must not release the new tail");
|
||||
drop(new);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_tail_allows_reader_while_writer_waits() {
|
||||
let manager = Arc::new(create_test_manager());
|
||||
let key = ObjectKey::new("test-bucket", "tail-handoff");
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), Arc::from("first-writer"))
|
||||
.await
|
||||
.expect("initial writer");
|
||||
let writer = tokio::spawn({
|
||||
let manager = Arc::clone(&manager);
|
||||
let key = key.clone();
|
||||
async move { manager.acquire_write_lock(key, Arc::from("second-writer")).await }
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while manager.current_lock_counts().1 == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second writer should be queued");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = tokio::time::timeout(Duration::from_secs(2), manager.acquire_read_lock(key, Arc::from("reader")))
|
||||
.await
|
||||
.expect("reader should not wait behind committed tail")
|
||||
.expect("reader should acquire");
|
||||
assert!(!writer.is_finished(), "tail and reader must exclude the next writer");
|
||||
drop(tail);
|
||||
assert!(!writer.is_finished(), "reader still excludes the next writer");
|
||||
drop(reader);
|
||||
let writer = tokio::time::timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("writer should resume")
|
||||
.expect("writer task should join")
|
||||
.expect("writer should acquire");
|
||||
drop(writer);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_lock_counts_report_live_holders_and_waiters() {
|
||||
let manager = Arc::new(create_test_manager());
|
||||
|
||||
@@ -107,38 +107,6 @@ impl NamespaceLockGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Permit reads while continuing to exclude competing writers.
|
||||
pub async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.downgrade_committed_tail().await,
|
||||
Self::Fast(guard) => guard.downgrade_committed_tail(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to restore exclusive access while retaining the writer fence.
|
||||
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.try_upgrade_committed_tail().await,
|
||||
Self::Fast(guard) => guard.try_upgrade_committed_tail(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether the held leases can safely use heal lock transitions.
|
||||
pub async fn supports_heal_lock_transition(&self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.supports_heal_lock_transition().await,
|
||||
Self::Fast(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe whether a writer is waiting on this object's held leases.
|
||||
pub async fn waiting_writer(&self) -> Option<bool> {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.waiting_writer().await,
|
||||
Self::Fast(guard) => guard.waiting_writer(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Manually release the lock early
|
||||
pub fn release(&mut self) -> bool {
|
||||
match self {
|
||||
|
||||
@@ -154,43 +154,6 @@ impl crate::client::LockClient for DelayedClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PendingTailClient(LocalClient);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::client::LockClient for PendingTailClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
|
||||
self.0.acquire_lock(request).await
|
||||
}
|
||||
async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.release(lock_id).await
|
||||
}
|
||||
async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.refresh(lock_id).await
|
||||
}
|
||||
async fn downgrade_committed_tail(&self, _: &LockId) -> crate::Result<bool> {
|
||||
std::future::pending().await
|
||||
}
|
||||
async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.force_release(lock_id).await
|
||||
}
|
||||
async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
|
||||
self.0.check_status(lock_id).await
|
||||
}
|
||||
async fn get_stats(&self) -> crate::Result<LockStats> {
|
||||
self.0.get_stats().await
|
||||
}
|
||||
async fn close(&self) -> crate::Result<()> {
|
||||
self.0.close().await
|
||||
}
|
||||
async fn is_online(&self) -> bool {
|
||||
self.0.is_online().await
|
||||
}
|
||||
async fn is_local(&self) -> bool {
|
||||
self.0.is_local().await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FlakyAcquireClient {
|
||||
inner: LocalClient,
|
||||
@@ -616,286 +579,6 @@ async fn test_namespace_lock_guard_lock_id() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tail_conversion_rejects_non_intersecting_quorums() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..4)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients_and_quorum("non-intersecting".to_owned(), clients, 2);
|
||||
let resource = ObjectKey::new("bucket", "non-intersecting-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("write lease");
|
||||
assert!(
|
||||
!tail.downgrade_committed_tail().await,
|
||||
"conversion requires intersecting read and write quorums"
|
||||
);
|
||||
assert!(tail.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_tail_conversion_keeps_writers_excluded() {
|
||||
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = Vec::new();
|
||||
for index in 0..4 {
|
||||
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
|
||||
if index < 2 {
|
||||
clients.push(inner);
|
||||
} else {
|
||||
clients.push(Arc::new(DelayedClient {
|
||||
inner,
|
||||
delay: Duration::ZERO,
|
||||
}));
|
||||
}
|
||||
}
|
||||
let lock = NamespaceLock::with_clients("mixed-peers".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "partial-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(!tail.downgrade_committed_tail().await, "mixed peers cannot all confirm conversion");
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"mixed leases still exclude writers"
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_tail_conversion_keeps_writers_excluded() {
|
||||
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
clients.push(Arc::new(PendingTailClient(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))));
|
||||
let lock = NamespaceLock::with_clients_and_quorum("cancel-tail".to_owned(), clients, 4);
|
||||
let resource = ObjectKey::new("bucket", "cancel-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), tail.downgrade_committed_tail())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("other peers converted before cancellation");
|
||||
assert!(reader.release());
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"cancelled conversion must still exclude writers"
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsupported_peer_keeps_exclusive_tail_lease() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| {
|
||||
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
|
||||
Arc::new(DelayedClient {
|
||||
inner,
|
||||
delay: Duration::ZERO,
|
||||
}) as Arc<dyn crate::client::LockClient>
|
||||
})
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("old-peers".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "old-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(!tail.supports_heal_lock_transition().await, "old peers must not claim support");
|
||||
assert_eq!(tail.waiting_writer().await, None, "old peers cannot report contention");
|
||||
assert!(!tail.downgrade_committed_tail().await, "old peers must not claim conversion");
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "old peers must not claim upgrade support");
|
||||
assert!(
|
||||
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "writer", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after tail");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_committed_tail_allows_reader_and_excludes_writer() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("tail-test".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "committed-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(tail.downgrade_committed_tail().await, "all held peer leases should convert");
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("committed reader");
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"tail and reader must exclude writer"
|
||||
);
|
||||
assert!(tail.release());
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"reader still excludes writer"
|
||||
);
|
||||
assert!(reader.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after drain");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_tail_upgrade_requires_all_leases_and_drained_readers() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("tail-upgrade".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "tail-upgrade");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("write lock");
|
||||
assert!(tail.supports_heal_lock_transition().await);
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("reader");
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "live readers prevent full upgrade");
|
||||
assert!(reader.release());
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while !tail.try_upgrade_committed_tail().await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retry must finish partial upgrades after asynchronous reader release");
|
||||
assert!(tail.try_upgrade_committed_tail().await, "upgrade retries must be idempotent");
|
||||
assert!(
|
||||
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_tail_observes_a_waiting_writer() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = Arc::new(NamespaceLock::with_clients("writer-observation".to_owned(), clients));
|
||||
let resource = ObjectKey::new("bucket", "writer-observation");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("heal lock");
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
assert_eq!(tail.waiting_writer().await, Some(false));
|
||||
let writer = {
|
||||
let lock = Arc::clone(&lock);
|
||||
let resource = resource.clone();
|
||||
tokio::spawn(async move { lock.get_write_lock(resource, "writer", Duration::from_secs(3)).await })
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while tail.waiting_writer().await != Some(true) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("waiting writer becomes visible on a held peer");
|
||||
assert!(tail.release());
|
||||
let mut guard = writer.await.expect("writer task").expect("writer succeeds after release");
|
||||
assert!(guard.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_partial_upgrade_remembers_contention_on_upgraded_peer() {
|
||||
use crate::client::LockClient;
|
||||
|
||||
let clients: Vec<Arc<LocalClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))))
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients(
|
||||
"partial-upgrade-writer".to_owned(),
|
||||
clients.iter().cloned().map(|client| client as Arc<dyn LockClient>).collect(),
|
||||
);
|
||||
let resource = ObjectKey::new("bucket", "partial-upgrade-writer");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("heal lock");
|
||||
let ids = match &tail {
|
||||
NamespaceLockGuard::Standard(guard) => guard.granted_lock_ids().cloned().collect::<Vec<_>>(),
|
||||
NamespaceLockGuard::Fast(_) => panic!("distributed guard expected"),
|
||||
};
|
||||
let mut holders = Vec::new();
|
||||
for client in &clients {
|
||||
for id in &ids {
|
||||
if client.check_status(id).await.expect("peer status").is_some() {
|
||||
holders.push(Arc::clone(client));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(holders.len() >= 2, "write quorum must hold multiple peers");
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
let reader = LockRequest::new(resource.clone(), LockType::Shared, "reader");
|
||||
assert!(holders[0].acquire_lock(&reader).await.expect("reader request").success);
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "reader prevents a full upgrade");
|
||||
let writer = LockRequest::new(resource, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_millis(100));
|
||||
assert!(!holders[1].acquire_lock(&writer).await.expect("writer request").success);
|
||||
assert_eq!(
|
||||
tail.waiting_writer().await,
|
||||
Some(true),
|
||||
"partial upgrade must retain the earlier contention"
|
||||
);
|
||||
assert!(holders[0].release(&reader.lock_id).await.expect("release reader"));
|
||||
assert!(tail.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_distributed_multi_node_simulation() {
|
||||
// Simulate a 3-node distributed environment where each node has its own lock backend
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1092,51 +1092,6 @@ impl_canonical_mutation_body!(
|
||||
// its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
|
||||
// the JSON fallback decode path, breaks the signature-bound digest.
|
||||
|
||||
pub fn canonical_metadata_reservation_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_str(&request.object)?;
|
||||
body.push_u64(request.timeout_ms);
|
||||
body.push_bytes(&request.bucket_incarnation_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_metadata_reservation_cancel_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationCancelRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-cancel-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_metadata_reservation_activate_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationActivateRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_str(&request.object)?;
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_reserved_rename_data_request_body(
|
||||
request: &proto_gen::node_service::ReservedRenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-reserved-rename-data-v1\0");
|
||||
if let Some(rename) = request.rename.as_ref() {
|
||||
body.push_bytes(&canonical_rename_data_request_body(rename)?)?;
|
||||
} else {
|
||||
body.push_bytes(&[])?;
|
||||
}
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_data_request_body(
|
||||
request: &proto_gen::node_service::RenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
@@ -1767,84 +1722,6 @@ mod disk_mutation_canonical_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_reservation_canonical_bodies_bind_scope_and_token() {
|
||||
use proto_gen::node_service::{
|
||||
MetadataReservationActivateRequest, MetadataReservationCancelRequest, MetadataReservationRequest,
|
||||
ReservedRenameDataRequest,
|
||||
};
|
||||
|
||||
let request = MetadataReservationRequest {
|
||||
disk: "disk".into(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
timeout_ms: 5000,
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_metadata_reservation_request_body(&request).expect("reservation body")];
|
||||
for mutate in [
|
||||
|r: &mut MetadataReservationRequest| r.disk = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.bucket = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.object = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.timeout_ms = 1,
|
||||
|r: &mut MetadataReservationRequest| r.bucket_incarnation_id = vec![2; 16].into(),
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_metadata_reservation_request_body(&changed).expect("changed body"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let activation = MetadataReservationActivateRequest {
|
||||
disk: "disk".into(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_metadata_reservation_activate_request_body(&activation).expect("activation body")];
|
||||
for mutate in [
|
||||
|r: &mut MetadataReservationActivateRequest| r.disk = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.bucket = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.object = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.token = vec![2; 16].into(),
|
||||
] {
|
||||
let mut changed = activation.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_metadata_reservation_activate_request_body(&changed).expect("changed activation"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let cancellation = MetadataReservationCancelRequest {
|
||||
disk: "disk".into(),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut changed = cancellation.clone();
|
||||
changed.token = vec![2; 16].into();
|
||||
assert_ne!(
|
||||
canonical_metadata_reservation_cancel_request_body(&cancellation).expect("cancel body"),
|
||||
canonical_metadata_reservation_cancel_request_body(&changed).expect("changed cancel body")
|
||||
);
|
||||
let rename = ReservedRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
dst_path: "object".into(),
|
||||
..Default::default()
|
||||
}),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut changed_token = rename.clone();
|
||||
changed_token.token = vec![2; 16].into();
|
||||
let mut changed_object = rename.clone();
|
||||
changed_object.rename.as_mut().expect("rename").dst_path = "other".into();
|
||||
assert_all_distinct(&[
|
||||
canonical_reserved_rename_data_request_body(&rename).expect("rename body"),
|
||||
canonical_reserved_rename_data_request_body(&changed_token).expect("changed token"),
|
||||
canonical_reserved_rename_data_request_body(&changed_object).expect("changed object"),
|
||||
canonical_reserved_rename_data_request_body(&ReservedRenameDataRequest {
|
||||
rename: None,
|
||||
token: rename.token,
|
||||
})
|
||||
.expect("missing rename"),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_canonical_body_binds_every_field() {
|
||||
let baseline = RenameDataRequest {
|
||||
|
||||
@@ -339,31 +339,6 @@ message RenameDataResponse {
|
||||
bytes rename_data_resp_bin = 4;
|
||||
}
|
||||
|
||||
message MetadataReservationRequest {
|
||||
string disk = 1;
|
||||
string bucket = 2;
|
||||
string object = 3;
|
||||
uint64 timeout_ms = 4;
|
||||
bytes bucket_incarnation_id = 5;
|
||||
}
|
||||
|
||||
message MetadataReservationCancelRequest {
|
||||
string disk = 1;
|
||||
bytes token = 2;
|
||||
}
|
||||
|
||||
message MetadataReservationActivateRequest {
|
||||
string disk = 1;
|
||||
string bucket = 2;
|
||||
string object = 3;
|
||||
bytes token = 4;
|
||||
}
|
||||
|
||||
message ReservedRenameDataRequest {
|
||||
RenameDataRequest rename = 1;
|
||||
bytes token = 2;
|
||||
}
|
||||
|
||||
// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
|
||||
// accepts an arbitrary volume or object path.
|
||||
message PoolMetaNativeRecoveryRequest {
|
||||
@@ -1339,10 +1314,6 @@ service NodeService {
|
||||
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
|
||||
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
|
||||
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc ReserveObjectMetadata(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
|
||||
rpc ActivateObjectMetadataReservation(MetadataReservationActivateRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
|
||||
rpc CancelObjectMetadataReservation(MetadataReservationCancelRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameReservedData(ReservedRenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
// The method itself requires source-preserving multipart publication on every peer.
|
||||
rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
|
||||
@@ -90,7 +90,9 @@ pub use bucket::BUCKET_LIFECYCLE_LOCK_OBJECT;
|
||||
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
|
||||
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
|
||||
pub use error::{StorageErrorCode, StorageResult};
|
||||
pub use multipart::MultipartUploadInspection;
|
||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||
pub use object::AuthoritativeListOperations;
|
||||
pub use object::DeleteAccounting;
|
||||
pub use object::ObjectLockDeleteOptions;
|
||||
pub use object::ObjectToDeleteIdentity;
|
||||
|
||||
@@ -15,6 +15,15 @@
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Checks upload absence before removing a bucket's data protections.
|
||||
#[async_trait::async_trait]
|
||||
pub trait MultipartUploadInspection: Send + Sync {
|
||||
type Error: std::error::Error + Send + Sync + 'static;
|
||||
|
||||
/// Unknown or unreadable upload ownership must fail this check.
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct MultipartUploadResult {
|
||||
pub upload_id: String,
|
||||
|
||||
@@ -566,6 +566,29 @@ pub trait ListOperations: Send + Sync + fmt::Debug {
|
||||
) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
/// Quorum-backed enumeration for decisions that require proving object absence.
|
||||
///
|
||||
/// Consumers must drain the result stream and await successful completion before
|
||||
/// trusting the scan. An error or cancellation can follow partial results and
|
||||
/// leaves absence unproven. Callers coordinate concurrent mutations separately;
|
||||
/// this capability does not provide a snapshot or retain a mutation lock.
|
||||
#[async_trait::async_trait]
|
||||
pub trait AuthoritativeListOperations: ListOperations {
|
||||
/// Resolves every set at its version read quorum, including reserved prefixes.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when a set cannot establish quorum, metadata cannot be
|
||||
/// resolved, or enumeration is interrupted. Retry starts a new scan.
|
||||
async fn walk_authoritative(
|
||||
self: Arc<Self>,
|
||||
rx: Self::WalkCancellation,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Self::WalkResultSender,
|
||||
opts: Self::WalkOptions,
|
||||
) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub trait MultipartOperations: Send + Sync + fmt::Debug {
|
||||
|
||||
@@ -91,6 +91,26 @@ Use `GET /iceberg/v1/buckets/{warehouse}` to check `enabled` and `disable-suppor
|
||||
|
||||
Review the bucket lifecycle configuration before disabling: clearing table-bucket protection allows existing object expiration rules to run again. Objects are preserved by the disable operation itself. Catalog disablement requires no namespaces, retained table/view resources, or objects under `.rustfs-table/`; an active rename or backing migration also blocks it. Clean up through supported catalog operations; do not manually delete internal metadata to bypass these checks.
|
||||
|
||||
The server persists an inactive catalog entry before clearing the bucket marker. If marker publication fails, retry the same DELETE; the marker retains object protections until the retry succeeds. A subsequent PUT explicitly enables the bucket again. Stale requests cannot initialize an inactive catalog entry. No table metadata or object data is deleted by either state change.
|
||||
The server persists an inactive catalog entry before clearing the bucket marker. If marker publication fails, retry the same DELETE; the marker retains object protections until the retry succeeds. A subsequent PUT explicitly enables the bucket again. Stale requests cannot initialize an inactive catalog entry. A plain DELETE and a subsequent PUT preserve table metadata and object data.
|
||||
|
||||
### Clean Up Deleted Empty Tables
|
||||
|
||||
For the object-backed catalog, `DELETE /iceberg/v1/buckets/{warehouse}?cleanup=true` removes the metadata and internal records left by deleted empty tables, then disables the table catalog. Ordinary table deletion continues to preserve files. Cleanup preserves existing table record formats and can process empty tables deleted before the server upgrade.
|
||||
|
||||
Complete the server upgrade across all serving nodes and set the existing `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` setting before requesting cleanup. Delete the empty tables and their namespaces through the catalog APIs, then send the cleanup request with `admin:SetTableBucket` and `s3:GetObject`/`s3:DeleteObject` permissions for the target metadata objects. The `/_iceberg/v1` alias accepts the same option. Omitting `cleanup` or setting it to `false` keeps the ordinary disable behavior; repeated or invalid values are rejected.
|
||||
|
||||
Cleanup accepts only recognized empty-table metadata in the default managed warehouse. Active catalog resources, snapshots or other data references, user objects, object versions, incomplete multipart uploads, corrupt or unknown metadata, and unfinished catalog operations block cleanup. The server validates the complete cleanup plan and object permissions before deleting any object. Durable-strong catalogs retain their existing disable procedure.
|
||||
|
||||
If a storage operation fails during cleanup, retry the same request. The server checks the remaining objects again and keeps the bucket's table protection enabled until cleanup and catalog disablement complete. No separate cleanup task file is created. After a successful response with `enabled: false`, delete the empty bucket through the normal S3 `DeleteBucket` operation.
|
||||
|
||||
## Object-Backed Registration Reads
|
||||
|
||||
Table registration checks table ID uniqueness and warehouse directory overlap using the existing catalog and warehouse index records. Each registration obtains a complete inventory under the existing publication and migration guards. System-bucket inventory records must satisfy the same metadata commit checks as ordinary catalog reads. An incomplete or failed inventory rejects registration.
|
||||
|
||||
Each server keeps a bounded memory cache of validated ownership records. A record can be reused only when its object identity and ETag match the current inventory. New or changed records are read and validated again. Cache eviction and process restarts cause additional reads; catalog state continues to come from the existing persisted records. The cache is automatic and requires no additional configuration or stored objects.
|
||||
|
||||
Registration still enumerates the namespace and warehouse index prefixes, so enumeration work grows with catalog size. Cache hits reduce object-body reads and JSON parsing. A cache hit verifies the current metadata identity and reuses previously validated content; it does not re-read the current body to check its physical readability. Conditional writes, rollback, and lock-loss fencing continue to control publication.
|
||||
|
||||
Existing catalog formats and Console and Iceberg client request formats are preserved. Concurrent writers must follow the existing publication, migration, and storage fencing protocol; the memory cache does not provide synchronization for writers that bypass those guards.
|
||||
|
||||
Before rolling back to a server version without this endpoint, re-enable disabled catalogs with PUT while the supporting version is running. Older enable handlers do not reactivate inactive catalog entries. Complete the server upgrade across the cluster before using disablement.
|
||||
|
||||
Generated
+6
-6
@@ -2,11 +2,11 @@
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1792812153,
|
||||
"narHash": "sha256-T2+XBfNbf9pecLUWS4+RtuVNZ3W8K5taG5QdV71mWlk=",
|
||||
"lastModified": 1793466162,
|
||||
"narHash": "sha256-pt+peAEoOK3lLLdtbnQ1AGIBTZbdB6jrkgmurfcQwig=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "674a2028661e550a97400c1cc992b9cca9a668a1",
|
||||
"rev": "a0788123ada57ec92a79ab2581cda6d3c6cca75f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -29,11 +29,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1792825530,
|
||||
"narHash": "sha256-aphXA2IFNd3q48++TKmfuuvDDjkrgGdJDnDIhij9684=",
|
||||
"lastModified": 1793434469,
|
||||
"narHash": "sha256-+sgVNI2c67di+XIeMaz5KSei4T65kX68s5Nl0E5Gzbc=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "5b9f1198b59bcb31c5c3250b332e7d77590e6306",
|
||||
"rev": "6ecf534a7c16e7c969f26838fdd8c93597ae93f3",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -50,17 +50,23 @@ pub struct DisableTableBucketHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for DisableTableBucketHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
let resource = TableCatalogResource::warehouse(&warehouse);
|
||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableBucketAction).await?;
|
||||
let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableBucketAction).await?;
|
||||
let cleanup = table_bucket_cleanup_requested_from_query(&req.uri)?;
|
||||
let app_context = runtime_sources::app_context_from_extensions(&req.extensions);
|
||||
let backend = table_catalog_backend_from_extensions(&req.extensions)?;
|
||||
let object_store = runtime_sources::object_store_from_req(&req)
|
||||
.ok_or_else(|| table_catalog_internal_error("request object store is not initialized"))?;
|
||||
let store = table_catalog_store_from_backend(backend.clone())?;
|
||||
let publication = TableCommitObjectBackend::preauthorized(backend);
|
||||
let response = disable_table_bucket_response(&store, &publication, object_store.as_ref(), &warehouse).await?;
|
||||
install_table_catalog_s3_request_info(&mut req, &principal)?;
|
||||
let publication = TableCommitObjectBackend::for_request(backend, req);
|
||||
let response = if cleanup {
|
||||
cleanup_empty_table_bucket_response(&store, &publication, object_store.as_ref(), &warehouse).await?
|
||||
} else {
|
||||
disable_table_bucket_response(&store, &publication, object_store.as_ref(), &warehouse).await?
|
||||
};
|
||||
crate::app::bucket_usecase::notify_table_bucket_metadata_reload(warehouse.clone(), "disable table bucket", app_context)
|
||||
.await;
|
||||
build_json_response(StatusCode::OK, &response)
|
||||
|
||||
@@ -2092,17 +2092,25 @@ fn warehouse_from_config_query(uri: &http::Uri) -> S3Result<Option<String>> {
|
||||
}
|
||||
|
||||
fn rest_purge_requested_from_query(uri: &http::Uri) -> S3Result<bool> {
|
||||
let mut purge_requested = None;
|
||||
table_catalog_boolean_query(uri, "purgeRequested")
|
||||
}
|
||||
|
||||
fn table_bucket_cleanup_requested_from_query(uri: &http::Uri) -> S3Result<bool> {
|
||||
table_catalog_boolean_query(uri, "cleanup")
|
||||
}
|
||||
|
||||
fn table_catalog_boolean_query(uri: &http::Uri, parameter: &str) -> S3Result<bool> {
|
||||
let mut requested = None;
|
||||
if let Some(query) = uri.query() {
|
||||
for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
|
||||
if key != "purgeRequested" {
|
||||
if key != parameter {
|
||||
continue;
|
||||
}
|
||||
if purge_requested.is_some() {
|
||||
if requested.is_some() {
|
||||
return Err(iceberg_rest_error(
|
||||
ICEBERG_ERROR_BAD_REQUEST,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"purgeRequested query parameter must not be repeated",
|
||||
format!("{parameter} query parameter must not be repeated"),
|
||||
));
|
||||
}
|
||||
let value = if value.eq_ignore_ascii_case("true") {
|
||||
@@ -2113,13 +2121,13 @@ fn rest_purge_requested_from_query(uri: &http::Uri) -> S3Result<bool> {
|
||||
return Err(iceberg_rest_error(
|
||||
ICEBERG_ERROR_BAD_REQUEST,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"purgeRequested query parameter must be true or false",
|
||||
format!("{parameter} query parameter must be true or false"),
|
||||
));
|
||||
};
|
||||
purge_requested = Some(value);
|
||||
requested = Some(value);
|
||||
}
|
||||
}
|
||||
Ok(purge_requested.unwrap_or(false))
|
||||
Ok(requested.unwrap_or(false))
|
||||
}
|
||||
|
||||
fn rest_table_snapshot_selection_from_query(uri: &http::Uri) -> S3Result<RestTableSnapshotSelection> {
|
||||
@@ -2417,9 +2425,10 @@ fn table_catalog_backend_from_extensions(
|
||||
) -> S3Result<crate::table_catalog::EcStoreTableCatalogObjectBackend<ECStore>> {
|
||||
let context = runtime_sources::app_context_from_extensions(extensions)
|
||||
.ok_or_else(|| table_catalog_internal_error("request application context is not initialized"))?;
|
||||
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_runtimes(
|
||||
context.object_store(),
|
||||
context.table_catalog_strong_runtime(),
|
||||
context.table_catalog_registration_cache(),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -2572,6 +2581,61 @@ where
|
||||
table_bucket_response(store, bucket, true).await
|
||||
}
|
||||
|
||||
async fn cleanup_empty_table_bucket_response<B>(
|
||||
store: &crate::table_catalog::ConfiguredTableCatalogStore<B>,
|
||||
publication: &TableCommitObjectBackend<B>,
|
||||
object_store: &ECStore,
|
||||
bucket: &str,
|
||||
) -> S3Result<TableBucketResponse>
|
||||
where
|
||||
B: crate::table_catalog::TableCatalogObjectBackend,
|
||||
{
|
||||
if !publication.publication_fence_fleet_confirmed {
|
||||
return Err(catalog_store_error(crate::table_catalog::TableCatalogStoreError::Conflict(
|
||||
"empty table bucket cleanup requires publication fences on every cluster node".to_string(),
|
||||
)));
|
||||
}
|
||||
crate::table_catalog::TableCommitPublication::begin_table_bucket(publication, bucket)
|
||||
.await
|
||||
.map_err(catalog_store_error)?;
|
||||
let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication);
|
||||
let incarnation = object_store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.map_err(|err| ApiError::other(format!("failed to read table bucket identity: {err}")))?;
|
||||
let prepared = store
|
||||
.prepare_empty_table_bucket_cleanup(table_bucket_entry_from_metadata_marker(bucket), publication)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
let mut cleanup = publication.finish(prepared).await?;
|
||||
for object in cleanup.metadata_objects() {
|
||||
if let Err(err) = publication.authorize(bucket, object, S3Action::DeleteObjectAction).await {
|
||||
return publication.finish(Err(catalog_store_error(err))).await;
|
||||
}
|
||||
}
|
||||
store
|
||||
.execute_empty_table_bucket_cleanup(&mut cleanup, incarnation)
|
||||
.await
|
||||
.map_err(catalog_store_error)?;
|
||||
let fence = cleanup.mutation_fence();
|
||||
object_store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
crate::table_catalog::TABLE_BUCKET_MARKER_CONFIG,
|
||||
Vec::new(),
|
||||
incarnation,
|
||||
|| {
|
||||
fence
|
||||
.ensure_held()
|
||||
.map_err(|err| crate::admin::storage_api::error::StorageError::other(err.to_string()))
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ApiError::other(format!("failed to disable table bucket: {err}")))?;
|
||||
drop(cleanup);
|
||||
table_bucket_response(store, bucket, false).await
|
||||
}
|
||||
|
||||
async fn disable_table_bucket_response<S, B>(
|
||||
store: &S,
|
||||
publication: &TableCommitObjectBackend<B>,
|
||||
|
||||
@@ -87,6 +87,15 @@ async fn call_load_table_handler(request: S3Request<Body>) -> S3Result<S3Respons
|
||||
RestLoadTableHandler {}.call(request, matched.params).await
|
||||
}
|
||||
|
||||
async fn call_disable_table_bucket_handler(request: S3Request<Body>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let mut router = matchit::Router::new();
|
||||
router.insert("/iceberg/v1/buckets/{warehouse}", ()).unwrap();
|
||||
router.insert("/_iceberg/v1/buckets/{warehouse}", ()).unwrap();
|
||||
let path = request.uri.path().to_string();
|
||||
let matched = router.at(&path).unwrap();
|
||||
DisableTableBucketHandler {}.call(request, matched.params).await
|
||||
}
|
||||
|
||||
async fn call_maintenance_worker_handler(request: S3Request<Body>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let mut router = matchit::Router::new();
|
||||
router
|
||||
@@ -1255,6 +1264,28 @@ fn drop_table_purge_query_is_explicit_and_strict() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_bucket_cleanup_query_is_explicit_and_strict() {
|
||||
for (uri, expected) in [
|
||||
("/iceberg/v1/buckets/analytics", false),
|
||||
("/iceberg/v1/buckets/analytics?cleanup=false", false),
|
||||
("/iceberg/v1/buckets/analytics?cleanup=true", true),
|
||||
("/_iceberg/v1/buckets/analytics?cleanup=true", true),
|
||||
] {
|
||||
assert_eq!(table_bucket_cleanup_requested_from_query(&uri.parse().unwrap()).unwrap(), expected);
|
||||
}
|
||||
for query in [
|
||||
"cleanup=",
|
||||
"cleanup=1",
|
||||
"cleanup=yes",
|
||||
"cleanup=true&cleanup=false",
|
||||
"cleanup=true&cleanup=true",
|
||||
] {
|
||||
let uri = format!("/iceberg/v1/buckets/analytics?{query}").parse().unwrap();
|
||||
assert!(table_bucket_cleanup_requested_from_query(&uri).is_err(), "invalid query: {query}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catalog_conflicts_use_operation_specific_iceberg_errors() {
|
||||
let already_exists = catalog_store_already_exists_error(crate::table_catalog::TableCatalogStoreError::Conflict(
|
||||
@@ -1689,9 +1720,7 @@ fn table_bucket_toggle_handlers_reload_peer_metadata_after_publication() {
|
||||
(enable, "enable_table_bucket_response", "enable table bucket"),
|
||||
(disable, "disable_table_bucket_response", "disable table bucket"),
|
||||
] {
|
||||
let response_offset = block
|
||||
.find(&format!("let response = {response}"))
|
||||
.expect("table bucket toggle should publish local state");
|
||||
let response_offset = block.find(response).expect("table bucket toggle should publish local state");
|
||||
let reload = block
|
||||
.find(&format!("notify_table_bucket_metadata_reload(warehouse.clone(), \"{operation}\""))
|
||||
.expect("table bucket toggle should reload peer metadata");
|
||||
@@ -16274,3 +16303,799 @@ async fn disable_table_bucket_preserves_objects_and_can_be_reenabled() {
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
}
|
||||
|
||||
async fn create_dropped_empty_tables_for_cleanup<S, B>(
|
||||
catalog: &S,
|
||||
backend: &B,
|
||||
object_store: &ECStore,
|
||||
bucket: &str,
|
||||
table_count: usize,
|
||||
) -> Vec<String>
|
||||
where
|
||||
S: TableCatalogStore + ?Sized,
|
||||
B: TableCatalogObjectBackend,
|
||||
{
|
||||
object_store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
enable_table_bucket_response(catalog, &TableCommitObjectBackend::trusted(backend.clone()), object_store, bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").unwrap();
|
||||
create_namespace_response(
|
||||
catalog,
|
||||
bucket,
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut metadata_objects = Vec::new();
|
||||
for number in 0..table_count {
|
||||
let name = format!("events_{number:02}");
|
||||
let request = serde_json::from_value(serde_json::json!({
|
||||
"name": name,
|
||||
"schema": {
|
||||
"type": "struct",
|
||||
"schema-id": 0,
|
||||
"fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
create_table_response(
|
||||
catalog,
|
||||
&TableCommitObjectBackend::trusted(backend.clone()),
|
||||
bucket,
|
||||
&namespace,
|
||||
request,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_objects.push(
|
||||
catalog
|
||||
.load_table(bucket, &namespace.public_name(), &name)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.metadata_location,
|
||||
);
|
||||
}
|
||||
for number in 0..table_count {
|
||||
catalog
|
||||
.drop_table(bucket, &namespace.public_name(), &format!("events_{number:02}"))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
catalog.drop_namespace(bucket, &namespace.public_name()).await.unwrap();
|
||||
metadata_objects
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn object_catalog_registration_cache_checks_real_records_without_additional_objects() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::{
|
||||
ConfiguredTableCatalogStore, EcStoreTableCatalogObjectBackend, StrongTableCatalogRuntime, TableCatalogBackingMode,
|
||||
TableCatalogObjectPaths, TableCatalogStoreError,
|
||||
};
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("registration-cache-{}", Uuid::new_v4().simple());
|
||||
let backend =
|
||||
EcStoreTableCatalogObjectBackend::new_with_strong_runtime(object_store.clone(), StrongTableCatalogRuntime::default());
|
||||
let catalog = ConfiguredTableCatalogStore::new_for_test(backend.clone(), TableCatalogBackingMode::ObjectBacked);
|
||||
object_store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
enable_table_bucket_response(&catalog, &TableCommitObjectBackend::trusted(backend.clone()), &object_store, &bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").unwrap();
|
||||
create_namespace_response(
|
||||
&catalog,
|
||||
&bucket,
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let paths = TableCatalogObjectPaths::default();
|
||||
let bucket_entry = paths.table_bucket_entry_path(&bucket);
|
||||
let catalog_prefix = format!("{}/", bucket_entry.rsplit_once('/').unwrap().0);
|
||||
let mut create_micros = Vec::new();
|
||||
for number in 0..100 {
|
||||
let store = ConfiguredTableCatalogStore::new_for_test(backend.clone(), TableCatalogBackingMode::ObjectBacked);
|
||||
let request = serde_json::from_value(serde_json::json!({
|
||||
"name": format!("events_{number:03}"),
|
||||
"schema": {
|
||||
"type": "struct",
|
||||
"schema-id": 0,
|
||||
"fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
let started = std::time::Instant::now();
|
||||
create_table_response(
|
||||
&store,
|
||||
&TableCommitObjectBackend::trusted(backend.clone()),
|
||||
&bucket,
|
||||
&namespace,
|
||||
request,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("fresh store instances share the runtime cache and create independent tables");
|
||||
create_micros.push(started.elapsed().as_micros());
|
||||
if matches!(number + 1, 20 | 100) {
|
||||
let before = backend.list_objects(RUSTFS_META_BUCKET, &catalog_prefix).await.unwrap();
|
||||
let cold_backend = EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
StrongTableCatalogRuntime::default(),
|
||||
);
|
||||
let probe = ConfiguredTableCatalogStore::new_for_test(cold_backend, TableCatalogBackingMode::ObjectBacked);
|
||||
let mut duplicate = catalog
|
||||
.load_table(&bucket, "analytics", &format!("events_{number:03}"))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
duplicate.table = "duplicate_probe".to_string();
|
||||
for cache_state in ["cold", "warm"] {
|
||||
let started = std::time::Instant::now();
|
||||
assert!(matches!(
|
||||
probe.create_table(duplicate.clone()).await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
eprintln!(
|
||||
"registration_check tables={} cache={} elapsed_us={}",
|
||||
number + 1,
|
||||
cache_state,
|
||||
started.elapsed().as_micros()
|
||||
);
|
||||
}
|
||||
assert_eq!(before, backend.list_objects(RUSTFS_META_BUCKET, &catalog_prefix).await.unwrap());
|
||||
let mut recent = create_micros[create_micros.len() - 5..].to_vec();
|
||||
recent.sort_unstable();
|
||||
eprintln!("create_table_response tables={} last_five_median_us={}", number + 1, recent[2]);
|
||||
}
|
||||
}
|
||||
assert_eq!(catalog.list_all_tables(&bucket).await.unwrap().len(), 100);
|
||||
let namespace_entry = paths.namespace_entry_path(&bucket, &namespace);
|
||||
let index_prefix = paths.warehouse_index_entries_prefix(&bucket);
|
||||
let objects = backend.list_objects(RUSTFS_META_BUCKET, &catalog_prefix).await.unwrap();
|
||||
assert!(objects.iter().all(|key| {
|
||||
key == &bucket_entry || key == &namespace_entry || key.starts_with(&index_prefix) || key.ends_with("/table-entry.json")
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_reclaims_twenty_dropped_tables_and_allows_bucket_deletion() {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-{}", Uuid::new_v4().simple());
|
||||
let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
);
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 20).await;
|
||||
assert!(
|
||||
disable_table_bucket_response(
|
||||
&catalog,
|
||||
&TableCommitObjectBackend::trusted(backend.clone()),
|
||||
object_store.as_ref(),
|
||||
&bucket,
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
|
||||
let authorizations = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let publication = TableCommitObjectBackend::test(backend.clone(), authorizations.clone(), None);
|
||||
let result = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket).await;
|
||||
assert!(!publication.finish(result).await.unwrap().enabled);
|
||||
for object in &metadata_objects {
|
||||
assert!(!backend.object_exists(&bucket, object).await.unwrap());
|
||||
let authorized = authorizations.lock().await;
|
||||
assert!(authorized.contains(&(object.clone(), S3Action::GetObjectAction)));
|
||||
assert!(authorized.contains(&(object.clone(), S3Action::DeleteObjectAction)));
|
||||
}
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
let publication = TableCommitObjectBackend::trusted(backend);
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_validates_authorization_and_metadata_before_deleting() {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-check-{}", Uuid::new_v4().simple());
|
||||
let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
);
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let publication = TableCommitObjectBackend::rolling_upgrade(backend.clone());
|
||||
assert!(
|
||||
cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
|
||||
let authorizations = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let denied_object = metadata_objects.last().unwrap().clone();
|
||||
let publication = TableCommitObjectBackend::test(backend.clone(), authorizations, Some(denied_object.clone()));
|
||||
let result = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket).await;
|
||||
assert_eq!(publication.finish(result).await.unwrap_err().code(), &S3ErrorCode::AccessDenied);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
|
||||
let original = backend.read_object(&bucket, &denied_object).await.unwrap().unwrap().data;
|
||||
backend
|
||||
.put_object(
|
||||
&bucket,
|
||||
&denied_object,
|
||||
b"{".to_vec(),
|
||||
crate::table_catalog::TableCatalogPutPrecondition::Any,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
backend
|
||||
.put_object(&bucket, &denied_object, original, crate::table_catalog::TableCatalogPutPrecondition::Any)
|
||||
.await
|
||||
.unwrap();
|
||||
let publication = TableCommitObjectBackend::trusted(backend);
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_handler_enforces_request_object_permissions() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(
|
||||
crate::table_catalog::ENV_TABLE_CATALOG_BACKING,
|
||||
Some(crate::table_catalog::TABLE_CATALOG_BACKING_OBJECT),
|
||||
),
|
||||
(crate::table_catalog::ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
async {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-request-{}", Uuid::new_v4().simple());
|
||||
rustfs_iam::store::object::ObjectStore::new(object_store.clone())
|
||||
.save_iam_config(
|
||||
serde_json::json!({"version": 1}),
|
||||
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let iam = rustfs_iam::build_iam_sys(object_store.clone()).await.unwrap();
|
||||
let access_key = "cleanup-request-user";
|
||||
let secret_key = "cleanup-request-user-secret";
|
||||
iam.create_user(
|
||||
access_key,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: secret_key.to_string(),
|
||||
policy: None,
|
||||
status: AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let context = Arc::new(AppContext::new(
|
||||
object_store.clone(),
|
||||
Arc::new(RequestIam { handle: iam.clone() }),
|
||||
Arc::new(RequestKms),
|
||||
));
|
||||
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
|
||||
access_key: "cleanup-request-root".to_string(),
|
||||
secret_key: "cleanup-request-root-secret".to_string(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
}));
|
||||
let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
context.table_catalog_strong_runtime(),
|
||||
);
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects =
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let request = |prefix: &str| {
|
||||
let mut request = table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
request.method = Method::DELETE;
|
||||
request.uri = format!("{prefix}/v1/buckets/{bucket}?cleanup=true").parse().unwrap();
|
||||
request
|
||||
};
|
||||
let object_resource = format!("arn:aws:s3:::{bucket}/*");
|
||||
let first_metadata_resource = format!("arn:aws:s3:::{bucket}/{}", metadata_objects[0]);
|
||||
for (policy_name, object_statements, allowed) in [
|
||||
("cleanup-request-no-read", Vec::new(), false),
|
||||
(
|
||||
"cleanup-request-partial-delete",
|
||||
vec![
|
||||
serde_json::json!({"Effect":"Allow","Action":["s3:GetObject"],"Resource":[object_resource]}),
|
||||
serde_json::json!({"Effect":"Allow","Action":["s3:DeleteObject"],"Resource":[first_metadata_resource]}),
|
||||
],
|
||||
false,
|
||||
),
|
||||
(
|
||||
"cleanup-request-full",
|
||||
vec![serde_json::json!({
|
||||
"Effect":"Allow",
|
||||
"Action":["s3:GetObject","s3:DeleteObject"],
|
||||
"Resource":[object_resource],
|
||||
})],
|
||||
true,
|
||||
),
|
||||
] {
|
||||
let mut statements = vec![serde_json::json!({"Effect":"Allow","Action":["admin:SetTableBucket"]})];
|
||||
statements.extend(object_statements);
|
||||
let policy = serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap();
|
||||
iam.set_policy(policy_name, Policy::parse_config(&policy).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
iam.policy_db_set(access_key, UserType::Reg, false, policy_name)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = call_disable_table_bucket_handler(request(if allowed { "/_iceberg" } else { "/iceberg" })).await;
|
||||
if allowed {
|
||||
let response = result.expect("request-scoped object permissions should authorize cleanup");
|
||||
assert_eq!(response.output.0, StatusCode::OK);
|
||||
let body: serde_json::Value = serde_json::from_slice(&response.output.1.bytes().unwrap()).unwrap();
|
||||
assert_eq!(body["enabled"], false);
|
||||
for object in &metadata_objects {
|
||||
assert!(!backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
} else {
|
||||
assert_eq!(result.unwrap_err().code(), &S3ErrorCode::AccessDenied);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
crate::table_catalog::TableCatalogEntryState::Active,
|
||||
);
|
||||
}
|
||||
}
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_retries_after_marker_write_rejection() {
|
||||
use crate::table_catalog::TableCommitPublication;
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-marker-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::default();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
TableCommitPublication::begin_table_bucket(&publication, &bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
let mut cleanup = catalog
|
||||
.prepare_empty_table_bucket_cleanup(table_bucket_entry_from_metadata_marker(&bucket), &publication)
|
||||
.await
|
||||
.unwrap();
|
||||
catalog
|
||||
.execute_empty_table_bucket_cleanup(&mut cleanup, incarnation)
|
||||
.await
|
||||
.unwrap();
|
||||
let marker_result = object_store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
&bucket,
|
||||
crate::table_catalog::TABLE_BUCKET_MARKER_CONFIG,
|
||||
Vec::new(),
|
||||
incarnation,
|
||||
|| {
|
||||
Err(crate::admin::storage_api::error::StorageError::other(
|
||||
"injected marker publication rejection",
|
||||
))
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(marker_result.is_err());
|
||||
drop(cleanup);
|
||||
TableCommitPublication::complete(&publication);
|
||||
for object in &metadata_objects {
|
||||
assert!(!backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
crate::table_catalog::TableCatalogEntryState::Deleted,
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend);
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_reads_incarnation_after_publication_wait() {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-recreated-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::default();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 0).await;
|
||||
let original_incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
let publication_lock = crate::table_catalog::default_table_bucket_publication_lock_path();
|
||||
let guard = backend.acquire_write_lock(&bucket, &publication_lock).await.unwrap();
|
||||
let attempts = backend.write_lock_acquisition_count(&bucket, &publication_lock).await;
|
||||
let cleanup_bucket = bucket.clone();
|
||||
let cleanup_store = object_store.clone();
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
let cleanup = tokio::spawn(async move {
|
||||
cleanup_empty_table_bucket_response(&catalog, &publication, cleanup_store.as_ref(), &cleanup_bucket).await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(5), async {
|
||||
while backend.write_lock_acquisition_count(&bucket, &publication_lock).await == attempts {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cleanup should wait for its publication lock");
|
||||
|
||||
// Replace the real bucket while the in-memory publication backend holds cleanup at its lock boundary.
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
object_store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
enable_table_bucket_marker(object_store.as_ref(), &bucket).await.unwrap();
|
||||
let replacement_incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
assert_ne!(replacement_incarnation, original_incarnation);
|
||||
drop(guard);
|
||||
let response = tokio::time::timeout(StdDuration::from_secs(10), cleanup)
|
||||
.await
|
||||
.expect("cleanup should resume after publication becomes available")
|
||||
.unwrap()
|
||||
.expect("cleanup must use the incarnation observed while holding its publication lock");
|
||||
assert!(!response.enabled);
|
||||
assert_eq!(
|
||||
object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap(),
|
||||
replacement_incarnation
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_retries_metadata_delete_failures_without_removing_protection() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::{TableCatalogEntryState, TableCatalogObjectPaths};
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
for fail_after_delete in [false, true] {
|
||||
let bucket = format!("cleanup-retry-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let mut metadata_objects =
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 3).await;
|
||||
metadata_objects.sort();
|
||||
let paths = TableCatalogObjectPaths::default();
|
||||
let index_prefix = paths.warehouse_index_entries_prefix(&bucket);
|
||||
let indexes = backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap();
|
||||
assert!(indexes.len() >= metadata_objects.len());
|
||||
if fail_after_delete {
|
||||
backend.fail_after_next_delete(&bucket, &metadata_objects[1]).await;
|
||||
} else {
|
||||
backend.fail_delete_attempt(&bucket, &metadata_objects[1], 1).await;
|
||||
}
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
let error = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("injected"));
|
||||
assert!(!backend.object_exists(&bucket, &metadata_objects[0]).await.unwrap());
|
||||
assert_eq!(backend.object_exists(&bucket, &metadata_objects[1]).await.unwrap(), !fail_after_delete);
|
||||
assert!(backend.object_exists(&bucket, &metadata_objects[2]).await.unwrap());
|
||||
assert_eq!(backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap(), indexes);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Active
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let restarted = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&restarted, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(backend.list_objects(&bucket, "").await.unwrap().is_empty());
|
||||
assert!(
|
||||
backend
|
||||
.list_objects(RUSTFS_META_BUCKET, &index_prefix)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
restarted.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Deleted
|
||||
);
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_retries_partial_warehouse_index_deletion() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::{TableCatalogEntryState, TableCatalogObjectPaths};
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-index-retry-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let index_prefix = TableCatalogObjectPaths::default().warehouse_index_entries_prefix(&bucket);
|
||||
let indexes = backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap();
|
||||
assert!(indexes.len() >= 2);
|
||||
let last_index = indexes.last().unwrap();
|
||||
backend.fail_delete_attempt(RUSTFS_META_BUCKET, last_index, 1).await;
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
let error = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("injected"));
|
||||
assert!(backend.list_objects(&bucket, "").await.unwrap().is_empty());
|
||||
assert_eq!(
|
||||
backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap(),
|
||||
vec![last_index.clone()]
|
||||
);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Active
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
.list_objects(RUSTFS_META_BUCKET, &index_prefix)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_stops_before_delete_when_publication_lease_is_lost() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::test_support::RecordingLeaseClient;
|
||||
use crate::table_catalog::{TableCatalogEntryState, TableCatalogLockGuard, TableCatalogObjectPaths, TableCommitPublication};
|
||||
use rustfs_lock::LockClient;
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-lease-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 1).await;
|
||||
let index_prefix = TableCatalogObjectPaths::default().warehouse_index_entries_prefix(&bucket);
|
||||
let indexes = backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap();
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
TableCommitPublication::begin_table_bucket(&publication, &bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let client = Arc::new(RecordingLeaseClient {
|
||||
inner: rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)))),
|
||||
acquired: std::sync::Mutex::new(None),
|
||||
});
|
||||
let lock = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_clients_and_quorum(
|
||||
"cleanup-publication-fence".to_string(),
|
||||
vec![client.clone() as Arc<dyn LockClient>],
|
||||
1,
|
||||
),
|
||||
rustfs_lock::ObjectKey::new(bucket.as_str(), "cleanup-publication-fence"),
|
||||
"cleanup-publication-owner".to_string(),
|
||||
);
|
||||
let guard = lock.get_read_lock(StdDuration::from_secs(5)).await.unwrap();
|
||||
let signal = guard.lock_lost_signal().unwrap();
|
||||
let lock_id = client.acquired.lock().unwrap().clone().unwrap();
|
||||
publication
|
||||
.publication
|
||||
.lock()
|
||||
.guards
|
||||
.push(TableCatalogLockGuard::test_namespace(guard));
|
||||
let mut cleanup = catalog
|
||||
.prepare_empty_table_bucket_cleanup(table_bucket_entry_from_metadata_marker(&bucket), &publication)
|
||||
.await
|
||||
.unwrap();
|
||||
let incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
let pause = backend.pause_next_delete(&bucket, &metadata_objects[0]).await;
|
||||
let operation = catalog.execute_empty_table_bucket_cleanup(&mut cleanup, incarnation);
|
||||
let interfere = async {
|
||||
tokio::time::timeout(StdDuration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(client.force_release(&lock_id).await.unwrap());
|
||||
tokio::time::timeout(StdDuration::from_secs(15), signal.notified())
|
||||
.await
|
||||
.expect("cleanup publication must observe lease loss");
|
||||
pause.release();
|
||||
};
|
||||
let (result, ()) = tokio::join!(operation, interfere);
|
||||
assert!(matches!(result, Err(crate::table_catalog::TableCatalogStoreError::Unavailable(_))));
|
||||
drop(cleanup);
|
||||
TableCommitPublication::complete(&publication);
|
||||
assert!(backend.object_exists(&bucket, &metadata_objects[0]).await.unwrap());
|
||||
assert_eq!(backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap(), indexes);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Active
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(!backend.object_exists(&bucket, &metadata_objects[0]).await.unwrap());
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ pub struct AppContext {
|
||||
object_data_cache: Arc<ObjectDataCacheAdapter>,
|
||||
object_traffic_health: Arc<ObjectTrafficHealth>,
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime,
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache,
|
||||
}
|
||||
|
||||
impl AppContext {
|
||||
@@ -127,6 +128,7 @@ impl AppContext {
|
||||
object_data_cache,
|
||||
object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,6 +152,10 @@ impl AppContext {
|
||||
self.table_catalog_strong_runtime.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn table_catalog_registration_cache(&self) -> crate::table_catalog::TableCatalogRegistrationCache {
|
||||
self.table_catalog_registration_cache.clone()
|
||||
}
|
||||
|
||||
pub fn iam(&self) -> Arc<dyn IamInterface> {
|
||||
self.iam.clone()
|
||||
}
|
||||
@@ -360,6 +366,7 @@ impl AppContext {
|
||||
object_data_cache: ObjectDataCacheAdapter::disabled_arc(),
|
||||
object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -691,6 +691,36 @@ impl DefaultMultipartUsecase {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
// A lost Complete response retries the same upload id after staging is
|
||||
// gone. `get_multipart_info` is NoSuchUpload in that case; completion
|
||||
// below replays the committed object instead of failing the retry.
|
||||
let multipart_info = match store.get_multipart_info(&bucket, &key, &upload_id, &opts).await {
|
||||
Ok(info) => Some(info),
|
||||
Err(err) if is_err_invalid_upload_id(&err) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
// A ciphertext-passthrough session stores encrypted parts verbatim and
|
||||
// completes without the customer key (the replication client has none),
|
||||
// so the SSE-C completion check must be skipped for it.
|
||||
if let Some(info) = multipart_info.as_ref() {
|
||||
if !contains_key_str(&info.user_defined, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT) {
|
||||
EncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
server_side_encryption: None,
|
||||
ssekms_key_id: None,
|
||||
ssekms_context: None,
|
||||
sse_customer_algorithm: sse_customer_algorithm.clone(),
|
||||
sse_customer_key: sse_customer_key.clone(),
|
||||
sse_customer_key_md5: sse_customer_key_md5.clone(),
|
||||
content_size: 0,
|
||||
principal: None,
|
||||
}
|
||||
.validate_complete_multipart_ssec(&info.user_defined)?;
|
||||
}
|
||||
validate_complete_multipart_checksum_type(&req.headers, &info.user_defined)?;
|
||||
}
|
||||
|
||||
let quota_metadata_sys = self.bucket_metadata_sys();
|
||||
let quota_tracking = quota_metadata_sys.is_some();
|
||||
let mut quota_enabled = false;
|
||||
@@ -732,35 +762,6 @@ impl DefaultMultipartUsecase {
|
||||
None
|
||||
};
|
||||
|
||||
// A lost Complete response retries the same upload id after staging is
|
||||
// gone. `get_multipart_info` is NoSuchUpload in that case; completion
|
||||
// below replays the committed object instead of failing the retry.
|
||||
let multipart_info = match store.get_multipart_info(&bucket, &key, &upload_id, &opts).await {
|
||||
Ok(info) => Some(info),
|
||||
Err(err) if is_err_invalid_upload_id(&err) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
// A ciphertext-passthrough session stores encrypted parts verbatim and
|
||||
// completes without the customer key (the replication client has none),
|
||||
// so the SSE-C completion check must be skipped for it.
|
||||
if let Some(info) = multipart_info.as_ref() {
|
||||
if !contains_key_str(&info.user_defined, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT) {
|
||||
EncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
server_side_encryption: None,
|
||||
ssekms_key_id: None,
|
||||
ssekms_context: None,
|
||||
sse_customer_algorithm: sse_customer_algorithm.clone(),
|
||||
sse_customer_key: sse_customer_key.clone(),
|
||||
sse_customer_key_md5: sse_customer_key_md5.clone(),
|
||||
content_size: 0,
|
||||
principal: None,
|
||||
}
|
||||
.validate_complete_multipart_ssec(&info.user_defined)?;
|
||||
}
|
||||
validate_complete_multipart_checksum_type(&req.headers, &info.user_defined)?;
|
||||
}
|
||||
let cache_adapter = self.object_data_cache();
|
||||
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
|
||||
|
||||
@@ -1171,18 +1172,22 @@ impl DefaultMultipartUsecase {
|
||||
);
|
||||
}
|
||||
|
||||
let current_opts: ObjectOptions = internal_object_info_lookup_opts(
|
||||
get_opts(&bucket, &key, opts.version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
match store.get_object_info(&bucket, &key, ¤t_opts).await {
|
||||
Ok(existing_obj_info) => {
|
||||
validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &opts)?
|
||||
}
|
||||
Err(err) => {
|
||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||
return Err(ApiError::from(err).into());
|
||||
if !super::object::shared::put_like_write_creates_new_version(&opts)
|
||||
&& super::object::shared::object_lock_checks_required_for_state(&object_lock_config_state)
|
||||
{
|
||||
let current_opts: ObjectOptions = internal_object_info_lookup_opts(
|
||||
get_opts(&bucket, &key, opts.version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
match store.get_object_info(&bucket, &key, ¤t_opts).await {
|
||||
Ok(existing_obj_info) => {
|
||||
validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &opts)?
|
||||
}
|
||||
Err(err) => {
|
||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1826,34 +1826,6 @@ impl Node for NodeService {
|
||||
self.handle_rename_data(request, false, true, false).await
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata(
|
||||
&self,
|
||||
request: Request<MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
self.handle_reserve_object_metadata(request).await
|
||||
}
|
||||
|
||||
async fn cancel_object_metadata_reservation(
|
||||
&self,
|
||||
request: Request<MetadataReservationCancelRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
self.handle_cancel_object_metadata_reservation(request).await
|
||||
}
|
||||
|
||||
async fn activate_object_metadata_reservation(
|
||||
&self,
|
||||
request: Request<MetadataReservationActivateRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
self.handle_activate_object_metadata_reservation(request).await
|
||||
}
|
||||
|
||||
async fn rename_reserved_data(
|
||||
&self,
|
||||
request: Request<ReservedRenameDataRequest>,
|
||||
) -> Result<Response<RenameDataResponse>, Status> {
|
||||
self.handle_rename_reserved_data(request).await
|
||||
}
|
||||
|
||||
async fn undo_pool_meta_write(
|
||||
&self,
|
||||
request: Request<DeleteVersionRequest>,
|
||||
@@ -3273,17 +3245,15 @@ mod tests {
|
||||
HealControlRequest, HealControlResponse, ListBucketRequest, ListDirRequest, ListVolumesRequest,
|
||||
LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest,
|
||||
LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest, MetadataReservationCancelRequest,
|
||||
MetadataReservationRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest, PreparePartTransactionRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest, PreparePartTransactionRequest,
|
||||
ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
|
||||
RenameFileRequest, RenamePartRequest, ReservedRenameDataRequest, ScannerActivityRequest,
|
||||
ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
|
||||
ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest,
|
||||
StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, TierMutationPeerState,
|
||||
TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, WriteRequest,
|
||||
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest,
|
||||
SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest,
|
||||
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest,
|
||||
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest,
|
||||
heal_control_service_client::HealControlServiceClient,
|
||||
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
|
||||
node_service_client::NodeServiceClient,
|
||||
@@ -3301,12 +3271,8 @@ mod tests {
|
||||
use tonic::{Request, Response, Status};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DISK_MUTATION_RPC_METHODS: [&str; 38] = [
|
||||
const DISK_MUTATION_RPC_METHODS: [&str; 34] = [
|
||||
"renamedata",
|
||||
"reserveobjectmetadata",
|
||||
"activateobjectmetadatareservation",
|
||||
"cancelobjectmetadatareservation",
|
||||
"renamereserveddata",
|
||||
"renamemultipartdata",
|
||||
"recovermultipartsource",
|
||||
"renamedataatincarnation",
|
||||
@@ -5004,43 +4970,6 @@ mod tests {
|
||||
},
|
||||
rustfs_protos::canonical_rename_data_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
reserve_object_metadata,
|
||||
MetadataReservationRequest {
|
||||
disk: disk.clone(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
timeout_ms: 5000,
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
},
|
||||
rustfs_protos::canonical_metadata_reservation_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
activate_object_metadata_reservation,
|
||||
MetadataReservationActivateRequest {
|
||||
disk: disk.clone(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
token: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
},
|
||||
rustfs_protos::canonical_metadata_reservation_activate_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
cancel_object_metadata_reservation,
|
||||
MetadataReservationCancelRequest {
|
||||
disk: disk.clone(),
|
||||
token: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
},
|
||||
rustfs_protos::canonical_metadata_reservation_cancel_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
rename_reserved_data,
|
||||
ReservedRenameDataRequest {
|
||||
rename: Some(RenameDataRequest::default()),
|
||||
token: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
},
|
||||
rustfs_protos::canonical_reserved_rename_data_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
recover_multipart_source,
|
||||
RecoverMultipartSourceRequest::default(),
|
||||
@@ -7546,15 +7475,11 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn target_rpc_fixture() -> TargetRpcFixture {
|
||||
target_rpc_fixture_with_bucket_metadata(false).await
|
||||
}
|
||||
|
||||
async fn target_rpc_fixture_with_bucket_metadata(initialize_bucket_metadata: bool) -> TargetRpcFixture {
|
||||
super::timeout(Duration::from_secs(90), async {
|
||||
let root = tempfile::tempdir().expect("target RPC root");
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(root.path())
|
||||
.init_bucket_metadata(initialize_bucket_metadata)
|
||||
.init_bucket_metadata(false)
|
||||
.build()
|
||||
.await;
|
||||
ObjectStore::new(env.ecstore.clone())
|
||||
@@ -8221,13 +8146,6 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn stage_target_rpc(fixture: &TargetRpcFixture) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec<u8>) {
|
||||
stage_target_rpc_inner(fixture, true).await
|
||||
}
|
||||
|
||||
async fn stage_target_rpc_inner(
|
||||
fixture: &TargetRpcFixture,
|
||||
create_volume: bool,
|
||||
) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec<u8>) {
|
||||
use crate::storage::storage_api::ecstore_disk::{DiskAPI, ReadOptions};
|
||||
let set = fixture
|
||||
.env
|
||||
@@ -8250,9 +8168,7 @@ mod tests {
|
||||
}];
|
||||
fi.data = Some(Bytes::from_static(b"target-rpc-inline"));
|
||||
fi.set_inline_data();
|
||||
if create_volume {
|
||||
disk.make_volume("target-rpc").await.expect("target volume");
|
||||
}
|
||||
disk.make_volume("target-rpc").await.expect("target volume");
|
||||
disk.write_metadata("target-rpc", "target-rpc", "staged", fi.clone())
|
||||
.await
|
||||
.expect("stage real inline body");
|
||||
@@ -8304,303 +8220,6 @@ mod tests {
|
||||
request
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_reservation_handlers_bind_and_publish_on_the_target_disk() {
|
||||
use crate::storage::storage_api::ecstore_disk::{DiskAPI, RUSTFS_META_TMP_BUCKET, ReadOptions};
|
||||
|
||||
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
|
||||
fixture.env.make_bucket("target-rpc", false).await;
|
||||
let incarnation = fixture
|
||||
.env
|
||||
.ecstore
|
||||
.bucket_incarnation_id("target-rpc")
|
||||
.await
|
||||
.expect("bucket incarnation");
|
||||
let (disk, fi, _) = stage_target_rpc_inner(&fixture, false).await;
|
||||
disk.write_metadata(RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET, "reserved-stage", fi.clone())
|
||||
.await
|
||||
.expect("stage reserved body");
|
||||
let service = make_server_for_context(Some(fixture.context.clone()));
|
||||
let mut reserve = Request::new(MetadataReservationRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
bucket: "target-rpc".into(),
|
||||
object: "destination".into(),
|
||||
timeout_ms: 5000,
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let mut stale = Request::new(reserve.get_ref().clone());
|
||||
stale.get_mut().bucket_incarnation_id = Uuid::new_v4().as_bytes().to_vec().into();
|
||||
let stale_body = rustfs_protos::canonical_metadata_reservation_request_body(stale.get_ref()).expect("stale body");
|
||||
set_tonic_canonical_body_digest(&mut stale, &stale_body).expect("stale digest");
|
||||
mark_v2_authenticated(&mut stale);
|
||||
assert!(
|
||||
!service
|
||||
.reserve_object_metadata(stale)
|
||||
.await
|
||||
.expect("stale handler")
|
||||
.into_inner()
|
||||
.success,
|
||||
"a token must not be issued for a different bucket incarnation"
|
||||
);
|
||||
let mut self_lock = Request::new(reserve.get_ref().clone());
|
||||
self_lock.get_mut().object = rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH.to_owned();
|
||||
let self_lock_body =
|
||||
rustfs_protos::canonical_metadata_reservation_request_body(self_lock.get_ref()).expect("self-lock body");
|
||||
set_tonic_canonical_body_digest(&mut self_lock, &self_lock_body).expect("self-lock digest");
|
||||
mark_v2_authenticated(&mut self_lock);
|
||||
assert!(
|
||||
!service
|
||||
.reserve_object_metadata(self_lock)
|
||||
.await
|
||||
.expect("self-lock handler")
|
||||
.into_inner()
|
||||
.success,
|
||||
"a token cannot hold the same lock that its object commit must acquire"
|
||||
);
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(reserve.get_ref()).expect("reserve body");
|
||||
set_tonic_canonical_body_digest(&mut reserve, &body).expect("reserve digest");
|
||||
mark_v2_authenticated(&mut reserve);
|
||||
let response = service
|
||||
.reserve_object_metadata(reserve)
|
||||
.await
|
||||
.expect("reserve RPC")
|
||||
.into_inner();
|
||||
assert!(response.success, "reservation must succeed: {:?}", response.error);
|
||||
assert_eq!(response.protocol_version, 2);
|
||||
let token = response.token;
|
||||
let mut activate = Request::new(MetadataReservationActivateRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
bucket: "target-rpc".into(),
|
||||
object: "destination".into(),
|
||||
token: token.clone(),
|
||||
});
|
||||
let body =
|
||||
rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activate body");
|
||||
set_tonic_canonical_body_digest(&mut activate, &body).expect("activate digest");
|
||||
mark_v2_authenticated(&mut activate);
|
||||
let activated = service
|
||||
.activate_object_metadata_reservation(activate)
|
||||
.await
|
||||
.expect("activate RPC")
|
||||
.into_inner();
|
||||
assert!(activated.success, "activation must succeed: {:?}", activated.error);
|
||||
let message = ReservedRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
src_volume: RUSTFS_META_TMP_BUCKET.into(),
|
||||
src_path: "reserved-stage".into(),
|
||||
file_info: serde_json::to_string(&fi).expect("encode metadata"),
|
||||
dst_volume: "target-rpc".into(),
|
||||
dst_path: "destination".into(),
|
||||
..Default::default()
|
||||
}),
|
||||
token,
|
||||
};
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(&message).expect("rename body");
|
||||
let mut tampered = Request::new(message.clone());
|
||||
set_tonic_canonical_body_digest(&mut tampered, &body).expect("rename digest");
|
||||
mark_v2_authenticated(&mut tampered);
|
||||
tampered.get_mut().rename.as_mut().expect("rename").dst_path = "other".into();
|
||||
assert_eq!(
|
||||
service
|
||||
.rename_reserved_data(tampered)
|
||||
.await
|
||||
.expect_err("tampered object")
|
||||
.code(),
|
||||
tonic::Code::PermissionDenied
|
||||
);
|
||||
let mut rename = Request::new(message);
|
||||
set_tonic_canonical_body_digest(&mut rename, &body).expect("rename digest");
|
||||
mark_v2_authenticated(&mut rename);
|
||||
let response = service.rename_reserved_data(rename).await.expect("rename RPC").into_inner();
|
||||
assert!(response.success, "reserved rename must succeed: {:?}", response.error);
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-rpc",
|
||||
"target-rpc",
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read committed object");
|
||||
assert_eq!(read.data, fi.data);
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn reserved_rename_rpc_cancellation_does_not_stop_physical_publication() {
|
||||
use crate::storage::storage_api::{
|
||||
LocalPublicationPause, LocalPublicationStage,
|
||||
contract::namespace::NamespaceLocking as _,
|
||||
ecstore_disk::{DiskAPI, RUSTFS_META_TMP_BUCKET, ReadOptions},
|
||||
};
|
||||
|
||||
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
|
||||
fixture.env.make_bucket("target-rpc", false).await;
|
||||
let incarnation = fixture
|
||||
.env
|
||||
.ecstore
|
||||
.bucket_incarnation_id("target-rpc")
|
||||
.await
|
||||
.expect("bucket incarnation");
|
||||
let (disk, fi, _) = stage_target_rpc_inner(&fixture, false).await;
|
||||
disk.write_metadata(RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET, "reserved-cancel", fi.clone())
|
||||
.await
|
||||
.expect("stage reserved body");
|
||||
let service = make_server_for_context(Some(fixture.context.clone()));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind RPC listener");
|
||||
let addr = listener.local_addr().expect("RPC listener address");
|
||||
let (shutdown, stopped) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
tonic::transport::Server::builder()
|
||||
.add_service(NodeServiceServer::new(service))
|
||||
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
|
||||
let _ = stopped.await;
|
||||
})
|
||||
.await
|
||||
.expect("serve RPC target");
|
||||
});
|
||||
let mut client = NodeServiceClient::connect(format!("http://{addr}"))
|
||||
.await
|
||||
.expect("connect RPC client");
|
||||
let mut prepare = Request::new(MetadataReservationRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
bucket: "target-rpc".into(),
|
||||
object: "destination".into(),
|
||||
timeout_ms: 5000,
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(prepare.get_ref()).expect("prepare body");
|
||||
set_tonic_canonical_body_digest(&mut prepare, &body).expect("prepare digest");
|
||||
mark_v2_authenticated(&mut prepare);
|
||||
let prepared = client
|
||||
.reserve_object_metadata(prepare)
|
||||
.await
|
||||
.expect("prepare handler")
|
||||
.into_inner();
|
||||
assert!(prepared.success, "prepare: {:?}", prepared.error);
|
||||
let mut activate = Request::new(MetadataReservationActivateRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
bucket: "target-rpc".into(),
|
||||
object: "destination".into(),
|
||||
token: prepared.token.clone(),
|
||||
});
|
||||
let body =
|
||||
rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activate body");
|
||||
set_tonic_canonical_body_digest(&mut activate, &body).expect("activate digest");
|
||||
mark_v2_authenticated(&mut activate);
|
||||
let activated = client
|
||||
.activate_object_metadata_reservation(activate)
|
||||
.await
|
||||
.expect("activate handler")
|
||||
.into_inner();
|
||||
assert!(activated.success, "activate: {:?}", activated.error);
|
||||
let mut rename = Request::new(ReservedRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
disk: disk.endpoint().to_string(),
|
||||
src_volume: RUSTFS_META_TMP_BUCKET.into(),
|
||||
src_path: "reserved-cancel".into(),
|
||||
dst_volume: "target-rpc".into(),
|
||||
dst_path: "destination".into(),
|
||||
file_info: serde_json::to_string(&fi).expect("encode metadata"),
|
||||
..Default::default()
|
||||
}),
|
||||
token: prepared.token,
|
||||
});
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(rename.get_ref()).expect("rename body");
|
||||
set_tonic_canonical_body_digest(&mut rename, &body).expect("rename digest");
|
||||
mark_v2_authenticated(&mut rename);
|
||||
let mut pause =
|
||||
LocalPublicationPause::install(&disk, "target-rpc", "destination/xl.meta", LocalPublicationStage::PreparedRename)
|
||||
.expect("install physical pause");
|
||||
let mut handler = Box::pin(client.rename_reserved_data(rename));
|
||||
super::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
result = &mut handler => panic!("handler completed before physical entry: {result:?}"),
|
||||
entered = pause.entered() => entered.expect("physical executor entered"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded physical entry");
|
||||
drop(handler);
|
||||
assert!(
|
||||
fixture.env.ecstore.namespace_commits_pending_for_test(),
|
||||
"physical publication remains active"
|
||||
);
|
||||
let lifecycle = fixture
|
||||
.env
|
||||
.ecstore
|
||||
.new_ns_lock("target-rpc", rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("bucket publication lock");
|
||||
let mut bucket_writer = Box::pin(lifecycle.get_write_lock(Duration::from_secs(10)));
|
||||
let early = super::timeout(Duration::from_millis(500), &mut bucket_writer).await;
|
||||
let old_lifecycle = fixture
|
||||
.env
|
||||
.ecstore
|
||||
.new_ns_lock("target-rpc", "")
|
||||
.await
|
||||
.expect("bucket lifecycle lock");
|
||||
let mut old_writer = Box::pin(old_lifecycle.get_write_lock(Duration::from_secs(10)));
|
||||
let old_early = super::timeout(Duration::from_millis(500), &mut old_writer).await;
|
||||
drop(pause);
|
||||
match early {
|
||||
Ok(result) => {
|
||||
drop(result.expect("early lifecycle result"));
|
||||
panic!("bucket publication writer acquired while the remote physical write was still paused");
|
||||
}
|
||||
Err(_) => drop(
|
||||
super::timeout(Duration::from_secs(10), bucket_writer)
|
||||
.await
|
||||
.expect("lifecycle owner drains")
|
||||
.expect("bucket writer acquires"),
|
||||
),
|
||||
}
|
||||
match old_early {
|
||||
Ok(result) => {
|
||||
drop(result.expect("early lifecycle result"));
|
||||
panic!("bucket lifecycle writer acquired while the remote physical write was still paused");
|
||||
}
|
||||
Err(_) => drop(
|
||||
super::timeout(Duration::from_secs(10), old_writer)
|
||||
.await
|
||||
.expect("lifecycle owner drains")
|
||||
.expect("bucket lifecycle writer acquires"),
|
||||
),
|
||||
}
|
||||
super::timeout(Duration::from_secs(10), async {
|
||||
while fixture.env.ecstore.namespace_commits_pending_for_test() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical publication drains");
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-rpc",
|
||||
"target-rpc",
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read published object");
|
||||
assert_eq!(read.data, fi.data);
|
||||
drop(client);
|
||||
let _ = shutdown.send(());
|
||||
super::timeout(Duration::from_secs(10), server)
|
||||
.await
|
||||
.expect("RPC server shuts down")
|
||||
.expect("RPC server task");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn quota_fence_handlers_commit_real_data_and_drain_before_revocation() {
|
||||
@@ -9355,47 +8974,28 @@ mod tests {
|
||||
))
|
||||
}
|
||||
|
||||
async fn internal_snapshot_once(
|
||||
root: &std::path::Path,
|
||||
) -> std::io::Result<std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>>> {
|
||||
async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>> {
|
||||
let mut snapshot = std::collections::BTreeMap::new();
|
||||
let mut directories = (0..4)
|
||||
.map(|index| std::path::PathBuf::from(format!("disk{index}/{RUSTFS_META_BUCKET}")))
|
||||
.collect::<Vec<_>>();
|
||||
while let Some(relative) = directories.pop() {
|
||||
let mut entries = tokio::fs::read_dir(root.join(&relative)).await?;
|
||||
let mut entries = tokio::fs::read_dir(root.join(&relative))
|
||||
.await
|
||||
.expect("read internal snapshot directory");
|
||||
snapshot.insert(relative.clone(), None);
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
while let Some(entry) = entries.next_entry().await.expect("read internal snapshot entry") {
|
||||
let path = relative.join(entry.file_name());
|
||||
let file_type = entry.file_type().await?;
|
||||
let file_type = entry.file_type().await.expect("read internal snapshot entry type");
|
||||
if file_type.is_dir() {
|
||||
directories.push(path);
|
||||
} else {
|
||||
assert!(file_type.is_file(), "fixture snapshot must contain only directories and regular files");
|
||||
snapshot.insert(path, Some(tokio::fs::read(entry.path()).await?));
|
||||
snapshot.insert(path, Some(tokio::fs::read(entry.path()).await.expect("read snapshot file bytes")));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>> {
|
||||
// Startup cleanup can remove temporary paths after the namespace
|
||||
// commit drains. Capture the complete tree only once it is stable.
|
||||
super::timeout(Duration::from_secs(10), async {
|
||||
let mut previous = None;
|
||||
loop {
|
||||
match internal_snapshot_once(root).await {
|
||||
Ok(snapshot) if previous.as_ref() == Some(&snapshot) => return snapshot,
|
||||
Ok(snapshot) => previous = Some(snapshot),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => previous = None,
|
||||
Err(error) => panic!("read internal snapshot: {error}"),
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("internal snapshot must stabilize within ten seconds")
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn file_info(object: &str, version: Uuid, body: Bytes) -> FileInfo {
|
||||
|
||||
@@ -507,161 +507,6 @@ impl NodeService {
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
pub(super) async fn handle_reserve_object_metadata(
|
||||
&self,
|
||||
request: Request<MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("invalid reservation body"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &body)
|
||||
.map_err(|_| Status::permission_denied("metadata reservation requires body-bound authentication"))?;
|
||||
let request = request.into_inner();
|
||||
if request.timeout_ms == 0 {
|
||||
return Err(Status::invalid_argument("reservation timeout must be positive"));
|
||||
}
|
||||
let incarnation = Uuid::from_slice(&request.bucket_incarnation_id)
|
||||
.ok()
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Status::invalid_argument("invalid bucket incarnation"))?;
|
||||
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
|
||||
return Err(Status::failed_precondition("storage instance is unavailable"));
|
||||
};
|
||||
let result = store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&request.disk,
|
||||
&request.bucket,
|
||||
&request.object,
|
||||
incarnation,
|
||||
std::time::Duration::from_millis(request.timeout_ms.min(5000)),
|
||||
)
|
||||
.await;
|
||||
Ok(Response::new(match result {
|
||||
Ok(token) => SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
protocol_version: 2,
|
||||
error: None,
|
||||
},
|
||||
Err(error) => SnapshotLeaseResponse {
|
||||
success: false,
|
||||
token: Bytes::new(),
|
||||
protocol_version: 2,
|
||||
error: Some(error.into()),
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) async fn handle_activate_object_metadata_reservation(
|
||||
&self,
|
||||
request: Request<MetadataReservationActivateRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("invalid reservation activation body"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &body)
|
||||
.map_err(|_| Status::permission_denied("reservation activation requires body-bound authentication"))?;
|
||||
let request = request.into_inner();
|
||||
let token = Uuid::from_slice(&request.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
|
||||
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
|
||||
return Err(Status::failed_precondition("storage instance is unavailable"));
|
||||
};
|
||||
Ok(Response::new(
|
||||
match store
|
||||
.activate_local_object_metadata_reservation(&request.disk, &request.bucket, &request.object, token)
|
||||
.await
|
||||
{
|
||||
Ok(()) => SnapshotLeaseMutationResponse {
|
||||
success: true,
|
||||
error: None,
|
||||
},
|
||||
Err(error) => SnapshotLeaseMutationResponse {
|
||||
success: false,
|
||||
error: Some(error.into()),
|
||||
},
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn handle_cancel_object_metadata_reservation(
|
||||
&self,
|
||||
request: Request<MetadataReservationCancelRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("invalid reservation cancellation body"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &body)
|
||||
.map_err(|_| Status::permission_denied("reservation cancellation requires body-bound authentication"))?;
|
||||
let request = request.into_inner();
|
||||
let token = Uuid::from_slice(&request.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
|
||||
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
|
||||
return Err(Status::failed_precondition("storage instance is unavailable"));
|
||||
};
|
||||
Ok(Response::new(
|
||||
match store.cancel_local_object_metadata_reservation(&request.disk, token).await {
|
||||
Ok(cancelled) => SnapshotLeaseMutationResponse {
|
||||
success: cancelled,
|
||||
error: None,
|
||||
},
|
||||
Err(error) => SnapshotLeaseMutationResponse {
|
||||
success: false,
|
||||
error: Some(error.into()),
|
||||
},
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn handle_rename_reserved_data(
|
||||
&self,
|
||||
request: Request<ReservedRenameDataRequest>,
|
||||
) -> Result<Response<RenameDataResponse>, Status> {
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("invalid reserved rename body"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &body)
|
||||
.map_err(|_| Status::permission_denied("reserved rename requires body-bound authentication"))?;
|
||||
let request = request.into_inner();
|
||||
let token = Uuid::from_slice(&request.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
|
||||
let rename = request.rename.ok_or_else(|| Status::invalid_argument("missing rename"))?;
|
||||
if !rename.scanner_publication_lease_token.is_empty() || !rename.bucket_incarnation_id.is_empty() {
|
||||
return Err(Status::invalid_argument("reserved rename does not support scanner or heal scope"));
|
||||
}
|
||||
let decoded = decode_rename_data_request_file_info(&rename.file_info_bin, &rename.file_info)
|
||||
.map_err(|_| Status::invalid_argument("invalid reserved rename FileInfo"))?;
|
||||
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
|
||||
return Err(Status::failed_precondition("storage instance is unavailable"));
|
||||
};
|
||||
let result = store
|
||||
.rename_local_data_reserved(
|
||||
&rename.disk,
|
||||
(&rename.src_volume, &rename.src_path),
|
||||
&decoded.value,
|
||||
(&rename.dst_volume, &rename.dst_path),
|
||||
token,
|
||||
)
|
||||
.await;
|
||||
let encoded = result.and_then(|value| encode_rename_data_response_payloads(&value, decoded.from_msgpack));
|
||||
Ok(Response::new(match encoded {
|
||||
Ok((rename_data_resp, rename_data_resp_bin)) => RenameDataResponse {
|
||||
success: true,
|
||||
rename_data_resp,
|
||||
rename_data_resp_bin: rename_data_resp_bin.into(),
|
||||
error: None,
|
||||
},
|
||||
Err(error) => RenameDataResponse {
|
||||
success: false,
|
||||
rename_data_resp: String::new(),
|
||||
rename_data_resp_bin: Bytes::new(),
|
||||
error: Some(error.into()),
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) async fn handle_acquire_snapshot_lease(
|
||||
&self,
|
||||
request: Request<SnapshotLeaseRequest>,
|
||||
|
||||
@@ -49,91 +49,6 @@ async fn refresh_lock(
|
||||
lock_client: &std::sync::Arc<dyn rustfs_lock::client::LockClient>,
|
||||
args: &LockRequest,
|
||||
) -> GenerallyLockResponse {
|
||||
let writer_marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
|
||||
if args
|
||||
.metadata
|
||||
.tags
|
||||
.get(writer_marker)
|
||||
.is_some_and(|value| value == writer_marker)
|
||||
{
|
||||
return match lock_client.waiting_writer(&args.lock_id).await {
|
||||
Ok(Some(waiting)) => GenerallyLockResponse {
|
||||
success: true,
|
||||
error_info: None,
|
||||
lock_info: Some(format!("{writer_marker}:{}", u8::from(waiting))),
|
||||
},
|
||||
Ok(None) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
},
|
||||
Err(error) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(error.to_string()),
|
||||
lock_info: None,
|
||||
},
|
||||
};
|
||||
}
|
||||
let probe_marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
|
||||
if args
|
||||
.metadata
|
||||
.tags
|
||||
.get(probe_marker)
|
||||
.is_some_and(|value| value == probe_marker)
|
||||
{
|
||||
return match lock_client.supports_heal_lock_transition(&args.lock_id).await {
|
||||
Ok(success) => GenerallyLockResponse {
|
||||
success,
|
||||
error_info: None,
|
||||
lock_info: success.then(|| probe_marker.to_owned()),
|
||||
},
|
||||
Err(error) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(error.to_string()),
|
||||
lock_info: None,
|
||||
},
|
||||
};
|
||||
}
|
||||
let upgrade_marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
|
||||
if args
|
||||
.metadata
|
||||
.tags
|
||||
.get(upgrade_marker)
|
||||
.is_some_and(|value| value == upgrade_marker)
|
||||
{
|
||||
return match lock_client.try_upgrade_committed_tail(&args.lock_id).await {
|
||||
Ok(success) => GenerallyLockResponse {
|
||||
success,
|
||||
error_info: None,
|
||||
lock_info: success.then(|| upgrade_marker.to_owned()),
|
||||
},
|
||||
Err(error) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(error.to_string()),
|
||||
lock_info: None,
|
||||
},
|
||||
};
|
||||
}
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
|
||||
if args.metadata.tags.get(marker).is_some_and(|value| value == marker) {
|
||||
return match lock_client.downgrade_committed_tail(&args.lock_id).await {
|
||||
Ok(true) => GenerallyLockResponse {
|
||||
success: true,
|
||||
error_info: None,
|
||||
lock_info: Some(marker.to_owned()),
|
||||
},
|
||||
Ok(false) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
},
|
||||
Err(error) => GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(error.to_string()),
|
||||
lock_info: None,
|
||||
},
|
||||
};
|
||||
}
|
||||
match lock_client.refresh(&args.lock_id).await {
|
||||
Ok(true) => GenerallyLockResponse {
|
||||
success: true,
|
||||
@@ -432,115 +347,6 @@ mod tests {
|
||||
assert!(result.lock_info.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_reports_waiting_writer_for_held_lease() {
|
||||
use super::refresh_lock;
|
||||
use rustfs_lock::client::{HEAL_WAITING_WRITER_MARKER, LockClient};
|
||||
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
|
||||
let key = ObjectKey::new("bucket", "writer-observation-rpc");
|
||||
let mut held = LockRequest::new(key.clone(), LockType::Exclusive, "heal");
|
||||
assert!(client.acquire_lock(&held).await.expect("acquire heal lease").success);
|
||||
held.metadata
|
||||
.tags
|
||||
.insert(HEAL_WAITING_WRITER_MARKER.into(), HEAL_WAITING_WRITER_MARKER.into());
|
||||
let writer = {
|
||||
let client = Arc::clone(&client);
|
||||
tokio::spawn(async move {
|
||||
let request = LockRequest::new(key, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_secs(3));
|
||||
let response = client.acquire_lock(&request).await.expect("writer request");
|
||||
(request, response)
|
||||
})
|
||||
};
|
||||
let expected = format!("{HEAL_WAITING_WRITER_MARKER}:1");
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
let observed = refresh_lock(&client, &held).await;
|
||||
if observed.lock_info.as_deref() == Some(expected.as_str()) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("waiting writer reported");
|
||||
assert!(client.release(&held.lock_id).await.expect("release heal lease"));
|
||||
let (request, response) = writer.await.expect("writer task");
|
||||
assert!(response.success);
|
||||
assert!(client.release(&request.lock_id).await.expect("release writer"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_upgrade_seals_lease_against_late_conversion() {
|
||||
use super::refresh_lock;
|
||||
use rustfs_lock::client::{
|
||||
COMMITTED_TAIL_CONVERSION_MARKER, COMMITTED_TAIL_UPGRADE_MARKER, HEAL_LOCK_TRANSITION_PROBE_MARKER,
|
||||
HEAL_WAITING_WRITER_MARKER, LockClient,
|
||||
};
|
||||
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
|
||||
use std::sync::Arc;
|
||||
|
||||
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
|
||||
let mut request = LockRequest::new(ObjectKey::new("bucket", "upgrade-rpc"), LockType::Exclusive, "owner");
|
||||
assert!(client.acquire_lock(&request).await.expect("acquire lease").success);
|
||||
request
|
||||
.metadata
|
||||
.tags
|
||||
.insert(HEAL_LOCK_TRANSITION_PROBE_MARKER.into(), HEAL_LOCK_TRANSITION_PROBE_MARKER.into());
|
||||
let probe = refresh_lock(&client, &request).await;
|
||||
assert!(probe.success);
|
||||
assert_eq!(probe.lock_info.as_deref(), Some(HEAL_LOCK_TRANSITION_PROBE_MARKER));
|
||||
request.metadata.tags.clear();
|
||||
request
|
||||
.metadata
|
||||
.tags
|
||||
.insert(HEAL_WAITING_WRITER_MARKER.into(), HEAL_WAITING_WRITER_MARKER.into());
|
||||
let observation = refresh_lock(&client, &request).await;
|
||||
assert!(observation.success);
|
||||
assert_eq!(observation.lock_info.as_deref(), Some(format!("{HEAL_WAITING_WRITER_MARKER}:0").as_str()));
|
||||
request.metadata.tags.clear();
|
||||
request
|
||||
.metadata
|
||||
.tags
|
||||
.insert(COMMITTED_TAIL_UPGRADE_MARKER.into(), COMMITTED_TAIL_UPGRADE_MARKER.into());
|
||||
let response = refresh_lock(&client, &request).await;
|
||||
assert!(response.success);
|
||||
assert_eq!(response.lock_info.as_deref(), Some(COMMITTED_TAIL_UPGRADE_MARKER));
|
||||
request.metadata.tags.clear();
|
||||
request
|
||||
.metadata
|
||||
.tags
|
||||
.insert(COMMITTED_TAIL_CONVERSION_MARKER.into(), COMMITTED_TAIL_CONVERSION_MARKER.into());
|
||||
assert!(!refresh_lock(&client, &request).await.success, "late conversion must be rejected");
|
||||
assert!(client.release(&request.lock_id).await.expect("release lease"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_can_convert_committed_tail_and_acknowledges_protocol_version() {
|
||||
use super::refresh_lock;
|
||||
use rustfs_lock::client::{COMMITTED_TAIL_CONVERSION_MARKER, LockClient};
|
||||
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
|
||||
let resource = ObjectKey::new("bucket", "tail-conversion-rpc");
|
||||
let mut request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner");
|
||||
assert!(client.acquire_lock(&request).await.expect("acquire write lease").success);
|
||||
request
|
||||
.metadata
|
||||
.tags
|
||||
.insert(COMMITTED_TAIL_CONVERSION_MARKER.into(), COMMITTED_TAIL_CONVERSION_MARKER.into());
|
||||
let response = refresh_lock(&client, &request).await;
|
||||
assert!(response.success);
|
||||
assert_eq!(response.lock_info.as_deref(), Some(COMMITTED_TAIL_CONVERSION_MARKER));
|
||||
let read = LockRequest::new(resource, LockType::Shared, "reader").with_acquire_timeout(Duration::from_secs(1));
|
||||
assert!(client.acquire_lock(&read).await.expect("read committed tail").success);
|
||||
assert!(client.release(&read.lock_id).await.expect("release reader"));
|
||||
assert!(client.release(&request.lock_id).await.expect("release tail"));
|
||||
}
|
||||
|
||||
// B1 -- refresh_lock delegates to LocalClient::refresh for a held lock, returns
|
||||
// success=true, and actually extends expires_at (inverts the "parse then drop" stub).
|
||||
#[tokio::test]
|
||||
|
||||
@@ -490,7 +490,8 @@ pub(crate) mod table {
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use super::super::super::storage_contracts::{
|
||||
ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr, WalkOptions,
|
||||
AuthoritativeListOperations, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr,
|
||||
WalkOptions,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -498,6 +499,10 @@ pub(crate) mod table {
|
||||
pub(crate) use super::super::super::storage_contracts::NamespaceLocking;
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use super::super::super::storage_contracts::MultipartUploadInspection;
|
||||
}
|
||||
|
||||
pub(crate) mod object {
|
||||
pub(crate) use super::super::super::storage_contracts::{ObjectIO, ObjectOperations};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Validates one metadata object; callers must also validate the complete object inventory.
|
||||
pub(crate) fn validate_empty_table_cleanup_metadata(
|
||||
table_bucket: &str,
|
||||
object_key: &str,
|
||||
metadata: &[u8],
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let (namespace, table) = table_identity_from_metadata_object_key(object_key).ok_or_else(|| {
|
||||
TableCatalogStoreError::Invalid("cleanup metadata must use the default table metadata directory".to_string())
|
||||
})?;
|
||||
let metadata = decode_table_metadata_json(object_key, metadata)?;
|
||||
validate_supported_table_metadata(&metadata)?;
|
||||
|
||||
// Future metadata fields may introduce additional object references.
|
||||
for field in metadata.as_object().expect("validated metadata is a JSON object").keys() {
|
||||
if !matches!(
|
||||
field.as_str(),
|
||||
"format-version"
|
||||
| "table-uuid"
|
||||
| "location"
|
||||
| "last-sequence-number"
|
||||
| "last-updated-ms"
|
||||
| "last-column-id"
|
||||
| "schema"
|
||||
| "schemas"
|
||||
| "current-schema-id"
|
||||
| "partition-spec"
|
||||
| "partition-specs"
|
||||
| "default-spec-id"
|
||||
| "last-partition-id"
|
||||
| "properties"
|
||||
| "current-snapshot-id"
|
||||
| "snapshots"
|
||||
| "snapshot-log"
|
||||
| "metadata-log"
|
||||
| "sort-orders"
|
||||
| "default-sort-order-id"
|
||||
| "refs"
|
||||
| "statistics"
|
||||
| "partition-statistics"
|
||||
) {
|
||||
return Err(TableCatalogStoreError::Unsupported(format!(
|
||||
"empty table cleanup does not support metadata field {field}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
for field in ["snapshots", "snapshot-log", "statistics", "partition-statistics"] {
|
||||
let Some(value) = metadata.get(field) else {
|
||||
continue;
|
||||
};
|
||||
let entries = value
|
||||
.as_array()
|
||||
.ok_or_else(|| TableCatalogStoreError::Invalid(format!("cleanup metadata field {field} must be an array")))?;
|
||||
if !entries.is_empty() {
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"empty table cleanup requires metadata field {field} to be empty"
|
||||
)));
|
||||
}
|
||||
}
|
||||
if let Some(value) = metadata.get("refs") {
|
||||
let refs = value
|
||||
.as_object()
|
||||
.ok_or_else(|| TableCatalogStoreError::Invalid("cleanup metadata refs must be an object".to_string()))?;
|
||||
if !refs.is_empty() {
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"empty table cleanup requires metadata refs to be empty".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let table_uuid = table_metadata_uuid(&metadata)?;
|
||||
if Uuid::parse_str(table_uuid).ok().filter(|uuid| !uuid.is_nil()).is_none() {
|
||||
return Err(TableCatalogStoreError::Invalid(
|
||||
"cleanup metadata table-uuid must be a non-nil UUID".to_string(),
|
||||
));
|
||||
}
|
||||
let warehouse_prefix = table_warehouse_object_prefix_from_location(table_bucket, table_metadata_location(&metadata)?)?;
|
||||
if warehouse_prefix
|
||||
.strip_prefix("tables/")
|
||||
.and_then(|prefix| prefix.strip_suffix('/'))
|
||||
.and_then(|table_id| Uuid::parse_str(table_id).ok())
|
||||
.filter(|table_id| !table_id.is_nil())
|
||||
.is_none()
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"empty table cleanup requires the default tables/<table-id> warehouse location".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(entries) = metadata.get("metadata-log").and_then(serde_json::Value::as_array) {
|
||||
for entry in entries {
|
||||
let entry = entry.as_object().expect("validated metadata log entries are JSON objects");
|
||||
if entry
|
||||
.keys()
|
||||
.any(|field| !matches!(field.as_str(), "timestamp-ms" | "metadata-file"))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Unsupported(
|
||||
"empty table cleanup does not support unknown metadata-log fields".to_string(),
|
||||
));
|
||||
}
|
||||
let location = entry["metadata-file"]
|
||||
.as_str()
|
||||
.expect("validated metadata log entries contain metadata-file");
|
||||
table_catalog_object_key_from_location(table_bucket, location)
|
||||
.filter(|location| is_valid_table_metadata_location(&namespace, &table, location))
|
||||
.filter(|location| location != object_key)
|
||||
.ok_or_else(|| {
|
||||
TableCatalogStoreError::Invalid(
|
||||
"cleanup metadata-log must reference another file in the same table metadata directory".to_string(),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::table_catalog::test_support::table_metadata_json;
|
||||
use serde_json::{Value, json};
|
||||
use std::io::Write;
|
||||
|
||||
const BUCKET: &str = "warehouse";
|
||||
const TABLE_UUID: &str = "f6dd46c2-4971-4d41-b188-452d5f5c2c01";
|
||||
const WAREHOUSE_LOCATION: &str = "s3://warehouse/tables/a9d8f3b1-abde-4a0b-b6fb-e36a30abff16";
|
||||
const OBJECT: &str =
|
||||
".rustfs-table/warehouses/default/namespaces/analytics/events/tables/orders/metadata/00002.metadata.json";
|
||||
const PREVIOUS: &str =
|
||||
".rustfs-table/warehouses/default/namespaces/analytics/events/tables/orders/metadata/00001.metadata.json";
|
||||
|
||||
fn empty_metadata() -> Value {
|
||||
table_metadata_json(TABLE_UUID, WAREHOUSE_LOCATION)
|
||||
}
|
||||
|
||||
fn validate(metadata: &Value) -> TableCatalogStoreResult<()> {
|
||||
validate_empty_table_cleanup_metadata(BUCKET, OBJECT, &serde_json::to_vec(metadata).unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_accepts_v2_and_same_directory_history() {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["metadata-log"] = json!([
|
||||
{"timestamp-ms": 1, "metadata-file": PREVIOUS},
|
||||
{"timestamp-ms": 1, "metadata-file": format!("s3://{BUCKET}/{PREVIOUS}")}
|
||||
]);
|
||||
assert!(validate(&metadata).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_accepts_v1_without_snapshots() {
|
||||
let mut metadata = json!({
|
||||
"format-version": 1,
|
||||
"table-uuid": TABLE_UUID,
|
||||
"location": WAREHOUSE_LOCATION,
|
||||
"last-updated-ms": 1,
|
||||
"last-column-id": 0,
|
||||
"schema": {"type": "struct", "fields": []},
|
||||
"partition-spec": [],
|
||||
"current-snapshot-id": -1
|
||||
});
|
||||
assert!(validate(&metadata).is_ok());
|
||||
metadata["snapshots"] = json!([{
|
||||
"snapshot-id": 1,
|
||||
"timestamp-ms": 1,
|
||||
"manifests": ["s3://warehouse/tables/table-id/metadata/manifest.avro"]
|
||||
}]);
|
||||
validate_supported_table_metadata(&metadata).unwrap();
|
||||
assert!(matches!(validate(&metadata), Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_unsupported_versions_and_unknown_references() {
|
||||
for version in [0, 3, 4] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["format-version"] = json!(version);
|
||||
assert!(validate(&metadata).is_err(), "version {version}");
|
||||
}
|
||||
for field in [
|
||||
"manifest-list",
|
||||
"manifests",
|
||||
"data-files",
|
||||
"delete-files",
|
||||
"next-row-id",
|
||||
"future-field",
|
||||
] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata[field] = json!([]);
|
||||
assert!(validate(&metadata).is_err(), "field {field}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_snapshot_and_history_references() {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["snapshots"] = json!([{
|
||||
"snapshot-id": 1,
|
||||
"sequence-number": 0,
|
||||
"timestamp-ms": 1,
|
||||
"manifest-list": "s3://warehouse/tables/table-id/metadata/list.avro",
|
||||
"summary": {"operation": "append"}
|
||||
}]);
|
||||
validate_supported_table_metadata(&metadata).unwrap();
|
||||
assert!(matches!(validate(&metadata), Err(TableCatalogStoreError::Conflict(_))));
|
||||
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["snapshot-log"] = json!([{"timestamp-ms": 1, "snapshot-id": 1}]);
|
||||
assert!(validate(&metadata).is_err());
|
||||
metadata = empty_metadata();
|
||||
metadata["refs"] = json!({"main": {"snapshot-id": 1, "type": "branch"}});
|
||||
assert!(validate(&metadata).is_err());
|
||||
metadata = empty_metadata();
|
||||
metadata["current-snapshot-id"] = json!(1);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_accepts_only_absent_null_or_minus_one_current_snapshot() {
|
||||
for version in [1, 2] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["format-version"] = json!(version);
|
||||
if version == 1 {
|
||||
metadata["schema"] = metadata["schemas"][0].clone();
|
||||
metadata["partition-spec"] = json!([]);
|
||||
}
|
||||
assert!(validate(&metadata).is_ok());
|
||||
for value in [Value::Null, json!(-1)] {
|
||||
metadata["current-snapshot-id"] = value;
|
||||
assert!(validate(&metadata).is_ok(), "version {version}");
|
||||
}
|
||||
for value in [json!(1), json!(0), json!(-2), json!("1"), json!(false), json!([])] {
|
||||
metadata["current-snapshot-id"] = value;
|
||||
assert!(validate(&metadata).is_err(), "version {version}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_statistics_references_and_malformed_empty_fields() {
|
||||
for field in ["snapshots", "snapshot-log", "statistics", "partition-statistics"] {
|
||||
for value in [Value::Null, json!({}), json!("empty"), json!([{}])] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata[field] = value;
|
||||
assert!(validate(&metadata).is_err(), "field {field}");
|
||||
}
|
||||
}
|
||||
for value in [Value::Null, json!([]), json!("empty")] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["refs"] = value;
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_unknown_or_cross_table_history() {
|
||||
for location in [
|
||||
format!("s3://other/{PREVIOUS}"),
|
||||
PREVIOUS.replace("/orders/", "/other/"),
|
||||
PREVIOUS.replace("/analytics/events/", "/another/"),
|
||||
PREVIOUS.replace("00001.metadata.json", "../00001.metadata.json"),
|
||||
PREVIOUS.replace("00001.metadata.json", "%2e%2e.json"),
|
||||
format!("{PREVIOUS}?versionId=previous"),
|
||||
format!("/{PREVIOUS}"),
|
||||
OBJECT.to_string(),
|
||||
] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["metadata-log"] = json!([{"timestamp-ms": 1, "metadata-file": location}]);
|
||||
assert!(validate(&metadata).is_err(), "location {location}");
|
||||
}
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["metadata-log"] = json!([{"timestamp-ms": 1, "metadata-file": PREVIOUS, "future-file": "data.parquet"}]);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_requires_valid_schema_uuid_and_warehouse() {
|
||||
for uuid in ["table-uuid", "00000000-0000-0000-0000-000000000000", ""] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["table-uuid"] = json!(uuid);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
for location in [
|
||||
"s3://other/tables/table-id",
|
||||
"s3://warehouse/",
|
||||
"s3://warehouse/.rustfs-table/metadata",
|
||||
"file:///table",
|
||||
"s3://warehouse/custom/table-id",
|
||||
"s3://warehouse/tables/table-id",
|
||||
"s3://warehouse/tables/00000000-0000-0000-0000-000000000000",
|
||||
"s3://warehouse/tables/a9d8f3b1-abde-4a0b-b6fb-e36a30abff16/nested",
|
||||
] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["location"] = json!(location);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["schemas"] = json!([]);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_non_default_metadata_paths() {
|
||||
let bytes = serde_json::to_vec(&empty_metadata()).unwrap();
|
||||
for object in [
|
||||
OBJECT.replace("/default/", "/custom/"),
|
||||
OBJECT.replace("/tables/", "/views/"),
|
||||
OBJECT.replace("/metadata/", "/data/"),
|
||||
OBJECT.replace("00002.metadata.json", "part.parquet"),
|
||||
OBJECT.replace("00002.metadata.json", "nested/00002.metadata.json"),
|
||||
format!("s3://{BUCKET}/{OBJECT}"),
|
||||
] {
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, &object, &bytes).is_err(), "object {object}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_decodes_gzip_and_rejects_invalid_json() {
|
||||
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
encoder.write_all(&serde_json::to_vec(&empty_metadata()).unwrap()).unwrap();
|
||||
let bytes = encoder.finish().unwrap();
|
||||
let object = OBJECT.replace(".metadata.json", ".gz.metadata.json");
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, &object, &bytes).is_ok());
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, OBJECT, b"{}").is_err());
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, OBJECT, b"[").is_err());
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, &object, b"invalid gzip").is_err());
|
||||
}
|
||||
}
|
||||
@@ -28,9 +28,11 @@ use std::{
|
||||
|
||||
use crate::storage_api::table::contract::http::HTTPPreconditions;
|
||||
use crate::storage_api::table::contract::list::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations as StorageListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
|
||||
AuthoritativeListOperations as StorageAuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as StorageListOperations,
|
||||
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use crate::storage_api::table::contract::multipart::MultipartUploadInspection as StorageMultipartUploadInspection;
|
||||
use crate::storage_api::table::contract::namespace::NamespaceLocking as StorageNamespaceLocking;
|
||||
use crate::storage_api::table::contract::object::{ObjectIO as StorageObjectIO, ObjectOperations as StorageObjectOperations};
|
||||
use crate::storage_api::table::contract::range::HTTPRangeSpec;
|
||||
@@ -58,6 +60,7 @@ use crate::storage_api::table::{
|
||||
StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader,
|
||||
};
|
||||
|
||||
mod cleanup_metadata;
|
||||
mod error;
|
||||
mod iceberg;
|
||||
mod identifier;
|
||||
@@ -246,7 +249,9 @@ pub(crate) trait TableCatalogStorage:
|
||||
WalkOptions = CatalogWalkOptions,
|
||||
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<CatalogObjectInfoOrErr>,
|
||||
> + StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
> + StorageAuthoritativeListOperations
|
||||
+ StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
+ StorageMultipartUploadInspection<Error = EcstoreError>
|
||||
{
|
||||
}
|
||||
|
||||
@@ -274,7 +279,9 @@ impl<T> TableCatalogStorage for T where
|
||||
WalkOptions = CatalogWalkOptions,
|
||||
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<CatalogObjectInfoOrErr>,
|
||||
> + StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
> + StorageAuthoritativeListOperations
|
||||
+ StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
+ StorageMultipartUploadInspection<Error = EcstoreError>
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::object::{
|
||||
next_table_catalog_update_time, table_warehouse_index_state_ready, validate_table_warehouse_index_entry_object,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
pub(super) struct TableBucketDisablement {
|
||||
pub(super) entry: TableBucketEntry,
|
||||
guards: Vec<TableCatalogLockGuard>,
|
||||
}
|
||||
|
||||
/// An in-memory, fenced cleanup plan. No mutation occurs until every object has
|
||||
/// been validated and the caller has authorized all metadata deletions.
|
||||
pub(crate) struct EmptyTableBucketCleanup {
|
||||
disablement: TableBucketDisablement,
|
||||
metadata: Vec<(String, String)>,
|
||||
indexes: Vec<(String, String)>,
|
||||
fence: TableCatalogObjectMutationFence,
|
||||
}
|
||||
|
||||
impl EmptyTableBucketCleanup {
|
||||
pub(crate) fn metadata_objects(&self) -> impl Iterator<Item = &str> {
|
||||
self.metadata.iter().map(|(key, _)| key.as_str())
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_held(&self) -> TableCatalogStoreResult<()> {
|
||||
self.fence.ensure_held()
|
||||
}
|
||||
|
||||
pub(crate) fn mutation_fence(&self) -> TableCatalogObjectMutationFence {
|
||||
self.fence.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_conflict(message: impl Into<String>) -> TableCatalogStoreError {
|
||||
TableCatalogStoreError::Conflict(message.into())
|
||||
}
|
||||
|
||||
impl<B: TableCatalogObjectBackend> ObjectTableCatalogStore<B> {
|
||||
pub(super) async fn begin_table_bucket_disablement(
|
||||
&self,
|
||||
mut entry: TableBucketEntry,
|
||||
publication: Option<&dyn TableCommitPublication>,
|
||||
) -> TableCatalogStoreResult<TableBucketDisablement> {
|
||||
validate_table_bucket_entry(&entry)?;
|
||||
let mut guards = Vec::new();
|
||||
if publication.is_some_and(|publication| publication.catalog_migration_read_permit_status() == Some(true)) {
|
||||
if self
|
||||
.backend
|
||||
.read_object(self.catalog_bucket(), &self.paths.backing_migration_global_fence_path())
|
||||
.await?
|
||||
.is_some()
|
||||
{
|
||||
return Err(cleanup_conflict("table bucket registry migration is in progress"));
|
||||
}
|
||||
self.ensure_object_backed_catalog_write_permit_after_lock(&entry.table_bucket)
|
||||
.await?;
|
||||
} else {
|
||||
guards.push(self.acquire_table_bucket_registry_write_permit().await?);
|
||||
guards.push(self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?);
|
||||
}
|
||||
let object = self.paths.table_bucket_entry_path(&entry.table_bucket);
|
||||
guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?);
|
||||
if let Some((current, _)) = self.read_table_bucket_with_etag_unlocked(&entry.table_bucket).await? {
|
||||
if current.active_rename_id.is_some()
|
||||
|| !matches!(current.state, TableCatalogEntryState::Active | TableCatalogEntryState::Deleted)
|
||||
{
|
||||
return Err(cleanup_conflict("table bucket has an operation in progress"));
|
||||
}
|
||||
entry = current;
|
||||
}
|
||||
Ok(TableBucketDisablement { entry, guards })
|
||||
}
|
||||
|
||||
pub(super) async fn finish_table_bucket_disablement(
|
||||
&self,
|
||||
disablement: &mut TableBucketDisablement,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let mut entry = disablement.entry.clone();
|
||||
entry.state = TableCatalogEntryState::Deleted;
|
||||
entry.updated_at = Some(next_table_catalog_update_time(entry.updated_at.as_deref()));
|
||||
self.write_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&self.paths.table_bucket_entry_path(&entry.table_bucket),
|
||||
&entry,
|
||||
TableCatalogPutPrecondition::Any,
|
||||
disablement
|
||||
.guards
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
entry: TableBucketEntry,
|
||||
publication: &impl TableCatalogObjectBackend,
|
||||
) -> TableCatalogStoreResult<EmptyTableBucketCleanup> {
|
||||
if !publication.holds_table_bucket(&entry.table_bucket) {
|
||||
return Err(cleanup_conflict("empty table bucket cleanup requires a publication fence"));
|
||||
}
|
||||
let disablement = self.begin_table_bucket_disablement(entry, Some(publication)).await?;
|
||||
let bucket = &disablement.entry.table_bucket;
|
||||
let mut fence = publication
|
||||
.object_mutation_fence()
|
||||
.ok_or_else(|| cleanup_conflict("cleanup requires storage fencing"))?;
|
||||
fence
|
||||
.signals
|
||||
.extend(disablement.guards.iter().filter_map(TableCatalogLockGuard::lock_lost_signal));
|
||||
fence.ensure_held()?;
|
||||
|
||||
let catalog_objects = self
|
||||
.backend
|
||||
.list_objects_for_cleanup(self.catalog_bucket(), &self.paths.table_bucket_root_prefix(bucket))
|
||||
.await?;
|
||||
let bucket_path = self.paths.table_bucket_entry_path(bucket);
|
||||
let index_state_path = self.paths.warehouse_index_state_path(bucket);
|
||||
let index_prefix = self.paths.warehouse_index_entries_prefix(bucket);
|
||||
let mut indexes = Vec::new();
|
||||
for object in catalog_objects {
|
||||
if object.delete_marker
|
||||
|| object
|
||||
.version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.is_empty() && version != "null")
|
||||
{
|
||||
return Err(cleanup_conflict("catalog contains versioned or deleted records"));
|
||||
}
|
||||
if object.key == bucket_path {
|
||||
continue;
|
||||
}
|
||||
let etag = if object.key == index_state_path {
|
||||
let (state, etag) = self
|
||||
.read_entry::<TableWarehouseIndexStateEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
.ok_or_else(|| cleanup_conflict("warehouse index state changed during cleanup"))?;
|
||||
table_warehouse_index_state_ready(&state, bucket)?;
|
||||
if !matches!(state.state, TableCatalogEntryState::Active | TableCatalogEntryState::Deleted) {
|
||||
return Err(cleanup_conflict("warehouse index has an operation in progress"));
|
||||
}
|
||||
etag
|
||||
} else if object.key.starts_with(&index_prefix) {
|
||||
let (index, etag) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
.ok_or_else(|| cleanup_conflict("warehouse index changed during cleanup"))?;
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object.key, &index)?;
|
||||
if index.table_bucket != *bucket || index.state != TableCatalogEntryState::Deleted {
|
||||
return Err(cleanup_conflict("table bucket contains an active warehouse index"));
|
||||
}
|
||||
Namespace::parse(&index.namespace).map_err(|err| cleanup_conflict(err.to_string()))?;
|
||||
IdentifierSegment::parse(index.table.clone()).map_err(|err| cleanup_conflict(err.to_string()))?;
|
||||
if Uuid::parse_str(&index.table_id).ok().filter(|id| !id.is_nil()).is_none() {
|
||||
return Err(cleanup_conflict("warehouse index table identity is invalid"));
|
||||
}
|
||||
let expected = format!("tables/{}/", index.table_id);
|
||||
if index.warehouse_object_prefix != expected {
|
||||
return Err(cleanup_conflict("cleanup requires the default table warehouse location"));
|
||||
}
|
||||
etag
|
||||
} else {
|
||||
return Err(cleanup_conflict("table bucket contains namespaces or retained catalog resources"));
|
||||
};
|
||||
let etag = etag.ok_or_else(|| cleanup_conflict("catalog cleanup object has no etag"))?;
|
||||
indexes.push((object.key, etag));
|
||||
}
|
||||
self.backend.ensure_no_multipart_uploads(bucket).await?;
|
||||
let mut metadata = Vec::new();
|
||||
for object in self.backend.list_objects_for_cleanup(bucket, "").await? {
|
||||
if object.delete_marker
|
||||
|| object
|
||||
.version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.is_empty() && version != "null")
|
||||
{
|
||||
return Err(cleanup_conflict("table bucket contains object versions or delete markers"));
|
||||
}
|
||||
let value = publication
|
||||
.read_object_limited(bucket, &object.key, TABLE_METADATA_JSON_MAX_SIZE)
|
||||
.await?
|
||||
.ok_or_else(|| cleanup_conflict("metadata object changed during cleanup"))?;
|
||||
crate::table_catalog::cleanup_metadata::validate_empty_table_cleanup_metadata(bucket, &object.key, &value.data)?;
|
||||
let etag = value
|
||||
.etag
|
||||
.ok_or_else(|| cleanup_conflict("metadata cleanup object has no etag"))?;
|
||||
metadata.push((object.key, etag));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
Ok(EmptyTableBucketCleanup {
|
||||
disablement,
|
||||
metadata,
|
||||
indexes,
|
||||
fence,
|
||||
})
|
||||
}
|
||||
|
||||
async fn execute_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
cleanup: &mut EmptyTableBucketCleanup,
|
||||
incarnation: Uuid,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let bucket = &cleanup.disablement.entry.table_bucket;
|
||||
for (key, etag) in &cleanup.metadata {
|
||||
cleanup.ensure_held()?;
|
||||
self.backend
|
||||
.delete_object_if_match_fenced_for_incarnation(bucket, key, etag, incarnation, &cleanup.fence)
|
||||
.await?;
|
||||
}
|
||||
// A retry inventories the remaining metadata. Keep warehouse protection
|
||||
// until all user objects and uploads have been checked again.
|
||||
if !self.backend.list_objects_for_cleanup(bucket, "").await?.is_empty() {
|
||||
return Err(cleanup_conflict("table bucket changed during cleanup"));
|
||||
}
|
||||
self.backend.ensure_no_multipart_uploads(bucket).await?;
|
||||
for (key, etag) in &cleanup.indexes {
|
||||
cleanup.ensure_held()?;
|
||||
self.backend
|
||||
.delete_object_if_match_fenced(self.catalog_bucket(), key, etag, &cleanup.fence)
|
||||
.await?;
|
||||
}
|
||||
cleanup.ensure_held()?;
|
||||
self.finish_table_bucket_disablement(&mut cleanup.disablement).await
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: TableCatalogObjectBackend> ConfiguredTableCatalogStore<B> {
|
||||
pub(crate) async fn prepare_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
entry: TableBucketEntry,
|
||||
publication: &impl TableCatalogObjectBackend,
|
||||
) -> TableCatalogStoreResult<EmptyTableBucketCleanup> {
|
||||
match self {
|
||||
Self::ObjectBacked(store) => store.prepare_empty_table_bucket_cleanup(entry, publication).await,
|
||||
Self::DurableStrong(_) => Err(TableCatalogStoreError::Unsupported(
|
||||
"empty table bucket cleanup requires object-backed catalog storage".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
cleanup: &mut EmptyTableBucketCleanup,
|
||||
incarnation: Uuid,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
match self {
|
||||
Self::ObjectBacked(store) => store.execute_empty_table_bucket_cleanup(cleanup, incarnation).await,
|
||||
Self::DurableStrong(_) => Err(TableCatalogStoreError::Unsupported(
|
||||
"empty table bucket cleanup requires object-backed catalog storage".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,8 +17,13 @@ use super::*;
|
||||
use crate::storage_api::table::{WriteCommitGuard, WriteCompletion};
|
||||
|
||||
mod backup;
|
||||
mod cleanup;
|
||||
mod migration;
|
||||
mod object;
|
||||
mod registration_cache;
|
||||
#[cfg(test)]
|
||||
mod registration_cache_tests;
|
||||
pub(crate) use registration_cache::TableCatalogRegistrationCache;
|
||||
mod strong;
|
||||
|
||||
pub(crate) use backup::{TableCatalogBackupReport, TableCatalogRestoreReport};
|
||||
@@ -775,6 +780,14 @@ pub(crate) struct TableCatalogObjectListPage {
|
||||
pub is_truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct TableCatalogCleanupObject {
|
||||
pub key: String,
|
||||
pub version_id: Option<String>,
|
||||
pub etag: Option<String>,
|
||||
pub delete_marker: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) enum TableCatalogPutPrecondition {
|
||||
Any,
|
||||
@@ -803,6 +816,10 @@ pub(in crate::table_catalog) fn catalog_list_next_continuation(
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static {
|
||||
fn registration_cache(&self) -> Option<TableCatalogRegistrationCache> {
|
||||
None
|
||||
}
|
||||
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
None
|
||||
}
|
||||
@@ -918,6 +935,30 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static {
|
||||
fence.ensure_held()
|
||||
}
|
||||
|
||||
async fn put_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: Vec<u8>,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
_write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
self.put_object_fenced(bucket, object, data, precondition, fence).await
|
||||
}
|
||||
|
||||
async fn delete_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
_write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
self.delete_object_unlocked(bucket, object).await?;
|
||||
fence.ensure_held()
|
||||
}
|
||||
|
||||
async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()>;
|
||||
|
||||
async fn delete_object_if_match(&self, bucket: &str, object: &str, expected_etag: &str) -> TableCatalogStoreResult<()>;
|
||||
@@ -934,12 +975,41 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static {
|
||||
fence.ensure_held()
|
||||
}
|
||||
|
||||
async fn delete_object_if_match_fenced_for_incarnation(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_expected_etag: &str,
|
||||
_expected_incarnation_id: Uuid,
|
||||
_fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
Err(TableCatalogStoreError::Unsupported(
|
||||
"incarnation-checked cleanup is not supported by this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> {
|
||||
self.delete_object(bucket, object).await
|
||||
}
|
||||
|
||||
async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult<Vec<String>>;
|
||||
|
||||
async fn list_objects_for_cleanup(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
) -> TableCatalogStoreResult<Vec<TableCatalogCleanupObject>> {
|
||||
Err(TableCatalogStoreError::Unsupported(
|
||||
"authoritative cleanup listing is not supported by this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, _bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
Err(TableCatalogStoreError::Unsupported(
|
||||
"multipart cleanup validation is not supported by this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn list_objects_page(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2204,6 +2274,7 @@ where
|
||||
pub(crate) struct EcStoreTableCatalogObjectBackend<S> {
|
||||
store: Arc<S>,
|
||||
strong_runtime: StrongTableCatalogRuntime,
|
||||
registration_cache: TableCatalogRegistrationCache,
|
||||
}
|
||||
|
||||
impl<S> Clone for EcStoreTableCatalogObjectBackend<S> {
|
||||
@@ -2211,6 +2282,7 @@ impl<S> Clone for EcStoreTableCatalogObjectBackend<S> {
|
||||
Self {
|
||||
store: self.store.clone(),
|
||||
strong_runtime: self.strong_runtime.clone(),
|
||||
registration_cache: self.registration_cache.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2220,7 +2292,19 @@ where
|
||||
S: TableCatalogStorage,
|
||||
{
|
||||
pub fn new_with_strong_runtime(store: Arc<S>, strong_runtime: StrongTableCatalogRuntime) -> Self {
|
||||
Self { store, strong_runtime }
|
||||
Self::new_with_runtimes(store, strong_runtime, TableCatalogRegistrationCache::default())
|
||||
}
|
||||
|
||||
pub fn new_with_runtimes(
|
||||
store: Arc<S>,
|
||||
strong_runtime: StrongTableCatalogRuntime,
|
||||
registration_cache: TableCatalogRegistrationCache,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
strong_runtime,
|
||||
registration_cache,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2231,6 +2315,10 @@ impl<S> TableCatalogObjectBackend for EcStoreTableCatalogObjectBackend<S>
|
||||
where
|
||||
S: TableCatalogStorage,
|
||||
{
|
||||
fn registration_cache(&self) -> Option<TableCatalogRegistrationCache> {
|
||||
Some(self.registration_cache.clone())
|
||||
}
|
||||
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
Some(self.strong_runtime.clone())
|
||||
}
|
||||
@@ -2390,6 +2478,46 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: Vec<u8>,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
self.put_object_with_options(bucket, object, data, precondition, Some(write_guards), Some(fence))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
if write_guards.is_empty() {
|
||||
return Err(TableCatalogStoreError::Internal("catalog delete has no namespace owner".to_string()));
|
||||
}
|
||||
let mut options = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
fence.add_to_object_options(&mut options);
|
||||
for guard in &write_guards {
|
||||
options.add_write_commit_guard(guard);
|
||||
}
|
||||
match self.store.delete_object(bucket, object, options).await {
|
||||
Ok(_) => fence.ensure_held(),
|
||||
Err(err) if is_missing_storage_error(&err) => fence.ensure_held(),
|
||||
Err(err) => Err(storage_error_to_catalog("delete fenced catalog object", err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn put_object_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2412,7 +2540,7 @@ where
|
||||
}
|
||||
|
||||
async fn delete_object_if_match(&self, bucket: &str, object: &str, expected_etag: &str) -> TableCatalogStoreResult<()> {
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, None)
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -2424,7 +2552,25 @@ where
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, Some(fence))
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, Some(fence), None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_object_if_match_fenced_for_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_etag: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
if expected_incarnation_id.is_nil() {
|
||||
return Err(TableCatalogStoreError::Invalid(
|
||||
"cleanup requires a non-nil bucket incarnation".to_string(),
|
||||
));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, Some(fence), Some(expected_incarnation_id))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -2498,6 +2644,57 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_objects_for_cleanup(
|
||||
&self,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
) -> TableCatalogStoreResult<Vec<TableCatalogCleanupObject>> {
|
||||
let cancel = tokio_util::sync::CancellationToken::new();
|
||||
let _cancel_on_drop = cancel.clone().drop_guard();
|
||||
let (sender, mut receiver) = tokio::sync::mpsc::channel::<CatalogObjectInfoOrErr>(100);
|
||||
let walk = self.store.clone().walk_authoritative(
|
||||
cancel,
|
||||
bucket,
|
||||
prefix,
|
||||
sender,
|
||||
CatalogWalkOptions {
|
||||
include_free_versions: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let collect = async move {
|
||||
let mut objects = Vec::new();
|
||||
while let Some(result) = receiver.recv().await {
|
||||
if let Some(err) = result.err {
|
||||
return Err(storage_error_to_catalog("inspect cleanup object versions", err));
|
||||
}
|
||||
let object = result.item.ok_or_else(|| {
|
||||
TableCatalogStoreError::Internal("cleanup object walk returned an empty result".to_string())
|
||||
})?;
|
||||
objects.push(TableCatalogCleanupObject {
|
||||
key: object.name,
|
||||
version_id: object.version_id.filter(|id| !id.is_nil()).map(|id| id.to_string()),
|
||||
etag: object.etag,
|
||||
delete_marker: object.delete_marker,
|
||||
});
|
||||
}
|
||||
Ok(objects)
|
||||
};
|
||||
let (walk_result, objects) = tokio::join!(walk, collect);
|
||||
let objects = objects?;
|
||||
walk_result.map_err(|err| storage_error_to_catalog("list cleanup object versions", err))?;
|
||||
Ok(objects)
|
||||
}
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
self.store.ensure_no_multipart_uploads(bucket).await.map_err(|err| match err {
|
||||
StorageError::BucketNotEmpty(_) => {
|
||||
TableCatalogStoreError::Conflict(format!("table bucket {bucket} contains unfinished multipart uploads"))
|
||||
}
|
||||
other => storage_error_to_catalog("inspect unfinished multipart uploads", other),
|
||||
})
|
||||
}
|
||||
|
||||
async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<TableCatalogLockGuard> {
|
||||
let lock = self
|
||||
.store
|
||||
@@ -2534,12 +2731,14 @@ where
|
||||
object: &str,
|
||||
expected_etag: &str,
|
||||
fence: Option<&TableCatalogObjectMutationFence>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let mut options = ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(expected_etag.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
expected_bucket_incarnation_id: expected_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(fence) = fence {
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
|
||||
pub(super) const TABLE_REGISTRATION_READ_CONCURRENCY: usize = 8;
|
||||
const TABLE_MAINTENANCE_RENAME_MAX_OBJECTS: usize = 4096;
|
||||
const TABLE_MAINTENANCE_SCHEDULER_HISTORY_MAX_OBJECTS: usize = 4096;
|
||||
|
||||
@@ -261,7 +263,7 @@ fn validate_table_rename_intent_object(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn next_table_catalog_update_time(previous: Option<&str>) -> String {
|
||||
pub(super) fn next_table_catalog_update_time(previous: Option<&str>) -> String {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let update_time = previous
|
||||
.and_then(|value| OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339).ok())
|
||||
@@ -360,7 +362,7 @@ pub(super) fn validate_table_maintenance_report_owner(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_table_warehouse_index_entry_object(
|
||||
pub(super) fn validate_table_warehouse_index_entry_object(
|
||||
paths: &TableCatalogObjectPaths,
|
||||
object: &str,
|
||||
index: &TableWarehouseIndexEntry,
|
||||
@@ -374,7 +376,10 @@ fn validate_table_warehouse_index_entry_object(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn table_warehouse_index_state_ready(state: &TableWarehouseIndexStateEntry, table_bucket: &str) -> TableCatalogStoreResult<bool> {
|
||||
pub(super) fn table_warehouse_index_state_ready(
|
||||
state: &TableWarehouseIndexStateEntry,
|
||||
table_bucket: &str,
|
||||
) -> TableCatalogStoreResult<bool> {
|
||||
if state.version == 0 || state.version > TABLE_WAREHOUSE_INDEX_STATE_VERSION {
|
||||
return Err(TableCatalogStoreError::Invalid(format!(
|
||||
"unsupported warehouse index state version: {}",
|
||||
@@ -1217,7 +1222,7 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_entry_unlocked<T>(
|
||||
pub(super) async fn write_entry_unlocked<T>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -1225,14 +1230,39 @@ where
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
) -> TableCatalogStoreResult<()>
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
self.write_entry_unlocked_with_fence(bucket, object, entry, precondition, write_guards, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_entry_unlocked_with_fence<T>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
entry: &T,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: Option<&TableCatalogObjectMutationFence>,
|
||||
) -> TableCatalogStoreResult<()>
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
let data = serde_json::to_vec(entry)
|
||||
.map_err(|err| TableCatalogStoreError::Internal(format!("failed to serialize catalog entry {object}: {err}")))?;
|
||||
self.backend
|
||||
.put_object_unlocked(bucket, object, data, precondition, write_guards)
|
||||
.await
|
||||
match fence {
|
||||
Some(fence) => {
|
||||
self.backend
|
||||
.put_object_unlocked_fenced(bucket, object, data, precondition, write_guards, fence)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
self.backend
|
||||
.put_object_unlocked(bucket, object, data, precondition, write_guards)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_exact_entry_unlocked<T>(
|
||||
@@ -1243,11 +1273,27 @@ where
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
) -> TableCatalogStoreResult<String>
|
||||
where
|
||||
T: DeserializeOwned + PartialEq + Serialize,
|
||||
{
|
||||
self.write_exact_entry_unlocked_with_fence(bucket, object, entry, precondition, write_guards, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_exact_entry_unlocked_with_fence<T>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
entry: &T,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: Option<&TableCatalogObjectMutationFence>,
|
||||
) -> TableCatalogStoreResult<String>
|
||||
where
|
||||
T: DeserializeOwned + PartialEq + Serialize,
|
||||
{
|
||||
let write_result = self
|
||||
.write_entry_unlocked(bucket, object, entry, precondition, write_guards)
|
||||
.write_entry_unlocked_with_fence(bucket, object, entry, precondition, write_guards, fence)
|
||||
.await;
|
||||
let current = self.read_entry_unlocked::<T>(bucket, object).await?;
|
||||
match current {
|
||||
@@ -1264,7 +1310,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_table_bucket_with_etag_unlocked(
|
||||
pub(super) async fn read_table_bucket_with_etag_unlocked(
|
||||
&self,
|
||||
table_bucket: &str,
|
||||
) -> TableCatalogStoreResult<Option<(TableBucketEntry, String)>> {
|
||||
@@ -1387,6 +1433,7 @@ where
|
||||
"table rename recovery requires a table-bucket publication fence".to_string(),
|
||||
));
|
||||
}
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
let bucket_object = self.paths.table_bucket_entry_path(table_bucket);
|
||||
let Some((observed_bucket, _)) = self
|
||||
.read_entry::<TableBucketEntry>(self.catalog_bucket(), &bucket_object)
|
||||
@@ -1511,7 +1558,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == source_fence || current == source_tombstone => {}
|
||||
Some((current, current_etag)) if current == intent.source && current_etag == intent.source_etag => {
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&source_object,
|
||||
&source_fence,
|
||||
@@ -1520,6 +1567,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1549,7 +1597,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == destination_fence || current == intent.destination => {}
|
||||
None if intent.destination_etag.is_none() => {
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&destination_object,
|
||||
&destination_fence,
|
||||
@@ -1558,6 +1606,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1565,7 +1614,7 @@ where
|
||||
if current.state == TableCatalogEntryState::Deleted
|
||||
&& Some(current_etag.as_str()) == intent.destination_etag.as_deref() =>
|
||||
{
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&destination_object,
|
||||
&destination_fence,
|
||||
@@ -1574,6 +1623,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1601,7 +1651,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == source_tombstone => {}
|
||||
Some((current, current_etag)) if current == source_fence => {
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&source_object,
|
||||
&source_tombstone,
|
||||
@@ -1610,6 +1660,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1647,7 +1698,7 @@ where
|
||||
"table rename warehouse index changed during recovery".to_string(),
|
||||
));
|
||||
}
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&index_object,
|
||||
&destination_index,
|
||||
@@ -1656,6 +1707,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1674,7 +1726,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == intent.destination => {}
|
||||
Some((current, current_etag)) if current == destination_fence => {
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&destination_object,
|
||||
&intent.destination,
|
||||
@@ -1683,6 +1735,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1720,12 +1773,13 @@ where
|
||||
}
|
||||
bucket_entry.active_rename_id = None;
|
||||
bucket_entry.updated_at = Some(next_table_catalog_update_time(bucket_entry.updated_at.as_deref()));
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&bucket_object,
|
||||
&bucket_entry,
|
||||
TableCatalogPutPrecondition::IfMatch(bucket_etag),
|
||||
_bucket_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
@@ -1873,7 +1927,12 @@ where
|
||||
let candidate = table_warehouse_index_entry(entry)?;
|
||||
validate_table_entry_version_and_id(entry)?;
|
||||
for existing in self
|
||||
.list_all_table_entries_with_limit(&candidate.table_bucket, None, locked_table_entry)
|
||||
.list_all_table_entries_with_limit(
|
||||
&candidate.table_bucket,
|
||||
None,
|
||||
locked_table_entry,
|
||||
TABLE_REGISTRATION_READ_CONCURRENCY,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
if existing.state != TableCatalogEntryState::Active {
|
||||
@@ -1896,34 +1955,38 @@ where
|
||||
}
|
||||
}
|
||||
let state_object = self.paths.warehouse_index_state_path(&candidate.table_bucket);
|
||||
for object in self
|
||||
let index_objects = self
|
||||
.backend
|
||||
.list_objects(self.catalog_bucket(), &self.paths.warehouse_index_entries_prefix(&candidate.table_bucket))
|
||||
.await?
|
||||
{
|
||||
if object == state_object {
|
||||
continue;
|
||||
}
|
||||
let Some((existing, _)) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object)
|
||||
.await?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object, &existing)?;
|
||||
if existing.table_id == candidate.table_id || existing.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
if warehouse_object_prefixes_overlap(&existing.warehouse_object_prefix, &candidate.warehouse_object_prefix)
|
||||
&& self.warehouse_index_entry_has_active_owner(&existing).await?
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"table warehouse location overlaps an active table: {}",
|
||||
candidate.warehouse_object_prefix
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
.await?;
|
||||
stream::iter(index_objects.into_iter().filter(|object| object != &state_object))
|
||||
.map(|object| {
|
||||
let candidate = &candidate;
|
||||
async move {
|
||||
let Some((existing, _)) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object)
|
||||
.await?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object, &existing)?;
|
||||
if existing.table_id == candidate.table_id || existing.state != TableCatalogEntryState::Active {
|
||||
return Ok(());
|
||||
}
|
||||
if warehouse_object_prefixes_overlap(&existing.warehouse_object_prefix, &candidate.warehouse_object_prefix)
|
||||
&& self.warehouse_index_entry_has_active_owner(&existing).await?
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"table warehouse location overlaps an active table: {}",
|
||||
candidate.warehouse_object_prefix
|
||||
)));
|
||||
}
|
||||
Ok::<_, TableCatalogStoreError>(())
|
||||
}
|
||||
})
|
||||
.buffered(TABLE_REGISTRATION_READ_CONCURRENCY)
|
||||
.try_for_each(|()| async { Ok(()) })
|
||||
.await
|
||||
}
|
||||
|
||||
async fn reserve_table_warehouse_index(
|
||||
@@ -2364,7 +2427,7 @@ where
|
||||
) -> TableCatalogStoreResult<Option<TableDataPlaneResource>> {
|
||||
let mut matched: Option<TableDataPlaneResource> = None;
|
||||
for table in self
|
||||
.list_all_table_entries_with_limit(table_bucket, Some(TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_CATALOG_OBJECTS), None)
|
||||
.list_all_table_entries_with_limit(table_bucket, Some(TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_CATALOG_OBJECTS), None, 1)
|
||||
.await?
|
||||
{
|
||||
if table.state != TableCatalogEntryState::Active {
|
||||
@@ -2456,7 +2519,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
let tables = self
|
||||
.list_all_table_entries_with_limit(table_bucket, max_catalog_objects, None)
|
||||
.list_all_table_entries_with_limit(table_bucket, max_catalog_objects, None, 1)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|table| table.state == TableCatalogEntryState::Active)
|
||||
@@ -2537,7 +2600,7 @@ where
|
||||
}
|
||||
|
||||
async fn list_all_table_entries(&self, table_bucket: &str) -> TableCatalogStoreResult<Vec<TableEntry>> {
|
||||
self.list_all_table_entries_with_limit(table_bucket, None, None).await
|
||||
self.list_all_table_entries_with_limit(table_bucket, None, None, 1).await
|
||||
}
|
||||
|
||||
async fn list_table_entry_objects_for_data_plane_scan(
|
||||
@@ -2586,8 +2649,8 @@ where
|
||||
table_bucket: &str,
|
||||
max_catalog_objects: Option<usize>,
|
||||
locked_table_entry: Option<&str>,
|
||||
read_concurrency: usize,
|
||||
) -> TableCatalogStoreResult<Vec<TableEntry>> {
|
||||
let mut entries = Vec::new();
|
||||
let table_objects = match max_catalog_objects {
|
||||
Some(max_catalog_objects) => {
|
||||
self.list_table_entry_objects_for_data_plane_scan(table_bucket, max_catalog_objects)
|
||||
@@ -2601,19 +2664,24 @@ where
|
||||
.filter(|object| object.ends_with(TABLE_ENTRY_FILE))
|
||||
.collect(),
|
||||
};
|
||||
for object in table_objects {
|
||||
// Only the exact entry protected by the caller's write lock can bypass a read lock.
|
||||
let entry = if locked_table_entry == Some(object.as_str()) {
|
||||
self.read_entry_unlocked::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
} else {
|
||||
self.read_entry::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
};
|
||||
let Some((entry, _)) = entry else {
|
||||
continue;
|
||||
};
|
||||
validate_table_entry_object(&self.paths, &object, &entry)?;
|
||||
entries.push(entry);
|
||||
}
|
||||
let mut entries = stream::iter(table_objects)
|
||||
.map(|object| async move {
|
||||
// Only the exact entry protected by the caller's write lock can bypass a read lock.
|
||||
let entry = if locked_table_entry == Some(object.as_str()) {
|
||||
self.read_entry_unlocked::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
} else {
|
||||
self.read_entry::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
};
|
||||
let Some((entry, _)) = entry else {
|
||||
return Ok(None);
|
||||
};
|
||||
validate_table_entry_object(&self.paths, &object, &entry)?;
|
||||
Ok::<_, TableCatalogStoreError>(Some(entry))
|
||||
})
|
||||
.buffered(read_concurrency)
|
||||
.try_filter_map(|entry| async move { Ok(entry) })
|
||||
.try_collect::<Vec<_>>()
|
||||
.await?;
|
||||
entries.sort_by(|left, right| (&left.namespace, &left.table).cmp(&(&right.namespace, &right.table)));
|
||||
Ok(entries)
|
||||
}
|
||||
@@ -2772,8 +2840,7 @@ where
|
||||
"table registration requires a table publication fence".to_string(),
|
||||
));
|
||||
}
|
||||
self.ensure_table_warehouse_prefix_available(&entry, Some(&table_path))
|
||||
.await?;
|
||||
self.check_registration_ownership(&entry, &table_path, publication).await?;
|
||||
let reservation = self.reserve_table_warehouse_index(&entry, true, &table_path).await?;
|
||||
if !publication.holds_table_bucket(&entry.table_bucket)
|
||||
|| !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table)
|
||||
@@ -2784,20 +2851,23 @@ where
|
||||
"table registration publication fence was lost before catalog update".to_string(),
|
||||
));
|
||||
}
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
let result = self
|
||||
.write_entry_unlocked(
|
||||
.write_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&table_path,
|
||||
&entry,
|
||||
precondition,
|
||||
_table_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await;
|
||||
if result.is_err() {
|
||||
self.rollback_table_warehouse_index_reservation(&entry, reservation, "table entry write failed")
|
||||
.await;
|
||||
}
|
||||
result
|
||||
result?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_view_entry(&self, entry: ViewEntry, precondition: TableCatalogPutPrecondition) -> TableCatalogStoreResult<()> {
|
||||
@@ -6258,27 +6328,15 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
async fn disable_empty_table_bucket(&self, mut entry: TableBucketEntry) -> TableCatalogStoreResult<()> {
|
||||
validate_table_bucket_entry(&entry)?;
|
||||
let _registry_guard = self.acquire_table_bucket_registry_write_permit().await?;
|
||||
let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?;
|
||||
let object = self.paths.table_bucket_entry_path(&entry.table_bucket);
|
||||
let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?;
|
||||
if let Some((current, _)) = self.read_table_bucket_with_etag_unlocked(&entry.table_bucket).await? {
|
||||
if current.active_rename_id.is_some()
|
||||
|| !matches!(current.state, TableCatalogEntryState::Active | TableCatalogEntryState::Deleted)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict("table bucket has an operation in progress".to_string()));
|
||||
}
|
||||
entry = current;
|
||||
}
|
||||
async fn disable_empty_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> {
|
||||
let mut disablement = self.begin_table_bucket_disablement(entry, None).await?;
|
||||
// Namespace creation holds this same bucket lock. Retained table, view, and
|
||||
// maintenance records also prevent removing their data-plane protections.
|
||||
let page = self
|
||||
.backend
|
||||
.list_objects_page(
|
||||
self.catalog_bucket(),
|
||||
&self.paths.namespace_entries_prefix(&entry.table_bucket),
|
||||
&self.paths.namespace_entries_prefix(&disablement.entry.table_bucket),
|
||||
None,
|
||||
NonZeroUsize::MIN,
|
||||
)
|
||||
@@ -6288,16 +6346,7 @@ where
|
||||
"table bucket contains namespaces or retained catalog resources".to_string(),
|
||||
));
|
||||
}
|
||||
entry.state = TableCatalogEntryState::Deleted;
|
||||
entry.updated_at = Some(next_table_catalog_update_time(entry.updated_at.as_deref()));
|
||||
self.write_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&object,
|
||||
&entry,
|
||||
TableCatalogPutPrecondition::Any,
|
||||
_guard.write_commit_guards(),
|
||||
)
|
||||
.await
|
||||
self.finish_table_bucket_disablement(&mut disablement).await
|
||||
}
|
||||
|
||||
async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> {
|
||||
@@ -6681,6 +6730,7 @@ where
|
||||
));
|
||||
}
|
||||
let _publication_completion = TableCommitPublicationCompletion::new(&publication);
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
self.recover_active_table_rename(table_bucket, &publication).await?;
|
||||
|
||||
{
|
||||
@@ -6820,12 +6870,13 @@ where
|
||||
));
|
||||
}
|
||||
None => {
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&index_object,
|
||||
&source_index,
|
||||
TableCatalogPutPrecondition::IfAbsent,
|
||||
_index_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
@@ -6846,22 +6897,24 @@ where
|
||||
validate_table_rename_intent_object(&self.paths, &intent_object, &intent)?;
|
||||
|
||||
// An orphan intent has no catalog effect; a published bucket fence without its intent cannot be recovered safely.
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&intent_object,
|
||||
&intent,
|
||||
TableCatalogPutPrecondition::IfAbsent,
|
||||
_intent_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
bucket_entry.active_rename_id = Some(rename_id);
|
||||
bucket_entry.updated_at = Some(now);
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&bucket_object,
|
||||
&bucket_entry,
|
||||
TableCatalogPutPrecondition::IfMatch(bucket_etag),
|
||||
_bucket_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -7311,13 +7364,15 @@ where
|
||||
}
|
||||
|
||||
let cas_started = Instant::now();
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
let cas_result = self
|
||||
.write_entry_unlocked(
|
||||
.write_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&table_path,
|
||||
&next,
|
||||
TableCatalogPutPrecondition::IfMatch(current_etag),
|
||||
_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await;
|
||||
record_table_commit_cas_result(&request.operation, cas_started, &cas_result);
|
||||
@@ -7388,6 +7443,7 @@ where
|
||||
table.as_str()
|
||||
)));
|
||||
};
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
self.tombstone_table_warehouse_index_for_drop(&entry, false).await?;
|
||||
if !publication.holds_table_bucket(table_bucket)
|
||||
|| !publication.holds_table(table_bucket, &namespace.public_name(), table.as_str())
|
||||
@@ -7398,7 +7454,11 @@ where
|
||||
"table drop publication fence was lost before catalog update".to_string(),
|
||||
));
|
||||
}
|
||||
if let Err(err) = self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await {
|
||||
if let Err(err) = self
|
||||
.backend
|
||||
.delete_object_unlocked_fenced(self.catalog_bucket(), &object, _guard.write_commit_guards(), &mutation_fence)
|
||||
.await
|
||||
{
|
||||
match self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await {
|
||||
Ok(None) => return Ok(()),
|
||||
Ok(Some((current, _))) if current == entry => {
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::object::{
|
||||
TABLE_REGISTRATION_READ_CONCURRENCY, table_warehouse_index_state_ready, validate_table_entry_object,
|
||||
validate_table_warehouse_index_entry_object,
|
||||
};
|
||||
use super::*;
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const REGISTRATION_CACHE_BYTES: u64 = 16 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CachedOwner {
|
||||
etag: String,
|
||||
owner: Arc<TableWarehouseIndexEntry>,
|
||||
}
|
||||
|
||||
/// Runtime-local object projections. Every lookup requires a fresh committed inventory.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct TableCatalogRegistrationCache {
|
||||
entries: Arc<OnceLock<moka::sync::Cache<String, CachedOwner>>>,
|
||||
capacity_bytes: u64,
|
||||
}
|
||||
|
||||
impl Default for TableCatalogRegistrationCache {
|
||||
fn default() -> Self {
|
||||
Self::with_capacity(REGISTRATION_CACHE_BYTES)
|
||||
}
|
||||
}
|
||||
|
||||
impl TableCatalogRegistrationCache {
|
||||
fn with_capacity(bytes: u64) -> Self {
|
||||
Self {
|
||||
entries: Arc::new(OnceLock::new()),
|
||||
capacity_bytes: bytes,
|
||||
}
|
||||
}
|
||||
|
||||
fn entries(&self) -> &moka::sync::Cache<String, CachedOwner> {
|
||||
self.entries.get_or_init(|| {
|
||||
moka::sync::Cache::builder()
|
||||
.max_capacity(self.capacity_bytes)
|
||||
.weigher(|key: &String, value: &CachedOwner| {
|
||||
let owner = &value.owner;
|
||||
let bytes = [
|
||||
key.len(),
|
||||
value.etag.len(),
|
||||
owner.table_bucket.len(),
|
||||
owner.namespace.len(),
|
||||
owner.table.len(),
|
||||
owner.table_id.len(),
|
||||
owner.warehouse_object_prefix.len(),
|
||||
]
|
||||
.into_iter()
|
||||
.fold(256_usize, usize::saturating_add);
|
||||
u32::try_from(bytes).unwrap_or(u32::MAX)
|
||||
})
|
||||
.build()
|
||||
})
|
||||
}
|
||||
|
||||
fn get(&self, object: &TableCatalogCleanupObject) -> Option<Arc<TableWarehouseIndexEntry>> {
|
||||
let etag = object.etag.as_deref().filter(|etag| !etag.is_empty())?;
|
||||
self.entries
|
||||
.get()?
|
||||
.get(&object.key)
|
||||
.filter(|cached| cached.etag == etag)
|
||||
.map(|cached| cached.owner)
|
||||
}
|
||||
|
||||
fn insert(&self, object: String, etag: Option<String>, owner: Arc<TableWarehouseIndexEntry>) {
|
||||
if let Some(etag) = etag.filter(|etag| !etag.is_empty()) {
|
||||
self.entries().insert(object, CachedOwner { etag, owner });
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::table_catalog) fn clear(&self) {
|
||||
if let Some(entries) = self.entries.get() {
|
||||
entries.invalidate_all();
|
||||
entries.run_pending_tasks();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_inventory(objects: &mut [TableCatalogCleanupObject], prefix: &str) -> TableCatalogStoreResult<()> {
|
||||
objects.sort_unstable_by(|left, right| left.key.cmp(&right.key));
|
||||
if objects.iter().any(|object| {
|
||||
!object.key.starts_with(prefix)
|
||||
|| object.delete_marker
|
||||
|| object
|
||||
.version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.is_empty() && version != "null")
|
||||
}) || objects.windows(2).any(|pair| pair[0].key == pair[1].key)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Invalid(
|
||||
"registration requires an unversioned, complete catalog inventory".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl<B: TableCatalogObjectBackend> ObjectTableCatalogStore<B> {
|
||||
async fn registration_owner(
|
||||
&self,
|
||||
object: TableCatalogCleanupObject,
|
||||
table_record: bool,
|
||||
locked_table_entry: &str,
|
||||
cache: Option<&TableCatalogRegistrationCache>,
|
||||
) -> TableCatalogStoreResult<Arc<TableWarehouseIndexEntry>> {
|
||||
if let Some(owner) = cache.and_then(|cache| cache.get(&object)) {
|
||||
return Ok(owner);
|
||||
}
|
||||
let (owner, etag) = if table_record {
|
||||
let value = if object.key == locked_table_entry {
|
||||
self.read_entry_unlocked::<TableEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
} else {
|
||||
self.read_entry::<TableEntry>(self.catalog_bucket(), &object.key).await?
|
||||
};
|
||||
let (entry, etag) = value.ok_or_else(|| {
|
||||
TableCatalogStoreError::Unavailable("table record changed during registration inventory".to_string())
|
||||
})?;
|
||||
validate_table_entry_object(&self.paths, &object.key, &entry)?;
|
||||
(table_warehouse_index_entry(&entry)?, etag)
|
||||
} else {
|
||||
let (entry, etag) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
TableCatalogStoreError::Unavailable("warehouse index changed during registration inventory".to_string())
|
||||
})?;
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object.key, &entry)?;
|
||||
(entry, etag)
|
||||
};
|
||||
let owner = Arc::new(owner);
|
||||
if let Some(cache) = cache {
|
||||
// A concurrent index repair may change its ETag after enumeration.
|
||||
// Associate the projection only with the object that GET validated.
|
||||
cache.insert(object.key, etag, Arc::clone(&owner));
|
||||
}
|
||||
Ok(owner)
|
||||
}
|
||||
|
||||
pub(super) async fn check_registration_ownership(
|
||||
&self,
|
||||
entry: &TableEntry,
|
||||
locked_table_entry: &str,
|
||||
publication: &(dyn TableCommitPublication + Sync),
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
if !publication.holds_table_bucket(&entry.table_bucket) {
|
||||
return Err(TableCatalogStoreError::Unavailable("registration publication fence was lost".to_string()));
|
||||
}
|
||||
validate_table_entry_version_and_id(entry)?;
|
||||
let candidate = table_warehouse_index_entry(entry)?;
|
||||
let namespace_prefix = self.paths.namespace_entries_prefix(&entry.table_bucket);
|
||||
let index_prefix = self.paths.warehouse_index_entries_prefix(&entry.table_bucket);
|
||||
// The backend must preserve system-bucket committed-read semantics as
|
||||
// well as completeness; ordinary S3 LIST is insufficient for cache hits.
|
||||
let mut tables = self
|
||||
.backend
|
||||
.list_objects_for_cleanup(self.catalog_bucket(), &namespace_prefix)
|
||||
.await?;
|
||||
let mut indexes = self
|
||||
.backend
|
||||
.list_objects_for_cleanup(self.catalog_bucket(), &index_prefix)
|
||||
.await?;
|
||||
validate_inventory(&mut tables, &namespace_prefix)?;
|
||||
validate_inventory(&mut indexes, &index_prefix)?;
|
||||
let cache = self.backend.registration_cache();
|
||||
let mut owners = BTreeMap::new();
|
||||
let mut table_ids = BTreeSet::new();
|
||||
let mut table_reads = stream::iter(tables.into_iter().filter(|object| object.key.ends_with(TABLE_ENTRY_FILE)))
|
||||
.map(|object| self.registration_owner(object, true, locked_table_entry, cache.as_ref()))
|
||||
.buffered(TABLE_REGISTRATION_READ_CONCURRENCY);
|
||||
while let Some(owner) = table_reads.try_next().await? {
|
||||
if owner.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
if !table_ids.insert(owner.table_id.clone())
|
||||
|| (owner.table_id == candidate.table_id
|
||||
&& (owner.namespace != candidate.namespace || owner.table != candidate.table))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table id is already registered in this table bucket".to_string(),
|
||||
));
|
||||
}
|
||||
if owner.table_id != candidate.table_id
|
||||
&& warehouse_object_prefixes_overlap(&owner.warehouse_object_prefix, &candidate.warehouse_object_prefix)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table warehouse location overlaps an active table".to_string(),
|
||||
));
|
||||
}
|
||||
owners.insert((owner.namespace.clone(), owner.table.clone()), owner);
|
||||
}
|
||||
let mut prefixes = owners
|
||||
.values()
|
||||
.map(|owner| owner.warehouse_object_prefix.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
prefixes.sort_unstable();
|
||||
if prefixes
|
||||
.windows(2)
|
||||
.any(|pair| warehouse_object_prefixes_overlap(pair[0], pair[1]))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict("active table warehouse locations overlap".to_string()));
|
||||
}
|
||||
let state_path = self.paths.warehouse_index_state_path(&entry.table_bucket);
|
||||
let has_state = indexes.iter().any(|object| object.key == state_path);
|
||||
let mut index_reads = stream::iter(indexes.into_iter().filter(|object| object.key != state_path))
|
||||
.map(|object| self.registration_owner(object, false, locked_table_entry, cache.as_ref()))
|
||||
.buffered(TABLE_REGISTRATION_READ_CONCURRENCY);
|
||||
while let Some(index) = index_reads.try_next().await? {
|
||||
if index.table_id == candidate.table_id || index.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
if warehouse_object_prefixes_overlap(&index.warehouse_object_prefix, &candidate.warehouse_object_prefix) {
|
||||
parse_namespace_for_store(&index.namespace)?;
|
||||
parse_table_for_store(&index.table)?;
|
||||
if owners
|
||||
.get(&(index.namespace.clone(), index.table.clone()))
|
||||
.is_some_and(|owner| {
|
||||
owner.table_id == index.table_id && owner.warehouse_object_prefix == index.warehouse_object_prefix
|
||||
})
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table warehouse location overlaps an active table".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
if has_state {
|
||||
let (state, _) = self
|
||||
.read_entry::<TableWarehouseIndexStateEntry>(self.catalog_bucket(), &state_path)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
TableCatalogStoreError::Unavailable("warehouse index state changed during registration inventory".to_string())
|
||||
})?;
|
||||
table_warehouse_index_state_ready(&state, &entry.table_bucket)?;
|
||||
}
|
||||
if !publication.holds_table_bucket(&entry.table_bucket) {
|
||||
return Err(TableCatalogStoreError::Unavailable("registration publication fence was lost".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::registration_cache_tests::{cache_fixture, registration_entry};
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_capacity_and_oversized_owners_preserve_checks() {
|
||||
let (backend, _, namespace) = cache_fixture().await;
|
||||
let cache = TableCatalogRegistrationCache::with_capacity(1024);
|
||||
let backend = crate::table_catalog::test_support::TestCatalogObjectBackend {
|
||||
registration_cache: cache.clone(),
|
||||
..backend
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let mut oversized = registration_entry(&namespace, "oversized");
|
||||
oversized.warehouse_location = format!("s3://analytics/{}", "directory".repeat(512));
|
||||
store.create_table(oversized.clone()).await.unwrap();
|
||||
for number in 0..12 {
|
||||
store
|
||||
.create_table(registration_entry(&namespace, &format!("table_{number}")))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
cache.entries().run_pending_tasks();
|
||||
assert!(cache.entries().weighted_size() <= 1024);
|
||||
backend.reset_call_counts().await;
|
||||
let mut candidate = registration_entry(&namespace, "conflicting");
|
||||
candidate.warehouse_location = format!("{}/child", oversized.warehouse_location);
|
||||
assert!(matches!(store.create_table(candidate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
assert!(
|
||||
backend.read_call_count().await > 0,
|
||||
"eviction must cause reads instead of accepting an absent owner"
|
||||
);
|
||||
cache.entries().run_pending_tasks();
|
||||
assert!(cache.entries().weighted_size() <= 1024);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::table_catalog::test_support::{RecordingLeaseClient, TestCatalogObjectBackend};
|
||||
use crate::table_catalog::tests::{test_bucket_entry, test_namespace_entry, test_table_entry};
|
||||
use rustfs_lock::LockClient;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
pub(super) fn registration_entry(namespace: &Namespace, name: &str) -> TableEntry {
|
||||
let table = IdentifierSegment::parse(name).expect("valid test table name");
|
||||
let mut entry = test_table_entry(
|
||||
"analytics",
|
||||
namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = format!("id-{name}");
|
||||
entry.warehouse_location = format!("s3://analytics/tables/id-{name}");
|
||||
entry
|
||||
}
|
||||
|
||||
pub(super) async fn cache_fixture() -> (TestCatalogObjectBackend, ObjectTableCatalogStore<TestCatalogObjectBackend>, Namespace) {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry("analytics")).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry("analytics", &namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
store.create_table(registration_entry(&namespace, "orders")).await.unwrap();
|
||||
store.create_table(registration_entry(&namespace, "warmup")).await.unwrap();
|
||||
(backend, store, namespace)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_binds_projection_to_the_get_etag() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
backend.registration_cache.clear();
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
let key = (RUSTFS_META_BUCKET.to_string(), path.clone());
|
||||
let original = backend.state.lock().await.objects.get(&key).unwrap().clone();
|
||||
let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &path).await;
|
||||
let task_store = store.clone();
|
||||
let entry = registration_entry(&namespace, "unrelated");
|
||||
let task = tokio::spawn(async move { task_store.create_table(entry).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut changed = registration_entry(&namespace, "orders");
|
||||
changed.warehouse_location = "s3://analytics/changed".to_string();
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&changed).unwrap())
|
||||
.await;
|
||||
pause.release();
|
||||
tokio::time::timeout(Duration::from_secs(5), task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
backend.state.lock().await.objects.insert(key, original);
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.warehouse_location = "s3://analytics/changed/child".to_string();
|
||||
store
|
||||
.create_table(candidate)
|
||||
.await
|
||||
.expect("a GET projection cannot inherit an earlier inventory ETag");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_observes_changed_and_removed_objects() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
let mut changed = registration_entry(&namespace, "orders");
|
||||
changed.warehouse_location = "s3://analytics/relocated".to_string();
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&changed).unwrap())
|
||||
.await;
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.warehouse_location = "s3://analytics/relocated/child".to_string();
|
||||
assert!(matches!(
|
||||
store.create_table(candidate.clone()).await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
backend.delete_object(RUSTFS_META_BUCKET, &path).await.unwrap();
|
||||
candidate.table_id = changed.table_id;
|
||||
store
|
||||
.create_table(candidate)
|
||||
.await
|
||||
.expect("removed records cannot remain active through the cache");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_revalidates_missing_etags_and_corrupt_changed_indexes() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
backend
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.etagless_objects
|
||||
.insert((RUSTFS_META_BUCKET.to_string(), path.clone()));
|
||||
backend.fail_next_read(RUSTFS_META_BUCKET, &path).await;
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
backend.state.lock().await.etagless_objects.clear();
|
||||
let index = store.paths.warehouse_index_entry_path("analytics", "tables/id-orders/");
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &index, b"invalid json".to_vec())
|
||||
.await;
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
assert!(store.load_table("analytics", "sales", "candidate").await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_rejects_incomplete_and_versioned_inventories() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let namespace_prefix = store.paths.namespace_entries_prefix("analytics");
|
||||
let index_prefix = store.paths.warehouse_index_entries_prefix("analytics");
|
||||
for prefix in [&namespace_prefix, &index_prefix] {
|
||||
backend.state.lock().await.failed_inventory_prefixes.insert(prefix.clone());
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
backend.state.lock().await.failed_inventory_prefixes.clear();
|
||||
}
|
||||
for (version_id, delete_marker) in [(Some(Uuid::new_v4().to_string()), false), (None, true)] {
|
||||
backend.state.lock().await.inventory_overrides.insert(
|
||||
namespace_prefix.clone(),
|
||||
vec![TableCatalogCleanupObject {
|
||||
key: format!("{namespace_prefix}unexpected"),
|
||||
version_id,
|
||||
etag: Some("etag".to_string()),
|
||||
delete_marker,
|
||||
}],
|
||||
);
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
}
|
||||
backend.state.lock().await.inventory_overrides.clear();
|
||||
assert!(store.load_table("analytics", "sales", "candidate").await.unwrap().is_none());
|
||||
let mut duplicate = registration_entry(&namespace, "candidate");
|
||||
duplicate.table_id = "id-orders".to_string();
|
||||
assert!(matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_cancellation_does_not_publish_a_partial_catalog() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
backend.registration_cache.clear();
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("warmup").unwrap());
|
||||
let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &path).await;
|
||||
let task_store = store.clone();
|
||||
let candidate = registration_entry(&namespace, "cancelled");
|
||||
let task = tokio::spawn(async move { task_store.create_table(candidate).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.unwrap();
|
||||
task.abort();
|
||||
assert!(task.await.unwrap_err().is_cancelled());
|
||||
pause.release();
|
||||
assert!(store.load_table("analytics", "sales", "cancelled").await.unwrap().is_none());
|
||||
let mut changed = registration_entry(&namespace, "warmup");
|
||||
changed.warehouse_location = "s3://analytics/changed".to_string();
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&changed).unwrap())
|
||||
.await;
|
||||
let mut candidate = registration_entry(&namespace, "after_cancel");
|
||||
candidate.warehouse_location = "s3://analytics/changed/child".to_string();
|
||||
assert!(matches!(store.create_table(candidate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_isolated_runtimes_do_not_share_matching_etags() {
|
||||
let (first, first_store, namespace) = cache_fixture().await;
|
||||
let (second, second_store, _) = cache_fixture().await;
|
||||
let path = first_store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
let mut changed = registration_entry(&namespace, "orders");
|
||||
changed.warehouse_location = "s3://analytics/second-only".to_string();
|
||||
let first_etag = first
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.objects
|
||||
.get(&(RUSTFS_META_BUCKET.to_string(), path.clone()))
|
||||
.unwrap()
|
||||
.etag
|
||||
.clone();
|
||||
second.registration_cache.clear();
|
||||
{
|
||||
let mut state = second.state.lock().await;
|
||||
let object = state.objects.get_mut(&(RUSTFS_META_BUCKET.to_string(), path)).unwrap();
|
||||
object.data = serde_json::to_vec(&changed).unwrap();
|
||||
object.etag = first_etag;
|
||||
}
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.warehouse_location = "s3://analytics/second-only/child".to_string();
|
||||
assert!(matches!(
|
||||
second_store.create_table(candidate.clone()).await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
first_store
|
||||
.create_table(candidate)
|
||||
.await
|
||||
.expect("the first runtime has no such owner");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_preserves_identity_after_lost_response_and_failed_rollback() {
|
||||
for lost_response in [false, true] {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let entry = registration_entry(&namespace, "ambiguous");
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse(&entry.table).unwrap());
|
||||
let index = store.paths.warehouse_index_entry_path("analytics", "tables/id-ambiguous/");
|
||||
if lost_response {
|
||||
backend.fail_after_next_put(RUSTFS_META_BUCKET, &path).await;
|
||||
} else {
|
||||
backend.fail_next_put(RUSTFS_META_BUCKET, &path).await;
|
||||
}
|
||||
backend.fail_delete_attempt(RUSTFS_META_BUCKET, &index, 1).await;
|
||||
assert!(store.create_table(entry.clone()).await.is_err());
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.table_id = entry.table_id;
|
||||
let result = store.create_table(candidate).await;
|
||||
if lost_response {
|
||||
assert!(matches!(result, Err(TableCatalogStoreError::Conflict(_))));
|
||||
} else {
|
||||
result.expect("failed primary writes cannot reserve identity through cached results");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn leased_registration_publication<'a>(
|
||||
backend: &'a TestCatalogObjectBackend,
|
||||
lock: &rustfs_lock::NamespaceLock,
|
||||
owner: &str,
|
||||
) -> (
|
||||
TableCommitLockPublication<'a, TestCatalogObjectBackend>,
|
||||
Arc<rustfs_lock::distributed_lock::LockLostSignal>,
|
||||
) {
|
||||
let guard = lock
|
||||
.lock_guard(
|
||||
rustfs_lock::ObjectKey::new("analytics", default_table_bucket_publication_lock_path()),
|
||||
owner,
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(3),
|
||||
)
|
||||
.await
|
||||
.expect("publication lease acquisition succeeds")
|
||||
.expect("publication lease is acquired");
|
||||
let signal = guard.lock_lost_signal().expect("distributed publication has a loss signal");
|
||||
let publication = TableCommitLockPublication::new(backend);
|
||||
{
|
||||
let mut state = publication.state.lock();
|
||||
state.table_bucket = Some("analytics".to_string());
|
||||
state.guards.push(TableCatalogLockGuard::namespace(guard));
|
||||
}
|
||||
(publication, signal)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn registration_cache_rejects_delayed_table_write_after_publication_lease_loss() {
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let stalled_namespace = Namespace::parse("stalled").unwrap();
|
||||
let active_namespace = Namespace::parse("active").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry("analytics")).await.unwrap();
|
||||
for namespace in [&stalled_namespace, &active_namespace] {
|
||||
store
|
||||
.create_namespace(test_namespace_entry("analytics", namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let initial = registration_entry(&active_namespace, "initial");
|
||||
store.create_table(initial.clone()).await.unwrap();
|
||||
let client = Arc::new(RecordingLeaseClient {
|
||||
inner: rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)))),
|
||||
acquired: Mutex::new(None),
|
||||
});
|
||||
let lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
|
||||
"registration-publication-loss".to_string(),
|
||||
vec![client.clone() as Arc<dyn LockClient>],
|
||||
1,
|
||||
);
|
||||
let (old_publication, old_signal) = leased_registration_publication(&backend, &lock, "old-writer").await;
|
||||
let old_lease = client.acquired.lock().unwrap().clone().unwrap();
|
||||
let delayed = registration_entry(&stalled_namespace, "delayed");
|
||||
let delayed_path =
|
||||
store
|
||||
.paths
|
||||
.table_entry_path("analytics", &stalled_namespace, &IdentifierSegment::parse(&delayed.table).unwrap());
|
||||
let pause = backend.pause_put_attempt(RUSTFS_META_BUCKET, &delayed_path, 1).await;
|
||||
let replacement = registration_entry(&active_namespace, "replacement");
|
||||
let delayed_write = store.register_table_with_publication(delayed.clone(), &old_publication);
|
||||
let takeover = async {
|
||||
tokio::time::timeout(Duration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.expect("old writer reaches the table-entry commit");
|
||||
assert!(client.force_release(&old_lease).await.unwrap());
|
||||
tokio::time::timeout(Duration::from_secs(10), old_signal.notified())
|
||||
.await
|
||||
.expect("heartbeat observes publication lease loss");
|
||||
assert!(!old_publication.holds_table_bucket("analytics"));
|
||||
|
||||
let (new_publication, _) = leased_registration_publication(&backend, &lock, "new-writer").await;
|
||||
backend.reset_call_counts().await;
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
store.register_table_with_publication(replacement.clone(), &new_publication),
|
||||
)
|
||||
.await
|
||||
.expect("replacement writer completes while the old put is paused")
|
||||
.expect("replacement writer publishes its table after a fresh inventory");
|
||||
assert!(backend.list_call_count().await > 0, "takeover inventories the catalog");
|
||||
pause.release();
|
||||
};
|
||||
let (delayed_result, ()) = tokio::time::timeout(Duration::from_secs(25), async { tokio::join!(delayed_write, takeover) })
|
||||
.await
|
||||
.expect("both writers finish after the paused put is resumed");
|
||||
assert!(
|
||||
matches!(delayed_result, Err(TableCatalogStoreError::Unavailable(_))),
|
||||
"the delayed storage commit must reject the lost publication fence: {delayed_result:?}"
|
||||
);
|
||||
assert_eq!(backend.put_attempt_count(RUSTFS_META_BUCKET, &delayed_path).await, 1);
|
||||
assert!(
|
||||
backend
|
||||
.read_object(RUSTFS_META_BUCKET, &delayed_path)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
for entry in [&initial, &replacement] {
|
||||
assert_eq!(
|
||||
store.load_table("analytics", &entry.namespace, &entry.table).await.unwrap(),
|
||||
Some(entry.clone())
|
||||
);
|
||||
}
|
||||
|
||||
let subsequent = registration_entry(&active_namespace, "subsequent");
|
||||
store.create_table(subsequent.clone()).await.unwrap();
|
||||
for entry in [&initial, &replacement, &subsequent] {
|
||||
let mut duplicate = registration_entry(&active_namespace, "duplicate");
|
||||
duplicate.table_id = entry.table_id.clone();
|
||||
assert!(matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
store
|
||||
.create_table(delayed)
|
||||
.await
|
||||
.expect("the rejected writer left no active table");
|
||||
}
|
||||
@@ -411,6 +411,7 @@ pub(crate) struct TestCatalogObjectBackend {
|
||||
pub(crate) state: Arc<tokio::sync::Mutex<TestCatalogObjectState>>,
|
||||
pub(crate) locks: TestCatalogObjectLocks,
|
||||
pub(crate) strong_runtime: Option<StrongTableCatalogRuntime>,
|
||||
pub(crate) registration_cache: TableCatalogRegistrationCache,
|
||||
// One-shot, path-keyed injection knobs from the admin handler tests'
|
||||
// former TestTableCatalogObjectBackend (backlog#1837 PR2): each fires
|
||||
// once for the named object and clears itself, mirroring the original
|
||||
@@ -543,8 +544,11 @@ pub(crate) struct TestCatalogObjectState {
|
||||
pub(crate) write_lock_acquisitions: BTreeMap<(String, String), usize>,
|
||||
pub(crate) read_lock_acquisitions: BTreeMap<(String, String), usize>,
|
||||
pub(crate) read_calls: usize,
|
||||
pub(crate) read_bytes: usize,
|
||||
pub(crate) metadata_calls: usize,
|
||||
pub(crate) list_calls: usize,
|
||||
pub(crate) failed_inventory_prefixes: BTreeSet<String>,
|
||||
pub(crate) inventory_overrides: BTreeMap<String, Vec<TableCatalogCleanupObject>>,
|
||||
pub(crate) next_etag: u64,
|
||||
}
|
||||
|
||||
@@ -608,6 +612,7 @@ impl TestCatalogObjectBackend {
|
||||
pub(crate) async fn reset_call_counts(&self) {
|
||||
let mut state = self.state.lock().await;
|
||||
state.read_calls = 0;
|
||||
state.read_bytes = 0;
|
||||
state.metadata_calls = 0;
|
||||
state.list_calls = 0;
|
||||
}
|
||||
@@ -955,6 +960,10 @@ impl TestCatalogObjectState {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
fn registration_cache(&self) -> Option<TableCatalogRegistrationCache> {
|
||||
Some(self.registration_cache.clone())
|
||||
}
|
||||
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
self.strong_runtime.clone()
|
||||
}
|
||||
@@ -1018,6 +1027,7 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
etag: (!etagless).then(|| record.etag.clone()),
|
||||
mod_time: record.mod_time,
|
||||
});
|
||||
state.read_bytes += result.as_ref().map_or(0, |object| object.data.len());
|
||||
let pause = state
|
||||
.pause_read_attempts
|
||||
.get_mut(&key)
|
||||
@@ -1142,6 +1152,17 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_object_if_match_fenced_for_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_etag: &str,
|
||||
_expected_incarnation_id: Uuid,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
self.delete_object_if_match_fenced(bucket, object, expected_etag, fence).await
|
||||
}
|
||||
|
||||
async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult<Vec<String>> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.list_calls += 1;
|
||||
@@ -1153,6 +1174,38 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn list_objects_for_cleanup(
|
||||
&self,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
) -> TableCatalogStoreResult<Vec<TableCatalogCleanupObject>> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.list_calls += 1;
|
||||
if state.failed_inventory_prefixes.contains(prefix) {
|
||||
return Err(TableCatalogStoreError::Unavailable(
|
||||
"injected authoritative inventory failure".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(objects) = state.inventory_overrides.get(prefix) {
|
||||
return Ok(objects.clone());
|
||||
}
|
||||
Ok(state
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|((entry_bucket, object), _)| entry_bucket == bucket && object.starts_with(prefix))
|
||||
.map(|(key, record)| TableCatalogCleanupObject {
|
||||
key: key.1.clone(),
|
||||
version_id: None,
|
||||
etag: (!state.etagless_objects.contains(key)).then(|| record.etag.clone()),
|
||||
delete_marker: false,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, _bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<TableCatalogLockGuard> {
|
||||
self.lock_attempts.lock().await.push((bucket.to_string(), object.to_string()));
|
||||
{
|
||||
|
||||
@@ -7145,6 +7145,228 @@ async fn table_data_plane_resource_does_not_match_sibling_prefix() {
|
||||
assert_eq!(backend.list_call_count().await, 1);
|
||||
}
|
||||
|
||||
async fn cached_registration_fixture() -> (TestCatalogObjectBackend, ObjectTableCatalogStore<TestCatalogObjectBackend>, Namespace)
|
||||
{
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let namespace = Namespace::parse("sales").expect("test namespace is valid");
|
||||
store
|
||||
.put_table_bucket(test_bucket_entry("analytics"))
|
||||
.await
|
||||
.expect("bucket is created");
|
||||
store
|
||||
.create_namespace(test_namespace_entry("analytics", &namespace))
|
||||
.await
|
||||
.expect("namespace is created");
|
||||
(backend, store, namespace)
|
||||
}
|
||||
|
||||
fn cached_registration_entry(namespace: &Namespace, name: &str) -> TableEntry {
|
||||
let table = IdentifierSegment::parse(name).expect("test table name is valid");
|
||||
let mut entry = test_table_entry(
|
||||
"analytics",
|
||||
namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = format!("id-{name}");
|
||||
entry.warehouse_location = format!("s3://analytics/tables/id-{name}");
|
||||
entry
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_bounds_reads_across_store_instances() {
|
||||
let (backend, _, namespace) = cached_registration_fixture().await;
|
||||
for number in 0..256 {
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, &format!("table_{number:04}")))
|
||||
.await
|
||||
.expect("each new store instance shares validated registration projections");
|
||||
if number >= 32 {
|
||||
let state = backend.state.lock().await;
|
||||
assert_eq!(state.list_calls, 2, "each registration inventories both catalog ranges");
|
||||
assert!(
|
||||
state.read_calls < 24,
|
||||
"registration used {} object reads at table {number}",
|
||||
state.read_calls
|
||||
);
|
||||
assert!(
|
||||
state.read_bytes < 256 * 1024,
|
||||
"registration read {} bytes at table {number}",
|
||||
state.read_bytes
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_preserves_identity_and_overlap_checks() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
let mut existing = cached_registration_entry(&namespace, "orders");
|
||||
existing.warehouse_location = "s3://analytics/shared/orders".to_string();
|
||||
store.create_table(existing.clone()).await.expect("initial table is created");
|
||||
backend
|
||||
.delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_entry_path("analytics", "shared/orders/"))
|
||||
.await
|
||||
.expect("simulate a missing materialized warehouse index");
|
||||
|
||||
let mut duplicate = cached_registration_entry(&namespace, "duplicate");
|
||||
duplicate.table_id = existing.table_id.clone();
|
||||
assert_matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
for location in [
|
||||
"s3://analytics/shared",
|
||||
"s3://analytics/shared/orders",
|
||||
"s3://analytics/shared/orders/child",
|
||||
] {
|
||||
let mut conflicting = cached_registration_entry(&namespace, "conflicting");
|
||||
conflicting.warehouse_location = location.to_string();
|
||||
assert_matches!(store.create_table(conflicting).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
}
|
||||
let mut adjacent = cached_registration_entry(&namespace, "adjacent");
|
||||
adjacent.warehouse_location = "s3://analytics/shared/orders2".to_string();
|
||||
store
|
||||
.create_table(adjacent)
|
||||
.await
|
||||
.expect("a different directory remains available");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_rechecks_after_failed_write_and_drop() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "orders"))
|
||||
.await
|
||||
.expect("initial table is created");
|
||||
let returns = IdentifierSegment::parse("returns").expect("test name is valid");
|
||||
let table_path = store.paths.table_entry_path("analytics", &namespace, &returns);
|
||||
backend.fail_next_put(RUSTFS_META_BUCKET, &table_path).await;
|
||||
assert!(
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.load_table("analytics", "sales", "returns")
|
||||
.await
|
||||
.expect("table lookup succeeds")
|
||||
.is_none()
|
||||
);
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.expect("retry reads the authoritative records");
|
||||
store
|
||||
.drop_table("analytics", "sales", "orders")
|
||||
.await
|
||||
.expect("drop removes the active owner");
|
||||
let mut replacement = cached_registration_entry(&namespace, "replacement");
|
||||
replacement.table_id = "id-orders".to_string();
|
||||
replacement.warehouse_location = "s3://analytics/tables/id-orders".to_string();
|
||||
store
|
||||
.create_table(replacement)
|
||||
.await
|
||||
.expect("a dropped owner no longer reserves its identity");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_observes_pointer_commits_and_rename() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
let entry = cached_registration_entry(&namespace, "orders");
|
||||
store.create_table(entry.clone()).await.expect("initial table is created");
|
||||
let metadata_directory = entry
|
||||
.metadata_location
|
||||
.rsplit_once('/')
|
||||
.expect("metadata has a parent directory")
|
||||
.0;
|
||||
let next_metadata = format!("{metadata_directory}/00002.metadata.json");
|
||||
backend
|
||||
.seed_object(
|
||||
"analytics",
|
||||
&next_metadata,
|
||||
serde_json::to_vec(&serde_json::json!({"location": entry.warehouse_location})).unwrap(),
|
||||
)
|
||||
.await;
|
||||
store
|
||||
.commit_table(TableCommitRequest {
|
||||
table_bucket: "analytics".to_string(),
|
||||
namespace: "sales".to_string(),
|
||||
table: "orders".to_string(),
|
||||
commit_id: "pointer-only".to_string(),
|
||||
idempotency_key: None,
|
||||
operation: "append".to_string(),
|
||||
expected_version_token: entry.version_token,
|
||||
expected_metadata_location: entry.metadata_location,
|
||||
new_metadata_location: next_metadata,
|
||||
requirements: Vec::new(),
|
||||
writer: None,
|
||||
})
|
||||
.await
|
||||
.expect("metadata pointer commit succeeds");
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
backend.list_call_count().await,
|
||||
2,
|
||||
"pointer changes are checked through fresh inventories"
|
||||
);
|
||||
|
||||
store
|
||||
.rename_table("analytics", "sales", "orders", "sales", "renamed")
|
||||
.await
|
||||
.unwrap();
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "shipments"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
backend.list_call_count().await > 0,
|
||||
"rename is visible in the next authoritative inventory"
|
||||
);
|
||||
let mut duplicate = cached_registration_entry(&namespace, "duplicate");
|
||||
duplicate.table_id = "id-orders".to_string();
|
||||
assert_matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_observes_an_independent_writer() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "orders"))
|
||||
.await
|
||||
.unwrap();
|
||||
let scan_store = ObjectTableCatalogStore::new(TestCatalogObjectBackend {
|
||||
registration_cache: TableCatalogRegistrationCache::default(),
|
||||
..backend.clone()
|
||||
});
|
||||
scan_store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.unwrap();
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "shipments"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
backend.list_call_count().await > 0,
|
||||
"an independent writer is visible in the next authoritative inventory"
|
||||
);
|
||||
let mut duplicate = cached_registration_entry(&namespace, "duplicate");
|
||||
duplicate.table_id = "id-returns".to_string();
|
||||
assert_matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_scans_warehouse_prefixes_once() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
@@ -7173,10 +7395,217 @@ async fn object_catalog_registration_scans_warehouse_prefixes_once() {
|
||||
reads.push(backend.read_call_count().await);
|
||||
assert_eq!(backend.list_call_count().await, 2, "one table scan and one warehouse index scan");
|
||||
}
|
||||
assert_eq!(reads[2] - reads[1], 2, "each existing table and its index should be read once");
|
||||
assert_eq!(reads[2], reads[1], "unchanged validated objects should not require another GET");
|
||||
assert_eq!(store.list_tables(bucket, &namespace.public_name()).await.unwrap().len(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_bounds_concurrent_catalog_reads() {
|
||||
const MAX_CONCURRENT_READS: usize = 8;
|
||||
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut table_objects = Vec::new();
|
||||
let mut index_objects = Vec::new();
|
||||
for number in 0..MAX_CONCURRENT_READS + 2 {
|
||||
let name = format!("orders_{number:02}");
|
||||
let table = IdentifierSegment::parse(&name).unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = format!("table-{name}");
|
||||
entry.warehouse_location = format!("s3://{bucket}/tables/{name}");
|
||||
store.create_table(entry).await.unwrap();
|
||||
table_objects.push(store.paths.table_entry_path(bucket, &namespace, &table));
|
||||
index_objects.push(store.paths.warehouse_index_entry_path(bucket, &format!("tables/{name}/")));
|
||||
}
|
||||
|
||||
backend.registration_cache.clear();
|
||||
let mut scans = Vec::new();
|
||||
for mut objects in [table_objects, index_objects] {
|
||||
objects.sort();
|
||||
let mut scan = Vec::new();
|
||||
for object in objects {
|
||||
let previous_reads = backend
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.read_attempts
|
||||
.get(&(RUSTFS_META_BUCKET.to_string(), object.clone()))
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &object).await;
|
||||
scan.push((object, pause, previous_reads));
|
||||
}
|
||||
scans.push(scan);
|
||||
}
|
||||
|
||||
let table = IdentifierSegment::parse("new_orders").unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = "new-table-id".to_string();
|
||||
entry.warehouse_location = format!("s3://{bucket}/tables/new_orders");
|
||||
let registering_store = store.clone();
|
||||
let registration = tokio::spawn(async move { registering_store.create_table(entry).await });
|
||||
|
||||
for scan in scans {
|
||||
for (_, pause, _) in scan.iter().take(MAX_CONCURRENT_READS) {
|
||||
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, pause.wait_started())
|
||||
.await
|
||||
.expect("independent catalog reads should start concurrently");
|
||||
}
|
||||
let state = backend.state.lock().await;
|
||||
let started = scan
|
||||
.iter()
|
||||
.filter(|(object, _, previous_reads)| {
|
||||
state
|
||||
.read_attempts
|
||||
.get(&(RUSTFS_META_BUCKET.to_string(), object.clone()))
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
> *previous_reads
|
||||
})
|
||||
.count();
|
||||
assert_eq!(started, MAX_CONCURRENT_READS, "catalog reads must respect the concurrency limit");
|
||||
drop(state);
|
||||
for (_, pause, _) in scan {
|
||||
pause.release();
|
||||
}
|
||||
}
|
||||
|
||||
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, registration)
|
||||
.await
|
||||
.expect("registration should finish after its reads are released")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.load_table(bucket, &namespace.public_name(), table.as_str())
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_propagates_catalog_scan_read_errors() {
|
||||
for fail_index_read in [false, true] {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
let existing = IdentifierSegment::parse("orders").unwrap();
|
||||
seed_table_for_metadata_maintenance(
|
||||
&store,
|
||||
bucket,
|
||||
&namespace,
|
||||
&existing,
|
||||
default_table_metadata_file_path(&namespace, &existing, "00001.metadata.json"),
|
||||
)
|
||||
.await;
|
||||
let failed_object = if fail_index_read {
|
||||
store.paths.warehouse_index_entry_path(bucket, "tables/table-id/")
|
||||
} else {
|
||||
store.paths.table_entry_path(bucket, &namespace, &existing)
|
||||
};
|
||||
backend.fail_next_read(RUSTFS_META_BUCKET, &failed_object).await;
|
||||
|
||||
let table = IdentifierSegment::parse("returns").unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = "new-table-id".to_string();
|
||||
entry.warehouse_location = format!("s3://{bucket}/tables/new-table-id");
|
||||
assert_matches!(
|
||||
store.create_table(entry.clone()).await,
|
||||
Err(TableCatalogStoreError::Internal(message)) if message.contains(&failed_object)
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.load_table(bucket, &namespace.public_name(), table.as_str())
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
.read_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&store.paths.warehouse_index_entry_path(bucket, "tables/new-table-id/")
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
store
|
||||
.create_table(entry)
|
||||
.await
|
||||
.expect("a failed scan must release its read locks for retry");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_reuses_locked_deleted_entry() {
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
let table = IdentifierSegment::parse("orders").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.state = TableCatalogEntryState::Deleted;
|
||||
backend
|
||||
.seed_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&store.paths.table_entry_path(bucket, &namespace, &table),
|
||||
serde_json::to_vec(&entry).unwrap(),
|
||||
)
|
||||
.await;
|
||||
entry.state = TableCatalogEntryState::Active;
|
||||
entry.table_id = "replacement-table-id".to_string();
|
||||
store.create_table(entry.clone()).await.unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.load_table(bucket, &namespace.public_name(), table.as_str())
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(entry)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_backings_reject_overlapping_registered_warehouse_prefixes() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
@@ -20526,9 +20955,9 @@ async fn object_table_catalog_store_reuses_old_prefix_after_failed_relocation_in
|
||||
let mut next_entry = test_table_entry(bucket, &namespace, &next_table, current_metadata);
|
||||
next_entry.table_id = "next-table-id".to_string();
|
||||
next_entry.warehouse_location = format!("s3://{bucket}/tables/table-id");
|
||||
store
|
||||
.create_table(next_entry)
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), store.create_table(next_entry))
|
||||
.await
|
||||
.expect("stale warehouse prefix reuse completes")
|
||||
.expect("stale old warehouse index should not block prefix reuse");
|
||||
|
||||
let (index, _) = store
|
||||
|
||||
Reference in New Issue
Block a user