mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 06:53:44 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 85c58c9abf | |||
| c8ad0be2fd | |||
| 96a16c697d |
@@ -1111,7 +1111,7 @@ impl RenameAdmissionGrant {
|
||||
drop(self.permit.take());
|
||||
}
|
||||
|
||||
fn quarantine_until(mut self, deadline: tokio::time::Instant) {
|
||||
fn quarantine_after_ambiguity(mut self) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
@@ -1119,12 +1119,16 @@ impl RenameAdmissionGrant {
|
||||
if items == 0 {
|
||||
return;
|
||||
}
|
||||
let hold = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
// The RPC deadline has already expired when timeout ambiguity is
|
||||
// detected. Hold admission for another operation timeout so detached
|
||||
// server-side disk work cannot immediately be replaced by new work.
|
||||
let hold = get_max_timeout_duration();
|
||||
let release_at = tokio::time::Instant::now() + hold;
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine(self.batch, items, hold);
|
||||
let batch = self.batch;
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
tokio::time::sleep_until(release_at).await;
|
||||
drop(permit);
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_released(batch, items);
|
||||
});
|
||||
@@ -1187,7 +1191,7 @@ impl Drop for RenameBatchCallGuard {
|
||||
if self.dispatch_state == RenameAdmissionDispatchState::MayHavePublished
|
||||
&& let Some(admission) = self.admission.take()
|
||||
{
|
||||
admission.quarantine_until(self.deadline);
|
||||
admission.quarantine_after_ambiguity();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1228,7 +1232,7 @@ impl RemoteRenameBatchStream {
|
||||
}
|
||||
self.terminal = true;
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
admission.quarantine_after_ambiguity();
|
||||
}
|
||||
error
|
||||
}
|
||||
@@ -1302,7 +1306,7 @@ impl Drop for RemoteRenameBatchStream {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
admission.quarantine_after_ambiguity();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2883,7 +2887,11 @@ fn encode_delete_options_msgpack(value: &DeleteOptions) -> Result<Vec<u8>> {
|
||||
}
|
||||
|
||||
fn encode_file_info_msgpack(value: &FileInfo) -> Result<Vec<u8>> {
|
||||
encode_msgpack_with_capacity(value, FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT)
|
||||
// Inline shards dominate the encoded payload. Reserving their known size
|
||||
// avoids repeated growth while retaining the bounded base hint for metadata.
|
||||
let inline_bytes = value.data.as_ref().map_or(0, Bytes::len);
|
||||
let capacity = FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT.saturating_add(inline_bytes);
|
||||
encode_msgpack_with_capacity(value, capacity)
|
||||
}
|
||||
|
||||
fn encode_file_info_versions_msgpack(value: &FileInfoVersions) -> Result<Vec<u8>> {
|
||||
@@ -8153,6 +8161,38 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_inline_file_info_msgpack_reserves_known_payloads() {
|
||||
for (object_size, inline_len) in [(32 * 1024, 2_763), (100 * 1024, 8_566), (12 * (32 * 1024 - 32), 32 * 1024)] {
|
||||
let mut file_info = sample_rename_data_file_info();
|
||||
file_info.size = object_size;
|
||||
file_info.erasure.data_blocks = 12;
|
||||
file_info.erasure.parity_blocks = 4;
|
||||
file_info.erasure.distribution = (1..=16).collect();
|
||||
file_info.data = Some(Bytes::from(vec![0x5a; inline_len]));
|
||||
|
||||
let fixed_hint = encode_msgpack_with_capacity(&file_info, FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT)
|
||||
.expect("fixed-hint inline file info should encode");
|
||||
assert!(
|
||||
fixed_hint.capacity() > Vec::<u8>::with_capacity(FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT).capacity(),
|
||||
"the fixed hint must grow for the representative inline shard"
|
||||
);
|
||||
let encoded = encode_file_info_msgpack(&file_info).expect("inline file info should encode");
|
||||
let reserved = FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT + inline_len;
|
||||
assert!(
|
||||
encoded.len() <= reserved,
|
||||
"the payload-aware capacity must cover a representative inline shard"
|
||||
);
|
||||
assert_eq!(
|
||||
encoded.capacity(),
|
||||
Vec::<u8>::with_capacity(reserved).capacity(),
|
||||
"encoding the inline shard must not grow the initial buffer"
|
||||
);
|
||||
let decoded: FileInfo = rmp_serde::from_slice(&encoded).expect("inline file info should decode");
|
||||
assert_eq!(decoded, file_info);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_resp_named_msgpack_is_smaller_than_json() {
|
||||
let response = RenameDataResp {
|
||||
@@ -13474,24 +13514,25 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_admission_quarantine_holds_until_absolute_deadline() {
|
||||
async fn rename_admission_quarantine_holds_after_ambiguity() {
|
||||
let gate = peer_rename_admission("http://rename-quarantine-peer", true).expect("enabled rename gate");
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}
|
||||
.quarantine_until(deadline);
|
||||
.quarantine_after_ambiguity();
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
|
||||
tokio::time::advance(Duration::from_millis(999)).await;
|
||||
let hold = get_max_timeout_duration();
|
||||
assert!(!hold.is_zero(), "the test requires a bounded positive ambiguity hold");
|
||||
tokio::time::advance(hold - Duration::from_nanos(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
tokio::time::advance(Duration::from_nanos(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
@@ -13523,7 +13564,7 @@ mod tests {
|
||||
guard.mark_dispatched();
|
||||
drop(guard);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
tokio::time::advance(get_max_timeout_duration()).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
@@ -13544,6 +13585,123 @@ mod tests {
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_call_guard_timeout_starts_a_fresh_ambiguity_hold() {
|
||||
let gate = peer_rename_admission("http://rename-expired-call-guard-peer", true).expect("enabled rename gate");
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("dispatched permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline: tokio::time::Instant::now(),
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
drop(guard);
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
gate.available_permits(),
|
||||
rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4,
|
||||
"an RPC timeout must not immediately readmit work over detached mutations"
|
||||
);
|
||||
|
||||
let hold = get_max_timeout_duration();
|
||||
assert!(!hold.is_zero(), "the test requires a bounded positive ambiguity hold");
|
||||
tokio::time::advance(hold - Duration::from_nanos(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_nanos(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_batch_stream_incomplete_eof_quarantines_only_missing_items() {
|
||||
use tonic::codec::Codec;
|
||||
|
||||
let gate = peer_rename_admission("http://rename-incomplete-stream-peer", true).expect("enabled rename gate");
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("batch permits");
|
||||
let mut admission = RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
};
|
||||
assert!(admission.release_one(), "the completed first item releases its permit");
|
||||
let mut codec = tonic_prost::ProstCodec::<RenameDataBatchResult, RenameDataBatchResult>::default();
|
||||
let inner = Streaming::new_empty(codec.decoder(), http_body_util::Empty::<Bytes>::new());
|
||||
let mut stream = RemoteRenameBatchStream {
|
||||
inner,
|
||||
expected_mask: 0b1111,
|
||||
seen_mask: 0b0001,
|
||||
admission: Some(admission),
|
||||
deadline: tokio::time::Instant::now(),
|
||||
terminal: false,
|
||||
stream_started_at: None,
|
||||
first_item_at: None,
|
||||
};
|
||||
|
||||
let error = match stream.next_outcome().await {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("incomplete EOF must be commit-ambiguous"),
|
||||
};
|
||||
assert!(matches!(error, DiskError::Unexpected));
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
gate.available_permits(),
|
||||
rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3,
|
||||
"only the three missing item permits remain quarantined"
|
||||
);
|
||||
|
||||
let hold = get_max_timeout_duration();
|
||||
tokio::time::advance(hold - Duration::from_nanos(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
|
||||
tokio::time::advance(Duration::from_nanos(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_batch_stream_drop_quarantines_only_missing_items() {
|
||||
use tonic::codec::Codec;
|
||||
|
||||
let gate = peer_rename_admission("http://rename-dropped-stream-peer", true).expect("enabled rename gate");
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("batch permits");
|
||||
let mut admission = RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
};
|
||||
assert!(admission.release_one(), "the completed first item releases its permit");
|
||||
let mut codec = tonic_prost::ProstCodec::<RenameDataBatchResult, RenameDataBatchResult>::default();
|
||||
let inner = Streaming::new_empty(codec.decoder(), http_body_util::Empty::<Bytes>::new());
|
||||
let stream = RemoteRenameBatchStream {
|
||||
inner,
|
||||
expected_mask: 0b1111,
|
||||
seen_mask: 0b0001,
|
||||
admission: Some(admission),
|
||||
deadline: tokio::time::Instant::now(),
|
||||
terminal: false,
|
||||
stream_started_at: None,
|
||||
first_item_at: None,
|
||||
};
|
||||
|
||||
drop(stream);
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
gate.available_permits(),
|
||||
rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3,
|
||||
"dropping an active stream quarantines only the three missing item permits"
|
||||
);
|
||||
|
||||
let hold = get_max_timeout_duration();
|
||||
tokio::time::advance(hold - Duration::from_nanos(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
|
||||
tokio::time::advance(Duration::from_nanos(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_disk_builds_optional_complete_body_gate_from_configuration() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, Some("3"))], async {
|
||||
|
||||
Reference in New Issue
Block a user