Compare commits

...

3 Commits

Author SHA1 Message Date
Hauser Zhi 85c58c9abf Merge origin/main into houseme/rename-quarantine
Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 13:14:26 +08:00
Hauser Zhi c8ad0be2fd Merge origin/main into houseme/rename-quarantine
Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 10:52:46 +08:00
Hauser Zhi 96a16c697d fix(ecstore): harden remote rename handling
Start a fresh bounded quarantine when a dispatched rename becomes ambiguous so detached disk work cannot be immediately replaced after the RPC deadline. Pre-size inline FileInfo MessagePack buffers from their resident payload to avoid proven growth without changing wire bytes or durability semantics.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 10:50:32 +08:00
+171 -13
View File
@@ -1111,7 +1111,7 @@ impl RenameAdmissionGrant {
drop(self.permit.take());
}
fn quarantine_until(mut self, deadline: tokio::time::Instant) {
fn quarantine_after_ambiguity(mut self) {
let Some(permit) = self.permit.take() else {
return;
};
@@ -1119,12 +1119,16 @@ impl RenameAdmissionGrant {
if items == 0 {
return;
}
let hold = deadline.saturating_duration_since(tokio::time::Instant::now());
// The RPC deadline has already expired when timeout ambiguity is
// detected. Hold admission for another operation timeout so detached
// server-side disk work cannot immediately be replaced by new work.
let hold = get_max_timeout_duration();
let release_at = tokio::time::Instant::now() + hold;
rustfs_io_metrics::record_ecstore_rename_quarantine(self.batch, items, hold);
let batch = self.batch;
if let Ok(handle) = tokio::runtime::Handle::try_current() {
handle.spawn(async move {
tokio::time::sleep_until(deadline).await;
tokio::time::sleep_until(release_at).await;
drop(permit);
rustfs_io_metrics::record_ecstore_rename_quarantine_released(batch, items);
});
@@ -1187,7 +1191,7 @@ impl Drop for RenameBatchCallGuard {
if self.dispatch_state == RenameAdmissionDispatchState::MayHavePublished
&& let Some(admission) = self.admission.take()
{
admission.quarantine_until(self.deadline);
admission.quarantine_after_ambiguity();
}
}
}
@@ -1228,7 +1232,7 @@ impl RemoteRenameBatchStream {
}
self.terminal = true;
if let Some(admission) = self.admission.take() {
admission.quarantine_until(self.deadline);
admission.quarantine_after_ambiguity();
}
error
}
@@ -1302,7 +1306,7 @@ impl Drop for RemoteRenameBatchStream {
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
}
if let Some(admission) = self.admission.take() {
admission.quarantine_until(self.deadline);
admission.quarantine_after_ambiguity();
}
}
}
@@ -2883,7 +2887,11 @@ fn encode_delete_options_msgpack(value: &DeleteOptions) -> Result<Vec<u8>> {
}
fn encode_file_info_msgpack(value: &FileInfo) -> Result<Vec<u8>> {
encode_msgpack_with_capacity(value, FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT)
// Inline shards dominate the encoded payload. Reserving their known size
// avoids repeated growth while retaining the bounded base hint for metadata.
let inline_bytes = value.data.as_ref().map_or(0, Bytes::len);
let capacity = FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT.saturating_add(inline_bytes);
encode_msgpack_with_capacity(value, capacity)
}
fn encode_file_info_versions_msgpack(value: &FileInfoVersions) -> Result<Vec<u8>> {
@@ -8153,6 +8161,38 @@ mod tests {
);
}
#[test]
fn rename_data_inline_file_info_msgpack_reserves_known_payloads() {
for (object_size, inline_len) in [(32 * 1024, 2_763), (100 * 1024, 8_566), (12 * (32 * 1024 - 32), 32 * 1024)] {
let mut file_info = sample_rename_data_file_info();
file_info.size = object_size;
file_info.erasure.data_blocks = 12;
file_info.erasure.parity_blocks = 4;
file_info.erasure.distribution = (1..=16).collect();
file_info.data = Some(Bytes::from(vec![0x5a; inline_len]));
let fixed_hint = encode_msgpack_with_capacity(&file_info, FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT)
.expect("fixed-hint inline file info should encode");
assert!(
fixed_hint.capacity() > Vec::<u8>::with_capacity(FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT).capacity(),
"the fixed hint must grow for the representative inline shard"
);
let encoded = encode_file_info_msgpack(&file_info).expect("inline file info should encode");
let reserved = FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT + inline_len;
assert!(
encoded.len() <= reserved,
"the payload-aware capacity must cover a representative inline shard"
);
assert_eq!(
encoded.capacity(),
Vec::<u8>::with_capacity(reserved).capacity(),
"encoding the inline shard must not grow the initial buffer"
);
let decoded: FileInfo = rmp_serde::from_slice(&encoded).expect("inline file info should decode");
assert_eq!(decoded, file_info);
}
}
#[test]
fn rename_data_resp_named_msgpack_is_smaller_than_json() {
let response = RenameDataResp {
@@ -13474,24 +13514,25 @@ mod tests {
}
#[tokio::test(start_paused = true)]
async fn rename_admission_quarantine_holds_until_absolute_deadline() {
async fn rename_admission_quarantine_holds_after_ambiguity() {
let gate = peer_rename_admission("http://rename-quarantine-peer", true).expect("enabled rename gate");
let permit = Arc::clone(&gate)
.acquire_many_owned(4)
.await
.expect("weighted batch admission");
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
RenameAdmissionGrant {
permit: Some(permit),
batch: true,
}
.quarantine_until(deadline);
.quarantine_after_ambiguity();
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
tokio::time::advance(Duration::from_millis(999)).await;
let hold = get_max_timeout_duration();
assert!(!hold.is_zero(), "the test requires a bounded positive ambiguity hold");
tokio::time::advance(hold - Duration::from_nanos(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
tokio::time::advance(Duration::from_millis(1)).await;
tokio::time::advance(Duration::from_nanos(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
}
@@ -13523,7 +13564,7 @@ mod tests {
guard.mark_dispatched();
drop(guard);
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
tokio::time::advance(Duration::from_secs(1)).await;
tokio::time::advance(get_max_timeout_duration()).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
@@ -13544,6 +13585,123 @@ mod tests {
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
}
#[tokio::test(start_paused = true)]
async fn rename_call_guard_timeout_starts_a_fresh_ambiguity_hold() {
let gate = peer_rename_admission("http://rename-expired-call-guard-peer", true).expect("enabled rename gate");
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("dispatched permits");
let mut guard = RenameBatchCallGuard {
admission: Some(RenameAdmissionGrant {
permit: Some(permit),
batch: true,
}),
deadline: tokio::time::Instant::now(),
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
};
guard.mark_dispatched();
drop(guard);
tokio::task::yield_now().await;
assert_eq!(
gate.available_permits(),
rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4,
"an RPC timeout must not immediately readmit work over detached mutations"
);
let hold = get_max_timeout_duration();
assert!(!hold.is_zero(), "the test requires a bounded positive ambiguity hold");
tokio::time::advance(hold - Duration::from_nanos(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
tokio::time::advance(Duration::from_nanos(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
}
#[tokio::test(start_paused = true)]
async fn rename_batch_stream_incomplete_eof_quarantines_only_missing_items() {
use tonic::codec::Codec;
let gate = peer_rename_admission("http://rename-incomplete-stream-peer", true).expect("enabled rename gate");
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("batch permits");
let mut admission = RenameAdmissionGrant {
permit: Some(permit),
batch: true,
};
assert!(admission.release_one(), "the completed first item releases its permit");
let mut codec = tonic_prost::ProstCodec::<RenameDataBatchResult, RenameDataBatchResult>::default();
let inner = Streaming::new_empty(codec.decoder(), http_body_util::Empty::<Bytes>::new());
let mut stream = RemoteRenameBatchStream {
inner,
expected_mask: 0b1111,
seen_mask: 0b0001,
admission: Some(admission),
deadline: tokio::time::Instant::now(),
terminal: false,
stream_started_at: None,
first_item_at: None,
};
let error = match stream.next_outcome().await {
Err(error) => error,
Ok(_) => panic!("incomplete EOF must be commit-ambiguous"),
};
assert!(matches!(error, DiskError::Unexpected));
tokio::task::yield_now().await;
assert_eq!(
gate.available_permits(),
rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3,
"only the three missing item permits remain quarantined"
);
let hold = get_max_timeout_duration();
tokio::time::advance(hold - Duration::from_nanos(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
tokio::time::advance(Duration::from_nanos(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
}
#[tokio::test(start_paused = true)]
async fn rename_batch_stream_drop_quarantines_only_missing_items() {
use tonic::codec::Codec;
let gate = peer_rename_admission("http://rename-dropped-stream-peer", true).expect("enabled rename gate");
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("batch permits");
let mut admission = RenameAdmissionGrant {
permit: Some(permit),
batch: true,
};
assert!(admission.release_one(), "the completed first item releases its permit");
let mut codec = tonic_prost::ProstCodec::<RenameDataBatchResult, RenameDataBatchResult>::default();
let inner = Streaming::new_empty(codec.decoder(), http_body_util::Empty::<Bytes>::new());
let stream = RemoteRenameBatchStream {
inner,
expected_mask: 0b1111,
seen_mask: 0b0001,
admission: Some(admission),
deadline: tokio::time::Instant::now(),
terminal: false,
stream_started_at: None,
first_item_at: None,
};
drop(stream);
tokio::task::yield_now().await;
assert_eq!(
gate.available_permits(),
rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3,
"dropping an active stream quarantines only the three missing item permits"
);
let hold = get_max_timeout_duration();
tokio::time::advance(hold - Duration::from_nanos(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
tokio::time::advance(Duration::from_nanos(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
}
#[tokio::test]
async fn remote_disk_builds_optional_complete_body_gate_from_configuration() {
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, Some("3"))], async {