mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-17 13:34:36 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d1e706d6f9 | |||
| cfe5d2cc72 | |||
| 42a8ddb6cf | |||
| e94b8637fa |
@@ -573,6 +573,15 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
type RenameDataBatchStream = ResponseStream<rustfs_protos::proto_gen::node_service::RenameDataBatchResult>;
|
||||
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataBatchRequest>,
|
||||
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_multipart_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -73,10 +73,11 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteAllResponse,
|
||||
WriteFormatControlRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataBatchItem, RenameDataBatchRequest,
|
||||
RenameDataBatchResult, RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest,
|
||||
VerifyFileRequest, WriteAllRequest, WriteAllResponse, WriteFormatControlRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
ChannelClass, ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, prewarm_channel_class,
|
||||
@@ -104,7 +105,7 @@ use tokio::{
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService};
|
||||
use tonic::{Code, Request, Streaming, service::interceptor::InterceptedService};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -1045,6 +1046,7 @@ pub struct RemoteDisk {
|
||||
/// Optional per-peer gate held through complete bulk-unary decoding and validation.
|
||||
bulk_body_admission: Option<Arc<Semaphore>>,
|
||||
bulk_body_wait_timeout: Duration,
|
||||
rename_admission: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
/// Shortest gap between two failure warnings of one kind from one remote disk.
|
||||
@@ -1060,6 +1062,208 @@ struct BulkBodyAdmissionRegistry {
|
||||
static BULK_BODY_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<BulkBodyAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(BulkBodyAdmissionRegistry::default()));
|
||||
|
||||
const ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH: &str = "RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH";
|
||||
const RENAME_ADMISSION_REGISTRY_CAPACITY: usize = 512;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RenameAdmissionRegistry {
|
||||
peers: HashMap<String, Weak<Semaphore>>,
|
||||
overflow: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
static RENAME_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<RenameAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(RenameAdmissionRegistry::default()));
|
||||
|
||||
fn peer_rename_admission(addr: &str, enabled: bool) -> Option<Arc<Semaphore>> {
|
||||
if !enabled {
|
||||
return None;
|
||||
}
|
||||
let mut registry = RENAME_ADMISSION_REGISTRY
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(semaphore) = registry.peers.get(addr).and_then(Weak::upgrade) {
|
||||
return Some(semaphore);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
registry.peers.retain(|_, semaphore| semaphore.strong_count() > 0);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
return Some(Arc::clone(registry.overflow.get_or_insert_with(|| {
|
||||
Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT))
|
||||
})));
|
||||
}
|
||||
let semaphore = Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT));
|
||||
registry.peers.insert(addr.to_owned(), Arc::downgrade(&semaphore));
|
||||
Some(semaphore)
|
||||
}
|
||||
|
||||
struct RenameAdmissionGrant {
|
||||
permit: Option<OwnedSemaphorePermit>,
|
||||
}
|
||||
|
||||
impl RenameAdmissionGrant {
|
||||
fn release_one(&mut self) -> bool {
|
||||
self.permit.as_mut().is_some_and(|permit| permit.split(1).is_some())
|
||||
}
|
||||
|
||||
fn release_all(mut self) {
|
||||
drop(self.permit.take());
|
||||
}
|
||||
|
||||
fn quarantine_until(mut self, deadline: tokio::time::Instant) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
drop(permit);
|
||||
});
|
||||
} else {
|
||||
// Without a runtime there is no safe deadline task. Leaking until
|
||||
// process exit is safer than admitting work over a still-running mutation.
|
||||
std::mem::forget(permit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RenameBatchCallGuard {
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
quarantine_on_drop: bool,
|
||||
}
|
||||
|
||||
impl RenameBatchCallGuard {
|
||||
fn complete(mut self) {
|
||||
self.quarantine_on_drop = false;
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn release_for_fallback(mut self) {
|
||||
self.quarantine_on_drop = false;
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn into_stream(mut self, inner: Streaming<RenameDataBatchResult>, expected_mask: u16) -> RemoteRenameBatchStream {
|
||||
self.quarantine_on_drop = false;
|
||||
RemoteRenameBatchStream {
|
||||
inner,
|
||||
expected_mask,
|
||||
seen_mask: 0,
|
||||
admission: self.admission.take(),
|
||||
deadline: self.deadline,
|
||||
terminal: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameBatchCallGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.quarantine_on_drop
|
||||
&& let Some(admission) = self.admission.take()
|
||||
{
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameBatchClientItem<'a> {
|
||||
pub slot: usize,
|
||||
pub disk: String,
|
||||
pub file_info: &'a FileInfo,
|
||||
}
|
||||
|
||||
pub(crate) enum RenameBatchStartError {
|
||||
NotDispatched(Error),
|
||||
UnsupportedBeforeStream,
|
||||
MayHavePublished(Error),
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchOutcome {
|
||||
pub slot: usize,
|
||||
pub result: Result<RenameDataResp>,
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchStream {
|
||||
inner: Streaming<RenameDataBatchResult>,
|
||||
expected_mask: u16,
|
||||
seen_mask: u16,
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
terminal: bool,
|
||||
}
|
||||
|
||||
impl RemoteRenameBatchStream {
|
||||
fn terminate_ambiguous(&mut self, error: Error) -> Error {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
self.terminal = true;
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
pub(crate) async fn next_outcome(&mut self) -> Result<Option<RemoteRenameBatchOutcome>> {
|
||||
if self.terminal {
|
||||
return Ok(None);
|
||||
}
|
||||
let response = match tokio::time::timeout_at(self.deadline, self.inner.message()).await {
|
||||
Err(_) => return Err(self.terminate_ambiguous(DiskError::Timeout)),
|
||||
Ok(Err(status)) => return Err(self.terminate_ambiguous(status.into())),
|
||||
Ok(Ok(None)) if self.seen_mask == self.expected_mask => {
|
||||
self.terminal = true;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Ok(None)) => return Err(self.terminate_ambiguous(DiskError::Unexpected)),
|
||||
Ok(Ok(Some(response))) => response,
|
||||
};
|
||||
let Ok(slot) = usize::try_from(response.item_index) else {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
};
|
||||
if slot >= 16 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << slot;
|
||||
if self.expected_mask & bit == 0 || self.seen_mask & bit != 0 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
self.seen_mask |= bit;
|
||||
if self.admission.as_mut().is_some_and(|admission| !admission.release_one()) {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_item_completed();
|
||||
let result = if response.success {
|
||||
if response.error.is_some() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
} else {
|
||||
decode_msgpack_strict(&response.rename_data_resp_msgpack)
|
||||
}
|
||||
} else {
|
||||
Err(response.error.unwrap_or_default().into())
|
||||
};
|
||||
Ok(Some(RemoteRenameBatchOutcome { slot, result }))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteRenameBatchStream {
|
||||
fn drop(&mut self) {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BulkBodyPermit {
|
||||
_metrics: rustfs_io_metrics::internode_metrics::InternodeBulkBodyAdmissionGuard,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
@@ -1226,6 +1430,10 @@ fn spawn_control_channel_prewarm(addr: String) {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
pub(crate) fn rename_stream_batch_enabled(&self) -> bool {
|
||||
self.rename_admission.is_some()
|
||||
}
|
||||
|
||||
async fn guarded_delete_versions_rpc(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1551,6 +1759,8 @@ impl RemoteDisk {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
|
||||
let bulk_body_admission = peer_bulk_body_admission(&addr, bulk_body_concurrency_limit()?);
|
||||
let bulk_body_wait_timeout = bulk_body_wait_timeout();
|
||||
let rename_admission =
|
||||
peer_rename_admission(&addr, rustfs_utils::get_env_bool(ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH, false));
|
||||
|
||||
let disk = Self {
|
||||
handle_id: Uuid::new_v4(),
|
||||
@@ -1573,6 +1783,7 @@ impl RemoteDisk {
|
||||
network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
|
||||
bulk_body_admission,
|
||||
bulk_body_wait_timeout,
|
||||
rename_admission,
|
||||
};
|
||||
record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
|
||||
|
||||
@@ -2494,6 +2705,29 @@ impl RemoteDisk {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn acquire_rename_admission_until(
|
||||
&self,
|
||||
item_count: usize,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<Option<RenameAdmissionGrant>> {
|
||||
let Some(semaphore) = &self.rename_admission else {
|
||||
return Ok(None);
|
||||
};
|
||||
if item_count == 0 || item_count > rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
|
||||
return Err(Error::other("invalid internode rename admission weight"));
|
||||
}
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
let weight = u32::try_from(item_count).map_err(Error::other)?;
|
||||
let permit = match tokio::time::timeout_at(deadline, Arc::clone(semaphore).acquire_many_owned(weight)).await {
|
||||
Ok(Ok(permit)) => permit,
|
||||
Ok(Err(_)) => return Err(Error::RemoteClientUnavailable("internode rename admission closed".to_owned())),
|
||||
Err(_) => return Err(DiskError::DiskOngoingReq),
|
||||
};
|
||||
Ok(Some(RenameAdmissionGrant { permit: Some(permit) }))
|
||||
}
|
||||
|
||||
async fn get_bulk_client(&self) -> Result<NodeRpcClient> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
@@ -2634,6 +2868,18 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_msgpack_strict<T: DeserializeOwned>(binary: &[u8]) -> Result<T> {
|
||||
if binary.is_empty() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let mut cursor = Cursor::new(binary);
|
||||
let value = rmp_serde::from_read(&mut cursor).map_err(|_| DiskError::FileCorrupt)?;
|
||||
if cursor.position() != u64::try_from(binary.len()).map_err(Error::other)? {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn decode_multipart_publication_response(binary: &[u8]) -> Result<RenameDataResp> {
|
||||
let result = if binary.len() > MultipartSourceOperation::max_wire_payload_bytes() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
@@ -2831,6 +3077,91 @@ impl RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn start_rename_data_batch(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
items: Vec<RenameBatchClientItem<'_>>,
|
||||
) -> std::result::Result<RemoteRenameBatchStream, RenameBatchStartError> {
|
||||
if self.rename_admission.is_none() || items.is_empty() || items.len() > 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::UnsupportedDisk));
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let mut expected_mask = 0_u16;
|
||||
let mut request_items = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
if item.slot >= 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << item.slot;
|
||||
if expected_mask & bit != 0 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
expected_mask |= bit;
|
||||
let item_index = u32::try_from(item.slot)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let file_info_msgpack = encode_file_info_msgpack(item.file_info).map_err(RenameBatchStartError::NotDispatched)?;
|
||||
request_items.push(RenameDataBatchItem {
|
||||
item_index,
|
||||
disk: item.disk,
|
||||
file_info_msgpack: file_info_msgpack.into(),
|
||||
});
|
||||
}
|
||||
let mut request = Request::new(RenameDataBatchRequest {
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
items: request_items,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_batch_request_body(request.get_ref())
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let mut client = match tokio::time::timeout_at(deadline, self.get_client()).await {
|
||||
Ok(result) => result.map_err(RenameBatchStartError::NotDispatched)?,
|
||||
Err(_) => return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout)),
|
||||
};
|
||||
let admission = self
|
||||
.acquire_rename_admission_until(request.get_ref().items.len(), deadline)
|
||||
.await
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let guard = RenameBatchCallGuard {
|
||||
admission,
|
||||
deadline,
|
||||
quarantine_on_drop: true,
|
||||
};
|
||||
let call_timeout = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
let response = match self
|
||||
.execute_with_timeout_for_op(
|
||||
"rename_data_batch",
|
||||
|| async {
|
||||
match client.rename_data_batch(request).await {
|
||||
Ok(response) => Ok(response),
|
||||
Err(status) if status.code() == Code::Unimplemented => Err(DiskError::UnsupportedDisk),
|
||||
Err(status) => Err(status.into()),
|
||||
}
|
||||
},
|
||||
call_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(DiskError::UnsupportedDisk) => {
|
||||
guard.release_for_fallback();
|
||||
return Err(RenameBatchStartError::UnsupportedBeforeStream);
|
||||
}
|
||||
Err(error) => return Err(RenameBatchStartError::MayHavePublished(error)),
|
||||
Ok(response) => response,
|
||||
};
|
||||
Ok(guard.into_stream(response.into_inner(), expected_mask))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
@@ -2902,6 +3233,27 @@ impl RemoteDisk {
|
||||
let scanner_publication_lease = scanner_publication_lease_token
|
||||
.map(|token| Bytes::copy_from_slice(token.as_bytes()))
|
||||
.unwrap_or_default();
|
||||
let rename_deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let rename_admission = self.acquire_rename_admission_until(1, rename_deadline).await?;
|
||||
let rename_admission_guard = RenameBatchCallGuard {
|
||||
admission: rename_admission,
|
||||
deadline: rename_deadline,
|
||||
quarantine_on_drop: true,
|
||||
};
|
||||
let rpc_kind = if multipart_rename {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_MULTIPART
|
||||
} else if pool_meta_cas {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_POOL_META_CAS
|
||||
} else if quota_fenced {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE
|
||||
} else if incarnation_bound {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_INCARNATION
|
||||
} else if scanner_publication_lease_token.is_some() {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE
|
||||
} else {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_ORDINARY
|
||||
};
|
||||
rustfs_io_metrics::record_ecstore_rename_remote_rpc(rpc_kind);
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let mut retried_dual_write = false;
|
||||
let response = loop {
|
||||
@@ -2988,6 +3340,7 @@ impl RemoteDisk {
|
||||
Err(status) => return Err(status.into()),
|
||||
}
|
||||
};
|
||||
rename_admission_guard.complete();
|
||||
let response_process_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_headers_started = rustfs_io_metrics::put_stage_timer();
|
||||
let _ = response.metadata().len();
|
||||
@@ -12963,6 +13316,72 @@ mod tests {
|
||||
.expect("shared gate remains open");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_admission_is_weighted_shared_and_released_per_item() {
|
||||
use std::future::Future;
|
||||
use std::task::{Context, Poll, Waker};
|
||||
|
||||
let first = peer_rename_admission("http://rename-peer-a", true).expect("enabled rename gate");
|
||||
let same = peer_rename_admission("http://rename-peer-a", true).expect("same peer rename gate");
|
||||
let other = peer_rename_admission("http://rename-peer-b", true).expect("other peer rename gate");
|
||||
assert!(Arc::ptr_eq(&first, &same));
|
||||
assert!(!Arc::ptr_eq(&first, &other));
|
||||
assert!(peer_rename_admission("http://rename-disabled", false).is_none());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&first)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let mut grant = RenameAdmissionGrant { permit: Some(permit) };
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
assert!(grant.release_one());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
|
||||
drop(grant);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let held = Arc::clone(&first)
|
||||
.acquire_many_owned(u32::try_from(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3).expect("small limit"))
|
||||
.await
|
||||
.expect("reserve all but three permits");
|
||||
let mut waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
assert!(matches!(waiting.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
|
||||
assert_eq!(first.available_permits(), 0, "the FIFO waiter reserves currently available permits");
|
||||
drop(waiting);
|
||||
assert_eq!(
|
||||
first.available_permits(),
|
||||
3,
|
||||
"cancelling a queued weighted acquire must return its reservation"
|
||||
);
|
||||
let waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
drop(held);
|
||||
let acquired = tokio::time::timeout(Duration::from_secs(1), waiting)
|
||||
.await
|
||||
.expect("weighted waiter wakes")
|
||||
.expect("rename gate remains open");
|
||||
drop(acquired);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_admission_quarantine_holds_until_absolute_deadline() {
|
||||
let gate = peer_rename_admission("http://rename-quarantine-peer", true).expect("enabled rename gate");
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
RenameAdmissionGrant { permit: Some(permit) }.quarantine_until(deadline);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
|
||||
tokio::time::advance(Duration::from_millis(999)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_disk_builds_optional_complete_body_gate_from_configuration() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, Some("3"))], async {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -205,6 +205,12 @@ fn heal_format_fence_lost_error() -> Error {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Whether this exact storage instance requires an incarnation-bound write
|
||||
/// for the object currently owned by a bucket-heal scope.
|
||||
pub fn object_requires_incarnation(&self, bucket: &str, object: &str) -> bool {
|
||||
bucket_heal_scope_for_object(bucket, object).is_some_and(|scope| std::ptr::eq(Arc::as_ptr(&scope.store), self))
|
||||
}
|
||||
|
||||
pub async fn purge_delete_marker_with_proof(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
|
||||
@@ -449,6 +449,73 @@ pub const PUT_RENAME_QUORUM_FANOUT_STATE_PANIC: &str = "panic";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST: &str = "quorum_first";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL: &str = "quorum_tail";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR: &str = "error";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_ORDINARY: &str = "ordinary";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE: &str = "quota_fence";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_INCARNATION: &str = "incarnation";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_MULTIPART: &str = "multipart";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE: &str = "scanner_lease";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_POOL_META_CAS: &str = "pool_meta_cas";
|
||||
pub const ECSTORE_RENAME_RPC_KINDS: [&str; 6] = [
|
||||
ECSTORE_RENAME_RPC_KIND_ORDINARY,
|
||||
ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE,
|
||||
ECSTORE_RENAME_RPC_KIND_INCARNATION,
|
||||
ECSTORE_RENAME_RPC_KIND_MULTIPART,
|
||||
ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE,
|
||||
ECSTORE_RENAME_RPC_KIND_POOL_META_CAS,
|
||||
];
|
||||
pub const ECSTORE_RENAME_PEER_WIDTHS: [&str; 18] = [
|
||||
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "gt16",
|
||||
];
|
||||
pub const ECSTORE_RENAME_COVERAGE: [&str; 3] = ["none", "partial", "full"];
|
||||
pub const ECSTORE_RENAME_BATCH_EXECUTION: [&str; 10] = [
|
||||
"planned_groups",
|
||||
"planned_items",
|
||||
"stream_groups",
|
||||
"stream_items",
|
||||
"completed_items",
|
||||
"fallback_groups",
|
||||
"fallback_items",
|
||||
"not_dispatched_items",
|
||||
"may_have_published_items",
|
||||
"terminal_missing_items",
|
||||
];
|
||||
|
||||
static ECSTORE_RENAME_RPC_COUNTS: [AtomicU64; 6] = [const { AtomicU64::new(0) }; 6];
|
||||
static ECSTORE_RENAME_REMOTE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
|
||||
static ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
|
||||
static ECSTORE_RENAME_COVERAGE_COUNTS: [AtomicU64; 3] = [const { AtomicU64::new(0) }; 3];
|
||||
static ECSTORE_RENAME_BATCH_EXECUTION_COUNTS: [AtomicU64; 10] = [const { AtomicU64::new(0) }; 10];
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct EcstoreRenameDiagnosticsSnapshot {
|
||||
pub rpc_counts: [u64; 6],
|
||||
pub remote_width_counts: [u64; 18],
|
||||
pub batchable_width_counts: [u64; 18],
|
||||
pub coverage_counts: [u64; 3],
|
||||
pub batch_execution_counts: [u64; 10],
|
||||
}
|
||||
|
||||
impl EcstoreRenameDiagnosticsSnapshot {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.rpc_counts
|
||||
.iter()
|
||||
.chain(self.remote_width_counts.iter())
|
||||
.chain(self.batchable_width_counts.iter())
|
||||
.chain(self.coverage_counts.iter())
|
||||
.chain(self.batch_execution_counts.iter())
|
||||
.all(|count| *count == 0)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot_ecstore_rename_diagnostics() -> EcstoreRenameDiagnosticsSnapshot {
|
||||
EcstoreRenameDiagnosticsSnapshot {
|
||||
rpc_counts: std::array::from_fn(|index| ECSTORE_RENAME_RPC_COUNTS[index].load(Ordering::Relaxed)),
|
||||
remote_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batchable_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
|
||||
coverage_counts: std::array::from_fn(|index| ECSTORE_RENAME_COVERAGE_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batch_execution_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].load(Ordering::Relaxed)),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn get_stage_metrics_enabled() -> bool {
|
||||
@@ -2663,6 +2730,91 @@ pub fn record_put_rename_quorum_wait_fanout(
|
||||
}
|
||||
}
|
||||
|
||||
/// Records a rename RPC after its canonical request digest has been attached.
|
||||
///
|
||||
/// This uses the PUT-stage diagnostics switch because it is temporary hot-path
|
||||
/// evidence, but the metric is ECStore-scoped: scanner and pool metadata
|
||||
/// renames are not S3 PutObject operations.
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_remote_rpc(kind: &'static str) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let Some(index) = ECSTORE_RENAME_RPC_KINDS.iter().position(|candidate| *candidate == kind) else {
|
||||
return;
|
||||
};
|
||||
ECSTORE_RENAME_RPC_COUNTS[index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn rename_peer_width_index(width: usize) -> usize {
|
||||
width.min(17)
|
||||
}
|
||||
|
||||
/// Records the per-peer opportunity for a future body-bound rename batch.
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_candidate_peer(remote_disks: usize, batchable_disks: usize) {
|
||||
if !put_stage_metrics_enabled() || remote_disks == 0 {
|
||||
return;
|
||||
}
|
||||
ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[rename_peer_width_index(remote_disks)].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[rename_peer_width_index(batchable_disks)].fetch_add(1, Ordering::Relaxed);
|
||||
let coverage_index = if batchable_disks == 0 {
|
||||
0
|
||||
} else if batchable_disks == remote_disks {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
};
|
||||
ECSTORE_RENAME_COVERAGE_COUNTS[coverage_index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn record_ecstore_rename_batch_execution(index: usize, count: usize) {
|
||||
if put_stage_metrics_enabled() {
|
||||
let count = u64::try_from(count).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_planned(items: usize) {
|
||||
record_ecstore_rename_batch_execution(0, 1);
|
||||
record_ecstore_rename_batch_execution(1, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_stream_started(items: usize) {
|
||||
record_ecstore_rename_batch_execution(2, 1);
|
||||
record_ecstore_rename_batch_execution(3, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_item_completed() {
|
||||
record_ecstore_rename_batch_execution(4, 1);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_fallback(items: usize) {
|
||||
record_ecstore_rename_batch_execution(5, 1);
|
||||
record_ecstore_rename_batch_execution(6, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_not_dispatched(items: usize) {
|
||||
record_ecstore_rename_batch_execution(7, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_may_have_published(items: usize) {
|
||||
record_ecstore_rename_batch_execution(8, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_terminal_missing(items: usize) {
|
||||
record_ecstore_rename_batch_execution(9, items);
|
||||
}
|
||||
|
||||
/// Record generic internal operation stage duration (non-PUT paths).
|
||||
/// Use this for metacache walks, listing, lifecycle, and other background
|
||||
/// operations that are NOT part of the PUT object hot path.
|
||||
@@ -4585,6 +4737,54 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_rename_batch_candidate_metrics_are_gated_and_bounded() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
set_put_stage_metrics_enabled(false);
|
||||
let disabled_before = snapshot_ecstore_rename_diagnostics();
|
||||
record_ecstore_rename_remote_rpc(ECSTORE_RENAME_RPC_KIND_ORDINARY);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 4);
|
||||
assert_eq!(snapshot_ecstore_rename_diagnostics(), disabled_before);
|
||||
|
||||
set_put_stage_metrics_enabled(true);
|
||||
let before = snapshot_ecstore_rename_diagnostics();
|
||||
for kind in ECSTORE_RENAME_RPC_KINDS {
|
||||
record_ecstore_rename_remote_rpc(kind);
|
||||
}
|
||||
record_ecstore_rename_batch_candidate_peer(4, 4);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 3);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 0);
|
||||
record_ecstore_rename_batch_candidate_peer(20, 20);
|
||||
record_ecstore_rename_batch_planned(4);
|
||||
record_ecstore_rename_batch_stream_started(4);
|
||||
record_ecstore_rename_batch_item_completed();
|
||||
record_ecstore_rename_batch_fallback(3);
|
||||
record_ecstore_rename_batch_not_dispatched(2);
|
||||
record_ecstore_rename_batch_may_have_published(4);
|
||||
record_ecstore_rename_batch_terminal_missing(1);
|
||||
set_put_stage_metrics_enabled(false);
|
||||
|
||||
let after = snapshot_ecstore_rename_diagnostics();
|
||||
for index in 0..ECSTORE_RENAME_RPC_KINDS.len() {
|
||||
assert_eq!(after.rpc_counts[index] - before.rpc_counts[index], 1);
|
||||
}
|
||||
assert_eq!(after.remote_width_counts[4] - before.remote_width_counts[4], 3);
|
||||
assert_eq!(after.remote_width_counts[17] - before.remote_width_counts[17], 1);
|
||||
assert_eq!(after.batchable_width_counts[0] - before.batchable_width_counts[0], 1);
|
||||
assert_eq!(after.batchable_width_counts[3] - before.batchable_width_counts[3], 1);
|
||||
assert_eq!(after.batchable_width_counts[4] - before.batchable_width_counts[4], 1);
|
||||
assert_eq!(after.batchable_width_counts[17] - before.batchable_width_counts[17], 1);
|
||||
assert_eq!(after.coverage_counts[0] - before.coverage_counts[0], 1);
|
||||
assert_eq!(after.coverage_counts[1] - before.coverage_counts[1], 1);
|
||||
assert_eq!(after.coverage_counts[2] - before.coverage_counts[2], 2);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 10, _>(|index| {
|
||||
after.batch_execution_counts[index] - before.batch_execution_counts[index]
|
||||
}),
|
||||
[1, 4, 1, 4, 1, 1, 3, 2, 4, 1]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_object_diagnostic_buckets() {
|
||||
assert_eq!(put_object_size_bucket(0), "unknown");
|
||||
|
||||
@@ -505,6 +505,45 @@ pub struct RenameDataResponse {
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub rename_data_resp_bin: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// Ordinary rename transport batch. Protected rename variants intentionally use
|
||||
/// their dedicated fail-closed RPCs and are never admitted here.
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct RenameDataBatchItem {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub item_index: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub file_info_msgpack: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct RenameDataBatchRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub src_volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub src_path: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub dst_volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "4")]
|
||||
pub dst_path: ::prost::alloc::string::String,
|
||||
#[prost(message, repeated, tag = "5")]
|
||||
pub items: ::prost::alloc::vec::Vec<RenameDataBatchItem>,
|
||||
}
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
#[prost(skip_debug)]
|
||||
pub struct RenameDataBatchResult {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub item_index: u32,
|
||||
#[prost(bool, tag = "2")]
|
||||
pub success: bool,
|
||||
#[prost(message, optional, tag = "3")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub rename_data_resp_msgpack: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
|
||||
/// accepts an arbitrary volume or object path.
|
||||
#[allow(clippy::all)]
|
||||
@@ -2894,6 +2933,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenameData"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn rename_data_batch(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::RenameDataBatchRequest>,
|
||||
) -> std::result::Result<tonic::Response<tonic::codec::Streaming<super::RenameDataBatchResult>>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameDataBatch");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenameDataBatch"));
|
||||
self.inner.server_streaming(req, path, codec).await
|
||||
}
|
||||
/// The method itself requires source-preserving multipart publication on every peer.
|
||||
pub async fn rename_multipart_data(
|
||||
&mut self,
|
||||
@@ -4252,6 +4306,14 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status>;
|
||||
/// Server streaming response type for the RenameDataBatch method.
|
||||
type RenameDataBatchStream: tonic::codegen::tokio_stream::Stream<Item = std::result::Result<super::RenameDataBatchResult, tonic::Status>>
|
||||
+ std::marker::Send
|
||||
+ 'static;
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataBatchRequest>,
|
||||
) -> std::result::Result<tonic::Response<Self::RenameDataBatchStream>, tonic::Status>;
|
||||
/// The method itself requires source-preserving multipart publication on every peer.
|
||||
async fn rename_multipart_data(
|
||||
&self,
|
||||
@@ -5573,6 +5635,35 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenameDataBatch" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenameDataBatchSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::ServerStreamingService<super::RenameDataBatchRequest> for RenameDataBatchSvc<T> {
|
||||
type Response = super::RenameDataBatchResult;
|
||||
type ResponseStream = T::RenameDataBatchStream;
|
||||
type Future = BoxFuture<tonic::Response<Self::ResponseStream>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::RenameDataBatchRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::rename_data_batch(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = RenameDataBatchSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.server_streaming(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenameMultipartData" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenameMultipartDataSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
@@ -67,6 +67,18 @@ impl std::fmt::Debug for proto_gen::node_service::RenameDataResponse {
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for proto_gen::node_service::RenameDataBatchResult {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("RenameDataBatchResult")
|
||||
.field("item_index", &self.item_index)
|
||||
.field("success", &self.success)
|
||||
.field("error", &self.error)
|
||||
.field("rename_data_resp_msgpack", &"[REDACTED]")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for proto_gen::node_service::RecoverMultipartSourceRequest {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
@@ -1171,6 +1183,23 @@ pub fn canonical_rename_data_request_body(
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_data_batch_request_body(
|
||||
request: &proto_gen::node_service::RenameDataBatchRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-rename-data-batch-request-v1\0");
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
body.push_count(request.items.len())?;
|
||||
for item in &request.items {
|
||||
body.push_u32(item.item_index);
|
||||
body.push_str(&item.disk)?;
|
||||
body.push_bytes(&item.file_info_msgpack)?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
/// The fixed-MPU recovery method authenticates every wire field in its own domain.
|
||||
pub fn canonical_recover_multipart_source_request_body(
|
||||
request: &proto_gen::node_service::RecoverMultipartSourceRequest,
|
||||
@@ -1690,9 +1719,10 @@ pub fn canonical_make_volumes_request_body(
|
||||
mod disk_mutation_canonical_tests {
|
||||
use super::proto_gen::node_service::{
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVolumeRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, RenameDataRequest,
|
||||
RenameFileRequest, RenamePartRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
DeleteVolumeRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, RenameDataBatchItem,
|
||||
RenameDataBatchRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, SettlePartTransactionRequest,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, UpdateMetadataRequest, WriteAllRequest,
|
||||
WriteMetadataRequest,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
@@ -1844,6 +1874,51 @@ mod disk_mutation_canonical_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_batch_canonical_body_binds_every_field_and_item_order() {
|
||||
let baseline = RenameDataBatchRequest {
|
||||
src_volume: "src-vol".into(),
|
||||
src_path: "src-path".into(),
|
||||
dst_volume: "dst-vol".into(),
|
||||
dst_path: "dst-path".into(),
|
||||
items: vec![
|
||||
RenameDataBatchItem {
|
||||
item_index: 1,
|
||||
disk: "disk-a".into(),
|
||||
file_info_msgpack: vec![0x81, 0x01].into(),
|
||||
},
|
||||
RenameDataBatchItem {
|
||||
item_index: 2,
|
||||
disk: "disk-b".into(),
|
||||
file_info_msgpack: vec![0x81, 0x02].into(),
|
||||
},
|
||||
],
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_data_batch_request_body(&baseline).expect("baseline batch body")];
|
||||
for mutate in [
|
||||
|r: &mut RenameDataBatchRequest| r.src_volume = "src-vol2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.src_path = "src-path2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.dst_volume = "dst-vol2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.dst_path = "dst-path2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.items[0].item_index = 3,
|
||||
|r: &mut RenameDataBatchRequest| r.items[0].disk = "disk-c".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.items[0].file_info_msgpack = vec![0x81, 0x03].into(),
|
||||
|r: &mut RenameDataBatchRequest| r.items.swap(0, 1),
|
||||
|r: &mut RenameDataBatchRequest| {
|
||||
r.items.pop();
|
||||
},
|
||||
] {
|
||||
let mut changed = baseline.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_rename_data_batch_request_body(&changed).expect("changed batch body"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
assert_ne!(
|
||||
canonical_rename_data_batch_request_body(&baseline).expect("batch body"),
|
||||
canonical_rename_data_request_body(&RenameDataRequest::default()).expect("unary body"),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_version_canonical_body_binds_every_field() {
|
||||
let baseline = DeleteVersionRequest {
|
||||
@@ -3655,7 +3730,7 @@ fn complete_pending_rebuild_on_cache_hit(dial_addr: &str, cache_key: &str) -> bo
|
||||
}
|
||||
|
||||
// Limits transport response futures; client RPC metrics separately cover unary body decoding.
|
||||
const INTERNODE_RPC_CONCURRENCY_LIMIT: usize = 64;
|
||||
pub const INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT: usize = 64;
|
||||
|
||||
// Quorum reads cancel surplus metadata RPCs in bursts. Keep bounded reset state
|
||||
// for their in-flight replies without spending the connection's protocol-error
|
||||
@@ -4109,7 +4184,7 @@ async fn dial_channel(
|
||||
rustfs_config::DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT,
|
||||
))?
|
||||
} else {
|
||||
INTERNODE_RPC_CONCURRENCY_LIMIT
|
||||
INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT
|
||||
};
|
||||
let stream_window = checked_rpc_window(stream_window)?;
|
||||
let conn_window = checked_rpc_window(conn_window)?;
|
||||
|
||||
@@ -69,6 +69,7 @@ fn main() -> Result<(), AnyError> {
|
||||
// Their hand-written Debug implementations redact the complete payload.
|
||||
.skip_debug([
|
||||
".node_service.RenameDataResponse",
|
||||
".node_service.RenameDataBatchResult",
|
||||
".node_service.RecoverMultipartSourceRequest",
|
||||
".node_service.RecoverMultipartSourceResponse",
|
||||
])
|
||||
|
||||
@@ -345,6 +345,29 @@ message RenameDataResponse {
|
||||
bytes rename_data_resp_bin = 4;
|
||||
}
|
||||
|
||||
// Ordinary rename transport batch. Protected rename variants intentionally use
|
||||
// their dedicated fail-closed RPCs and are never admitted here.
|
||||
message RenameDataBatchItem {
|
||||
uint32 item_index = 1;
|
||||
string disk = 2;
|
||||
bytes file_info_msgpack = 3;
|
||||
}
|
||||
|
||||
message RenameDataBatchRequest {
|
||||
string src_volume = 1;
|
||||
string src_path = 2;
|
||||
string dst_volume = 3;
|
||||
string dst_path = 4;
|
||||
repeated RenameDataBatchItem items = 5;
|
||||
}
|
||||
|
||||
message RenameDataBatchResult {
|
||||
uint32 item_index = 1;
|
||||
bool success = 2;
|
||||
optional Error error = 3;
|
||||
bytes rename_data_resp_msgpack = 4;
|
||||
}
|
||||
|
||||
// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
|
||||
// accepts an arbitrary volume or object path.
|
||||
message PoolMetaNativeRecoveryRequest {
|
||||
@@ -1394,6 +1417,7 @@ service NodeService {
|
||||
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
|
||||
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
|
||||
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataBatch(RenameDataBatchRequest) returns (stream RenameDataBatchResult) {}; // auth-policy: body-bound
|
||||
// The method itself requires source-preserving multipart publication on every peer.
|
||||
rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
|
||||
@@ -72,7 +72,8 @@ fn init_get_stage_local_summary_runtime(ctx: CancellationToken) {
|
||||
_ = ctx.cancelled() => break,
|
||||
_ = interval.tick() => {
|
||||
let rows = rustfs_io_metrics::take_get_stage_local_summary();
|
||||
if rows.is_empty() {
|
||||
let rename = rustfs_io_metrics::snapshot_ecstore_rename_diagnostics();
|
||||
if rows.is_empty() && rename.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let summary = rows.iter().map(|row| {
|
||||
@@ -83,9 +84,14 @@ fn init_get_stage_local_summary_runtime(ctx: CancellationToken) {
|
||||
.unwrap_or_default()
|
||||
.as_millis();
|
||||
let line = format!(
|
||||
"timestamp_ms={timestamp_ms}\tsample_rate={}\tstage_count={}\tstages={summary}\n",
|
||||
"timestamp_ms={timestamp_ms}\tsample_rate={}\tstage_count={}\tstages={summary}\trename_diag_version=2\trename_rpc={}\trename_remote_width={}\trename_batchable_width={}\trename_coverage={}\trename_batch_execution={}\n",
|
||||
rustfs_io_metrics::get_stage_local_summary_sample_rate(),
|
||||
rows.len(),
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KINDS.iter().zip(rename.rpc_counts).map(|(label, count)| format!("{label}:{count}")).collect::<Vec<_>>().join(","),
|
||||
rustfs_io_metrics::ECSTORE_RENAME_PEER_WIDTHS.iter().zip(rename.remote_width_counts).map(|(label, count)| format!("{label}:{count}")).collect::<Vec<_>>().join(","),
|
||||
rustfs_io_metrics::ECSTORE_RENAME_PEER_WIDTHS.iter().zip(rename.batchable_width_counts).map(|(label, count)| format!("{label}:{count}")).collect::<Vec<_>>().join(","),
|
||||
rustfs_io_metrics::ECSTORE_RENAME_COVERAGE.iter().zip(rename.coverage_counts).map(|(label, count)| format!("{label}:{count}")).collect::<Vec<_>>().join(","),
|
||||
rustfs_io_metrics::ECSTORE_RENAME_BATCH_EXECUTION.iter().zip(rename.batch_execution_counts).map(|(label, count)| format!("{label}:{count}")).collect::<Vec<_>>().join(","),
|
||||
);
|
||||
let Ok(mut file) = tokio::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
|
||||
@@ -605,6 +605,7 @@ struct QuotaTopologyCacheEntry {
|
||||
fingerprint: String,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum LocalMutationTarget {
|
||||
Ready(Arc<ECStore>),
|
||||
Bootstrap(BootstrapLocalTarget),
|
||||
@@ -1927,6 +1928,19 @@ impl Node for NodeService {
|
||||
self.handle_rename_data(request, false, false, false).await
|
||||
}
|
||||
|
||||
type RenameDataBatchStream = ResponseStream<RenameDataBatchResult>;
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
request: Request<RenameDataBatchRequest>,
|
||||
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
|
||||
if !rustfs_utils::get_env_bool("RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH", false) {
|
||||
// The feature gate is checked before authentication or mutation so
|
||||
// mixed-version clients may safely use call-level UNIMPLEMENTED.
|
||||
return Err(unimplemented_rpc("rename_data_batch"));
|
||||
}
|
||||
self.handle_rename_data_batch(request).await
|
||||
}
|
||||
|
||||
async fn rename_multipart_data(&self, request: Request<RenameDataRequest>) -> Result<Response<RenameDataResponse>, Status> {
|
||||
self.handle_rename_data(request, false, false, true).await
|
||||
}
|
||||
@@ -3489,13 +3503,13 @@ mod tests {
|
||||
MakeVolumeRequest, MakeVolumesRequest, MrfLifecycleRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest,
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest, ReloadPoolMetaRequest,
|
||||
ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, ScannerActivityRequest,
|
||||
ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
|
||||
ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest,
|
||||
StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, TierMutationPeerState,
|
||||
TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteFormatControlRequest, WriteMetadataRequest, WriteRequest,
|
||||
ReloadSiteReplicationConfigRequest, RenameDataBatchItem, RenameDataBatchRequest, RenameDataRequest, RenameFileRequest,
|
||||
RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest,
|
||||
ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest,
|
||||
StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass,
|
||||
TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest,
|
||||
VerifyFileRequest, WriteAllRequest, WriteFormatControlRequest, WriteMetadataRequest, WriteRequest,
|
||||
heal_control_service_client::HealControlServiceClient,
|
||||
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
|
||||
node_service_client::NodeServiceClient,
|
||||
@@ -3513,8 +3527,9 @@ mod tests {
|
||||
use tonic::{Request, Response, Status};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DISK_MUTATION_RPC_METHODS: [&str; 35] = [
|
||||
const DISK_MUTATION_RPC_METHODS: [&str; 36] = [
|
||||
"renamedata",
|
||||
"renamedatabatch",
|
||||
"renamemultipartdata",
|
||||
"recovermultipartsource",
|
||||
"renamedataatincarnation",
|
||||
@@ -3581,6 +3596,41 @@ mod tests {
|
||||
policies
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rename_data_batch_requires_explicit_enable_and_body_digest() {
|
||||
let request = || {
|
||||
Request::new(RenameDataBatchRequest {
|
||||
src_volume: "src".to_owned(),
|
||||
src_path: "object".to_owned(),
|
||||
dst_volume: "dst".to_owned(),
|
||||
dst_path: "object".to_owned(),
|
||||
items: vec![RenameDataBatchItem {
|
||||
item_index: 0,
|
||||
disk: "disk".to_owned(),
|
||||
file_info_msgpack: vec![0x80].into(),
|
||||
}],
|
||||
})
|
||||
};
|
||||
|
||||
temp_env::async_with_vars([("RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH", Some("false"))], async {
|
||||
let error = match make_server().rename_data_batch(request()).await {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("disabled batch must be unimplemented"),
|
||||
};
|
||||
assert_eq!(error.code(), tonic::Code::Unimplemented);
|
||||
})
|
||||
.await;
|
||||
temp_env::async_with_vars([("RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH", Some("true"))], async {
|
||||
let error = match make_server().rename_data_batch(request()).await {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("unsigned batch must fail closed"),
|
||||
};
|
||||
assert_eq!(error.code(), tonic::Code::PermissionDenied);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_node_service_rpc_declares_an_auth_policy() {
|
||||
const VALID_POLICIES: [&str; 4] = ["body-bound", "read-only", "streaming", "unimplemented"];
|
||||
|
||||
@@ -32,6 +32,7 @@ use crate::storage::storage_api::{
|
||||
verify_tonic_mutation_body_digest,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures_util::{FutureExt as _, StreamExt as _, stream::FuturesUnordered};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_io_metrics::internode_metrics::InternodeRpcDurationGuard;
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
@@ -77,6 +78,13 @@ fn startup_cas_rename_observation(
|
||||
}
|
||||
|
||||
impl LocalMutationTarget {
|
||||
fn object_requires_incarnation(&self, bucket: &str, object: &str) -> bool {
|
||||
match self {
|
||||
Self::Ready(store) => store.object_requires_incarnation(bucket, object),
|
||||
Self::Bootstrap(_) | Self::Unbound => false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_local_format_control(&self, disk_ref: &str, path: &str, data: Bytes) -> Result<(), DiskError> {
|
||||
match self {
|
||||
Self::Ready(store) => store.write_local_format_control(disk_ref, path, data).await,
|
||||
@@ -498,6 +506,18 @@ fn decode_multipart_rename_request_file_info(binary: &[u8]) -> Result<DecodedRpc
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_batch_rename_file_info(binary: &[u8]) -> Result<FileInfo, DiskError> {
|
||||
if binary.is_empty() || binary.len() > rustfs_protos::DEFAULT_GRPC_SERVER_MESSAGE_LEN {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let mut cursor = Cursor::new(binary);
|
||||
let value = rmp_serde::from_read(&mut cursor).map_err(|_| DiskError::FileCorrupt)?;
|
||||
if cursor.position() != u64::try_from(binary.len()).map_err(DiskError::other)? {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn encode_rename_data_response_payloads(
|
||||
rename_data_resp: &RenameDataResp,
|
||||
request_decoded_from_msgpack: bool,
|
||||
@@ -2112,6 +2132,126 @@ impl NodeService {
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
pub(super) async fn handle_rename_data_batch(
|
||||
&self,
|
||||
request: Request<RenameDataBatchRequest>,
|
||||
) -> Result<Response<super::ResponseStream<RenameDataBatchResult>>, Status> {
|
||||
const MAX_BATCH_ITEMS: usize = 16;
|
||||
|
||||
if request.get_ref().items.is_empty() || request.get_ref().items.len() > MAX_BATCH_ITEMS {
|
||||
return Err(Status::invalid_argument("rename batch item count must be between 1 and 16"));
|
||||
}
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_batch_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("rename batch body length cannot be represented"))?;
|
||||
let has_body_digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| !value.is_empty() && value != "UNSIGNED-PAYLOAD");
|
||||
if !has_body_digest {
|
||||
return Err(Status::permission_denied("rename batch requires a body-bound digest"));
|
||||
}
|
||||
crate::storage::storage_api::verify_tonic_canonical_body_digest(&request, &canonical_body)
|
||||
.map_err(|_| Status::permission_denied("rename batch requires body-bound authentication"))?;
|
||||
let request = request.into_inner();
|
||||
if request.src_volume == RUSTFS_META_MULTIPART_BUCKET {
|
||||
return Err(Status::invalid_argument("multipart source requires RenameMultipartData"));
|
||||
}
|
||||
|
||||
let target = self.local_mutation_target();
|
||||
if target.object_requires_incarnation(&request.dst_volume, &request.dst_path) {
|
||||
return Err(Status::failed_precondition(
|
||||
"incarnation-bound rename cannot use ordinary batch transport",
|
||||
));
|
||||
}
|
||||
let mut seen_indexes = 0_u16;
|
||||
let mut prepared = Vec::with_capacity(request.items.len());
|
||||
for item in request.items {
|
||||
let item_index =
|
||||
usize::try_from(item.item_index).map_err(|_| Status::invalid_argument("invalid rename batch item index"))?;
|
||||
if item_index >= MAX_BATCH_ITEMS || seen_indexes & (1_u16 << item_index) != 0 {
|
||||
return Err(Status::invalid_argument("rename batch item indexes must be unique and below 16"));
|
||||
}
|
||||
seen_indexes |= 1_u16 << item_index;
|
||||
if item.disk.is_empty() || prepared.iter().any(|(_, disk, _)| disk == &item.disk) {
|
||||
return Err(Status::invalid_argument("rename batch disk references must be non-empty and unique"));
|
||||
}
|
||||
let file_info = decode_batch_rename_file_info(&item.file_info_msgpack)
|
||||
.map_err(|_| Status::invalid_argument("invalid rename batch FileInfo"))?;
|
||||
if file_info.is_canonical_delete_marker() {
|
||||
file_info
|
||||
.validate_for_metadata_read()
|
||||
.map_err(|_| Status::invalid_argument("invalid rename batch delete marker"))?;
|
||||
} else {
|
||||
file_info
|
||||
.validate_for_erasure_write()
|
||||
.map_err(|_| Status::invalid_argument("invalid rename batch FileInfo geometry"))?;
|
||||
}
|
||||
if rustfs_utils::http::metadata_compat::contains_key_str(&file_info.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX)
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&file_info.metadata,
|
||||
POOL_META_NATIVE_CAS_METADATA_SUFFIX,
|
||||
)
|
||||
{
|
||||
return Err(Status::invalid_argument("protected rename cannot use ordinary batch transport"));
|
||||
}
|
||||
prepared.push((item.item_index, item.disk, file_info));
|
||||
}
|
||||
let src_volume = Arc::<str>::from(request.src_volume);
|
||||
let src_path = Arc::<str>::from(request.src_path);
|
||||
let dst_volume = Arc::<str>::from(request.dst_volume);
|
||||
let dst_path = Arc::<str>::from(request.dst_path);
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(prepared.len());
|
||||
tokio::spawn(async move {
|
||||
let mut pending = FuturesUnordered::new();
|
||||
for (item_index, disk, file_info) in prepared {
|
||||
let target = target.clone();
|
||||
let src_volume = Arc::clone(&src_volume);
|
||||
let src_path = Arc::clone(&src_path);
|
||||
let dst_volume = Arc::clone(&dst_volume);
|
||||
let dst_path = Arc::clone(&dst_path);
|
||||
pending.push(async move {
|
||||
let result = std::panic::AssertUnwindSafe(async {
|
||||
let result = target
|
||||
.rename_local_data(&disk, (&src_volume, &src_path), &file_info, (&dst_volume, &dst_path), None, None)
|
||||
.await;
|
||||
match result.and_then(|value| encode_msgpack_named(&value, "RenameDataResp")) {
|
||||
Ok(payload) => RenameDataBatchResult {
|
||||
item_index,
|
||||
success: true,
|
||||
error: None,
|
||||
rename_data_resp_msgpack: payload.into(),
|
||||
},
|
||||
Err(error) => RenameDataBatchResult {
|
||||
item_index,
|
||||
success: false,
|
||||
error: Some(error.into()),
|
||||
rename_data_resp_msgpack: Vec::new().into(),
|
||||
},
|
||||
}
|
||||
})
|
||||
.catch_unwind()
|
||||
.await;
|
||||
(item_index, result)
|
||||
});
|
||||
}
|
||||
while let Some(outcome) = pending.next().await {
|
||||
let (item_index, result) = outcome;
|
||||
let response = match result {
|
||||
Ok(response) => response,
|
||||
Err(_) => RenameDataBatchResult {
|
||||
item_index,
|
||||
success: false,
|
||||
error: Some(DiskError::Unexpected.into()),
|
||||
rename_data_resp_msgpack: Vec::new().into(),
|
||||
},
|
||||
};
|
||||
let _ = tx.send(Ok(response)).await;
|
||||
}
|
||||
});
|
||||
Ok(Response::new(Box::pin(tokio_stream::wrappers::ReceiverStream::new(rx))))
|
||||
}
|
||||
|
||||
pub(super) async fn handle_list_dir(&self, request: Request<ListDirRequest>) -> Result<Response<ListDirResponse>, Status> {
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await? {
|
||||
|
||||
Executable
+133
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed stability gate for fixed-rate benchmark warmup windows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def positive_number(row: dict, name: str) -> float:
|
||||
value = row.get(name)
|
||||
if (
|
||||
not isinstance(value, (int, float))
|
||||
or isinstance(value, bool)
|
||||
or not math.isfinite(value)
|
||||
or value <= 0
|
||||
):
|
||||
raise ValueError(f"window field {name} must be a positive number")
|
||||
return float(value)
|
||||
|
||||
|
||||
def validate_window(row: dict, target_rps: float, duration_secs: float, concurrency: int, tolerance_pct: float) -> dict:
|
||||
successful_puts = positive_number(row, "successful_puts")
|
||||
errors = row.get("errors")
|
||||
if not isinstance(errors, int) or isinstance(errors, bool) or errors < 0:
|
||||
raise ValueError("window field errors must be a non-negative integer")
|
||||
if errors:
|
||||
raise ValueError("warmup window contains request errors")
|
||||
reqps = positive_number(row, "reqps")
|
||||
scheduled = target_rps * duration_secs
|
||||
tolerance = tolerance_pct / 100
|
||||
minimum_puts = scheduled * (1 - tolerance)
|
||||
maximum_puts = (scheduled + concurrency - 1) * (1 + tolerance)
|
||||
if not minimum_puts <= successful_puts <= maximum_puts:
|
||||
raise ValueError("warmup successful PUT count is outside tolerance")
|
||||
if abs(reqps - target_rps) / target_rps * 100 > tolerance_pct:
|
||||
raise ValueError("warmup request rate is outside tolerance")
|
||||
host_counts = row.get("host_counts")
|
||||
if not isinstance(host_counts, dict) or len(host_counts) < 2:
|
||||
raise ValueError("warmup host_counts must contain at least two hosts")
|
||||
counts = [value for value in host_counts.values() if isinstance(value, int) and not isinstance(value, bool) and value >= 0]
|
||||
if len(counts) != len(host_counts) or sum(counts) <= 0:
|
||||
raise ValueError("warmup host_counts contains invalid values")
|
||||
if sum(counts) != successful_puts:
|
||||
raise ValueError("warmup host counts do not match successful PUT count")
|
||||
ideal = sum(counts) / len(counts)
|
||||
host_deviation_pct = max(abs(value - ideal) / ideal * 100 for value in counts)
|
||||
if host_deviation_pct > tolerance_pct:
|
||||
raise ValueError("warmup host distribution is outside tolerance")
|
||||
return {
|
||||
"successful_puts": int(successful_puts),
|
||||
"errors": errors,
|
||||
"reqps": reqps,
|
||||
"avg_ms": positive_number(row, "avg_ms"),
|
||||
"p90_ms": positive_number(row, "p90_ms"),
|
||||
"p99_ms": positive_number(row, "p99_ms"),
|
||||
"host_deviation_pct": host_deviation_pct,
|
||||
}
|
||||
|
||||
|
||||
def spread_pct(values: list[float]) -> float:
|
||||
median = statistics.median(values)
|
||||
return (max(values) - min(values)) / median * 100
|
||||
|
||||
|
||||
def evaluate(rows: list[dict], required_windows: int, max_windows: int, max_spread_pct: float) -> dict:
|
||||
if len(rows) < required_windows:
|
||||
return {"status": "continue", "reason": "insufficient_windows", "windows": rows}
|
||||
recent = rows[-required_windows:]
|
||||
spreads = {name: spread_pct([row[name] for row in recent]) for name in ("reqps", "avg_ms", "p90_ms", "p99_ms")}
|
||||
stable = all(value <= max_spread_pct for value in spreads.values())
|
||||
if stable:
|
||||
return {"status": "pass", "reason": "stable", "windows": rows, "recent_spread_pct": spreads}
|
||||
if len(rows) >= max_windows:
|
||||
return {"status": "fail", "reason": "max_windows_unstable", "windows": rows, "recent_spread_pct": spreads}
|
||||
return {"status": "continue", "reason": "recent_windows_unstable", "windows": rows, "recent_spread_pct": spreads}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input", required=True)
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--target-rps", type=float, default=250)
|
||||
parser.add_argument("--duration-secs", type=float, default=20)
|
||||
parser.add_argument("--concurrency", type=int, default=128)
|
||||
parser.add_argument("--required-windows", type=int, default=3)
|
||||
parser.add_argument("--max-windows", type=int, default=12)
|
||||
parser.add_argument("--quality-tolerance-pct", type=float, default=2)
|
||||
parser.add_argument("--max-spread-pct", type=float, default=5)
|
||||
args = parser.parse_args()
|
||||
output = Path(args.output)
|
||||
try:
|
||||
raw = json.loads(Path(args.input).read_text())
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError("warmup input must be a JSON array")
|
||||
if not 1 <= args.required_windows <= args.max_windows:
|
||||
raise ValueError("required windows must be between one and max windows")
|
||||
numeric_args = (args.target_rps, args.duration_secs, args.quality_tolerance_pct, args.max_spread_pct)
|
||||
if not all(math.isfinite(value) for value in numeric_args):
|
||||
raise ValueError("warmup numeric arguments must be finite")
|
||||
if args.target_rps <= 0 or args.duration_secs <= 0 or args.concurrency <= 0:
|
||||
raise ValueError("target RPS, duration, and concurrency must be positive")
|
||||
if args.quality_tolerance_pct < 0 or args.max_spread_pct < 0:
|
||||
raise ValueError("tolerances must be non-negative")
|
||||
if len(raw) > args.max_windows:
|
||||
raise ValueError("warmup input exceeds max windows")
|
||||
windows = [
|
||||
validate_window(row, args.target_rps, args.duration_secs, args.concurrency, args.quality_tolerance_pct)
|
||||
for row in raw
|
||||
]
|
||||
result = evaluate(windows, args.required_windows, args.max_windows, args.max_spread_pct)
|
||||
except (OSError, json.JSONDecodeError, ValueError) as error:
|
||||
result = {"status": "fail", "reason": str(error), "windows": []}
|
||||
result["schema_version"] = 1
|
||||
result["target_rps"] = args.target_rps
|
||||
result["duration_secs"] = args.duration_secs
|
||||
result["concurrency"] = args.concurrency
|
||||
result["required_windows"] = args.required_windows
|
||||
result["max_windows"] = args.max_windows
|
||||
output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
||||
if result["status"] == "pass":
|
||||
return 0
|
||||
if result["status"] == "continue":
|
||||
return 2
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+206
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run fixed-rate Warp warmup windows until the stability gate passes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import adaptive_warmup_gate
|
||||
import warp_raw_quality
|
||||
|
||||
WARMUP_UNSTABLE_EXIT = 95
|
||||
|
||||
|
||||
def result_exit_code(result: dict) -> int:
|
||||
if result.get("status") == "pass":
|
||||
return 0
|
||||
if result.get("reason") == "max_windows_unstable":
|
||||
return WARMUP_UNSTABLE_EXIT
|
||||
return 1
|
||||
|
||||
|
||||
def extract_latency_metrics(extractor: Path, log_path: Path) -> dict:
|
||||
result = subprocess.run(
|
||||
[
|
||||
str(extractor),
|
||||
"--tool",
|
||||
"warp",
|
||||
"--warp-mode",
|
||||
"put",
|
||||
"--extract-metrics-from-log",
|
||||
str(log_path),
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError("Warp metrics extractor failed")
|
||||
lines = [line for line in result.stdout.splitlines() if line.strip()]
|
||||
if len(lines) != 2:
|
||||
raise ValueError("Warp metrics extractor returned an unexpected schema")
|
||||
rows = list(csv.DictReader(lines))
|
||||
if len(rows) != 1:
|
||||
raise ValueError("Warp metrics extractor returned an unexpected row count")
|
||||
row = rows[0]
|
||||
try:
|
||||
return {
|
||||
"avg_ms": float(row["latency_ms"]),
|
||||
"p90_ms": float(row["req_p90_ms"]),
|
||||
"p99_ms": float(row["req_p99_ms"]),
|
||||
}
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise ValueError("Warp metrics extractor omitted a latency field") from error
|
||||
|
||||
|
||||
def write_gate(path: Path, result: dict, args: argparse.Namespace) -> None:
|
||||
output = dict(result)
|
||||
output.update(
|
||||
schema_version=1,
|
||||
target_rps=args.target_rps,
|
||||
duration_secs=args.duration_secs,
|
||||
concurrency=args.concurrency,
|
||||
required_windows=args.required_windows,
|
||||
max_windows=args.max_windows,
|
||||
bucket=args.bucket,
|
||||
hosts=args.host,
|
||||
)
|
||||
path.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--warp-bin", required=True)
|
||||
parser.add_argument("--metrics-extractor", required=True)
|
||||
parser.add_argument("--host", action="append", required=True)
|
||||
parser.add_argument("--access-key-env", default="RUSTFS_ACCESS_KEY")
|
||||
parser.add_argument("--secret-key-env", default="RUSTFS_SECRET_KEY")
|
||||
parser.add_argument("--bucket", required=True)
|
||||
parser.add_argument("--region", default="us-east-1")
|
||||
parser.add_argument("--size", default="100KiB")
|
||||
parser.add_argument("--concurrency", type=int, default=128)
|
||||
parser.add_argument("--target-rps", type=int, default=250)
|
||||
parser.add_argument("--duration-secs", type=int, default=20)
|
||||
parser.add_argument("--required-windows", type=int, default=3)
|
||||
parser.add_argument("--max-windows", type=int, default=12)
|
||||
parser.add_argument("--quality-tolerance-pct", type=float, default=2)
|
||||
parser.add_argument("--max-spread-pct", type=float, default=5)
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.required_windows <= args.max_windows:
|
||||
parser.error("required windows must be between one and max windows")
|
||||
if not math.isfinite(args.quality_tolerance_pct) or not math.isfinite(args.max_spread_pct):
|
||||
parser.error("tolerances must be finite")
|
||||
if args.target_rps <= 0 or args.duration_secs <= 0 or args.concurrency <= 0:
|
||||
parser.error("target RPS, duration, and concurrency must be positive")
|
||||
if args.quality_tolerance_pct < 0 or args.max_spread_pct < 0:
|
||||
parser.error("tolerances must be non-negative")
|
||||
access_key = os.environ.get(args.access_key_env)
|
||||
secret_key = os.environ.get(args.secret_key_env)
|
||||
if not access_key or not secret_key:
|
||||
parser.error("credential environment variables must be set")
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=False)
|
||||
windows_path = output_dir / "windows.json"
|
||||
gate_path = output_dir / "warmup-gate.json"
|
||||
windows = []
|
||||
for index in range(1, args.max_windows + 1):
|
||||
window_dir = output_dir / f"window-{index:02d}"
|
||||
window_dir.mkdir()
|
||||
log_path = window_dir / "warp.log"
|
||||
benchdata_prefix = window_dir / "warp"
|
||||
command = [
|
||||
args.warp_bin,
|
||||
"put",
|
||||
"--host",
|
||||
",".join(args.host),
|
||||
"--access-key",
|
||||
access_key,
|
||||
"--secret-key",
|
||||
secret_key,
|
||||
"--bucket",
|
||||
args.bucket,
|
||||
"--region",
|
||||
args.region,
|
||||
"--obj.size",
|
||||
args.size,
|
||||
"--concurrent",
|
||||
str(args.concurrency),
|
||||
"--duration",
|
||||
f"{args.duration_secs}s",
|
||||
"--rps-limit",
|
||||
str(args.target_rps),
|
||||
"--host-select",
|
||||
"roundrobin",
|
||||
"--full",
|
||||
"--noclear",
|
||||
"--no-color",
|
||||
"--analyze.v",
|
||||
"--benchdata",
|
||||
str(benchdata_prefix),
|
||||
]
|
||||
try:
|
||||
with log_path.open("w") as log:
|
||||
run = subprocess.run(
|
||||
command,
|
||||
stdout=log,
|
||||
stderr=subprocess.STDOUT,
|
||||
check=False,
|
||||
timeout=args.duration_secs + 60,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
result = {"status": "fail", "reason": "warp_command_timeout", "windows": windows}
|
||||
write_gate(gate_path, result, args)
|
||||
return 1
|
||||
if run.returncode:
|
||||
result = {"status": "fail", "reason": "warp_command_failed", "windows": windows}
|
||||
write_gate(gate_path, result, args)
|
||||
return 1
|
||||
try:
|
||||
raw = warp_raw_quality.parse(Path(f"{benchdata_prefix}.csv.zst"), set(args.host))
|
||||
latency = extract_latency_metrics(Path(args.metrics_extractor), log_path)
|
||||
row = {
|
||||
"successful_puts": raw["successful_puts"],
|
||||
"errors": raw["errors"],
|
||||
"reqps": raw["reqps"],
|
||||
"host_counts": raw["host_counts"],
|
||||
**latency,
|
||||
"raw_quality": str(window_dir / "raw-quality.json"),
|
||||
}
|
||||
validated = adaptive_warmup_gate.validate_window(
|
||||
row,
|
||||
args.target_rps,
|
||||
args.duration_secs,
|
||||
args.concurrency,
|
||||
args.quality_tolerance_pct,
|
||||
)
|
||||
(window_dir / "raw-quality.json").write_text(json.dumps(raw, indent=2, sort_keys=True) + "\n")
|
||||
windows.append(validated)
|
||||
windows_path.write_text(json.dumps(windows, indent=2, sort_keys=True) + "\n")
|
||||
result = adaptive_warmup_gate.evaluate(
|
||||
windows,
|
||||
args.required_windows,
|
||||
args.max_windows,
|
||||
args.max_spread_pct,
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError, csv.Error) as error:
|
||||
result = {"status": "fail", "reason": str(error), "windows": windows}
|
||||
write_gate(gate_path, result, args)
|
||||
return 1
|
||||
write_gate(gate_path, result, args)
|
||||
if result["status"] == "pass":
|
||||
return result_exit_code(result)
|
||||
if result["status"] == "fail":
|
||||
return result_exit_code(result)
|
||||
return WARMUP_UNSTABLE_EXIT
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("adaptive_warmup_gate.py")
|
||||
|
||||
|
||||
def window(avg: float, p90: float, p99: float, *, errors: int = 0, successful_puts: int = 5127) -> dict:
|
||||
per_host, remainder = divmod(successful_puts, 4)
|
||||
return {
|
||||
"successful_puts": successful_puts,
|
||||
"errors": errors,
|
||||
"reqps": 250,
|
||||
"avg_ms": avg,
|
||||
"p90_ms": p90,
|
||||
"p99_ms": p99,
|
||||
"host_counts": {
|
||||
"a": per_host + (remainder > 0),
|
||||
"b": per_host + (remainder > 1),
|
||||
"c": per_host + (remainder > 2),
|
||||
"d": per_host,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class AdaptiveWarmupGateTest(unittest.TestCase):
|
||||
def run_gate(self, rows: list[dict], *extra: str) -> tuple[int, dict]:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
input_path = Path(directory) / "input.json"
|
||||
output_path = Path(directory) / "output.json"
|
||||
input_path.write_text(json.dumps(rows))
|
||||
result = subprocess.run(
|
||||
[str(SCRIPT), "--input", str(input_path), "--output", str(output_path), *extra],
|
||||
check=False,
|
||||
)
|
||||
return result.returncode, json.loads(output_path.read_text())
|
||||
|
||||
def test_three_stable_windows_pass(self) -> None:
|
||||
code, result = self.run_gate([window(14, 20, 100), window(14.2, 20.5, 102), window(13.9, 20.2, 99)])
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(result["status"], "pass")
|
||||
|
||||
def test_unstable_tail_continues_before_limit(self) -> None:
|
||||
code, result = self.run_gate([window(16, 20, 126), window(14, 15, 98), window(13, 15, 70)])
|
||||
self.assertEqual(code, 2)
|
||||
self.assertEqual(result["reason"], "recent_windows_unstable")
|
||||
|
||||
def test_unstable_tail_fails_at_limit(self) -> None:
|
||||
rows = [window(16 if index % 2 == 0 else 13, 20, 126 if index % 2 == 0 else 70) for index in range(12)]
|
||||
code, result = self.run_gate(rows)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(result["reason"], "max_windows_unstable")
|
||||
|
||||
def test_request_error_fails_closed(self) -> None:
|
||||
code, result = self.run_gate([window(14, 20, 100, errors=1)])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(result["status"], "fail")
|
||||
|
||||
def test_exact_scheduled_count_is_valid_without_tail_completions(self) -> None:
|
||||
code, result = self.run_gate(
|
||||
[window(14, 20, 100, successful_puts=5000)] * 3,
|
||||
)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(result["status"], "pass")
|
||||
|
||||
def test_host_count_mismatch_fails_closed(self) -> None:
|
||||
row = window(14, 20, 100)
|
||||
row["host_counts"]["a"] -= 1
|
||||
code, result = self.run_gate([row])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(result["reason"], "warmup host counts do not match successful PUT count")
|
||||
|
||||
def test_non_finite_threshold_fails_closed(self) -> None:
|
||||
code, result = self.run_gate([window(14, 20, 100)], "--max-spread-pct", "inf")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(result["reason"], "warmup numeric arguments must be finite")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODULE_PATH = Path(__file__).with_name("adaptive_warmup_runner.py")
|
||||
sys.path.insert(0, str(MODULE_PATH.parent))
|
||||
SPEC = importlib.util.spec_from_file_location("adaptive_warmup_runner", MODULE_PATH)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
assert SPEC.loader is not None
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
class AdaptiveWarmupRunnerTest(unittest.TestCase):
|
||||
def test_latency_extractor_rejects_unknown_schema(self) -> None:
|
||||
with self.assertRaises((OSError, RuntimeError, ValueError)):
|
||||
MODULE.extract_latency_metrics(Path("/does/not/exist"), Path("/does/not/exist"))
|
||||
|
||||
def test_gate_evaluator_requires_recent_consecutive_windows(self) -> None:
|
||||
stable = {"reqps": 250.0, "avg_ms": 14.0, "p90_ms": 20.0, "p99_ms": 100.0}
|
||||
unstable = {"reqps": 250.0, "avg_ms": 20.0, "p90_ms": 30.0, "p99_ms": 180.0}
|
||||
result = MODULE.adaptive_warmup_gate.evaluate([stable, stable, unstable], 3, 12, 5)
|
||||
self.assertEqual(result["status"], "continue")
|
||||
|
||||
def test_unstable_exhaustion_has_non_retryable_exit_code(self) -> None:
|
||||
result = {"status": "fail", "reason": "max_windows_unstable"}
|
||||
self.assertEqual(MODULE.result_exit_code(result), 95)
|
||||
|
||||
def test_quality_failure_keeps_generic_failure_exit_code(self) -> None:
|
||||
result = {"status": "fail", "reason": "warmup request rate is outside tolerance"}
|
||||
self.assertEqual(MODULE.result_exit_code(result), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import importlib.util
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODULE_PATH = Path(__file__).with_name("warp_raw_quality.py")
|
||||
SPEC = importlib.util.spec_from_file_location("warp_raw_quality", MODULE_PATH)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
assert SPEC.loader is not None
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
HEADER = "thread\top\tn_objects\tbytes\tendpoint\terror\tstart\tend\tduration_ns\n"
|
||||
|
||||
|
||||
def row(thread: int, endpoint: str, start: int, end: int, error: str = "") -> str:
|
||||
def stamp(second: int) -> str:
|
||||
return f"2026-01-01T00:00:{second:02d}Z"
|
||||
|
||||
return f"{thread}\tPUT\t1\t100\t{endpoint}\t{error}\t{stamp(start)}\t{stamp(end)}\t{(end-start)*1_000_000_000}\n"
|
||||
|
||||
|
||||
class WarpRawQualityTest(unittest.TestCase):
|
||||
def fixture(self, error: bool = False) -> Path:
|
||||
directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(directory.cleanup)
|
||||
path = Path(directory.name) / "warp.tsv"
|
||||
content = HEADER
|
||||
content += row(0, "a", 0, 1)
|
||||
content += row(0, "a", 1, 3)
|
||||
content += row(0, "a", 3, 5)
|
||||
content += row(0, "a", 5, 10)
|
||||
content += row(1, "b", 0, 2, "boom" if error else "")
|
||||
content += row(1, "b", 2, 4)
|
||||
content += row(1, "b", 4, 6)
|
||||
content += row(1, "b", 6, 9)
|
||||
path.write_text(content)
|
||||
return path
|
||||
|
||||
def test_common_window_and_host_counts(self) -> None:
|
||||
result = MODULE.parse(self.fixture(), {"a", "b"})
|
||||
self.assertEqual(result["measure_duration_secs"], 3)
|
||||
self.assertEqual(result["completed_objects"], 3)
|
||||
self.assertEqual(result["completed_requests"], 3)
|
||||
self.assertEqual(result["host_counts"], {"a": 4, "b": 4})
|
||||
self.assertAlmostEqual(result["reqps"], 1.0)
|
||||
|
||||
def test_any_raw_error_fails_closed(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "contains request errors"):
|
||||
MODULE.parse(self.fixture(error=True), {"a", "b"})
|
||||
|
||||
def test_expected_hosts_are_exact(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "expected host set"):
|
||||
MODULE.parse(self.fixture(), {"a", "b", "c"})
|
||||
|
||||
def test_timestamp_parser_preserves_nanoseconds(self) -> None:
|
||||
self.assertEqual(
|
||||
MODULE.timestamp_ns("1970-01-01T00:00:01.123456789Z"),
|
||||
1_123_456_789,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Executable
+149
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Recompute fixed-rate Warp quality evidence from raw TSV benchdata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
TIMESTAMP = re.compile(r"^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?Z$")
|
||||
REQUIRED = {"thread", "op", "n_objects", "bytes", "endpoint", "error", "start", "end", "duration_ns"}
|
||||
|
||||
|
||||
def timestamp_ns(value: str) -> int:
|
||||
match = TIMESTAMP.fullmatch(value)
|
||||
if not match:
|
||||
raise ValueError(f"invalid RFC3339Nano timestamp: {value}")
|
||||
import calendar
|
||||
import datetime
|
||||
|
||||
year, month, day, hour, minute, second = map(int, match.groups()[:6])
|
||||
base = calendar.timegm(datetime.datetime(year, month, day, hour, minute, second).timetuple())
|
||||
fraction = (match.group(7) or "").ljust(9, "0")
|
||||
return base * 1_000_000_000 + int(fraction or "0")
|
||||
|
||||
|
||||
def open_text(path: Path):
|
||||
if path.suffix != ".zst":
|
||||
return path.open(newline="")
|
||||
process = subprocess.Popen(
|
||||
["zstd", "--decompress", "--stdout", "--quiet", str(path)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if process.stdout is None:
|
||||
raise RuntimeError("zstd stdout is unavailable")
|
||||
return io.TextIOWrapper(process.stdout, newline=""), process
|
||||
|
||||
|
||||
def parse(path: Path, expected_hosts: set[str] | None) -> dict:
|
||||
opened = open_text(path)
|
||||
process = None
|
||||
if isinstance(opened, tuple):
|
||||
stream, process = opened
|
||||
else:
|
||||
stream = opened
|
||||
try:
|
||||
reader = csv.DictReader((line for line in stream if not line.startswith("#")), delimiter="\t", strict=True)
|
||||
if reader.fieldnames is None or not REQUIRED.issubset(reader.fieldnames):
|
||||
raise ValueError("Warp raw header is missing required fields")
|
||||
rows = []
|
||||
first_end: dict[int, int] = {}
|
||||
last_start: dict[int, int] = {}
|
||||
raw_errors = 0
|
||||
raw_successful_objects = 0
|
||||
host_counts: dict[str, int] = defaultdict(int)
|
||||
for row in reader:
|
||||
if None in row:
|
||||
raise ValueError("Warp raw row width differs from header")
|
||||
if row["op"] != "PUT":
|
||||
raise ValueError("Warp raw input contains a non-PUT operation")
|
||||
thread = int(row["thread"])
|
||||
objects = int(row["n_objects"])
|
||||
size_bytes = int(row["bytes"])
|
||||
duration_ns = int(row["duration_ns"])
|
||||
if objects <= 0 or size_bytes < 0 or duration_ns < 0 or not row["endpoint"]:
|
||||
raise ValueError("Warp raw row contains an invalid numeric or endpoint field")
|
||||
start = timestamp_ns(row["start"])
|
||||
end = timestamp_ns(row["end"])
|
||||
if end < start:
|
||||
raise ValueError("Warp raw operation ends before it starts")
|
||||
error = bool(row["error"])
|
||||
raw_errors += int(error)
|
||||
if not error:
|
||||
raw_successful_objects += objects
|
||||
host_counts[row["endpoint"]] += objects
|
||||
first_end[thread] = min(first_end.get(thread, end), end)
|
||||
last_start[thread] = max(last_start.get(thread, start), start)
|
||||
rows.append((start, end, objects, error))
|
||||
if not rows or raw_errors:
|
||||
raise ValueError("Warp raw input is empty or contains request errors")
|
||||
if expected_hosts is not None and set(host_counts) != expected_hosts:
|
||||
raise ValueError("Warp raw endpoints differ from the expected host set")
|
||||
window_start = max(first_end.values())
|
||||
window_end = min(last_start.values())
|
||||
if window_end <= window_start:
|
||||
raise ValueError("Warp common active window is not positive")
|
||||
completed_objects = sum(objects for _, end, objects, error in rows if not error and window_start <= end < window_end)
|
||||
completed_requests = sum(1 for _, end, _, error in rows if not error and window_start <= end < window_end)
|
||||
fractional_objects = 0.0
|
||||
for start, end, objects, error in rows:
|
||||
if error or end <= start:
|
||||
continue
|
||||
overlap = max(0, min(end, window_end) - max(start, window_start))
|
||||
fractional_objects += objects * overlap / (end - start)
|
||||
duration_secs = (window_end - window_start) / 1_000_000_000
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"source": str(path),
|
||||
"source_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
"operation": "PUT",
|
||||
"threads": len(first_end),
|
||||
"raw_requests": len(rows),
|
||||
"successful_puts": raw_successful_objects,
|
||||
"errors": raw_errors,
|
||||
"host_counts": dict(sorted(host_counts.items())),
|
||||
"completed_objects": completed_objects,
|
||||
"completed_requests": completed_requests,
|
||||
"fractional_objects": fractional_objects,
|
||||
"measure_start_ns": window_start,
|
||||
"measure_end_exclusive_ns": window_end,
|
||||
"measure_duration_secs": duration_secs,
|
||||
"reqps": fractional_objects / duration_secs,
|
||||
"window_status": "warp_v1.8.2_common_active_window",
|
||||
}
|
||||
finally:
|
||||
stream.close()
|
||||
if process is not None:
|
||||
stderr = process.stderr.read().decode() if process.stderr is not None else ""
|
||||
if process.wait() != 0:
|
||||
raise RuntimeError(f"zstd failed: {stderr.strip()}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input", required=True)
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--expected-host", action="append")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = parse(Path(args.input), set(args.expected_host) if args.expected_host else None)
|
||||
except (OSError, RuntimeError, ValueError, csv.Error) as error:
|
||||
Path(args.output).write_text(json.dumps({"schema_version": 1, "status": "fail", "reason": str(error)}, indent=2) + "\n")
|
||||
return 1
|
||||
result["status"] = "pass"
|
||||
Path(args.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user