Compare commits

...

4 Commits

Author SHA1 Message Date
Hauser Zhi d1e706d6f9 fix(rpc): complete rename batch diagnostics coverage
Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-17 21:01:23 +08:00
Hauser Zhi cfe5d2cc72 perf(ecstore): expose rename batch execution counters
Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-17 20:25:38 +08:00
Hauser Zhi 42a8ddb6cf merge: integrate latest main into rename batch work
Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-17 20:20:16 +08:00
Hauser Zhi e94b8637fa perf(ecstore): batch remote rename fanout
Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-17 19:28:53 +08:00
18 changed files with 2456 additions and 283 deletions
@@ -573,6 +573,15 @@ impl NodeService for MinimalLockNodeService {
Err(Status::unimplemented("lock-only test server"))
}
type RenameDataBatchStream = ResponseStream<rustfs_protos::proto_gen::node_service::RenameDataBatchResult>;
async fn rename_data_batch(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataBatchRequest>,
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_multipart_data(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
+424 -5
View File
@@ -73,10 +73,11 @@ use rustfs_protos::proto_gen::node_service::{
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteAllResponse,
WriteFormatControlRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataBatchItem, RenameDataBatchRequest,
RenameDataBatchResult, RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest,
VerifyFileRequest, WriteAllRequest, WriteAllResponse, WriteFormatControlRequest, WriteMetadataRequest,
node_service_client::NodeServiceClient,
};
use rustfs_protos::{
ChannelClass, ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, prewarm_channel_class,
@@ -104,7 +105,7 @@ use tokio::{
time::timeout,
};
use tokio_util::sync::CancellationToken;
use tonic::{Code, Request, service::interceptor::InterceptedService};
use tonic::{Code, Request, Streaming, service::interceptor::InterceptedService};
use tracing::{debug, trace, warn};
use uuid::Uuid;
@@ -1045,6 +1046,7 @@ pub struct RemoteDisk {
/// Optional per-peer gate held through complete bulk-unary decoding and validation.
bulk_body_admission: Option<Arc<Semaphore>>,
bulk_body_wait_timeout: Duration,
rename_admission: Option<Arc<Semaphore>>,
}
/// Shortest gap between two failure warnings of one kind from one remote disk.
@@ -1060,6 +1062,208 @@ struct BulkBodyAdmissionRegistry {
static BULK_BODY_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<BulkBodyAdmissionRegistry>> =
std::sync::LazyLock::new(|| StdMutex::new(BulkBodyAdmissionRegistry::default()));
const ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH: &str = "RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH";
const RENAME_ADMISSION_REGISTRY_CAPACITY: usize = 512;
#[derive(Default)]
struct RenameAdmissionRegistry {
peers: HashMap<String, Weak<Semaphore>>,
overflow: Option<Arc<Semaphore>>,
}
static RENAME_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<RenameAdmissionRegistry>> =
std::sync::LazyLock::new(|| StdMutex::new(RenameAdmissionRegistry::default()));
fn peer_rename_admission(addr: &str, enabled: bool) -> Option<Arc<Semaphore>> {
if !enabled {
return None;
}
let mut registry = RENAME_ADMISSION_REGISTRY
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if let Some(semaphore) = registry.peers.get(addr).and_then(Weak::upgrade) {
return Some(semaphore);
}
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
registry.peers.retain(|_, semaphore| semaphore.strong_count() > 0);
}
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
return Some(Arc::clone(registry.overflow.get_or_insert_with(|| {
Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT))
})));
}
let semaphore = Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT));
registry.peers.insert(addr.to_owned(), Arc::downgrade(&semaphore));
Some(semaphore)
}
struct RenameAdmissionGrant {
permit: Option<OwnedSemaphorePermit>,
}
impl RenameAdmissionGrant {
fn release_one(&mut self) -> bool {
self.permit.as_mut().is_some_and(|permit| permit.split(1).is_some())
}
fn release_all(mut self) {
drop(self.permit.take());
}
fn quarantine_until(mut self, deadline: tokio::time::Instant) {
let Some(permit) = self.permit.take() else {
return;
};
if let Ok(handle) = tokio::runtime::Handle::try_current() {
handle.spawn(async move {
tokio::time::sleep_until(deadline).await;
drop(permit);
});
} else {
// Without a runtime there is no safe deadline task. Leaking until
// process exit is safer than admitting work over a still-running mutation.
std::mem::forget(permit);
}
}
}
struct RenameBatchCallGuard {
admission: Option<RenameAdmissionGrant>,
deadline: tokio::time::Instant,
quarantine_on_drop: bool,
}
impl RenameBatchCallGuard {
fn complete(mut self) {
self.quarantine_on_drop = false;
if let Some(admission) = self.admission.take() {
admission.release_all();
}
}
fn release_for_fallback(mut self) {
self.quarantine_on_drop = false;
if let Some(admission) = self.admission.take() {
admission.release_all();
}
}
fn into_stream(mut self, inner: Streaming<RenameDataBatchResult>, expected_mask: u16) -> RemoteRenameBatchStream {
self.quarantine_on_drop = false;
RemoteRenameBatchStream {
inner,
expected_mask,
seen_mask: 0,
admission: self.admission.take(),
deadline: self.deadline,
terminal: false,
}
}
}
impl Drop for RenameBatchCallGuard {
fn drop(&mut self) {
if self.quarantine_on_drop
&& let Some(admission) = self.admission.take()
{
admission.quarantine_until(self.deadline);
}
}
}
pub(crate) struct RenameBatchClientItem<'a> {
pub slot: usize,
pub disk: String,
pub file_info: &'a FileInfo,
}
pub(crate) enum RenameBatchStartError {
NotDispatched(Error),
UnsupportedBeforeStream,
MayHavePublished(Error),
}
pub(crate) struct RemoteRenameBatchOutcome {
pub slot: usize,
pub result: Result<RenameDataResp>,
}
pub(crate) struct RemoteRenameBatchStream {
inner: Streaming<RenameDataBatchResult>,
expected_mask: u16,
seen_mask: u16,
admission: Option<RenameAdmissionGrant>,
deadline: tokio::time::Instant,
terminal: bool,
}
impl RemoteRenameBatchStream {
fn terminate_ambiguous(&mut self, error: Error) -> Error {
if !self.terminal {
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
}
self.terminal = true;
if let Some(admission) = self.admission.take() {
admission.quarantine_until(self.deadline);
}
error
}
pub(crate) async fn next_outcome(&mut self) -> Result<Option<RemoteRenameBatchOutcome>> {
if self.terminal {
return Ok(None);
}
let response = match tokio::time::timeout_at(self.deadline, self.inner.message()).await {
Err(_) => return Err(self.terminate_ambiguous(DiskError::Timeout)),
Ok(Err(status)) => return Err(self.terminate_ambiguous(status.into())),
Ok(Ok(None)) if self.seen_mask == self.expected_mask => {
self.terminal = true;
return Ok(None);
}
Ok(Ok(None)) => return Err(self.terminate_ambiguous(DiskError::Unexpected)),
Ok(Ok(Some(response))) => response,
};
let Ok(slot) = usize::try_from(response.item_index) else {
return Err(self.terminate_ambiguous(DiskError::Unexpected));
};
if slot >= 16 {
return Err(self.terminate_ambiguous(DiskError::Unexpected));
}
let bit = 1_u16 << slot;
if self.expected_mask & bit == 0 || self.seen_mask & bit != 0 {
return Err(self.terminate_ambiguous(DiskError::Unexpected));
}
self.seen_mask |= bit;
if self.admission.as_mut().is_some_and(|admission| !admission.release_one()) {
return Err(self.terminate_ambiguous(DiskError::Unexpected));
}
rustfs_io_metrics::record_ecstore_rename_batch_item_completed();
let result = if response.success {
if response.error.is_some() {
Err(DiskError::FileCorrupt)
} else {
decode_msgpack_strict(&response.rename_data_resp_msgpack)
}
} else {
Err(response.error.unwrap_or_default().into())
};
Ok(Some(RemoteRenameBatchOutcome { slot, result }))
}
}
impl Drop for RemoteRenameBatchStream {
fn drop(&mut self) {
if !self.terminal {
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
}
if let Some(admission) = self.admission.take() {
admission.quarantine_until(self.deadline);
}
}
}
struct BulkBodyPermit {
_metrics: rustfs_io_metrics::internode_metrics::InternodeBulkBodyAdmissionGuard,
_permit: OwnedSemaphorePermit,
@@ -1226,6 +1430,10 @@ fn spawn_control_channel_prewarm(addr: String) {
}
impl RemoteDisk {
pub(crate) fn rename_stream_batch_enabled(&self) -> bool {
self.rename_admission.is_some()
}
async fn guarded_delete_versions_rpc(
&self,
volume: &str,
@@ -1551,6 +1759,8 @@ impl RemoteDisk {
rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
let bulk_body_admission = peer_bulk_body_admission(&addr, bulk_body_concurrency_limit()?);
let bulk_body_wait_timeout = bulk_body_wait_timeout();
let rename_admission =
peer_rename_admission(&addr, rustfs_utils::get_env_bool(ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH, false));
let disk = Self {
handle_id: Uuid::new_v4(),
@@ -1573,6 +1783,7 @@ impl RemoteDisk {
network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
bulk_body_admission,
bulk_body_wait_timeout,
rename_admission,
};
record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
@@ -2494,6 +2705,29 @@ impl RemoteDisk {
}))
}
async fn acquire_rename_admission_until(
&self,
item_count: usize,
deadline: tokio::time::Instant,
) -> Result<Option<RenameAdmissionGrant>> {
let Some(semaphore) = &self.rename_admission else {
return Ok(None);
};
if item_count == 0 || item_count > rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
return Err(Error::other("invalid internode rename admission weight"));
}
if let Some(err) = self.offline_bypass_error() {
return Err(err);
}
let weight = u32::try_from(item_count).map_err(Error::other)?;
let permit = match tokio::time::timeout_at(deadline, Arc::clone(semaphore).acquire_many_owned(weight)).await {
Ok(Ok(permit)) => permit,
Ok(Err(_)) => return Err(Error::RemoteClientUnavailable("internode rename admission closed".to_owned())),
Err(_) => return Err(DiskError::DiskOngoingReq),
};
Ok(Some(RenameAdmissionGrant { permit: Some(permit) }))
}
async fn get_bulk_client(&self) -> Result<NodeRpcClient> {
if let Some(err) = self.offline_bypass_error() {
return Err(err);
@@ -2634,6 +2868,18 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_
}
}
fn decode_msgpack_strict<T: DeserializeOwned>(binary: &[u8]) -> Result<T> {
if binary.is_empty() {
return Err(DiskError::FileCorrupt);
}
let mut cursor = Cursor::new(binary);
let value = rmp_serde::from_read(&mut cursor).map_err(|_| DiskError::FileCorrupt)?;
if cursor.position() != u64::try_from(binary.len()).map_err(Error::other)? {
return Err(DiskError::FileCorrupt);
}
Ok(value)
}
fn decode_multipart_publication_response(binary: &[u8]) -> Result<RenameDataResp> {
let result = if binary.len() > MultipartSourceOperation::max_wire_payload_bytes() {
Err(DiskError::FileCorrupt)
@@ -2831,6 +3077,91 @@ impl RemoteDisk {
.await
}
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn start_rename_data_batch(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
items: Vec<RenameBatchClientItem<'_>>,
) -> std::result::Result<RemoteRenameBatchStream, RenameBatchStartError> {
if self.rename_admission.is_none() || items.is_empty() || items.len() > 16 {
return Err(RenameBatchStartError::NotDispatched(DiskError::UnsupportedDisk));
}
let deadline = tokio::time::Instant::now() + get_max_timeout_duration();
let mut expected_mask = 0_u16;
let mut request_items = Vec::with_capacity(items.len());
for item in items {
if item.slot >= 16 {
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
}
let bit = 1_u16 << item.slot;
if expected_mask & bit != 0 {
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
}
expected_mask |= bit;
let item_index = u32::try_from(item.slot)
.map_err(Error::other)
.map_err(RenameBatchStartError::NotDispatched)?;
let file_info_msgpack = encode_file_info_msgpack(item.file_info).map_err(RenameBatchStartError::NotDispatched)?;
request_items.push(RenameDataBatchItem {
item_index,
disk: item.disk,
file_info_msgpack: file_info_msgpack.into(),
});
}
let mut request = Request::new(RenameDataBatchRequest {
src_volume: src_volume.to_owned(),
src_path: src_path.to_owned(),
dst_volume: dst_volume.to_owned(),
dst_path: dst_path.to_owned(),
items: request_items,
});
let canonical_body = rustfs_protos::canonical_rename_data_batch_request_body(request.get_ref())
.map_err(Error::other)
.map_err(RenameBatchStartError::NotDispatched)?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body)
.map_err(Error::other)
.map_err(RenameBatchStartError::NotDispatched)?;
let mut client = match tokio::time::timeout_at(deadline, self.get_client()).await {
Ok(result) => result.map_err(RenameBatchStartError::NotDispatched)?,
Err(_) => return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout)),
};
let admission = self
.acquire_rename_admission_until(request.get_ref().items.len(), deadline)
.await
.map_err(RenameBatchStartError::NotDispatched)?;
let guard = RenameBatchCallGuard {
admission,
deadline,
quarantine_on_drop: true,
};
let call_timeout = deadline.saturating_duration_since(tokio::time::Instant::now());
let response = match self
.execute_with_timeout_for_op(
"rename_data_batch",
|| async {
match client.rename_data_batch(request).await {
Ok(response) => Ok(response),
Err(status) if status.code() == Code::Unimplemented => Err(DiskError::UnsupportedDisk),
Err(status) => Err(status.into()),
}
},
call_timeout,
)
.await
{
Err(DiskError::UnsupportedDisk) => {
guard.release_for_fallback();
return Err(RenameBatchStartError::UnsupportedBeforeStream);
}
Err(error) => return Err(RenameBatchStartError::MayHavePublished(error)),
Ok(response) => response,
};
Ok(guard.into_stream(response.into_inner(), expected_mask))
}
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn rename_data_borrowed_with_fence(
&self,
@@ -2902,6 +3233,27 @@ impl RemoteDisk {
let scanner_publication_lease = scanner_publication_lease_token
.map(|token| Bytes::copy_from_slice(token.as_bytes()))
.unwrap_or_default();
let rename_deadline = tokio::time::Instant::now() + get_max_timeout_duration();
let rename_admission = self.acquire_rename_admission_until(1, rename_deadline).await?;
let rename_admission_guard = RenameBatchCallGuard {
admission: rename_admission,
deadline: rename_deadline,
quarantine_on_drop: true,
};
let rpc_kind = if multipart_rename {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_MULTIPART
} else if pool_meta_cas {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_POOL_META_CAS
} else if quota_fenced {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE
} else if incarnation_bound {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_INCARNATION
} else if scanner_publication_lease_token.is_some() {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE
} else {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_ORDINARY
};
rustfs_io_metrics::record_ecstore_rename_remote_rpc(rpc_kind);
let rpc_started = rustfs_io_metrics::put_stage_timer();
let mut retried_dual_write = false;
let response = loop {
@@ -2988,6 +3340,7 @@ impl RemoteDisk {
Err(status) => return Err(status.into()),
}
};
rename_admission_guard.complete();
let response_process_started = rustfs_io_metrics::put_stage_timer();
let response_headers_started = rustfs_io_metrics::put_stage_timer();
let _ = response.metadata().len();
@@ -12963,6 +13316,72 @@ mod tests {
.expect("shared gate remains open");
}
#[tokio::test]
async fn rename_admission_is_weighted_shared_and_released_per_item() {
use std::future::Future;
use std::task::{Context, Poll, Waker};
let first = peer_rename_admission("http://rename-peer-a", true).expect("enabled rename gate");
let same = peer_rename_admission("http://rename-peer-a", true).expect("same peer rename gate");
let other = peer_rename_admission("http://rename-peer-b", true).expect("other peer rename gate");
assert!(Arc::ptr_eq(&first, &same));
assert!(!Arc::ptr_eq(&first, &other));
assert!(peer_rename_admission("http://rename-disabled", false).is_none());
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
let permit = Arc::clone(&first)
.acquire_many_owned(4)
.await
.expect("weighted batch admission");
let mut grant = RenameAdmissionGrant { permit: Some(permit) };
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
assert!(grant.release_one());
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
drop(grant);
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
let held = Arc::clone(&first)
.acquire_many_owned(u32::try_from(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3).expect("small limit"))
.await
.expect("reserve all but three permits");
let mut waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
assert!(matches!(waiting.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
assert_eq!(first.available_permits(), 0, "the FIFO waiter reserves currently available permits");
drop(waiting);
assert_eq!(
first.available_permits(),
3,
"cancelling a queued weighted acquire must return its reservation"
);
let waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
drop(held);
let acquired = tokio::time::timeout(Duration::from_secs(1), waiting)
.await
.expect("weighted waiter wakes")
.expect("rename gate remains open");
drop(acquired);
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
}
#[tokio::test(start_paused = true)]
async fn rename_admission_quarantine_holds_until_absolute_deadline() {
let gate = peer_rename_admission("http://rename-quarantine-peer", true).expect("enabled rename gate");
let permit = Arc::clone(&gate)
.acquire_many_owned(4)
.await
.expect("weighted batch admission");
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
RenameAdmissionGrant { permit: Some(permit) }.quarantine_until(deadline);
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
tokio::time::advance(Duration::from_millis(999)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
tokio::time::advance(Duration::from_millis(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
}
#[tokio::test]
async fn remote_disk_builds_optional_complete_body_gate_from_configuration() {
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, Some("3"))], async {
File diff suppressed because it is too large Load Diff
+6
View File
@@ -205,6 +205,12 @@ fn heal_format_fence_lost_error() -> Error {
}
impl ECStore {
/// Whether this exact storage instance requires an incarnation-bound write
/// for the object currently owned by a bucket-heal scope.
pub fn object_requires_incarnation(&self, bucket: &str, object: &str) -> bool {
bucket_heal_scope_for_object(bucket, object).is_some_and(|scope| std::ptr::eq(Arc::as_ptr(&scope.store), self))
}
pub async fn purge_delete_marker_with_proof(
self: &Arc<Self>,
bucket: &str,
+200
View File
@@ -449,6 +449,73 @@ pub const PUT_RENAME_QUORUM_FANOUT_STATE_PANIC: &str = "panic";
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST: &str = "quorum_first";
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL: &str = "quorum_tail";
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR: &str = "error";
pub const ECSTORE_RENAME_RPC_KIND_ORDINARY: &str = "ordinary";
pub const ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE: &str = "quota_fence";
pub const ECSTORE_RENAME_RPC_KIND_INCARNATION: &str = "incarnation";
pub const ECSTORE_RENAME_RPC_KIND_MULTIPART: &str = "multipart";
pub const ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE: &str = "scanner_lease";
pub const ECSTORE_RENAME_RPC_KIND_POOL_META_CAS: &str = "pool_meta_cas";
pub const ECSTORE_RENAME_RPC_KINDS: [&str; 6] = [
ECSTORE_RENAME_RPC_KIND_ORDINARY,
ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE,
ECSTORE_RENAME_RPC_KIND_INCARNATION,
ECSTORE_RENAME_RPC_KIND_MULTIPART,
ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE,
ECSTORE_RENAME_RPC_KIND_POOL_META_CAS,
];
pub const ECSTORE_RENAME_PEER_WIDTHS: [&str; 18] = [
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "gt16",
];
pub const ECSTORE_RENAME_COVERAGE: [&str; 3] = ["none", "partial", "full"];
pub const ECSTORE_RENAME_BATCH_EXECUTION: [&str; 10] = [
"planned_groups",
"planned_items",
"stream_groups",
"stream_items",
"completed_items",
"fallback_groups",
"fallback_items",
"not_dispatched_items",
"may_have_published_items",
"terminal_missing_items",
];
static ECSTORE_RENAME_RPC_COUNTS: [AtomicU64; 6] = [const { AtomicU64::new(0) }; 6];
static ECSTORE_RENAME_REMOTE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
static ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
static ECSTORE_RENAME_COVERAGE_COUNTS: [AtomicU64; 3] = [const { AtomicU64::new(0) }; 3];
static ECSTORE_RENAME_BATCH_EXECUTION_COUNTS: [AtomicU64; 10] = [const { AtomicU64::new(0) }; 10];
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct EcstoreRenameDiagnosticsSnapshot {
pub rpc_counts: [u64; 6],
pub remote_width_counts: [u64; 18],
pub batchable_width_counts: [u64; 18],
pub coverage_counts: [u64; 3],
pub batch_execution_counts: [u64; 10],
}
impl EcstoreRenameDiagnosticsSnapshot {
pub fn is_empty(&self) -> bool {
self.rpc_counts
.iter()
.chain(self.remote_width_counts.iter())
.chain(self.batchable_width_counts.iter())
.chain(self.coverage_counts.iter())
.chain(self.batch_execution_counts.iter())
.all(|count| *count == 0)
}
}
pub fn snapshot_ecstore_rename_diagnostics() -> EcstoreRenameDiagnosticsSnapshot {
EcstoreRenameDiagnosticsSnapshot {
rpc_counts: std::array::from_fn(|index| ECSTORE_RENAME_RPC_COUNTS[index].load(Ordering::Relaxed)),
remote_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
batchable_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
coverage_counts: std::array::from_fn(|index| ECSTORE_RENAME_COVERAGE_COUNTS[index].load(Ordering::Relaxed)),
batch_execution_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].load(Ordering::Relaxed)),
}
}
#[inline(always)]
pub fn get_stage_metrics_enabled() -> bool {
@@ -2663,6 +2730,91 @@ pub fn record_put_rename_quorum_wait_fanout(
}
}
/// Records a rename RPC after its canonical request digest has been attached.
///
/// This uses the PUT-stage diagnostics switch because it is temporary hot-path
/// evidence, but the metric is ECStore-scoped: scanner and pool metadata
/// renames are not S3 PutObject operations.
#[inline(always)]
pub fn record_ecstore_rename_remote_rpc(kind: &'static str) {
if !put_stage_metrics_enabled() {
return;
}
let Some(index) = ECSTORE_RENAME_RPC_KINDS.iter().position(|candidate| *candidate == kind) else {
return;
};
ECSTORE_RENAME_RPC_COUNTS[index].fetch_add(1, Ordering::Relaxed);
}
#[inline(always)]
fn rename_peer_width_index(width: usize) -> usize {
width.min(17)
}
/// Records the per-peer opportunity for a future body-bound rename batch.
#[inline(always)]
pub fn record_ecstore_rename_batch_candidate_peer(remote_disks: usize, batchable_disks: usize) {
if !put_stage_metrics_enabled() || remote_disks == 0 {
return;
}
ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[rename_peer_width_index(remote_disks)].fetch_add(1, Ordering::Relaxed);
ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[rename_peer_width_index(batchable_disks)].fetch_add(1, Ordering::Relaxed);
let coverage_index = if batchable_disks == 0 {
0
} else if batchable_disks == remote_disks {
2
} else {
1
};
ECSTORE_RENAME_COVERAGE_COUNTS[coverage_index].fetch_add(1, Ordering::Relaxed);
}
#[inline(always)]
fn record_ecstore_rename_batch_execution(index: usize, count: usize) {
if put_stage_metrics_enabled() {
let count = u64::try_from(count).unwrap_or(u64::MAX);
ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].fetch_add(count, Ordering::Relaxed);
}
}
#[inline(always)]
pub fn record_ecstore_rename_batch_planned(items: usize) {
record_ecstore_rename_batch_execution(0, 1);
record_ecstore_rename_batch_execution(1, items);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_stream_started(items: usize) {
record_ecstore_rename_batch_execution(2, 1);
record_ecstore_rename_batch_execution(3, items);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_item_completed() {
record_ecstore_rename_batch_execution(4, 1);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_fallback(items: usize) {
record_ecstore_rename_batch_execution(5, 1);
record_ecstore_rename_batch_execution(6, items);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_not_dispatched(items: usize) {
record_ecstore_rename_batch_execution(7, items);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_may_have_published(items: usize) {
record_ecstore_rename_batch_execution(8, items);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_terminal_missing(items: usize) {
record_ecstore_rename_batch_execution(9, items);
}
/// Record generic internal operation stage duration (non-PUT paths).
/// Use this for metacache walks, listing, lifecycle, and other background
/// operations that are NOT part of the PUT object hot path.
@@ -4585,6 +4737,54 @@ mod tests {
);
}
#[test]
fn ecstore_rename_batch_candidate_metrics_are_gated_and_bounded() {
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
set_put_stage_metrics_enabled(false);
let disabled_before = snapshot_ecstore_rename_diagnostics();
record_ecstore_rename_remote_rpc(ECSTORE_RENAME_RPC_KIND_ORDINARY);
record_ecstore_rename_batch_candidate_peer(4, 4);
assert_eq!(snapshot_ecstore_rename_diagnostics(), disabled_before);
set_put_stage_metrics_enabled(true);
let before = snapshot_ecstore_rename_diagnostics();
for kind in ECSTORE_RENAME_RPC_KINDS {
record_ecstore_rename_remote_rpc(kind);
}
record_ecstore_rename_batch_candidate_peer(4, 4);
record_ecstore_rename_batch_candidate_peer(4, 3);
record_ecstore_rename_batch_candidate_peer(4, 0);
record_ecstore_rename_batch_candidate_peer(20, 20);
record_ecstore_rename_batch_planned(4);
record_ecstore_rename_batch_stream_started(4);
record_ecstore_rename_batch_item_completed();
record_ecstore_rename_batch_fallback(3);
record_ecstore_rename_batch_not_dispatched(2);
record_ecstore_rename_batch_may_have_published(4);
record_ecstore_rename_batch_terminal_missing(1);
set_put_stage_metrics_enabled(false);
let after = snapshot_ecstore_rename_diagnostics();
for index in 0..ECSTORE_RENAME_RPC_KINDS.len() {
assert_eq!(after.rpc_counts[index] - before.rpc_counts[index], 1);
}
assert_eq!(after.remote_width_counts[4] - before.remote_width_counts[4], 3);
assert_eq!(after.remote_width_counts[17] - before.remote_width_counts[17], 1);
assert_eq!(after.batchable_width_counts[0] - before.batchable_width_counts[0], 1);
assert_eq!(after.batchable_width_counts[3] - before.batchable_width_counts[3], 1);
assert_eq!(after.batchable_width_counts[4] - before.batchable_width_counts[4], 1);
assert_eq!(after.batchable_width_counts[17] - before.batchable_width_counts[17], 1);
assert_eq!(after.coverage_counts[0] - before.coverage_counts[0], 1);
assert_eq!(after.coverage_counts[1] - before.coverage_counts[1], 1);
assert_eq!(after.coverage_counts[2] - before.coverage_counts[2], 2);
assert_eq!(
std::array::from_fn::<_, 10, _>(|index| {
after.batch_execution_counts[index] - before.batch_execution_counts[index]
}),
[1, 4, 1, 4, 1, 1, 3, 2, 4, 1]
);
}
#[test]
fn test_put_object_diagnostic_buckets() {
assert_eq!(put_object_size_bucket(0), "unknown");
@@ -505,6 +505,45 @@ pub struct RenameDataResponse {
#[prost(bytes = "bytes", tag = "4")]
pub rename_data_resp_bin: ::prost::bytes::Bytes,
}
/// Ordinary rename transport batch. Protected rename variants intentionally use
/// their dedicated fail-closed RPCs and are never admitted here.
#[allow(clippy::all)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RenameDataBatchItem {
#[prost(uint32, tag = "1")]
pub item_index: u32,
#[prost(string, tag = "2")]
pub disk: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "3")]
pub file_info_msgpack: ::prost::bytes::Bytes,
}
#[allow(clippy::all)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct RenameDataBatchRequest {
#[prost(string, tag = "1")]
pub src_volume: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub src_path: ::prost::alloc::string::String,
#[prost(string, tag = "3")]
pub dst_volume: ::prost::alloc::string::String,
#[prost(string, tag = "4")]
pub dst_path: ::prost::alloc::string::String,
#[prost(message, repeated, tag = "5")]
pub items: ::prost::alloc::vec::Vec<RenameDataBatchItem>,
}
#[allow(clippy::all)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
#[prost(skip_debug)]
pub struct RenameDataBatchResult {
#[prost(uint32, tag = "1")]
pub item_index: u32,
#[prost(bool, tag = "2")]
pub success: bool,
#[prost(message, optional, tag = "3")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", tag = "4")]
pub rename_data_resp_msgpack: ::prost::bytes::Bytes,
}
/// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
/// accepts an arbitrary volume or object path.
#[allow(clippy::all)]
@@ -2894,6 +2933,21 @@ pub mod node_service_client {
.insert(GrpcMethod::new("node_service.NodeService", "RenameData"));
self.inner.unary(req, path, codec).await
}
pub async fn rename_data_batch(
&mut self,
request: impl tonic::IntoRequest<super::RenameDataBatchRequest>,
) -> std::result::Result<tonic::Response<tonic::codec::Streaming<super::RenameDataBatchResult>>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameDataBatch");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "RenameDataBatch"));
self.inner.server_streaming(req, path, codec).await
}
/// The method itself requires source-preserving multipart publication on every peer.
pub async fn rename_multipart_data(
&mut self,
@@ -4252,6 +4306,14 @@ pub mod node_service_server {
&self,
request: tonic::Request<super::RenameDataRequest>,
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status>;
/// Server streaming response type for the RenameDataBatch method.
type RenameDataBatchStream: tonic::codegen::tokio_stream::Stream<Item = std::result::Result<super::RenameDataBatchResult, tonic::Status>>
+ std::marker::Send
+ 'static;
async fn rename_data_batch(
&self,
request: tonic::Request<super::RenameDataBatchRequest>,
) -> std::result::Result<tonic::Response<Self::RenameDataBatchStream>, tonic::Status>;
/// The method itself requires source-preserving multipart publication on every peer.
async fn rename_multipart_data(
&self,
@@ -5573,6 +5635,35 @@ pub mod node_service_server {
};
Box::pin(fut)
}
"/node_service.NodeService/RenameDataBatch" => {
#[allow(non_camel_case_types)]
struct RenameDataBatchSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::ServerStreamingService<super::RenameDataBatchRequest> for RenameDataBatchSvc<T> {
type Response = super::RenameDataBatchResult;
type ResponseStream = T::RenameDataBatchStream;
type Future = BoxFuture<tonic::Response<Self::ResponseStream>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::RenameDataBatchRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::rename_data_batch(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = RenameDataBatchSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.server_streaming(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/RenameMultipartData" => {
#[allow(non_camel_case_types)]
struct RenameMultipartDataSvc<T: NodeService>(pub Arc<T>);
+80 -5
View File
@@ -67,6 +67,18 @@ impl std::fmt::Debug for proto_gen::node_service::RenameDataResponse {
}
}
impl std::fmt::Debug for proto_gen::node_service::RenameDataBatchResult {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("RenameDataBatchResult")
.field("item_index", &self.item_index)
.field("success", &self.success)
.field("error", &self.error)
.field("rename_data_resp_msgpack", &"[REDACTED]")
.finish()
}
}
impl std::fmt::Debug for proto_gen::node_service::RecoverMultipartSourceRequest {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
@@ -1171,6 +1183,23 @@ pub fn canonical_rename_data_request_body(
Ok(body.finish())
}
pub fn canonical_rename_data_batch_request_body(
request: &proto_gen::node_service::RenameDataBatchRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-rename-data-batch-request-v1\0");
body.push_str(&request.src_volume)?;
body.push_str(&request.src_path)?;
body.push_str(&request.dst_volume)?;
body.push_str(&request.dst_path)?;
body.push_count(request.items.len())?;
for item in &request.items {
body.push_u32(item.item_index);
body.push_str(&item.disk)?;
body.push_bytes(&item.file_info_msgpack)?;
}
Ok(body.finish())
}
/// The fixed-MPU recovery method authenticates every wire field in its own domain.
pub fn canonical_recover_multipart_source_request_body(
request: &proto_gen::node_service::RecoverMultipartSourceRequest,
@@ -1690,9 +1719,10 @@ pub fn canonical_make_volumes_request_body(
mod disk_mutation_canonical_tests {
use super::proto_gen::node_service::{
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
DeleteVolumeRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, RenameDataRequest,
RenameFileRequest, RenamePartRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
DeleteVolumeRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, RenameDataBatchItem,
RenameDataBatchRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, SettlePartTransactionRequest,
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, UpdateMetadataRequest, WriteAllRequest,
WriteMetadataRequest,
};
use super::*;
@@ -1844,6 +1874,51 @@ mod disk_mutation_canonical_tests {
);
}
#[test]
fn rename_data_batch_canonical_body_binds_every_field_and_item_order() {
let baseline = RenameDataBatchRequest {
src_volume: "src-vol".into(),
src_path: "src-path".into(),
dst_volume: "dst-vol".into(),
dst_path: "dst-path".into(),
items: vec![
RenameDataBatchItem {
item_index: 1,
disk: "disk-a".into(),
file_info_msgpack: vec![0x81, 0x01].into(),
},
RenameDataBatchItem {
item_index: 2,
disk: "disk-b".into(),
file_info_msgpack: vec![0x81, 0x02].into(),
},
],
};
let mut bodies = vec![canonical_rename_data_batch_request_body(&baseline).expect("baseline batch body")];
for mutate in [
|r: &mut RenameDataBatchRequest| r.src_volume = "src-vol2".into(),
|r: &mut RenameDataBatchRequest| r.src_path = "src-path2".into(),
|r: &mut RenameDataBatchRequest| r.dst_volume = "dst-vol2".into(),
|r: &mut RenameDataBatchRequest| r.dst_path = "dst-path2".into(),
|r: &mut RenameDataBatchRequest| r.items[0].item_index = 3,
|r: &mut RenameDataBatchRequest| r.items[0].disk = "disk-c".into(),
|r: &mut RenameDataBatchRequest| r.items[0].file_info_msgpack = vec![0x81, 0x03].into(),
|r: &mut RenameDataBatchRequest| r.items.swap(0, 1),
|r: &mut RenameDataBatchRequest| {
r.items.pop();
},
] {
let mut changed = baseline.clone();
mutate(&mut changed);
bodies.push(canonical_rename_data_batch_request_body(&changed).expect("changed batch body"));
}
assert_all_distinct(&bodies);
assert_ne!(
canonical_rename_data_batch_request_body(&baseline).expect("batch body"),
canonical_rename_data_request_body(&RenameDataRequest::default()).expect("unary body"),
);
}
#[test]
fn delete_version_canonical_body_binds_every_field() {
let baseline = DeleteVersionRequest {
@@ -3655,7 +3730,7 @@ fn complete_pending_rebuild_on_cache_hit(dial_addr: &str, cache_key: &str) -> bo
}
// Limits transport response futures; client RPC metrics separately cover unary body decoding.
const INTERNODE_RPC_CONCURRENCY_LIMIT: usize = 64;
pub const INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT: usize = 64;
// Quorum reads cancel surplus metadata RPCs in bursts. Keep bounded reset state
// for their in-flight replies without spending the connection's protocol-error
@@ -4109,7 +4184,7 @@ async fn dial_channel(
rustfs_config::DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT,
))?
} else {
INTERNODE_RPC_CONCURRENCY_LIMIT
INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT
};
let stream_window = checked_rpc_window(stream_window)?;
let conn_window = checked_rpc_window(conn_window)?;
+1
View File
@@ -69,6 +69,7 @@ fn main() -> Result<(), AnyError> {
// Their hand-written Debug implementations redact the complete payload.
.skip_debug([
".node_service.RenameDataResponse",
".node_service.RenameDataBatchResult",
".node_service.RecoverMultipartSourceRequest",
".node_service.RecoverMultipartSourceResponse",
])
+24
View File
@@ -345,6 +345,29 @@ message RenameDataResponse {
bytes rename_data_resp_bin = 4;
}
// Ordinary rename transport batch. Protected rename variants intentionally use
// their dedicated fail-closed RPCs and are never admitted here.
message RenameDataBatchItem {
uint32 item_index = 1;
string disk = 2;
bytes file_info_msgpack = 3;
}
message RenameDataBatchRequest {
string src_volume = 1;
string src_path = 2;
string dst_volume = 3;
string dst_path = 4;
repeated RenameDataBatchItem items = 5;
}
message RenameDataBatchResult {
uint32 item_index = 1;
bool success = 2;
optional Error error = 3;
bytes rename_data_resp_msgpack = 4;
}
// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
// accepts an arbitrary volume or object path.
message PoolMetaNativeRecoveryRequest {
@@ -1394,6 +1417,7 @@ service NodeService {
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
rpc RenameDataBatch(RenameDataBatchRequest) returns (stream RenameDataBatchResult) {}; // auth-policy: body-bound
// The method itself requires source-preserving multipart publication on every peer.
rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
+8 -2
View File
@@ -72,7 +72,8 @@ fn init_get_stage_local_summary_runtime(ctx: CancellationToken) {
_ = ctx.cancelled() => break,
_ = interval.tick() => {
let rows = rustfs_io_metrics::take_get_stage_local_summary();
if rows.is_empty() {
let rename = rustfs_io_metrics::snapshot_ecstore_rename_diagnostics();
if rows.is_empty() && rename.is_empty() {
continue;
}
let summary = rows.iter().map(|row| {
@@ -83,9 +84,14 @@ fn init_get_stage_local_summary_runtime(ctx: CancellationToken) {
.unwrap_or_default()
.as_millis();
let line = format!(
"timestamp_ms={timestamp_ms}\tsample_rate={}\tstage_count={}\tstages={summary}\n",
"timestamp_ms={timestamp_ms}\tsample_rate={}\tstage_count={}\tstages={summary}\trename_diag_version=2\trename_rpc={}\trename_remote_width={}\trename_batchable_width={}\trename_coverage={}\trename_batch_execution={}\n",
rustfs_io_metrics::get_stage_local_summary_sample_rate(),
rows.len(),
rustfs_io_metrics::ECSTORE_RENAME_RPC_KINDS.iter().zip(rename.rpc_counts).map(|(label, count)| format!("{label}:{count}")).collect::<Vec<_>>().join(","),
rustfs_io_metrics::ECSTORE_RENAME_PEER_WIDTHS.iter().zip(rename.remote_width_counts).map(|(label, count)| format!("{label}:{count}")).collect::<Vec<_>>().join(","),
rustfs_io_metrics::ECSTORE_RENAME_PEER_WIDTHS.iter().zip(rename.batchable_width_counts).map(|(label, count)| format!("{label}:{count}")).collect::<Vec<_>>().join(","),
rustfs_io_metrics::ECSTORE_RENAME_COVERAGE.iter().zip(rename.coverage_counts).map(|(label, count)| format!("{label}:{count}")).collect::<Vec<_>>().join(","),
rustfs_io_metrics::ECSTORE_RENAME_BATCH_EXECUTION.iter().zip(rename.batch_execution_counts).map(|(label, count)| format!("{label}:{count}")).collect::<Vec<_>>().join(","),
);
let Ok(mut file) = tokio::fs::OpenOptions::new()
.create(true)
+58 -8
View File
@@ -605,6 +605,7 @@ struct QuotaTopologyCacheEntry {
fingerprint: String,
}
#[derive(Clone)]
enum LocalMutationTarget {
Ready(Arc<ECStore>),
Bootstrap(BootstrapLocalTarget),
@@ -1927,6 +1928,19 @@ impl Node for NodeService {
self.handle_rename_data(request, false, false, false).await
}
type RenameDataBatchStream = ResponseStream<RenameDataBatchResult>;
async fn rename_data_batch(
&self,
request: Request<RenameDataBatchRequest>,
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
if !rustfs_utils::get_env_bool("RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH", false) {
// The feature gate is checked before authentication or mutation so
// mixed-version clients may safely use call-level UNIMPLEMENTED.
return Err(unimplemented_rpc("rename_data_batch"));
}
self.handle_rename_data_batch(request).await
}
async fn rename_multipart_data(&self, request: Request<RenameDataRequest>) -> Result<Response<RenameDataResponse>, Status> {
self.handle_rename_data(request, false, false, true).await
}
@@ -3489,13 +3503,13 @@ mod tests {
MakeVolumeRequest, MakeVolumesRequest, MrfLifecycleRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest,
PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest,
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest, ReloadPoolMetaRequest,
ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, ScannerActivityRequest,
ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest,
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest,
StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, TierMutationPeerState,
TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteFormatControlRequest, WriteMetadataRequest, WriteRequest,
ReloadSiteReplicationConfigRequest, RenameDataBatchItem, RenameDataBatchRequest, RenameDataRequest, RenameFileRequest,
RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest,
ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest,
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest,
StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass,
TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest,
VerifyFileRequest, WriteAllRequest, WriteFormatControlRequest, WriteMetadataRequest, WriteRequest,
heal_control_service_client::HealControlServiceClient,
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
node_service_client::NodeServiceClient,
@@ -3513,8 +3527,9 @@ mod tests {
use tonic::{Request, Response, Status};
use uuid::Uuid;
const DISK_MUTATION_RPC_METHODS: [&str; 35] = [
const DISK_MUTATION_RPC_METHODS: [&str; 36] = [
"renamedata",
"renamedatabatch",
"renamemultipartdata",
"recovermultipartsource",
"renamedataatincarnation",
@@ -3581,6 +3596,41 @@ mod tests {
policies
}
#[tokio::test]
#[serial_test::serial]
async fn rename_data_batch_requires_explicit_enable_and_body_digest() {
let request = || {
Request::new(RenameDataBatchRequest {
src_volume: "src".to_owned(),
src_path: "object".to_owned(),
dst_volume: "dst".to_owned(),
dst_path: "object".to_owned(),
items: vec![RenameDataBatchItem {
item_index: 0,
disk: "disk".to_owned(),
file_info_msgpack: vec![0x80].into(),
}],
})
};
temp_env::async_with_vars([("RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH", Some("false"))], async {
let error = match make_server().rename_data_batch(request()).await {
Err(error) => error,
Ok(_) => panic!("disabled batch must be unimplemented"),
};
assert_eq!(error.code(), tonic::Code::Unimplemented);
})
.await;
temp_env::async_with_vars([("RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH", Some("true"))], async {
let error = match make_server().rename_data_batch(request()).await {
Err(error) => error,
Ok(_) => panic!("unsigned batch must fail closed"),
};
assert_eq!(error.code(), tonic::Code::PermissionDenied);
})
.await;
}
#[test]
fn every_node_service_rpc_declares_an_auth_policy() {
const VALID_POLICIES: [&str; 4] = ["body-bound", "read-only", "streaming", "unimplemented"];
+140
View File
@@ -32,6 +32,7 @@ use crate::storage::storage_api::{
verify_tonic_mutation_body_digest,
};
use bytes::Bytes;
use futures_util::{FutureExt as _, StreamExt as _, stream::FuturesUnordered};
use rustfs_filemeta::FileInfo;
use rustfs_io_metrics::internode_metrics::InternodeRpcDurationGuard;
use rustfs_io_metrics::internode_metrics::{
@@ -77,6 +78,13 @@ fn startup_cas_rename_observation(
}
impl LocalMutationTarget {
fn object_requires_incarnation(&self, bucket: &str, object: &str) -> bool {
match self {
Self::Ready(store) => store.object_requires_incarnation(bucket, object),
Self::Bootstrap(_) | Self::Unbound => false,
}
}
async fn write_local_format_control(&self, disk_ref: &str, path: &str, data: Bytes) -> Result<(), DiskError> {
match self {
Self::Ready(store) => store.write_local_format_control(disk_ref, path, data).await,
@@ -498,6 +506,18 @@ fn decode_multipart_rename_request_file_info(binary: &[u8]) -> Result<DecodedRpc
}
}
fn decode_batch_rename_file_info(binary: &[u8]) -> Result<FileInfo, DiskError> {
if binary.is_empty() || binary.len() > rustfs_protos::DEFAULT_GRPC_SERVER_MESSAGE_LEN {
return Err(DiskError::FileCorrupt);
}
let mut cursor = Cursor::new(binary);
let value = rmp_serde::from_read(&mut cursor).map_err(|_| DiskError::FileCorrupt)?;
if cursor.position() != u64::try_from(binary.len()).map_err(DiskError::other)? {
return Err(DiskError::FileCorrupt);
}
Ok(value)
}
fn encode_rename_data_response_payloads(
rename_data_resp: &RenameDataResp,
request_decoded_from_msgpack: bool,
@@ -2112,6 +2132,126 @@ impl NodeService {
Ok(response)
}
pub(super) async fn handle_rename_data_batch(
&self,
request: Request<RenameDataBatchRequest>,
) -> Result<Response<super::ResponseStream<RenameDataBatchResult>>, Status> {
const MAX_BATCH_ITEMS: usize = 16;
if request.get_ref().items.is_empty() || request.get_ref().items.len() > MAX_BATCH_ITEMS {
return Err(Status::invalid_argument("rename batch item count must be between 1 and 16"));
}
let canonical_body = rustfs_protos::canonical_rename_data_batch_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("rename batch body length cannot be represented"))?;
let has_body_digest = request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok())
.is_some_and(|value| !value.is_empty() && value != "UNSIGNED-PAYLOAD");
if !has_body_digest {
return Err(Status::permission_denied("rename batch requires a body-bound digest"));
}
crate::storage::storage_api::verify_tonic_canonical_body_digest(&request, &canonical_body)
.map_err(|_| Status::permission_denied("rename batch requires body-bound authentication"))?;
let request = request.into_inner();
if request.src_volume == RUSTFS_META_MULTIPART_BUCKET {
return Err(Status::invalid_argument("multipart source requires RenameMultipartData"));
}
let target = self.local_mutation_target();
if target.object_requires_incarnation(&request.dst_volume, &request.dst_path) {
return Err(Status::failed_precondition(
"incarnation-bound rename cannot use ordinary batch transport",
));
}
let mut seen_indexes = 0_u16;
let mut prepared = Vec::with_capacity(request.items.len());
for item in request.items {
let item_index =
usize::try_from(item.item_index).map_err(|_| Status::invalid_argument("invalid rename batch item index"))?;
if item_index >= MAX_BATCH_ITEMS || seen_indexes & (1_u16 << item_index) != 0 {
return Err(Status::invalid_argument("rename batch item indexes must be unique and below 16"));
}
seen_indexes |= 1_u16 << item_index;
if item.disk.is_empty() || prepared.iter().any(|(_, disk, _)| disk == &item.disk) {
return Err(Status::invalid_argument("rename batch disk references must be non-empty and unique"));
}
let file_info = decode_batch_rename_file_info(&item.file_info_msgpack)
.map_err(|_| Status::invalid_argument("invalid rename batch FileInfo"))?;
if file_info.is_canonical_delete_marker() {
file_info
.validate_for_metadata_read()
.map_err(|_| Status::invalid_argument("invalid rename batch delete marker"))?;
} else {
file_info
.validate_for_erasure_write()
.map_err(|_| Status::invalid_argument("invalid rename batch FileInfo geometry"))?;
}
if rustfs_utils::http::metadata_compat::contains_key_str(&file_info.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX)
|| rustfs_utils::http::metadata_compat::contains_key_str(
&file_info.metadata,
POOL_META_NATIVE_CAS_METADATA_SUFFIX,
)
{
return Err(Status::invalid_argument("protected rename cannot use ordinary batch transport"));
}
prepared.push((item.item_index, item.disk, file_info));
}
let src_volume = Arc::<str>::from(request.src_volume);
let src_path = Arc::<str>::from(request.src_path);
let dst_volume = Arc::<str>::from(request.dst_volume);
let dst_path = Arc::<str>::from(request.dst_path);
let (tx, rx) = tokio::sync::mpsc::channel(prepared.len());
tokio::spawn(async move {
let mut pending = FuturesUnordered::new();
for (item_index, disk, file_info) in prepared {
let target = target.clone();
let src_volume = Arc::clone(&src_volume);
let src_path = Arc::clone(&src_path);
let dst_volume = Arc::clone(&dst_volume);
let dst_path = Arc::clone(&dst_path);
pending.push(async move {
let result = std::panic::AssertUnwindSafe(async {
let result = target
.rename_local_data(&disk, (&src_volume, &src_path), &file_info, (&dst_volume, &dst_path), None, None)
.await;
match result.and_then(|value| encode_msgpack_named(&value, "RenameDataResp")) {
Ok(payload) => RenameDataBatchResult {
item_index,
success: true,
error: None,
rename_data_resp_msgpack: payload.into(),
},
Err(error) => RenameDataBatchResult {
item_index,
success: false,
error: Some(error.into()),
rename_data_resp_msgpack: Vec::new().into(),
},
}
})
.catch_unwind()
.await;
(item_index, result)
});
}
while let Some(outcome) = pending.next().await {
let (item_index, result) = outcome;
let response = match result {
Ok(response) => response,
Err(_) => RenameDataBatchResult {
item_index,
success: false,
error: Some(DiskError::Unexpected.into()),
rename_data_resp_msgpack: Vec::new().into(),
},
};
let _ = tx.send(Ok(response)).await;
}
});
Ok(Response::new(Box::pin(tokio_stream::wrappers::ReceiverStream::new(rx))))
}
pub(super) async fn handle_list_dir(&self, request: Request<ListDirRequest>) -> Result<Response<ListDirResponse>, Status> {
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await? {
+133
View File
@@ -0,0 +1,133 @@
#!/usr/bin/env python3
"""Fail-closed stability gate for fixed-rate benchmark warmup windows."""
from __future__ import annotations
import argparse
import json
import math
import statistics
import sys
from pathlib import Path
def positive_number(row: dict, name: str) -> float:
value = row.get(name)
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not math.isfinite(value)
or value <= 0
):
raise ValueError(f"window field {name} must be a positive number")
return float(value)
def validate_window(row: dict, target_rps: float, duration_secs: float, concurrency: int, tolerance_pct: float) -> dict:
successful_puts = positive_number(row, "successful_puts")
errors = row.get("errors")
if not isinstance(errors, int) or isinstance(errors, bool) or errors < 0:
raise ValueError("window field errors must be a non-negative integer")
if errors:
raise ValueError("warmup window contains request errors")
reqps = positive_number(row, "reqps")
scheduled = target_rps * duration_secs
tolerance = tolerance_pct / 100
minimum_puts = scheduled * (1 - tolerance)
maximum_puts = (scheduled + concurrency - 1) * (1 + tolerance)
if not minimum_puts <= successful_puts <= maximum_puts:
raise ValueError("warmup successful PUT count is outside tolerance")
if abs(reqps - target_rps) / target_rps * 100 > tolerance_pct:
raise ValueError("warmup request rate is outside tolerance")
host_counts = row.get("host_counts")
if not isinstance(host_counts, dict) or len(host_counts) < 2:
raise ValueError("warmup host_counts must contain at least two hosts")
counts = [value for value in host_counts.values() if isinstance(value, int) and not isinstance(value, bool) and value >= 0]
if len(counts) != len(host_counts) or sum(counts) <= 0:
raise ValueError("warmup host_counts contains invalid values")
if sum(counts) != successful_puts:
raise ValueError("warmup host counts do not match successful PUT count")
ideal = sum(counts) / len(counts)
host_deviation_pct = max(abs(value - ideal) / ideal * 100 for value in counts)
if host_deviation_pct > tolerance_pct:
raise ValueError("warmup host distribution is outside tolerance")
return {
"successful_puts": int(successful_puts),
"errors": errors,
"reqps": reqps,
"avg_ms": positive_number(row, "avg_ms"),
"p90_ms": positive_number(row, "p90_ms"),
"p99_ms": positive_number(row, "p99_ms"),
"host_deviation_pct": host_deviation_pct,
}
def spread_pct(values: list[float]) -> float:
median = statistics.median(values)
return (max(values) - min(values)) / median * 100
def evaluate(rows: list[dict], required_windows: int, max_windows: int, max_spread_pct: float) -> dict:
if len(rows) < required_windows:
return {"status": "continue", "reason": "insufficient_windows", "windows": rows}
recent = rows[-required_windows:]
spreads = {name: spread_pct([row[name] for row in recent]) for name in ("reqps", "avg_ms", "p90_ms", "p99_ms")}
stable = all(value <= max_spread_pct for value in spreads.values())
if stable:
return {"status": "pass", "reason": "stable", "windows": rows, "recent_spread_pct": spreads}
if len(rows) >= max_windows:
return {"status": "fail", "reason": "max_windows_unstable", "windows": rows, "recent_spread_pct": spreads}
return {"status": "continue", "reason": "recent_windows_unstable", "windows": rows, "recent_spread_pct": spreads}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--input", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--target-rps", type=float, default=250)
parser.add_argument("--duration-secs", type=float, default=20)
parser.add_argument("--concurrency", type=int, default=128)
parser.add_argument("--required-windows", type=int, default=3)
parser.add_argument("--max-windows", type=int, default=12)
parser.add_argument("--quality-tolerance-pct", type=float, default=2)
parser.add_argument("--max-spread-pct", type=float, default=5)
args = parser.parse_args()
output = Path(args.output)
try:
raw = json.loads(Path(args.input).read_text())
if not isinstance(raw, list):
raise ValueError("warmup input must be a JSON array")
if not 1 <= args.required_windows <= args.max_windows:
raise ValueError("required windows must be between one and max windows")
numeric_args = (args.target_rps, args.duration_secs, args.quality_tolerance_pct, args.max_spread_pct)
if not all(math.isfinite(value) for value in numeric_args):
raise ValueError("warmup numeric arguments must be finite")
if args.target_rps <= 0 or args.duration_secs <= 0 or args.concurrency <= 0:
raise ValueError("target RPS, duration, and concurrency must be positive")
if args.quality_tolerance_pct < 0 or args.max_spread_pct < 0:
raise ValueError("tolerances must be non-negative")
if len(raw) > args.max_windows:
raise ValueError("warmup input exceeds max windows")
windows = [
validate_window(row, args.target_rps, args.duration_secs, args.concurrency, args.quality_tolerance_pct)
for row in raw
]
result = evaluate(windows, args.required_windows, args.max_windows, args.max_spread_pct)
except (OSError, json.JSONDecodeError, ValueError) as error:
result = {"status": "fail", "reason": str(error), "windows": []}
result["schema_version"] = 1
result["target_rps"] = args.target_rps
result["duration_secs"] = args.duration_secs
result["concurrency"] = args.concurrency
result["required_windows"] = args.required_windows
result["max_windows"] = args.max_windows
output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
if result["status"] == "pass":
return 0
if result["status"] == "continue":
return 2
return 1
if __name__ == "__main__":
sys.exit(main())
+206
View File
@@ -0,0 +1,206 @@
#!/usr/bin/env python3
"""Run fixed-rate Warp warmup windows until the stability gate passes."""
from __future__ import annotations
import argparse
import csv
import json
import math
import os
import subprocess
import sys
from pathlib import Path
import adaptive_warmup_gate
import warp_raw_quality
WARMUP_UNSTABLE_EXIT = 95
def result_exit_code(result: dict) -> int:
if result.get("status") == "pass":
return 0
if result.get("reason") == "max_windows_unstable":
return WARMUP_UNSTABLE_EXIT
return 1
def extract_latency_metrics(extractor: Path, log_path: Path) -> dict:
result = subprocess.run(
[
str(extractor),
"--tool",
"warp",
"--warp-mode",
"put",
"--extract-metrics-from-log",
str(log_path),
],
check=False,
capture_output=True,
text=True,
)
if result.returncode:
raise RuntimeError("Warp metrics extractor failed")
lines = [line for line in result.stdout.splitlines() if line.strip()]
if len(lines) != 2:
raise ValueError("Warp metrics extractor returned an unexpected schema")
rows = list(csv.DictReader(lines))
if len(rows) != 1:
raise ValueError("Warp metrics extractor returned an unexpected row count")
row = rows[0]
try:
return {
"avg_ms": float(row["latency_ms"]),
"p90_ms": float(row["req_p90_ms"]),
"p99_ms": float(row["req_p99_ms"]),
}
except (KeyError, TypeError, ValueError) as error:
raise ValueError("Warp metrics extractor omitted a latency field") from error
def write_gate(path: Path, result: dict, args: argparse.Namespace) -> None:
output = dict(result)
output.update(
schema_version=1,
target_rps=args.target_rps,
duration_secs=args.duration_secs,
concurrency=args.concurrency,
required_windows=args.required_windows,
max_windows=args.max_windows,
bucket=args.bucket,
hosts=args.host,
)
path.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--warp-bin", required=True)
parser.add_argument("--metrics-extractor", required=True)
parser.add_argument("--host", action="append", required=True)
parser.add_argument("--access-key-env", default="RUSTFS_ACCESS_KEY")
parser.add_argument("--secret-key-env", default="RUSTFS_SECRET_KEY")
parser.add_argument("--bucket", required=True)
parser.add_argument("--region", default="us-east-1")
parser.add_argument("--size", default="100KiB")
parser.add_argument("--concurrency", type=int, default=128)
parser.add_argument("--target-rps", type=int, default=250)
parser.add_argument("--duration-secs", type=int, default=20)
parser.add_argument("--required-windows", type=int, default=3)
parser.add_argument("--max-windows", type=int, default=12)
parser.add_argument("--quality-tolerance-pct", type=float, default=2)
parser.add_argument("--max-spread-pct", type=float, default=5)
parser.add_argument("--output-dir", required=True)
args = parser.parse_args()
if not 1 <= args.required_windows <= args.max_windows:
parser.error("required windows must be between one and max windows")
if not math.isfinite(args.quality_tolerance_pct) or not math.isfinite(args.max_spread_pct):
parser.error("tolerances must be finite")
if args.target_rps <= 0 or args.duration_secs <= 0 or args.concurrency <= 0:
parser.error("target RPS, duration, and concurrency must be positive")
if args.quality_tolerance_pct < 0 or args.max_spread_pct < 0:
parser.error("tolerances must be non-negative")
access_key = os.environ.get(args.access_key_env)
secret_key = os.environ.get(args.secret_key_env)
if not access_key or not secret_key:
parser.error("credential environment variables must be set")
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=False)
windows_path = output_dir / "windows.json"
gate_path = output_dir / "warmup-gate.json"
windows = []
for index in range(1, args.max_windows + 1):
window_dir = output_dir / f"window-{index:02d}"
window_dir.mkdir()
log_path = window_dir / "warp.log"
benchdata_prefix = window_dir / "warp"
command = [
args.warp_bin,
"put",
"--host",
",".join(args.host),
"--access-key",
access_key,
"--secret-key",
secret_key,
"--bucket",
args.bucket,
"--region",
args.region,
"--obj.size",
args.size,
"--concurrent",
str(args.concurrency),
"--duration",
f"{args.duration_secs}s",
"--rps-limit",
str(args.target_rps),
"--host-select",
"roundrobin",
"--full",
"--noclear",
"--no-color",
"--analyze.v",
"--benchdata",
str(benchdata_prefix),
]
try:
with log_path.open("w") as log:
run = subprocess.run(
command,
stdout=log,
stderr=subprocess.STDOUT,
check=False,
timeout=args.duration_secs + 60,
)
except subprocess.TimeoutExpired:
result = {"status": "fail", "reason": "warp_command_timeout", "windows": windows}
write_gate(gate_path, result, args)
return 1
if run.returncode:
result = {"status": "fail", "reason": "warp_command_failed", "windows": windows}
write_gate(gate_path, result, args)
return 1
try:
raw = warp_raw_quality.parse(Path(f"{benchdata_prefix}.csv.zst"), set(args.host))
latency = extract_latency_metrics(Path(args.metrics_extractor), log_path)
row = {
"successful_puts": raw["successful_puts"],
"errors": raw["errors"],
"reqps": raw["reqps"],
"host_counts": raw["host_counts"],
**latency,
"raw_quality": str(window_dir / "raw-quality.json"),
}
validated = adaptive_warmup_gate.validate_window(
row,
args.target_rps,
args.duration_secs,
args.concurrency,
args.quality_tolerance_pct,
)
(window_dir / "raw-quality.json").write_text(json.dumps(raw, indent=2, sort_keys=True) + "\n")
windows.append(validated)
windows_path.write_text(json.dumps(windows, indent=2, sort_keys=True) + "\n")
result = adaptive_warmup_gate.evaluate(
windows,
args.required_windows,
args.max_windows,
args.max_spread_pct,
)
except (OSError, RuntimeError, ValueError, csv.Error) as error:
result = {"status": "fail", "reason": str(error), "windows": windows}
write_gate(gate_path, result, args)
return 1
write_gate(gate_path, result, args)
if result["status"] == "pass":
return result_exit_code(result)
if result["status"] == "fail":
return result_exit_code(result)
return WARMUP_UNSTABLE_EXIT
if __name__ == "__main__":
sys.exit(main())
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
import json
import subprocess
import tempfile
import unittest
from pathlib import Path
SCRIPT = Path(__file__).with_name("adaptive_warmup_gate.py")
def window(avg: float, p90: float, p99: float, *, errors: int = 0, successful_puts: int = 5127) -> dict:
per_host, remainder = divmod(successful_puts, 4)
return {
"successful_puts": successful_puts,
"errors": errors,
"reqps": 250,
"avg_ms": avg,
"p90_ms": p90,
"p99_ms": p99,
"host_counts": {
"a": per_host + (remainder > 0),
"b": per_host + (remainder > 1),
"c": per_host + (remainder > 2),
"d": per_host,
},
}
class AdaptiveWarmupGateTest(unittest.TestCase):
def run_gate(self, rows: list[dict], *extra: str) -> tuple[int, dict]:
with tempfile.TemporaryDirectory() as directory:
input_path = Path(directory) / "input.json"
output_path = Path(directory) / "output.json"
input_path.write_text(json.dumps(rows))
result = subprocess.run(
[str(SCRIPT), "--input", str(input_path), "--output", str(output_path), *extra],
check=False,
)
return result.returncode, json.loads(output_path.read_text())
def test_three_stable_windows_pass(self) -> None:
code, result = self.run_gate([window(14, 20, 100), window(14.2, 20.5, 102), window(13.9, 20.2, 99)])
self.assertEqual(code, 0)
self.assertEqual(result["status"], "pass")
def test_unstable_tail_continues_before_limit(self) -> None:
code, result = self.run_gate([window(16, 20, 126), window(14, 15, 98), window(13, 15, 70)])
self.assertEqual(code, 2)
self.assertEqual(result["reason"], "recent_windows_unstable")
def test_unstable_tail_fails_at_limit(self) -> None:
rows = [window(16 if index % 2 == 0 else 13, 20, 126 if index % 2 == 0 else 70) for index in range(12)]
code, result = self.run_gate(rows)
self.assertEqual(code, 1)
self.assertEqual(result["reason"], "max_windows_unstable")
def test_request_error_fails_closed(self) -> None:
code, result = self.run_gate([window(14, 20, 100, errors=1)])
self.assertEqual(code, 1)
self.assertEqual(result["status"], "fail")
def test_exact_scheduled_count_is_valid_without_tail_completions(self) -> None:
code, result = self.run_gate(
[window(14, 20, 100, successful_puts=5000)] * 3,
)
self.assertEqual(code, 0)
self.assertEqual(result["status"], "pass")
def test_host_count_mismatch_fails_closed(self) -> None:
row = window(14, 20, 100)
row["host_counts"]["a"] -= 1
code, result = self.run_gate([row])
self.assertEqual(code, 1)
self.assertEqual(result["reason"], "warmup host counts do not match successful PUT count")
def test_non_finite_threshold_fails_closed(self) -> None:
code, result = self.run_gate([window(14, 20, 100)], "--max-spread-pct", "inf")
self.assertEqual(code, 1)
self.assertEqual(result["reason"], "warmup numeric arguments must be finite")
if __name__ == "__main__":
unittest.main()
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
import importlib.util
import sys
import unittest
from pathlib import Path
MODULE_PATH = Path(__file__).with_name("adaptive_warmup_runner.py")
sys.path.insert(0, str(MODULE_PATH.parent))
SPEC = importlib.util.spec_from_file_location("adaptive_warmup_runner", MODULE_PATH)
MODULE = importlib.util.module_from_spec(SPEC)
assert SPEC.loader is not None
SPEC.loader.exec_module(MODULE)
class AdaptiveWarmupRunnerTest(unittest.TestCase):
def test_latency_extractor_rejects_unknown_schema(self) -> None:
with self.assertRaises((OSError, RuntimeError, ValueError)):
MODULE.extract_latency_metrics(Path("/does/not/exist"), Path("/does/not/exist"))
def test_gate_evaluator_requires_recent_consecutive_windows(self) -> None:
stable = {"reqps": 250.0, "avg_ms": 14.0, "p90_ms": 20.0, "p99_ms": 100.0}
unstable = {"reqps": 250.0, "avg_ms": 20.0, "p90_ms": 30.0, "p99_ms": 180.0}
result = MODULE.adaptive_warmup_gate.evaluate([stable, stable, unstable], 3, 12, 5)
self.assertEqual(result["status"], "continue")
def test_unstable_exhaustion_has_non_retryable_exit_code(self) -> None:
result = {"status": "fail", "reason": "max_windows_unstable"}
self.assertEqual(MODULE.result_exit_code(result), 95)
def test_quality_failure_keeps_generic_failure_exit_code(self) -> None:
result = {"status": "fail", "reason": "warmup request rate is outside tolerance"}
self.assertEqual(MODULE.result_exit_code(result), 1)
if __name__ == "__main__":
unittest.main()
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env python3
import importlib.util
import tempfile
import unittest
from pathlib import Path
MODULE_PATH = Path(__file__).with_name("warp_raw_quality.py")
SPEC = importlib.util.spec_from_file_location("warp_raw_quality", MODULE_PATH)
MODULE = importlib.util.module_from_spec(SPEC)
assert SPEC.loader is not None
SPEC.loader.exec_module(MODULE)
HEADER = "thread\top\tn_objects\tbytes\tendpoint\terror\tstart\tend\tduration_ns\n"
def row(thread: int, endpoint: str, start: int, end: int, error: str = "") -> str:
def stamp(second: int) -> str:
return f"2026-01-01T00:00:{second:02d}Z"
return f"{thread}\tPUT\t1\t100\t{endpoint}\t{error}\t{stamp(start)}\t{stamp(end)}\t{(end-start)*1_000_000_000}\n"
class WarpRawQualityTest(unittest.TestCase):
def fixture(self, error: bool = False) -> Path:
directory = tempfile.TemporaryDirectory()
self.addCleanup(directory.cleanup)
path = Path(directory.name) / "warp.tsv"
content = HEADER
content += row(0, "a", 0, 1)
content += row(0, "a", 1, 3)
content += row(0, "a", 3, 5)
content += row(0, "a", 5, 10)
content += row(1, "b", 0, 2, "boom" if error else "")
content += row(1, "b", 2, 4)
content += row(1, "b", 4, 6)
content += row(1, "b", 6, 9)
path.write_text(content)
return path
def test_common_window_and_host_counts(self) -> None:
result = MODULE.parse(self.fixture(), {"a", "b"})
self.assertEqual(result["measure_duration_secs"], 3)
self.assertEqual(result["completed_objects"], 3)
self.assertEqual(result["completed_requests"], 3)
self.assertEqual(result["host_counts"], {"a": 4, "b": 4})
self.assertAlmostEqual(result["reqps"], 1.0)
def test_any_raw_error_fails_closed(self) -> None:
with self.assertRaisesRegex(ValueError, "contains request errors"):
MODULE.parse(self.fixture(error=True), {"a", "b"})
def test_expected_hosts_are_exact(self) -> None:
with self.assertRaisesRegex(ValueError, "expected host set"):
MODULE.parse(self.fixture(), {"a", "b", "c"})
def test_timestamp_parser_preserves_nanoseconds(self) -> None:
self.assertEqual(
MODULE.timestamp_ns("1970-01-01T00:00:01.123456789Z"),
1_123_456_789,
)
if __name__ == "__main__":
unittest.main()
+149
View File
@@ -0,0 +1,149 @@
#!/usr/bin/env python3
"""Recompute fixed-rate Warp quality evidence from raw TSV benchdata."""
from __future__ import annotations
import argparse
import csv
import hashlib
import io
import json
import re
import subprocess
import sys
from collections import defaultdict
from pathlib import Path
TIMESTAMP = re.compile(r"^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?Z$")
REQUIRED = {"thread", "op", "n_objects", "bytes", "endpoint", "error", "start", "end", "duration_ns"}
def timestamp_ns(value: str) -> int:
match = TIMESTAMP.fullmatch(value)
if not match:
raise ValueError(f"invalid RFC3339Nano timestamp: {value}")
import calendar
import datetime
year, month, day, hour, minute, second = map(int, match.groups()[:6])
base = calendar.timegm(datetime.datetime(year, month, day, hour, minute, second).timetuple())
fraction = (match.group(7) or "").ljust(9, "0")
return base * 1_000_000_000 + int(fraction or "0")
def open_text(path: Path):
if path.suffix != ".zst":
return path.open(newline="")
process = subprocess.Popen(
["zstd", "--decompress", "--stdout", "--quiet", str(path)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if process.stdout is None:
raise RuntimeError("zstd stdout is unavailable")
return io.TextIOWrapper(process.stdout, newline=""), process
def parse(path: Path, expected_hosts: set[str] | None) -> dict:
opened = open_text(path)
process = None
if isinstance(opened, tuple):
stream, process = opened
else:
stream = opened
try:
reader = csv.DictReader((line for line in stream if not line.startswith("#")), delimiter="\t", strict=True)
if reader.fieldnames is None or not REQUIRED.issubset(reader.fieldnames):
raise ValueError("Warp raw header is missing required fields")
rows = []
first_end: dict[int, int] = {}
last_start: dict[int, int] = {}
raw_errors = 0
raw_successful_objects = 0
host_counts: dict[str, int] = defaultdict(int)
for row in reader:
if None in row:
raise ValueError("Warp raw row width differs from header")
if row["op"] != "PUT":
raise ValueError("Warp raw input contains a non-PUT operation")
thread = int(row["thread"])
objects = int(row["n_objects"])
size_bytes = int(row["bytes"])
duration_ns = int(row["duration_ns"])
if objects <= 0 or size_bytes < 0 or duration_ns < 0 or not row["endpoint"]:
raise ValueError("Warp raw row contains an invalid numeric or endpoint field")
start = timestamp_ns(row["start"])
end = timestamp_ns(row["end"])
if end < start:
raise ValueError("Warp raw operation ends before it starts")
error = bool(row["error"])
raw_errors += int(error)
if not error:
raw_successful_objects += objects
host_counts[row["endpoint"]] += objects
first_end[thread] = min(first_end.get(thread, end), end)
last_start[thread] = max(last_start.get(thread, start), start)
rows.append((start, end, objects, error))
if not rows or raw_errors:
raise ValueError("Warp raw input is empty or contains request errors")
if expected_hosts is not None and set(host_counts) != expected_hosts:
raise ValueError("Warp raw endpoints differ from the expected host set")
window_start = max(first_end.values())
window_end = min(last_start.values())
if window_end <= window_start:
raise ValueError("Warp common active window is not positive")
completed_objects = sum(objects for _, end, objects, error in rows if not error and window_start <= end < window_end)
completed_requests = sum(1 for _, end, _, error in rows if not error and window_start <= end < window_end)
fractional_objects = 0.0
for start, end, objects, error in rows:
if error or end <= start:
continue
overlap = max(0, min(end, window_end) - max(start, window_start))
fractional_objects += objects * overlap / (end - start)
duration_secs = (window_end - window_start) / 1_000_000_000
return {
"schema_version": 1,
"source": str(path),
"source_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
"operation": "PUT",
"threads": len(first_end),
"raw_requests": len(rows),
"successful_puts": raw_successful_objects,
"errors": raw_errors,
"host_counts": dict(sorted(host_counts.items())),
"completed_objects": completed_objects,
"completed_requests": completed_requests,
"fractional_objects": fractional_objects,
"measure_start_ns": window_start,
"measure_end_exclusive_ns": window_end,
"measure_duration_secs": duration_secs,
"reqps": fractional_objects / duration_secs,
"window_status": "warp_v1.8.2_common_active_window",
}
finally:
stream.close()
if process is not None:
stderr = process.stderr.read().decode() if process.stderr is not None else ""
if process.wait() != 0:
raise RuntimeError(f"zstd failed: {stderr.strip()}")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--input", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--expected-host", action="append")
args = parser.parse_args()
try:
result = parse(Path(args.input), set(args.expected_host) if args.expected_host else None)
except (OSError, RuntimeError, ValueError, csv.Error) as error:
Path(args.output).write_text(json.dumps({"schema_version": 1, "status": "fail", "reason": str(error)}, indent=2) + "\n")
return 1
result["status"] = "pass"
Path(args.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
return 0
if __name__ == "__main__":
sys.exit(main())