Merge origin/main into houseme/perf-internode-grpc

Integrate da6058e37 and retain the updated Prost lock entry alongside the ephemeral TLS test certificate dependency.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

.config/make/tests.mak

@@ -38,6 +38,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_entrypoint_credentials.sh
 	./scripts/test_entrypoint_directories.sh
 	./scripts/test_internode_grpc_ab_bench.sh
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
 	./scripts/test_object_batch_bench_enhanced.sh
 	./scripts/test_package_service_scripts.sh
 	./scripts/test_hotpath_warp_ab_gate.sh

Cargo.lock

@@ -10766,6 +10766,7 @@ dependencies = [
  "flatbuffers",
  "hotpath",
  "prost 0.14.5",
+ "rcgen",
  "rmp-serde",
  "rustfs-common",
  "rustfs-config",

crates/config/src/constants/internode.rs

@@ -50,10 +50,8 @@ const _: () = assert!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS <= MAX_ADMIN_PEER_PR
 
 // ── Client-side internode gRPC channel tuning (P0) ──
 // These mirror the server-side HTTP/2 transport tuning in `rustfs/src/server/http.rs`
-// on the *client* `tonic` `Endpoint` used for internode control-plane RPCs. Prior to
-// this, the client channel only set timeouts/keepalive, leaving Nagle enabled and the
-// default 64KiB HTTP/2 window in place — hurting small lock-RPC latency and large
-// metadata-response throughput respectively.
+// on the *client* `tonic` `Endpoint` used for internode RPCs. Explicit values make
+// the operator's receive-credit policy independent of changes to library defaults.
 
 /// Disable Nagle's algorithm on internode gRPC client sockets.
 ///
@@ -69,8 +67,8 @@ const _: () = assert!(DEFAULT_INTERNODE_RPC_TCP_NODELAY);
 
 /// HTTP/2 initial stream window size (bytes) for internode gRPC client channels.
 ///
-/// The library default (64KiB) throttles larger unary responses (e.g. `ReadMultiple`,
-/// `BatchReadVersion`) by the bandwidth-delay product. Set to 0 to fall back to the
+/// Size this for the bandwidth-delay product and concurrent response budget of
+/// unary responses such as `ReadMultiple` and `BatchReadVersion`. Set to 0 to use the
 /// `tonic`/`hyper` default. Defaults to 1MiB.
 pub const ENV_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE";
 pub const DEFAULT_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE: u32 = 1024 * 1024;
@@ -284,6 +282,18 @@ const _: () = assert!(!DEFAULT_INTERNODE_CHANNEL_ISOLATION);
 pub const ENV_INTERNODE_BULK_CHANNELS: &str = "RUSTFS_INTERNODE_BULK_CHANNELS";
 pub const DEFAULT_INTERNODE_BULK_CHANNELS: usize = 2;
 
+/// Optional receive-window overrides for isolated bulk channels. When unset, inherit the
+/// corresponding `RUSTFS_INTERNODE_RPC_HTTP2_*` setting. Zero selects the library default.
+pub const ENV_INTERNODE_BULK_HTTP2_STREAM_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_BULK_HTTP2_STREAM_WINDOW_SIZE";
+pub const ENV_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE";
+
+/// Transport admission and pending-buffer limits for isolated bulk channels. These limits
+/// apply to the tonic service; complete unary bodies are observed separately by RPC metrics.
+pub const ENV_INTERNODE_BULK_CONCURRENCY_LIMIT: &str = "RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT";
+pub const DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT: usize = 64;
+pub const ENV_INTERNODE_BULK_BUFFER_SIZE: &str = "RUSTFS_INTERNODE_BULK_BUFFER_SIZE";
+pub const DEFAULT_INTERNODE_BULK_BUFFER_SIZE: usize = 1024;
+
 /// Profile selector for conservative internode HTTP data-plane client tuning.
 pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
 pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";

crates/ecstore/Cargo.toml

@@ -29,6 +29,10 @@ documentation = "https://docs.rs/rustfs-ecstore/latest/rustfs_ecstore/"
 [lints]
 workspace = true
 
+[[example]]
+name = "internode_rpc_bench"
+test = true
+
 [features]
 internode-transport = []
 default = []

crates/ecstore/examples/internode_rpc_bench.rs

@@ -0,0 +1,866 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+//! Read-only authenticated ReadAll/Ping probe against explicitly prepared test disks.
+//! Usage: internode_rpc_bench manifest.json output.json
+//! Fixtures live under each test disk's .rustfs.sys/tmp/<fixture_id>/ directory.
+
+use rustfs_ecstore::api::{
+    disk::{DiskAPI, DiskOption, DiskStore, endpoint::Endpoint, new_disk},
+    rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client},
+};
+use rustfs_protos::proto_gen::node_service::PingRequest;
+use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration, ensure_crypto_provider, publish_global_outbound_tls_state};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use std::collections::BTreeSet;
+use std::io::{Error, ErrorKind, Result, Write};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use tokio::io::AsyncReadExt;
+use tokio::sync::Semaphore;
+use tokio::task::JoinSet;
+use tokio::time::{Instant, sleep, sleep_until, timeout};
+use uuid::Uuid;
+
+const MAX_PAYLOAD: usize = 8 * 1024 * 1024;
+const MAX_BULK_ROWS: usize = 4_000_000;
+const MAX_CONTROL_ROWS: u32 = 1_000_000;
+
+#[derive(Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Config {
+    namespace: String,
+    targets: Vec<String>,
+    disk_ids: Option<Vec<Uuid>>,
+    fixture_id: Uuid,
+    fixture_file: PathBuf,
+    duration_secs: u32,
+    bulk_workers: usize,
+    control_rps: u32,
+    rpc_timeout_ms: u32,
+    tls_ca: Option<PathBuf>,
+    start_marker: Option<PathBuf>,
+    end_marker: Option<PathBuf>,
+}
+
+impl Config {
+    fn endpoints(&self) -> Result<Vec<Endpoint>> {
+        let valid_namespace = self.namespace.starts_with("grpc-bench-")
+            && (12..=64).contains(&self.namespace.len())
+            && self
+                .namespace
+                .bytes()
+                .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-');
+        if !valid_namespace || self.fixture_id.is_nil() || !(1..=64).contains(&self.targets.len()) {
+            return Err(Error::new(ErrorKind::InvalidInput, "invalid test namespace, fixture ID or target count"));
+        }
+        if self
+            .disk_ids
+            .as_ref()
+            .is_some_and(|ids| ids.len() != self.targets.len() || ids.iter().any(Uuid::is_nil))
+        {
+            return Err(Error::new(
+                ErrorKind::InvalidInput,
+                "disk IDs must match every declared target and be non-nil",
+            ));
+        }
+        if !(1..=1800).contains(&self.duration_secs)
+            || !(1..=128).contains(&self.bulk_workers)
+            || !(1..=10_000).contains(&self.control_rps)
+            || self
+                .duration_secs
+                .checked_mul(self.control_rps)
+                .is_none_or(|count| count > MAX_CONTROL_ROWS)
+            || !(1..=30_000).contains(&self.rpc_timeout_ms)
+        {
+            return Err(Error::new(ErrorKind::InvalidInput, "workload exceeds the probe's resource bounds"));
+        }
+        if self.start_marker.is_some() != self.end_marker.is_some()
+            || self.start_marker == self.end_marker && self.start_marker.is_some()
+        {
+            return Err(Error::new(ErrorKind::InvalidInput, "provide distinct start/end markers or neither"));
+        }
+        let mut seen = BTreeSet::new();
+        self.targets
+            .iter()
+            .map(|target| {
+                let mut endpoint = Endpoint::try_from(target.as_str())
+                    .map_err(|_| Error::new(ErrorKind::InvalidInput, "invalid test endpoint URL"))?;
+                let url = &endpoint.url;
+                if target.len() > 1024
+                    || !matches!(url.scheme(), "http" | "https")
+                    || url.host_str().is_none()
+                    || url.port().is_none_or(|port| port < 1024)
+                    || !url.username().is_empty()
+                    || url.password().is_some()
+                    || url.query().is_some()
+                    || url.fragment().is_some()
+                    || url.path().contains('%')
+                    || !url.path().ends_with(&format!("/{}", self.namespace))
+                    || !seen.insert(url.to_string())
+                    || url.scheme() == "https" && self.tls_ca.is_none()
+                {
+                    return Err(Error::new(ErrorKind::InvalidInput, "target must name an explicit test-disk URL"));
+                }
+                // This probe never constructs a local disk or invokes its startup cleanup.
+                endpoint.is_local = false;
+                Ok(endpoint)
+            })
+            .collect()
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Marker {
+    time_ns: u64,
+}
+
+#[derive(Serialize)]
+struct Sample {
+    operation: &'static str,
+    peer: usize,
+    scheduled_ns: u64,
+    dispatched_ns: Option<u64>,
+    completed_ns: u64,
+    outcome: &'static str,
+    bytes: usize,
+}
+
+#[derive(Serialize)]
+struct Report {
+    complete: bool,
+    errors: usize,
+    manifest_sha256: String,
+    fixture_sha256: String,
+    start_unix_ns: String,
+    observation_end_unix_ns: String,
+    offered_duration_secs: u32,
+    control_rps: u32,
+    bulk_bytes_completed_during_offering: u64,
+    bulk_coverage_verified: bool,
+    bulk_workers: usize,
+    control_expected: u32,
+    targets: usize,
+    peers: usize,
+    samples: Vec<Sample>,
+    external_markers: bool,
+    scope: &'static str,
+}
+
+fn now_ns() -> Result<u64> {
+    u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).map_err(Error::other)?.as_nanos()).map_err(Error::other)
+}
+
+fn elapsed_ns(origin: Instant) -> Result<u64> {
+    u64::try_from(origin.elapsed().as_nanos()).map_err(Error::other)
+}
+
+fn measurement_origin(start_ns: u64, current_ns: u64, monotonic_now: Instant) -> Result<Instant> {
+    let lag = current_ns
+        .checked_sub(start_ns)
+        .filter(|lag| *lag <= 1_000_000_000)
+        .ok_or_else(|| Error::other("measurement start must be within the preceding second"))?;
+    monotonic_now
+        .checked_sub(Duration::from_nanos(lag))
+        .ok_or_else(|| Error::other("invalid monotonic origin"))
+}
+
+fn validate_observation_end(start_ns: u64, end_ns: u64, finished_ns: u64, current_ns: u64, elapsed: u64) -> Result<()> {
+    let clock_end = start_ns
+        .checked_add(elapsed)
+        .ok_or_else(|| Error::other("observation window overflow"))?;
+    if end_ns < finished_ns || end_ns > current_ns || clock_end.abs_diff(current_ns) > 100_000_000 {
+        return Err(Error::other("invalid measurement-end acknowledgement or clock drift"));
+    }
+    Ok(())
+}
+
+async fn bounded_file(path: &Path, limit: usize) -> Result<Vec<u8>> {
+    let file = tokio::fs::File::open(path).await?;
+    if !file.metadata().await?.is_file() {
+        return Err(Error::new(ErrorKind::InvalidInput, "probe inputs must be regular files"));
+    }
+    let mut data = Vec::new();
+    file.take(u64::try_from(limit).map_err(Error::other)? + 1)
+        .read_to_end(&mut data)
+        .await?;
+    if data.len() > limit {
+        return Err(Error::new(ErrorKind::InvalidData, "probe input exceeds its size bound"));
+    }
+    Ok(data)
+}
+
+async fn marker(path: &Path) -> Result<u64> {
+    timeout(Duration::from_secs(30), async {
+        loop {
+            match bounded_file(path, 512).await {
+                Ok(data) => {
+                    return serde_json::from_slice::<Marker>(&data)
+                        .map(|value| value.time_ns)
+                        .map_err(Error::other);
+                }
+                Err(error) if error.kind() == ErrorKind::NotFound => sleep(Duration::from_millis(5)).await,
+                Err(error) => return Err(error),
+            }
+        }
+    })
+    .await
+    .map_err(|_| Error::new(ErrorKind::TimedOut, "measurement marker did not arrive"))?
+}
+
+fn phase(name: &str, time_ns: u64) -> Result<()> {
+    println!(
+        "{}",
+        serde_json::json!({"phase":name,"time_ns":time_ns.to_string(),"pid":std::process::id()})
+    );
+    std::io::stdout().flush()
+}
+
+async fn ping(peer: &str) -> Result<(u64, bytes::Bytes)> {
+    let mut client = node_service_time_out_client(peer, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
+        .await
+        .map_err(|_| Error::other("authenticated control client acquisition failed"))?
+        .max_decoding_message_size(4096);
+    let response = client
+        .ping(PingRequest {
+            version: 1,
+            body: bytes::Bytes::new(),
+        })
+        .await
+        .map_err(|_| Error::other("authenticated Ping failed"))?
+        .into_inner();
+    Ok((response.version, response.body))
+}
+
+struct Context {
+    config: Config,
+    disks: Vec<DiskStore>,
+    peers: Vec<String>,
+    echoes: Vec<bytes::Bytes>,
+    payload: Vec<u8>,
+    path: String,
+}
+
+async fn bulk(context: Arc<Context>, worker: usize, origin: Instant, end: Instant) -> Result<Vec<Sample>> {
+    let mut rows = Vec::with_capacity(256);
+    let limit = MAX_BULK_ROWS / context.config.bulk_workers;
+    for sequence in 0..limit {
+        if Instant::now() >= end {
+            return Ok(rows);
+        }
+        let peer = (worker + sequence) % context.disks.len();
+        let dispatched = elapsed_ns(origin)?;
+        let result = timeout(
+            Duration::from_millis(u64::from(context.config.rpc_timeout_ms)),
+            context.disks[peer].read_all(".rustfs.sys", &context.path),
+        )
+        .await;
+        let completed = elapsed_ns(origin)?;
+        let outcome = match result {
+            Ok(Ok(data)) if data.as_ref() == context.payload.as_slice() => "ok",
+            Ok(Ok(_)) => "payload_mismatch",
+            Ok(Err(_)) => "rpc_error",
+            Err(_) => "deadline",
+        };
+        rows.push(Sample {
+            operation: "ReadAll",
+            peer,
+            scheduled_ns: dispatched,
+            dispatched_ns: Some(dispatched),
+            completed_ns: completed,
+            outcome,
+            bytes: if outcome == "ok" { context.payload.len() } else { 0 },
+        });
+    }
+    if Instant::now() >= end {
+        Ok(rows)
+    } else {
+        Err(Error::other("bulk raw-sample limit reached; attribution is invalid"))
+    }
+}
+
+async fn control(context: Arc<Context>, origin: Instant) -> Result<Vec<Sample>> {
+    let count = context.config.duration_secs * context.config.control_rps;
+    let capacity = usize::try_from(count).map_err(Error::other)?;
+    let mut rows = Vec::with_capacity(capacity);
+    let permits = Arc::new(Semaphore::new(128));
+    let end = origin + Duration::from_secs(u64::from(context.config.duration_secs));
+    let mut tasks = JoinSet::new();
+    for sequence in 0..count {
+        // Configuration bounds count to one million, so this product fits u64.
+        let scheduled = u64::from(sequence) * 1_000_000_000 / u64::from(context.config.control_rps);
+        sleep_until(origin + Duration::from_nanos(scheduled)).await;
+        while let Some(result) = tasks.try_join_next() {
+            rows.push(result.map_err(Error::other)??);
+        }
+        let peer = usize::try_from(sequence).map_err(Error::other)? % context.peers.len();
+        let permit = (Instant::now() < end)
+            .then(|| permits.clone().try_acquire_owned())
+            .and_then(std::result::Result::ok);
+        let Some(permit) = permit else {
+            rows.push(Sample {
+                operation: "Ping",
+                peer,
+                scheduled_ns: scheduled,
+                dispatched_ns: None,
+                completed_ns: elapsed_ns(origin)?,
+                outcome: "schedule_drop",
+                bytes: 0,
+            });
+            continue;
+        };
+        let context = context.clone();
+        tasks.spawn(async move {
+            let _permit = permit;
+            let dispatched = elapsed_ns(origin)?;
+            let result = timeout(
+                Duration::from_millis(u64::from(context.config.rpc_timeout_ms)),
+                ping(&context.peers[peer]),
+            )
+            .await;
+            let completed = elapsed_ns(origin)?;
+            let outcome = match result {
+                Ok(Ok((1, body))) if body == context.echoes[peer] => "ok",
+                Ok(Ok(_)) => "payload_mismatch",
+                Ok(Err(_)) => "rpc_error",
+                Err(_) => "deadline",
+            };
+            Ok::<_, Error>(Sample {
+                operation: "Ping",
+                peer,
+                scheduled_ns: scheduled,
+                dispatched_ns: Some(dispatched),
+                completed_ns: completed,
+                outcome,
+                bytes: 0,
+            })
+        });
+    }
+    while let Some(result) = tasks.join_next().await {
+        rows.push(result.map_err(Error::other)??);
+    }
+    if rows.len() != capacity {
+        return Err(Error::other("control arrival accounting is incomplete"));
+    }
+    Ok(rows)
+}
+
+async fn run(config_bytes: &[u8], base: &Path, secret: String) -> Result<Report> {
+    let mut config: Config = serde_json::from_slice(config_bytes).map_err(|_| Error::other("invalid probe manifest"))?;
+    if config.fixture_file.is_relative() {
+        config.fixture_file = base.join(&config.fixture_file);
+    }
+    for path in [&mut config.tls_ca, &mut config.start_marker, &mut config.end_marker]
+        .into_iter()
+        .flatten()
+    {
+        if path.is_relative() {
+            *path = base.join(&*path);
+        }
+    }
+    let endpoints = config.endpoints()?;
+    for path in [&config.start_marker, &config.end_marker].into_iter().flatten() {
+        if tokio::fs::try_exists(path).await? {
+            return Err(Error::other("measurement markers must be fresh"));
+        }
+    }
+    // Require the test deployment's explicit secret; never use a default credential.
+    if secret.trim().is_empty() {
+        return Err(Error::other("RUSTFS_RPC_SECRET is required"));
+    }
+    rustfs_credentials::set_global_rpc_secret(secret).map_err(|_| Error::other("RPC secret already initialized"))?;
+    ensure_crypto_provider().map_err(Error::other)?;
+    if let Some(path) = &config.tls_ca {
+        let ca = bounded_file(path, 1024 * 1024).await?;
+        if ca.is_empty() {
+            return Err(Error::other("the explicit test CA must not be empty"));
+        }
+        publish_global_outbound_tls_state(
+            TlsGeneration(1),
+            &OutboundTlsMaterial {
+                root_ca_pem: ca,
+                mtls_identity: None,
+            },
+        )
+        .await;
+    }
+    let payload = bounded_file(&config.fixture_file, MAX_PAYLOAD).await?;
+    if payload.is_empty()
+        || payload
+            .len()
+            .checked_mul(config.bulk_workers)
+            .is_none_or(|bytes| bytes > 512 * 1024 * 1024)
+    {
+        return Err(Error::other("fixture and worker count exceed the payload memory budget"));
+    }
+    let path = format!("tmp/{}/grpc-bench-payload.bin", config.fixture_id);
+    let mut disks = Vec::with_capacity(endpoints.len());
+    let mut peers = BTreeSet::new();
+    for (index, endpoint) in endpoints.into_iter().enumerate() {
+        peers.insert(endpoint.grid_host());
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: true,
+            },
+        )
+        .await
+        .map_err(Error::other)?;
+        if let Some(ids) = &config.disk_ids {
+            disk.set_disk_id(Some(ids[index])).await.map_err(Error::other)?;
+        }
+        let data = timeout(Duration::from_secs(30), disk.read_all(".rustfs.sys", &path))
+            .await
+            .map_err(Error::other)?
+            .map_err(|error| Error::other(format!("ReadAll fixture preflight failed: disk_error_code={}", error.to_u32())))?;
+        if data.as_ref() != payload.as_slice() {
+            return Err(Error::other("ReadAll fixture preflight mismatch"));
+        }
+        disks.push(disk);
+    }
+    let peers: Vec<_> = peers.into_iter().collect();
+    let mut echoes = Vec::with_capacity(peers.len());
+    for peer in &peers {
+        let (version, body) = timeout(Duration::from_secs(30), ping(peer)).await.map_err(Error::other)??;
+        if version != 1 || body.is_empty() {
+            return Err(Error::other("Ping preflight mismatch"));
+        }
+        echoes.push(body);
+    }
+    let context = Arc::new(Context {
+        config,
+        disks,
+        peers,
+        echoes,
+        payload,
+        path,
+    });
+    phase("ready", now_ns()?)?;
+    let start_ns = match &context.config.start_marker {
+        Some(path) => marker(path).await?,
+        None => now_ns()?,
+    };
+    let current_ns = now_ns()?;
+    let origin = measurement_origin(start_ns, current_ns, Instant::now())?;
+    let offered = Duration::from_secs(u64::from(context.config.duration_secs));
+    let end = origin + offered;
+    phase("measure_start", start_ns)?;
+    let mut tasks = JoinSet::new();
+    for worker in 0..context.config.bulk_workers {
+        tasks.spawn(bulk(context.clone(), worker, origin, end));
+    }
+    tasks.spawn(control(context.clone(), origin));
+    let mut samples = Vec::new();
+    while let Some(result) = tasks.join_next().await {
+        samples.extend(result.map_err(Error::other)??);
+    }
+    let finished_ns = now_ns()?;
+    phase("load_finished", finished_ns)?;
+    let end_ns = match &context.config.end_marker {
+        Some(path) => marker(path).await?,
+        None => now_ns()?,
+    };
+    let current_ns = now_ns()?;
+    validate_observation_end(start_ns, end_ns, finished_ns, current_ns, elapsed_ns(origin)?)?;
+    let offered_ns = u64::try_from(offered.as_nanos()).map_err(Error::other)?;
+    let errors = samples.iter().filter(|sample| sample.outcome != "ok").count();
+    let measured_bytes = samples
+        .iter()
+        .filter(|sample| sample.operation == "ReadAll" && sample.outcome == "ok" && sample.completed_ns <= offered_ns)
+        .try_fold(0_u64, |total, sample| {
+            total
+                .checked_add(u64::try_from(sample.bytes).map_err(Error::other)?)
+                .ok_or_else(|| Error::other("measured payload byte counter overflow"))
+        })?;
+    let report = Report {
+        complete: true,
+        errors,
+        manifest_sha256: hex_simd::encode_to_string(Sha256::digest(config_bytes), hex_simd::AsciiCase::Lower),
+        fixture_sha256: hex_simd::encode_to_string(Sha256::digest(&context.payload), hex_simd::AsciiCase::Lower),
+        start_unix_ns: start_ns.to_string(),
+        observation_end_unix_ns: end_ns.to_string(),
+        offered_duration_secs: context.config.duration_secs,
+        control_rps: context.config.control_rps,
+        bulk_bytes_completed_during_offering: measured_bytes,
+        bulk_coverage_verified: measured_bytes > 0,
+        bulk_workers: context.config.bulk_workers,
+        control_expected: context.config.duration_secs * context.config.control_rps,
+        targets: context.disks.len(),
+        peers: context.peers.len(),
+        samples,
+        external_markers: context.config.start_marker.is_some(),
+        scope: "Authenticated production RPC clients and ReadAll temporary fixtures; client telemetry export is disabled. Storage S3/durability, mTLS/rotation and ABBA qualification remain separate.",
+    };
+    for disk in &context.disks {
+        disk.close().await.map_err(Error::other)?;
+    }
+    phase("measure_end", end_ns)?;
+    Ok(report)
+}
+
+#[tokio::main(worker_threads = 8)]
+async fn main() -> Result<()> {
+    let mut args = std::env::args_os().skip(1);
+    let config = args
+        .next()
+        .map(PathBuf::from)
+        .ok_or_else(|| Error::other("usage: internode_rpc_bench manifest.json output.json"))?;
+    let output = args
+        .next()
+        .map(PathBuf::from)
+        .ok_or_else(|| Error::other("output path is required"))?;
+    if args.next().is_some() {
+        return Err(Error::other("unexpected probe arguments"));
+    }
+    let input = bounded_file(&config, 64 * 1024).await?;
+    let file = tokio::fs::OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(output)
+        .await?
+        .into_std()
+        .await;
+    let secret = std::env::var("RUSTFS_RPC_SECRET").map_err(|_| Error::other("RUSTFS_RPC_SECRET is required"))?;
+    let report = run(&input, config.parent().unwrap_or_else(|| Path::new(".")), secret).await?;
+    let success = report.errors == 0 && report.bulk_coverage_verified;
+    tokio::task::spawn_blocking(move || {
+        let mut output = std::io::BufWriter::new(file);
+        serde_json::to_writer(&mut output, &report).map_err(Error::other)?;
+        output.flush()
+    })
+    .await
+    .map_err(Error::other)??;
+    if success {
+        Ok(())
+    } else {
+        Err(Error::other(
+            "RPC failures or missing bulk coverage invalidate attribution; see raw samples",
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_protos::proto_gen::node_service::{PingResponse, ReadAllRequest, ReadAllResponse};
+    use std::convert::Infallible;
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+    use std::task::{Context as TaskContext, Poll};
+    use tonic::codegen::{Body, BoxFuture, Service, StdError, http};
+
+    fn config() -> Config {
+        Config {
+            namespace: "grpc-bench-test".into(),
+            targets: vec!["http://127.0.0.1:19000/data/grpc-bench-test".into()],
+            disk_ids: None,
+            fixture_id: Uuid::from_u128(1),
+            fixture_file: PathBuf::from("fixture.bin"),
+            duration_secs: 1,
+            bulk_workers: 2,
+            control_rps: 20,
+            rpc_timeout_ms: 1000,
+            tls_ca: None,
+            start_marker: None,
+            end_marker: None,
+        }
+    }
+
+    #[test]
+    fn probe_rejects_non_test_and_credential_bearing_targets() {
+        for target in [
+            "file:///data/grpc-bench-test",
+            "http://user:private@127.0.0.1:19000/data/grpc-bench-test",
+            "http://127.0.0.1:19000/data/mnmd",
+            "http://127.0.0.1:80/data/grpc-bench-test",
+            "http://127.0.0.1:19000/data/grpc-bench-test?token=private",
+            "http://127.0.0.1:19000/data/%67rpc-bench-test",
+            "https://127.0.0.1:19000/data/grpc-bench-test",
+        ] {
+            let mut config = config();
+            config.targets = vec![target.into()];
+            assert!(config.endpoints().is_err(), "unsafe target was accepted");
+        }
+    }
+
+    #[test]
+    fn probe_forces_remote_client_and_rejects_duplicate_disks() {
+        let mut config = config();
+        assert!(!config.endpoints().expect("valid loopback test URL")[0].is_local);
+        config.targets.push(config.targets[0].clone());
+        assert!(config.endpoints().is_err(), "duplicate target must not inflate the inventory");
+    }
+
+    #[test]
+    fn probe_requires_a_complete_non_nil_disk_id_mapping() {
+        let mut config = config();
+        config.disk_ids = Some(vec![]);
+        assert!(config.endpoints().is_err());
+        config.disk_ids = Some(vec![Uuid::nil()]);
+        assert!(config.endpoints().is_err());
+        config.disk_ids = Some(vec![Uuid::from_u128(42)]);
+        assert!(config.endpoints().is_ok());
+    }
+
+    #[test]
+    fn probe_bounds_work_and_requires_distinct_measurement_markers() {
+        for (seconds, workers, rate, deadline) in [
+            (0, 2, 20, 1000),
+            (1, 0, 20, 1000),
+            (1, 129, 20, 1000),
+            (1800, 2, 10_000, 1000),
+            (1, 2, 0, 1000),
+            (1, 2, 20, 0),
+        ] {
+            let mut config = config();
+            config.duration_secs = seconds;
+            config.bulk_workers = workers;
+            config.control_rps = rate;
+            config.rpc_timeout_ms = deadline;
+            assert!(config.endpoints().is_err());
+        }
+        let mut config = config();
+        config.start_marker = Some("marker.json".into());
+        assert!(config.endpoints().is_err());
+        config.end_marker = config.start_marker.clone();
+        assert!(config.endpoints().is_err());
+    }
+
+    #[test]
+    fn probe_rejects_stale_future_or_truncated_measurement_markers() {
+        let now = Instant::now();
+        assert_eq!(
+            measurement_origin(10_000_000_000, 11_000_000_000, now).expect("one-second lag bound"),
+            now - Duration::from_secs(1)
+        );
+        assert!(measurement_origin(10_000_000_000, 11_000_000_001, now).is_err());
+        assert!(measurement_origin(11_000_000_001, 11_000_000_000, now).is_err());
+        assert!(validate_observation_end(100, 200, 200, 250, 150).is_ok());
+        assert!(validate_observation_end(100, 199, 200, 250, 150).is_err());
+        assert!(validate_observation_end(100, 251, 200, 250, 150).is_err());
+        assert!(validate_observation_end(100, 200, 200, 250_000_000, 150).is_err());
+    }
+
+    #[tokio::test]
+    async fn probe_marker_input_has_an_explicit_schema_and_size_bound() {
+        let directory = tempfile::tempdir().expect("marker directory");
+        let path = directory.path().join("marker.json");
+        tokio::fs::write(&path, br#"{"time_ns":100}"#).await.expect("marker");
+        assert_eq!(marker(&path).await.expect("valid marker"), 100);
+        tokio::fs::write(&path, br#"{"time_ns":100,"unknown":true}"#)
+            .await
+            .expect("invalid marker");
+        assert!(marker(&path).await.is_err());
+        tokio::fs::write(&path, vec![b' '; 513]).await.expect("oversized marker");
+        assert!(marker(&path).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn probe_bounds_fixture_reads_and_rejects_directories() {
+        let directory = tempfile::tempdir().expect("temporary fixture directory");
+        let path = directory.path().join("fixture.bin");
+        tokio::fs::write(&path, [7_u8; 32]).await.expect("fixture write");
+        assert_eq!(bounded_file(&path, 32).await.expect("exact-size fixture"), vec![7_u8; 32]);
+        tokio::fs::write(&path, [7_u8; 33]).await.expect("oversized fixture write");
+        assert!(bounded_file(&path, 32).await.is_err());
+        assert!(bounded_file(directory.path(), 32).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn probe_accounts_for_expired_control_arrivals_without_sending_them() {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("owned control socket");
+        let context = Arc::new(super::Context {
+            config: config(),
+            disks: vec![],
+            peers: vec![format!("http://{}", listener.local_addr().expect("owned address"))],
+            echoes: vec![bytes::Bytes::from_static(b"hello")],
+            payload: vec![],
+            path: String::new(),
+        });
+        let rows = control(context, Instant::now() - Duration::from_secs(2))
+            .await
+            .expect("arrival accounting");
+        assert_eq!(rows.len(), 20);
+        assert!(
+            rows.iter()
+                .all(|row| row.outcome == "schedule_drop" && row.dispatched_ns.is_none())
+        );
+    }
+
+    #[derive(Clone)]
+    struct ReadFixture(bytes::Bytes, Arc<AtomicBool>);
+
+    impl tonic::server::UnaryService<ReadAllRequest> for ReadFixture {
+        type Response = ReadAllResponse;
+        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+        fn call(&mut self, request: tonic::Request<ReadAllRequest>) -> Self::Future {
+            assert_eq!(request.get_ref().volume, ".rustfs.sys");
+            assert_eq!(request.get_ref().disk, Uuid::from_u128(42).to_string());
+            assert!(request.get_ref().path.ends_with("/grpc-bench-payload.bin"));
+            let data = if self.1.load(Ordering::SeqCst) {
+                bytes::Bytes::from_static(b"corrupt fixture")
+            } else {
+                self.0.clone()
+            };
+            Box::pin(async move {
+                Ok(tonic::Response::new(ReadAllResponse {
+                    success: true,
+                    data,
+                    error: None,
+                }))
+            })
+        }
+    }
+
+    #[derive(Clone)]
+    struct ReplyPing;
+
+    impl tonic::server::UnaryService<PingRequest> for ReplyPing {
+        type Response = PingResponse;
+        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+        fn call(&mut self, _: tonic::Request<PingRequest>) -> Self::Future {
+            Box::pin(async {
+                Ok(tonic::Response::new(PingResponse {
+                    version: 1,
+                    body: bytes::Bytes::from_static(b"hello, caller"),
+                }))
+            })
+        }
+    }
+
+    #[derive(Clone)]
+    struct FixtureService {
+        data: bytes::Bytes,
+        signed: Arc<AtomicUsize>,
+        corrupt: Arc<AtomicBool>,
+    }
+
+    impl tonic::server::NamedService for FixtureService {
+        const NAME: &'static str = "node_service.NodeService";
+    }
+
+    impl<B> Service<http::Request<B>> for FixtureService
+    where
+        B: Body + Send + 'static,
+        B::Error: Into<StdError> + Send + 'static,
+    {
+        type Response = http::Response<tonic::body::Body>;
+        type Error = Infallible;
+        type Future = BoxFuture<Self::Response, Self::Error>;
+        fn poll_ready(&mut self, _: &mut TaskContext<'_>) -> Poll<std::result::Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+        fn call(&mut self, request: http::Request<B>) -> Self::Future {
+            assert_eq!(
+                request
+                    .headers()
+                    .get("x-rustfs-rpc-auth-version")
+                    .expect("signed production client"),
+                "2"
+            );
+            assert!(request.headers().contains_key("x-rustfs-rpc-signature-v2"));
+            self.signed.fetch_add(1, Ordering::Relaxed);
+            let data = self.data.clone();
+            let corrupt = self.corrupt.clone();
+            Box::pin(async move {
+                if request.uri().path().ends_with("/ReadAll") {
+                    Ok(tonic::server::Grpc::new(tonic_prost::ProstCodec::default())
+                        .unary(ReadFixture(data, corrupt), request)
+                        .await)
+                } else {
+                    assert!(request.uri().path().ends_with("/Ping"));
+                    Ok(tonic::server::Grpc::new(tonic_prost::ProstCodec::default())
+                        .unary(ReplyPing, request)
+                        .await)
+                }
+            })
+        }
+    }
+
+    #[tokio::test]
+    async fn probe_exercises_production_readall_and_fixed_rate_ping_clients() {
+        let directory = tempfile::tempdir().expect("probe fixtures");
+        let payload = vec![17_u8; 4096];
+        let file = directory.path().join("fixture.bin");
+        tokio::fs::write(&file, &payload).await.expect("fixture bytes");
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("owned RPC listener");
+        let address = listener.local_addr().expect("owned RPC address");
+        let incoming = futures_util::stream::unfold(listener, |listener| async move {
+            Some((listener.accept().await.map(|(stream, _)| stream), listener))
+        });
+        let (stop, stopped) = tokio::sync::oneshot::channel();
+        let signed = Arc::new(AtomicUsize::new(0));
+        let corrupt = Arc::new(AtomicBool::new(false));
+        let service = FixtureService {
+            data: payload.into(),
+            signed: signed.clone(),
+            corrupt: corrupt.clone(),
+        };
+        let mut server = tokio::spawn(
+            tonic::transport::Server::builder()
+                .add_service(service)
+                .serve_with_incoming_shutdown(Box::pin(incoming), async {
+                    let _ = stopped.await;
+                }),
+        );
+        let mut config = config();
+        config.targets = vec![format!("http://{address}/data/grpc-bench-test")];
+        config.disk_ids = Some(vec![Uuid::from_u128(42)]);
+        config.fixture_file = file;
+        let input = serde_json::to_vec(&config).expect("probe manifest");
+        let result = run(&input, directory.path(), "test-bench-rpc-secret".into()).await;
+        let negative = async {
+            let endpoint = config.endpoints()?.remove(0);
+            let disk = new_disk(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: true,
+                },
+            )
+            .await
+            .map_err(Error::other)?;
+            disk.set_disk_id(Some(Uuid::from_u128(42))).await.map_err(Error::other)?;
+            corrupt.store(true, Ordering::SeqCst);
+            let context = Arc::new(super::Context {
+                path: format!("tmp/{}/grpc-bench-payload.bin", config.fixture_id),
+                config: config.clone(),
+                disks: vec![disk.clone()],
+                peers: vec![],
+                echoes: vec![],
+                payload: tokio::fs::read(&config.fixture_file).await?,
+            });
+            let origin = Instant::now();
+            let rows = bulk(context, 0, origin, origin + Duration::from_millis(50)).await?;
+            disk.close().await.map_err(Error::other)?;
+            Ok::<_, Error>(rows)
+        }
+        .await;
+        let _ = stop.send(());
+        let retirement = timeout(Duration::from_secs(5), &mut server).await;
+        if retirement.is_err() {
+            server.abort();
+            let _ = server.await;
+        }
+        assert!(matches!(retirement, Ok(Ok(Ok(())))), "owned RPC fixture must retire successfully");
+        let result = result.expect("production client probe");
+        assert!(result.bulk_coverage_verified);
+        let report = serde_json::to_value(result).expect("raw sample report");
+        assert_eq!(report["errors"], 0);
+        assert_eq!(report["control_expected"], 20);
+        let rows = report["samples"].as_array().expect("raw rows");
+        assert_eq!(rows.iter().filter(|row| row["operation"] == "Ping").count(), 20);
+        assert!(rows.iter().any(|row| row["operation"] == "ReadAll"));
+        assert!(signed.load(Ordering::Relaxed) > 20);
+        let rejected = negative.expect("corrupted-response probe");
+        assert!(!rejected.is_empty());
+        assert!(rejected.iter().all(|row| row.outcome == "payload_mismatch" && row.bytes == 0));
+    }
+}

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -54,6 +54,10 @@ use bytes::Bytes;
 use futures::lock::Mutex;
 use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
+use rustfs_io_metrics::internode_metrics::{
+    INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+    INTERNODE_OPERATION_GRPC_WRITE_ALL, measure_client_rpc,
+};
 use rustfs_io_metrics::internode_metrics::{
     INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
     INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
@@ -3737,13 +3741,17 @@ impl DiskAPI for RemoteDisk {
                     }
                     Err(err) => {
                         record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
-                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(
+                            crate::cluster::rpc::runtime_sources::ReadVersionErrorStage::RpcStatus,
+                        );
                         return Err(err.into());
                     }
                 };
 
                 if !response.success {
-                    crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
+                    crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(
+                        crate::cluster::rpc::runtime_sources::ReadVersionErrorStage::RemoteResponse,
+                    );
                     return Err(response.error.unwrap_or_default().into());
                 }
 
@@ -3762,7 +3770,9 @@ impl DiskAPI for RemoteDisk {
                     }
                     Err(err) => {
                         record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, decode_started);
-                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(
+                            crate::cluster::rpc::runtime_sources::ReadVersionErrorStage::ResponseDecode,
+                        );
                         return Err(err);
                     }
                 };
@@ -3824,12 +3834,8 @@ impl DiskAPI for RemoteDisk {
                         );
                     }
                     let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
-                    let response_result = client.batch_read_version(request).await;
-                    if let Some(rpc_started) = rpc_started {
-                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_duration(
-                            rpc_started.elapsed(),
-                        );
-                    }
+                    let response_result =
+                        measure_client_rpc(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, client.batch_read_version(request)).await;
                     let response = match response_result {
                         Ok(response) => {
                             record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
@@ -4855,9 +4861,7 @@ impl DiskAPI for RemoteDisk {
                     read_multiple_req_bin: read_multiple_req_bin.into(),
                 });
 
-                let rpc_started = Instant::now();
-                let response = client.read_multiple(request).await;
-                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_multiple_duration(rpc_started.elapsed());
+                let response = measure_client_rpc(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, client.read_multiple(request)).await;
                 let response = response?.into_inner();
 
                 if !response.success {
@@ -4909,8 +4913,7 @@ impl DiskAPI for RemoteDisk {
                 attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
 
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
-                let rpc_started = Instant::now();
-                let response = match client.write_all(request).await {
+                let response = match measure_client_rpc(INTERNODE_OPERATION_GRPC_WRITE_ALL, client.write_all(request)).await {
                     Ok(response) => response.into_inner(),
                     Err(err) => {
                         crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
@@ -4918,7 +4921,6 @@ impl DiskAPI for RemoteDisk {
                     }
                 };
 
-                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_duration(rpc_started.elapsed());
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
 
                 if !response.success {
@@ -5016,8 +5018,7 @@ impl DiskAPI for RemoteDisk {
                 });
 
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_request();
-                let rpc_started = Instant::now();
-                let response = match client.read_all(request).await {
+                let response = match measure_client_rpc(INTERNODE_OPERATION_GRPC_READ_ALL, client.read_all(request)).await {
                     Ok(response) => response.into_inner(),
                     Err(err) => {
                         crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
@@ -5025,8 +5026,6 @@ impl DiskAPI for RemoteDisk {
                     }
                 };
 
-                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_duration(rpc_started.elapsed());
-
                 if !response.success {
                     crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
                     return Err(response.error.unwrap_or_default().into());

crates/ecstore/src/cluster/rpc/runtime_sources.rs

@@ -481,14 +481,6 @@ pub(crate) fn record_remote_disk_grpc_write_all_request() {
         .record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
 }
 
-pub(crate) fn record_remote_disk_grpc_write_all_duration(duration: Duration) {
-    global_internode_metrics().record_duration_for_operation_and_backend(
-        INTERNODE_OPERATION_GRPC_WRITE_ALL,
-        INTERNODE_TRANSPORT_BACKEND_GRPC,
-        duration,
-    );
-}
-
 pub(crate) fn record_remote_disk_grpc_write_all_sent_bytes(bytes: usize) {
     global_internode_metrics().record_sent_bytes_for_operation_and_backend(
         INTERNODE_OPERATION_GRPC_WRITE_ALL,
@@ -529,14 +521,6 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
         .record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
 }
 
-pub(crate) fn record_remote_disk_grpc_read_all_duration(duration: Duration) {
-    global_internode_metrics().record_duration_for_operation_and_backend(
-        INTERNODE_OPERATION_GRPC_READ_ALL,
-        INTERNODE_TRANSPORT_BACKEND_GRPC,
-        duration,
-    );
-}
-
 #[cfg(feature = "diagnostic-read-version-caller")]
 fn read_version_request_mode(opts: &ReadOptions) -> ReadVersionRequestMode {
     match (opts.read_data, opts.healing, opts.incl_free_versions) {
@@ -614,17 +598,6 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static s
     );
 }
 
-pub(crate) fn record_remote_disk_grpc_batch_read_version_duration(duration: Duration) {
-    if !rustfs_io_metrics::get_stage_metrics_enabled() {
-        return;
-    }
-    global_internode_metrics().record_duration_for_operation_and_backend(
-        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
-        INTERNODE_TRANSPORT_BACKEND_GRPC,
-        duration,
-    );
-}
-
 pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
     if !rustfs_io_metrics::get_stage_metrics_enabled() {
         return;
@@ -656,12 +629,34 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize
     record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
 }
 
-pub(crate) fn record_remote_disk_grpc_read_version_error() {
+#[derive(Clone, Copy)]
+pub(crate) enum ReadVersionErrorStage {
+    RpcStatus,
+    RemoteResponse,
+    ResponseDecode,
+}
+
+impl ReadVersionErrorStage {
+    fn classification(self) -> &'static str {
+        match self {
+            Self::RpcStatus => "grpc_status",
+            Self::RemoteResponse => "remote_response",
+            Self::ResponseDecode => "response_decode",
+        }
+    }
+}
+
+pub(crate) fn record_remote_disk_grpc_read_version_error(stage: ReadVersionErrorStage) {
     if !rustfs_io_metrics::get_stage_metrics_enabled() {
         return;
     }
-    global_internode_metrics()
-        .record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
+    let metrics = global_internode_metrics();
+    metrics.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
+    metrics.record_classified_error_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        stage.classification(),
+    );
 }
 
 pub(crate) fn record_remote_disk_grpc_read_version_sent_bytes(bytes: usize) {
@@ -717,14 +712,6 @@ pub(crate) fn record_remote_disk_grpc_read_multiple_recv_bytes(bytes: usize) {
     record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, bytes);
 }
 
-pub(crate) fn record_remote_disk_grpc_read_multiple_duration(duration: Duration) {
-    global_internode_metrics().record_duration_for_operation_and_backend(
-        INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
-        INTERNODE_TRANSPORT_BACKEND_GRPC,
-        duration,
-    );
-}
-
 /// Payload-size threshold (bytes) above which a unary internode gRPC response is counted as a
 /// "large payload" for alerting. Env-overridable via `RUSTFS_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES`.
 fn internode_rpc_large_payload_warn_bytes() -> usize {
@@ -804,6 +791,53 @@ mod tests {
     #[cfg(feature = "diagnostic-read-version-caller")]
     use super::*;
 
+    #[test]
+    fn read_version_errors_preserve_totals_and_classify_failure_boundaries() {
+        use super::{ReadVersionErrorStage, record_remote_disk_grpc_read_version_error};
+        use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        rustfs_io_metrics::set_get_stage_metrics_enabled(true);
+        metrics::with_local_recorder(&recorder, || {
+            for stage in [
+                ReadVersionErrorStage::RpcStatus,
+                ReadVersionErrorStage::RemoteResponse,
+                ReadVersionErrorStage::ResponseDecode,
+            ] {
+                record_remote_disk_grpc_read_version_error(stage);
+            }
+        });
+        rustfs_io_metrics::set_get_stage_metrics_enabled(false);
+        let entries = snapshotter.snapshot().into_vec();
+        let total = entries
+            .iter()
+            .find(|entry| {
+                entry.0.key().name() == "rustfs_system_network_internode_operation_errors_total"
+                    && entry
+                        .0
+                        .key()
+                        .labels()
+                        .any(|label| label.key() == "operation" && label.value() == "grpc_read_version")
+            })
+            .expect("existing operation error counter");
+        assert!(matches!(total.3, DebugValue::Counter(3)));
+        for classification in ["grpc_status", "remote_response", "response_decode"] {
+            let count = entries
+                .iter()
+                .find(|entry| {
+                    entry.0.key().name() == "rustfs_system_network_internode_operation_classified_errors_total"
+                        && entry
+                            .0
+                            .key()
+                            .labels()
+                            .any(|label| label.key() == "classification" && label.value() == classification)
+                })
+                .expect("bounded failure classification");
+            assert!(matches!(count.3, DebugValue::Counter(1)));
+        }
+    }
+
     #[cfg(feature = "diagnostic-read-version-caller")]
     fn diagnostic_operation(operation: rustfs_s3_ops::S3Operation) -> S3HttpOperationContext {
         let mut request = rustfs_io_metrics::s3_http_metrics::S3HttpRequestGuard::new("GET");

crates/ecstore/src/disk/os.rs

@@ -611,6 +611,7 @@ pub(crate) mod prepare_rename_test_hooks {
             .push_back(Box::new(hook));
     }
 
+    #[cfg(target_os = "linux")]
     pub(crate) fn has_hooks() -> bool {
         AFTER_COMPONENT_OPENED.lock().values().any(|q| !q.is_empty())
     }

crates/io-metrics/src/internode_metrics.rs

@@ -430,6 +430,8 @@ const INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network
 const INTERNODE_OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total";
 const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_operation_duration_ms";
 const INTERNODE_OPERATION_SERVER_DURATION_MS: &str = "rustfs_system_network_internode_operation_server_duration_ms";
+const INTERNODE_RPC_ACTIVE: &str = "rustfs_system_network_internode_rpc_active";
+const INTERNODE_RPC_COMPLETIONS_TOTAL: &str = "rustfs_system_network_internode_rpc_completions_total";
 const INTERNODE_OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms";
 const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total";
 const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total";
@@ -463,6 +465,8 @@ pub struct InternodeOperationMetricDescriptor {
 }
 
 const SERVER_OPERATION_BACKEND_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL];
+const SERVER_OPERATION_BACKEND_SIDE_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, "side"];
+const SERVER_OPERATION_BACKEND_OUTCOME_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, "outcome"];
 const SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] =
     &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
 const SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL];
@@ -567,6 +571,14 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
         name: INTERNODE_OPERATION_SERVER_DURATION_MS,
         labels: SERVER_OPERATION_BACKEND_LABELS,
     },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_RPC_ACTIVE,
+        labels: SERVER_OPERATION_BACKEND_SIDE_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_RPC_COMPLETIONS_TOTAL,
+        labels: SERVER_OPERATION_BACKEND_OUTCOME_LABELS,
+    },
 ];
 
 static STABLE_SERVER_LABEL: OnceLock<String> = OnceLock::new();
@@ -597,6 +609,148 @@ impl InternodeServerMetricHandles {
 static INTERNODE_SERVER_METRIC_HANDLES: LazyLock<InternodeServerMetricHandles> =
     LazyLock::new(|| InternodeServerMetricHandles::new(current_server_label()));
 
+/// Measures a unary handler from entry through its returned response or cancellation. The
+/// handler metric is separate from client round trips, which include transport and body decode.
+pub struct InternodeRpcDurationGuard {
+    started_at: Instant,
+    handles: UnaryRpcMetricHandles,
+    outcome: RpcOutcome,
+}
+
+#[derive(Clone)]
+struct UnaryRpcMetricHandles {
+    duration: metrics::Histogram,
+    active: metrics::Gauge,
+    completion: Option<(metrics::Counter, metrics::Counter, metrics::Counter)>,
+}
+
+#[derive(Clone, Copy)]
+enum RpcOutcome {
+    Success,
+    Error,
+    Cancelled,
+}
+
+impl UnaryRpcMetricHandles {
+    fn new(operation: &'static str, client: bool, detailed: bool) -> Self {
+        let server = current_server_label();
+        let duration_name = if client {
+            INTERNODE_OPERATION_DURATION_MS
+        } else {
+            INTERNODE_OPERATION_SERVER_DURATION_MS
+        };
+        let duration = metrics::histogram!(
+            duration_name,
+            SERVER_LABEL => server, OPERATION_LABEL => operation, BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+        );
+        let active = if detailed {
+            metrics::gauge!(
+                INTERNODE_RPC_ACTIVE,
+                SERVER_LABEL => server, OPERATION_LABEL => operation, BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC,
+                "side" => if client { "client" } else { "server" }
+            )
+        } else {
+            metrics::Gauge::noop()
+        };
+        let completion = (client && detailed).then(|| {
+            let counter = |outcome| {
+                metrics::counter!(
+                    INTERNODE_RPC_COMPLETIONS_TOTAL,
+                    SERVER_LABEL => server, OPERATION_LABEL => operation,
+                    BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC, "outcome" => outcome
+                )
+            };
+            (counter("success"), counter("error"), counter("cancelled"))
+        });
+        Self {
+            duration,
+            active,
+            completion,
+        }
+    }
+}
+
+// Register once after recorder installation and node identity publication. Before that point,
+// use uncached handles so startup no-op instruments cannot poison the process-lifetime cache.
+#[cfg(not(test))]
+fn cached_unary_rpc_handles(operation: &'static str, client: bool) -> Option<UnaryRpcMetricHandles> {
+    const OPERATIONS: [&str; 4] = [
+        INTERNODE_OPERATION_GRPC_READ_ALL,
+        INTERNODE_OPERATION_GRPC_WRITE_ALL,
+        INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+    ];
+    static CLIENT: LazyLock<[UnaryRpcMetricHandles; 4]> =
+        LazyLock::new(|| OPERATIONS.map(|op| UnaryRpcMetricHandles::new(op, true, true)));
+    static SERVER: LazyLock<[UnaryRpcMetricHandles; 4]> =
+        LazyLock::new(|| OPERATIONS.map(|op| UnaryRpcMetricHandles::new(op, false, true)));
+    if !crate::metrics_enabled() {
+        return None;
+    }
+    STABLE_SERVER_LABEL.get()?;
+    let index = OPERATIONS.iter().position(|candidate| *candidate == operation)?;
+    Some(if client {
+        CLIENT[index].clone()
+    } else {
+        SERVER[index].clone()
+    })
+}
+
+impl InternodeRpcDurationGuard {
+    /// Start timing a server handler. `operation` must be a bounded RPC operation constant.
+    pub fn handler(operation: &'static str) -> Self {
+        Self::new(operation, false)
+    }
+
+    fn new(operation: &'static str, client: bool) -> Self {
+        #[cfg(not(test))]
+        let handles = cached_unary_rpc_handles(operation, client)
+            .unwrap_or_else(|| UnaryRpcMetricHandles::new(operation, client, crate::metrics_enabled()));
+        #[cfg(test)]
+        let handles = UnaryRpcMetricHandles::new(operation, client, crate::metrics_enabled());
+        handles.active.increment(1.0);
+        Self {
+            started_at: Instant::now(),
+            handles,
+            outcome: RpcOutcome::Cancelled,
+        }
+    }
+}
+
+impl Drop for InternodeRpcDurationGuard {
+    fn drop(&mut self) {
+        self.handles.duration.record(self.started_at.elapsed().as_secs_f64() * 1000.0);
+        self.handles.active.decrement(1.0);
+        if let Some((success, error, cancelled)) = &self.handles.completion {
+            match self.outcome {
+                RpcOutcome::Success => success,
+                RpcOutcome::Error => error,
+                RpcOutcome::Cancelled => cancelled,
+            }
+            .increment(1);
+        }
+    }
+}
+
+/// Measure one complete client unary attempt, including body consumption by the supplied
+/// future. Dropping a pending attempt records its elapsed time. General metrics enable the
+/// active gauge and success/error/cancelled completion counters alongside the duration.
+/// Transport deadlines returned by the future are errors; outer deadlines cancel the attempt.
+/// Retries should call this once per attempt. `operation` must be a bounded RPC constant.
+pub async fn measure_client_rpc<T, E>(
+    operation: &'static str,
+    future: impl std::future::Future<Output = Result<T, E>>,
+) -> Result<T, E> {
+    let mut guard = InternodeRpcDurationGuard::new(operation, true);
+    let result = future.await;
+    guard.outcome = if result.is_ok() {
+        RpcOutcome::Success
+    } else {
+        RpcOutcome::Error
+    };
+    result
+}
+
 #[cfg(not(test))]
 struct GrpcReadVersionMetricHandles {
     sent_bytes: metrics::Counter,
@@ -1617,6 +1771,83 @@ mod tests {
     use metrics_util::debugging::{DebugValue, DebuggingRecorder};
     use std::collections::{HashMap, HashSet};
 
+    #[test]
+    fn unary_rpc_metrics_separate_handler_and_client_and_cover_cancellation() {
+        use std::future::{Future, pending};
+        use std::task::{Context, Poll, Waker};
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let previously_enabled = crate::metrics_enabled();
+        crate::set_metrics_enabled(true);
+        with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("test runtime");
+            runtime.block_on(async {
+                let operation = INTERNODE_OPERATION_GRPC_READ_MULTIPLE;
+                assert_eq!(measure_client_rpc(operation, async { Ok::<_, ()>(7) }).await, Ok(7));
+                assert_eq!(measure_client_rpc(operation, async { Err::<(), _>("transport") }).await, Err("transport"));
+                let mut cancelled = Box::pin(measure_client_rpc(operation, pending::<Result<(), ()>>()));
+                assert!(matches!(cancelled.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
+                drop(cancelled);
+                drop(InternodeRpcDurationGuard::handler(operation));
+            });
+        });
+        crate::set_metrics_enabled(previously_enabled);
+        let entries = snapshotter.snapshot().into_vec();
+        let histogram_samples = |name| {
+            entries
+                .iter()
+                .find_map(|(key, _, _, value)| {
+                    if key.key().name() != name {
+                        return None;
+                    }
+                    match value {
+                        DebugValue::Histogram(values) => Some(values.len()),
+                        _ => None,
+                    }
+                })
+                .expect("duration histogram")
+        };
+        assert_eq!(histogram_samples(INTERNODE_OPERATION_DURATION_MS), 3);
+        assert_eq!(histogram_samples(INTERNODE_OPERATION_SERVER_DURATION_MS), 1);
+        let mut outcomes = HashSet::new();
+        let mut active_sides = HashSet::new();
+        for (key, _, _, value) in entries {
+            match key.key().name() {
+                INTERNODE_RPC_COMPLETIONS_TOTAL => {
+                    assert!(matches!(value, DebugValue::Counter(1)));
+                    outcomes.insert(
+                        key.key()
+                            .labels()
+                            .find(|label| label.key() == "outcome")
+                            .expect("outcome")
+                            .value()
+                            .to_owned(),
+                    );
+                }
+                INTERNODE_RPC_ACTIVE => {
+                    assert!(matches!(value, DebugValue::Gauge(value) if value.into_inner() == 0.0));
+                    active_sides.insert(
+                        key.key()
+                            .labels()
+                            .find(|label| label.key() == "side")
+                            .expect("side")
+                            .value()
+                            .to_owned(),
+                    );
+                }
+                _ => {}
+            }
+        }
+        assert_eq!(
+            outcomes,
+            HashSet::from(["success".to_owned(), "error".to_owned(), "cancelled".to_owned()])
+        );
+        assert_eq!(active_sides, HashSet::from(["client".to_owned(), "server".to_owned()]));
+    }
+
     #[test]
     fn snapshot_reports_recorded_values() {
         let metrics = global_internode_metrics();
@@ -1787,7 +2018,11 @@ mod tests {
 
     #[test]
     fn operation_metric_descriptors_include_backend_and_operation_labels() {
-        assert_eq!(INTERNODE_OPERATION_METRICS.len(), 23);
+        assert_eq!(INTERNODE_OPERATION_METRICS.len(), 25);
+        assert_eq!(INTERNODE_OPERATION_METRICS[23].name, INTERNODE_RPC_ACTIVE);
+        assert_eq!(INTERNODE_OPERATION_METRICS[23].labels, SERVER_OPERATION_BACKEND_SIDE_LABELS);
+        assert_eq!(INTERNODE_OPERATION_METRICS[24].name, INTERNODE_RPC_COMPLETIONS_TOTAL);
+        assert_eq!(INTERNODE_OPERATION_METRICS[24].labels, SERVER_OPERATION_BACKEND_OUTCOME_LABELS);
         for metric in &INTERNODE_OPERATION_METRICS[..6] {
             assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
         }

crates/io-metrics/src/lib.rs

@@ -179,8 +179,9 @@ fn record_get_stage_local_summary(
 /// Enable or disable general (non GET/PUT-stage) metric emission.
 ///
 /// Called once during startup, typically gated by `rustfs_obs::observability_metric_enabled()`.
+/// Enable after installing the recorder; the release/acquire pair publishes it to handle caches.
 pub fn set_metrics_enabled(enabled: bool) {
-    METRICS_ENABLED.store(enabled, Ordering::Relaxed);
+    METRICS_ENABLED.store(enabled, Ordering::Release);
 }
 
 /// Returns `true` if detailed per-stage PUT metrics are enabled.
@@ -447,7 +448,7 @@ pub fn get_stage_metrics_enabled() -> bool {
 /// Returns `true` if general (non GET/PUT-stage) metric emission is enabled.
 #[inline(always)]
 pub fn metrics_enabled() -> bool {
-    METRICS_ENABLED.load(Ordering::Relaxed)
+    METRICS_ENABLED.load(Ordering::Acquire)
 }
 
 // Handle-cached emission for hot, LABEL-LESS metrics. The `metrics` macros

crates/io-metrics/tests/unary_rpc_recorder.rs

@@ -0,0 +1,48 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+use rustfs_io_metrics::internode_metrics::{INTERNODE_OPERATION_GRPC_READ_ALL, measure_client_rpc, set_internode_server_label};
+
+#[tokio::test]
+async fn unary_rpc_handles_wait_for_recorder_installation() {
+    set_internode_server_label("unary-recorder-test");
+    // Exercise the real non-test library path before a process recorder exists.
+    measure_client_rpc(INTERNODE_OPERATION_GRPC_READ_ALL, async { Ok::<_, ()>(()) })
+        .await
+        .expect("startup call");
+
+    let recorder = DebuggingRecorder::new();
+    let snapshotter = recorder.snapshotter();
+    metrics::set_global_recorder(recorder).expect("install test recorder once");
+    measure_client_rpc(INTERNODE_OPERATION_GRPC_READ_ALL, async { Ok::<_, ()>(()) })
+        .await
+        .expect("legacy duration before detailed metrics enablement");
+    let before_enable = snapshotter.snapshot().into_vec();
+    assert_eq!(
+        before_enable.len(),
+        1,
+        "disabled detailed metrics register only the existing duration instrument"
+    );
+    assert_eq!(before_enable[0].0.key().name(), "rustfs_system_network_internode_operation_duration_ms");
+    assert!(matches!(&before_enable[0].3, DebugValue::Histogram(samples) if samples.len() == 1));
+    rustfs_io_metrics::set_metrics_enabled(true);
+    for _ in 0..3 {
+        measure_client_rpc(INTERNODE_OPERATION_GRPC_READ_ALL, async { Ok::<_, ()>(()) })
+            .await
+            .expect("recorded call");
+    }
+    let entries = snapshotter.snapshot().into_vec();
+    let duration = entries
+        .iter()
+        .find(|(key, _, _, _)| {
+            key.key().name() == "rustfs_system_network_internode_operation_duration_ms"
+                && key
+                    .key()
+                    .labels()
+                    .any(|label| label.key() == "operation" && label.value() == INTERNODE_OPERATION_GRPC_READ_ALL)
+        })
+        .expect("installed recorder receives the cached RPC duration handle");
+    // Snapshotter drains histogram observations, so this snapshot contains the three later calls.
+    assert!(matches!(&duration.3, DebugValue::Histogram(samples) if samples.len() == 3));
+}

crates/obs/src/telemetry/otel.rs

@@ -104,6 +104,16 @@ const OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
     0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0,
 ];
 
+const INTERNODE_DURATION_HISTOGRAM_METRICS: &[&str] = &[
+    "rustfs_system_network_internode_operation_duration_ms",
+    "rustfs_system_network_internode_operation_server_duration_ms",
+];
+// Internode instruments record milliseconds, including sub-millisecond control RPCs.
+const INTERNODE_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
+    0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 50.0, 75.0, 100.0, 250.0,
+    500.0, 1000.0, 2500.0, 5000.0, 10000.0, 30000.0, 60000.0, 120000.0,
+];
+
 const DEFAULT_OTLP_HTTP_TIMEOUT: Duration = Duration::from_secs(10);
 const OTEL_EXPORTER_OTLP_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_TIMEOUT";
 const OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT";
@@ -516,6 +526,8 @@ fn duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
         Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS)
     } else if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) {
         Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS)
+    } else if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) {
+        Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS)
     } else {
         None
     }
@@ -1133,6 +1145,9 @@ mod tests {
         for name in OBJECT_LOCK_DURATION_HISTOGRAM_METRICS {
             assert_eq!(duration_histogram_buckets(name), Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS));
         }
+        for name in INTERNODE_DURATION_HISTOGRAM_METRICS {
+            assert_eq!(duration_histogram_buckets(name), Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS));
+        }
         for name in [
             "rustfs_io_put_object_request_duration_seconds",
             "rustfs_io_get_object_response_size_bytes",
@@ -1164,6 +1179,8 @@ mod tests {
         for (name, observations) in [
             (OBJECT_LOCK_DURATION_HISTOGRAM_METRICS[0], [0.002, 0.004, 0.012]),
             (OBJECT_LOCK_DURATION_HISTOGRAM_METRICS[1], [0.002, 12.9, 31.0]),
+            (INTERNODE_DURATION_HISTOGRAM_METRICS[0], [0.04, 3.2, 31000.0]),
+            (INTERNODE_DURATION_HISTOGRAM_METRICS[1], [0.02, 1.2, 1300.0]),
         ] {
             let histogram = recorder.register_histogram(&metrics::Key::from_name(name), &metadata);
             for seconds in observations {
@@ -1181,6 +1198,8 @@ mod tests {
             let expected = match metric.name() {
                 "rustfs_object_lock_diag_acquire_duration_seconds" => [(0.002, 1), (0.004, 1), (0.015, 1)],
                 "rustfs_object_lock_diag_hold_duration_seconds" => [(0.002, 1), (15.0, 1), (60.0, 1)],
+                "rustfs_system_network_internode_operation_duration_ms" => [(0.05, 1), (4.0, 1), (60000.0, 1)],
+                "rustfs_system_network_internode_operation_server_duration_ms" => [(0.025, 1), (1.5, 1), (2500.0, 1)],
                 _ => continue,
             };
             let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
@@ -1196,7 +1215,7 @@ mod tests {
             assert_eq!(occupied, expected, "seconds remain distinct at millisecond and long-hold scales");
             checked += 1;
         }
-        assert_eq!(checked, 2, "both acquisition and hold histograms were exported");
+        assert_eq!(checked, 4, "lock and internode duration histograms were exported");
         provider.shutdown().expect("stop lock histogram test provider");
     }
 

crates/protos/Cargo.toml

@@ -85,5 +85,7 @@ uuid = { workspace = true }
 doctest = false
 
 [dev-dependencies]
+rcgen = { workspace = true }
 serde_json = { workspace = true }
-temp-env = { workspace = true }
+temp-env = { workspace = true, features = ["async_closure"] }
+tokio = { workspace = true, features = ["macros"] }

crates/protos/examples/internode_channel_bench.rs

@@ -0,0 +1,233 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+//! Local transport experiment using the production channel pool and two loopback tonic peers.
+//! Arguments: bulk workers (16), requests per bulk worker (64), payload bytes (262144), control workers (0).
+
+use rustfs_protos::proto_gen::node_service::{PingRequest, PingResponse, node_service_client::NodeServiceClient};
+use rustfs_protos::{ChannelClass, get_channel_for_class};
+use std::convert::Infallible;
+use std::io::Error;
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+use std::task::{Context, Poll};
+use std::time::{Duration, Instant};
+use tonic::codegen::{
+    Body, BoxFuture, Service, StdError, http,
+    tokio_stream::{StreamExt, wrappers::TcpListenerStream},
+};
+
+#[derive(Clone)]
+struct Echo;
+
+impl tonic::server::NamedService for Echo {
+    const NAME: &'static str = "node_service.NodeService";
+}
+
+impl tonic::server::UnaryService<PingRequest> for Echo {
+    type Response = PingResponse;
+    type Future = BoxFuture<tonic::Response<PingResponse>, tonic::Status>;
+
+    fn call(&mut self, request: tonic::Request<PingRequest>) -> Self::Future {
+        Box::pin(async move {
+            let request = request.into_inner();
+            Ok(tonic::Response::new(PingResponse {
+                version: request.version,
+                body: request.body,
+            }))
+        })
+    }
+}
+
+impl<B> Service<http::Request<B>> for Echo
+where
+    B: Body + Send + 'static,
+    B::Error: Into<StdError> + Send + 'static,
+{
+    type Response = http::Response<tonic::body::Body>;
+    type Error = Infallible;
+    type Future = BoxFuture<Self::Response, Self::Error>;
+
+    fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        Poll::Ready(Ok(()))
+    }
+
+    fn call(&mut self, request: http::Request<B>) -> Self::Future {
+        Box::pin(async move {
+            let codec = tonic_prost::ProstCodec::<PingResponse, PingRequest>::default();
+            let mut grpc = tonic::server::Grpc::new(codec)
+                .max_decoding_message_size(rustfs_protos::DEFAULT_GRPC_SERVER_MESSAGE_LEN)
+                .max_encoding_message_size(rustfs_protos::DEFAULT_GRPC_SERVER_MESSAGE_LEN);
+            Ok(grpc.unary(Echo, request).await)
+        })
+    }
+}
+
+async fn roundtrip(address: &str, class: ChannelClass, payload: prost::bytes::Bytes) -> Result<(), StdError> {
+    let channel = get_channel_for_class(address, class)
+        .await
+        .map_err(|error| Error::other(error.to_string()))?;
+    let response = NodeServiceClient::new(channel)
+        .max_decoding_message_size(rustfs_protos::DEFAULT_GRPC_SERVER_MESSAGE_LEN)
+        .max_encoding_message_size(rustfs_protos::DEFAULT_GRPC_SERVER_MESSAGE_LEN)
+        .ping(PingRequest {
+            version: 1,
+            body: payload.clone(),
+        })
+        .await?
+        .into_inner();
+    if response.version != 1 || response.body != payload {
+        return Err(Error::other("echo response content mismatch").into());
+    }
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() -> Result<(), StdError> {
+    let mut args = std::env::args().skip(1);
+    let mut argument =
+        |default| -> Result<usize, StdError> { Ok(args.next().map(|value| value.parse()).transpose()?.unwrap_or(default)) };
+    let workers = argument(16)?;
+    let requests = argument(64)?;
+    let bytes = argument(262_144)?;
+    let control_workers = argument(0)?;
+    let total_workers = workers
+        .checked_add(control_workers)
+        .ok_or_else(|| Error::other("worker count overflow"))?;
+    if args.next().is_some()
+        || workers == 0
+        || !(1..=256).contains(&total_workers)
+        || requests == 0
+        || bytes == 0
+        || bytes > 32 * 1024 * 1024
+        || total_workers.checked_mul(requests).is_none_or(|count| count > 1_000_000)
+        || workers.checked_mul(bytes).is_none_or(|size| size > 512 * 1024 * 1024)
+    {
+        return Err(Error::other("usage: internode_channel_bench [bulk-workers] [requests-per-bulk-worker] [payload-bytes] [control-workers]; maximum 256 workers, 1M samples, 32 MiB/message and 512 MiB concurrent payload").into());
+    }
+
+    let mut addresses = Vec::new();
+    let mut servers = Vec::new();
+    let mut connection_counts = Vec::new();
+    for _ in 0..2 {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
+        addresses.push(format!("http://{}", listener.local_addr()?));
+        let accepted = Arc::new(AtomicUsize::new(0));
+        connection_counts.push(Arc::clone(&accepted));
+        let incoming = TcpListenerStream::new(listener).map(move |socket| {
+            if socket.is_ok() {
+                accepted.fetch_add(1, Ordering::Relaxed);
+            }
+            socket
+        });
+        servers.push(tokio::spawn(async move {
+            tonic::transport::Server::builder()
+                .initial_stream_window_size(rustfs_config::DEFAULT_H2_INITIAL_STREAM_WINDOW_SIZE)
+                .initial_connection_window_size(rustfs_config::DEFAULT_H2_INITIAL_CONN_WINDOW_SIZE)
+                .max_concurrent_streams(Some(rustfs_config::DEFAULT_H2_MAX_CONCURRENT_STREAMS))
+                .max_frame_size(rustfs_config::DEFAULT_H2_MAX_FRAME_SIZE)
+                .http2_max_header_list_size(rustfs_config::DEFAULT_H2_MAX_HEADER_LIST_SIZE)
+                .http2_keepalive_interval(Some(Duration::from_secs(rustfs_config::DEFAULT_H2_KEEP_ALIVE_INTERVAL)))
+                .http2_keepalive_timeout(Some(Duration::from_secs(rustfs_config::DEFAULT_H2_KEEP_ALIVE_TIMEOUT)))
+                .add_service(Echo)
+                .serve_with_incoming(incoming)
+                .await
+        }));
+    }
+    let payload = prost::bytes::Bytes::from(vec![0x5a; bytes]);
+    // Consecutive warmups fill both default bulk slots without depending on peer interleaving.
+    for address in &addresses {
+        for _ in 0..4 {
+            roundtrip(address, ChannelClass::Bulk, payload.clone()).await?;
+        }
+        if control_workers > 0 {
+            roundtrip(address, ChannelClass::Control, prost::bytes::Bytes::from_static(b"control")).await?;
+        }
+    }
+    let addresses = Arc::new(addresses);
+    let barrier = Arc::new(tokio::sync::Barrier::new(total_workers + 1));
+    let mut tasks = tokio::task::JoinSet::new();
+    let finished_bulk = Arc::new(AtomicUsize::new(0));
+    let control_sample_limit = (1_000_000 - workers * requests) / control_workers.max(1);
+    for worker in 0..total_workers {
+        let addresses = Arc::clone(&addresses);
+        let barrier = Arc::clone(&barrier);
+        let control = worker >= workers;
+        let class = if control { ChannelClass::Control } else { ChannelClass::Bulk };
+        let payload = if control {
+            prost::bytes::Bytes::from_static(b"control")
+        } else {
+            payload.clone()
+        };
+        let finished_bulk = Arc::clone(&finished_bulk);
+        tasks.spawn(async move {
+            let mut latencies = Vec::with_capacity(requests);
+            barrier.wait().await;
+            let started = Instant::now();
+            loop {
+                if control {
+                    if !latencies.is_empty() && finished_bulk.load(Ordering::Acquire) == workers {
+                        break;
+                    }
+                    if latencies.len() >= control_sample_limit {
+                        return Err(
+                            Error::other("control sample budget exhausted; reduce the workload or control workers").into()
+                        );
+                    }
+                } else if latencies.len() == requests {
+                    break;
+                }
+                let start = Instant::now();
+                roundtrip(&addresses[(worker + latencies.len()) % addresses.len()], class, payload.clone()).await?;
+                latencies.push(start.elapsed().as_secs_f64() * 1000.0);
+            }
+            if !control {
+                finished_bulk.fetch_add(1, Ordering::Release);
+            }
+            Ok::<_, StdError>((control, started, Instant::now(), latencies))
+        });
+    }
+    barrier.wait().await;
+    let mut bulk_latencies = Vec::with_capacity(workers * requests);
+    let mut control_latencies = Vec::new();
+    let mut first = None::<Instant>;
+    let mut last = None::<Instant>;
+    while let Some(result) = tasks.join_next().await {
+        let (control, started, finished, samples) = result??;
+        first = Some(first.map_or(started, |previous| previous.min(started)));
+        last = Some(last.map_or(finished, |previous| previous.max(finished)));
+        if control {
+            control_latencies.extend(samples);
+        } else {
+            bulk_latencies.extend(samples);
+        }
+    }
+    let first = first.ok_or_else(|| Error::other("no worker started"))?;
+    let last = last.ok_or_else(|| Error::other("no worker completed"))?;
+    let seconds = last.duration_since(first).as_secs_f64();
+    let summary = |latencies: &mut [f64]| {
+        latencies.sort_unstable_by(f64::total_cmp);
+        let percentile =
+            |percent: usize| (!latencies.is_empty()).then(|| latencies[(latencies.len() * percent).div_ceil(100) - 1]);
+        serde_json::json!({"requests": latencies.len(), "p50_ms": percentile(50), "p95_ms": percentile(95), "p99_ms": percentile(99)})
+    };
+    let payload_mib = f64::from(u32::try_from(bytes)?) / (1024.0 * 1024.0);
+    println!(
+        "{}",
+        serde_json::json!({
+            "bulk_workers": workers, "control_workers": control_workers, "payload_bytes": bytes, "control_payload_bytes": 7,
+            "elapsed_seconds": seconds,
+            "roundtrip_payload_mib_per_second": payload_mib * f64::from(u32::try_from(bulk_latencies.len())?) / seconds,
+            "quantile_method": "nearest_rank", "bulk": summary(&mut bulk_latencies), "control": summary(&mut control_latencies),
+            "connections_per_peer": connection_counts.iter().map(|count| count.load(Ordering::Relaxed)).collect::<Vec<_>>()
+        })
+    );
+    rustfs_common::clear_all_connections().await;
+    for server in servers {
+        server.abort();
+        let _ = server.await;
+    }
+    Ok(())
+}

crates/protos/src/lib.rs

@@ -23,13 +23,14 @@ mod runtime_sources;
 use proto_gen::node_service::node_service_client::NodeServiceClient;
 use rustfs_common::{cache_connection, cached_connection, evict_connection_with_log_level};
 use std::{
+    borrow::Cow,
     collections::HashMap,
     error::Error,
     sync::atomic::{AtomicU8, AtomicUsize, Ordering},
-    sync::{Arc, LazyLock, Mutex as StdMutex, Weak},
+    sync::{Arc, LazyLock, Mutex as StdMutex, RwLock as StdRwLock, Weak},
     time::{Duration, Instant},
 };
-use tokio::sync::Mutex;
+use tokio::sync::{Mutex, RwLock};
 use tonic::{
     Request, Status,
     service::interceptor::InterceptedService,
@@ -89,7 +90,7 @@ const TLS_GENERATION_CACHE_MAX_SIZE: usize = 512;
 const INTERNODE_RPC_MSGPACK_ONLY_CACHE_UNSET: u8 = 0;
 const INTERNODE_RPC_MSGPACK_ONLY_CACHE_FALSE: u8 = 1;
 const INTERNODE_RPC_MSGPACK_ONLY_CACHE_TRUE: u8 = 2;
-static TLS_GENERATION_CACHE: LazyLock<Mutex<HashMap<String, u64>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+static TLS_GENERATION_CACHE: LazyLock<RwLock<HashMap<String, u64>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
 // Lock order: generation metadata, then the common connection cache. The dial
 // registry's standard mutex is released before waiting on any async lock.
 static DIAL_LOCK_REGISTRY: LazyLock<StdMutex<HashMap<String, Weak<Mutex<()>>>>> = LazyLock::new(|| StdMutex::new(HashMap::new()));
@@ -3276,7 +3277,7 @@ pub enum ChannelClass {
     Bulk,
 }
 
-// Keep multiplexed unary RPCs below h2's per-connection small-frame budget.
+// Limits transport response futures; client RPC metrics separately cover unary body decoding.
 const INTERNODE_RPC_CONCURRENCY_LIMIT: usize = 64;
 
 /// Whether control/bulk channel isolation is enabled (env-gated, default off for safe rollout).
@@ -3292,10 +3293,64 @@ fn bulk_channel_pool_size() -> usize {
     rustfs_utils::get_env_usize(rustfs_config::ENV_INTERNODE_BULK_CHANNELS, rustfs_config::DEFAULT_INTERNODE_BULK_CHANNELS).max(1)
 }
 
-/// Round-robin cursor for selecting a bulk channel within a peer's pool. A single global cursor
-/// is sufficient: it advances once per bulk acquisition and `% pool_size` spreads consecutive
-/// acquisitions across the pool.
-static BULK_CHANNEL_CURSOR: AtomicUsize = AtomicUsize::new(0);
+/// Bounds peer-selection bookkeeping across topology changes. Evicting a cursor
+/// only resets its next slot; established channels keep their existing cache identity.
+const BULK_CHANNEL_CURSOR_CACHE_CAPACITY: usize = 512;
+// Cover the default pool and common tuning pools without allocating
+// from an arbitrarily large environment-configured pool size.
+const BULK_CHANNEL_CACHED_KEYS: usize = 4;
+static BULK_CHANNEL_CURSORS: LazyLock<BulkChannelCursors> = LazyLock::new(BulkChannelCursors::default);
+
+#[derive(Default)]
+struct BulkChannelCursors {
+    peers: StdRwLock<HashMap<String, Arc<BulkChannelCursor>>>,
+}
+
+impl BulkChannelCursors {
+    fn for_peer(&self, addr: &str) -> Arc<BulkChannelCursor> {
+        let cached = self
+            .peers
+            .read()
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
+            .get(addr)
+            .cloned();
+        cached.unwrap_or_else(|| {
+            let mut peers = self.peers.write().unwrap_or_else(|poisoned| poisoned.into_inner());
+            if let Some(cursor) = peers.get(addr) {
+                return Arc::clone(cursor);
+            }
+            if peers.len() >= BULK_CHANNEL_CURSOR_CACHE_CAPACITY
+                && let Some(victim) = peers.keys().next().cloned()
+            {
+                peers.remove(&victim);
+            }
+            let cursor = Arc::new(BulkChannelCursor {
+                next: AtomicUsize::new(0),
+                keys: std::array::from_fn(|index| bulk_cache_key(addr, index)),
+            });
+            peers.insert(addr.to_owned(), Arc::clone(&cursor));
+            cursor
+        })
+    }
+}
+
+struct BulkChannelCursor {
+    next: AtomicUsize,
+    keys: [String; BULK_CHANNEL_CACHED_KEYS],
+}
+
+impl BulkChannelCursor {
+    fn next(&self, pool_size: usize) -> usize {
+        self.next.fetch_add(1, Ordering::Relaxed) % pool_size
+    }
+
+    fn cache_key(&self, addr: &str, index: usize) -> Cow<'_, str> {
+        match self.keys.get(index) {
+            Some(key) => Cow::Borrowed(key),
+            None => Cow::Owned(bulk_cache_key(addr, index)),
+        }
+    }
+}
 
 /// Connection-cache key for the `idx`-th bulk channel to `addr`. The NUL separator cannot appear
 /// in a URL, so a bulk key never collides with the control key (the bare `addr`).
@@ -3307,7 +3362,19 @@ async fn cached_channel_for_tls_generation(dial_addr: &str, cache_key: &str) ->
     if !dial_addr.starts_with(RUSTFS_HTTPS_PREFIX) {
         return cached_connection(cache_key).await;
     }
-    let mut generations = TLS_GENERATION_CACHE.lock().await;
+    // Cache readers share the generation lock. Both publication and eviction take it before
+    // the common connection cache, preserving the lock order and the generation/channel pair.
+    {
+        let generations = TLS_GENERATION_CACHE.read().await;
+        let generation = runtime_sources::outbound_tls_generation();
+        if generations.get(cache_key).copied() == Some(generation)
+            && let Some(channel) = cached_connection(cache_key).await
+            && runtime_sources::outbound_tls_generation() == generation
+        {
+            return Some(channel);
+        }
+    }
+    let mut generations = TLS_GENERATION_CACHE.write().await;
     let Some(channel) = cached_connection(cache_key).await else {
         generations.remove(cache_key);
         return None;
@@ -3326,7 +3393,7 @@ async fn cache_dialed_channel(dial_addr: &str, cache_key: &str, channel: &Channe
         cache_connection(cache_key.to_string(), channel.clone()).await;
         return true;
     }
-    let mut generations = TLS_GENERATION_CACHE.lock().await;
+    let mut generations = TLS_GENERATION_CACHE.write().await;
     if generation != Some(runtime_sources::outbound_tls_generation()) {
         runtime_sources::record_stale_grpc_channel_tls_generation();
         return false;
@@ -3350,6 +3417,33 @@ fn channel_acquisition_timeout(_: tokio::time::error::Elapsed) -> std::io::Error
     std::io::Error::new(std::io::ErrorKind::TimedOut, "internode channel acquisition deadline elapsed")
 }
 
+fn checked_rpc_window(value: Option<u32>) -> std::io::Result<Option<u32>> {
+    if value.is_some_and(|bytes| bytes > 0x7fff_ffff) {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "internode HTTP/2 receive window exceeds the 31-bit flow-control limit",
+        ));
+    }
+    Ok(value.filter(|bytes| *bytes != 0))
+}
+
+fn bulk_window_override(name: &str, inherited: Option<u32>) -> Option<u32> {
+    match rustfs_utils::get_env_u32(name, inherited.unwrap_or(0)) {
+        0 => None,
+        value => Some(value),
+    }
+}
+
+fn checked_rpc_capacity(value: usize) -> std::io::Result<usize> {
+    if value > tokio::sync::Semaphore::MAX_PERMITS {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "internode bulk admission exceeds the semaphore capacity limit",
+        ));
+    }
+    Ok(value.max(1))
+}
+
 /// Acquire a cached-or-newly-dialed channel for the given peer and channel class.
 ///
 /// For [`ChannelClass::Control`] (or whenever isolation is disabled) this is exactly the legacy
@@ -3362,17 +3456,16 @@ pub async fn get_channel_for_class(addr: &str, class: ChannelClass) -> Result<Ch
     tokio::time::timeout_at(deadline, async {
         if class == ChannelClass::Control || !channel_isolation_enabled() {
             if let Some(channel) = cached_channel_for_tls_generation(addr, addr).await {
-                debug!("Using cached control gRPC channel for: {}", addr);
                 return Ok(channel);
             }
             return build_channel_until(addr, addr, deadline, internode_rpc_timeout()).await;
         }
 
         let pool_size = bulk_channel_pool_size();
-        let idx = BULK_CHANNEL_CURSOR.fetch_add(1, Ordering::Relaxed) % pool_size;
-        let cache_key = bulk_cache_key(addr, idx);
+        let cursor = BULK_CHANNEL_CURSORS.for_peer(addr);
+        let idx = cursor.next(pool_size);
+        let cache_key = cursor.cache_key(addr, idx);
         if let Some(channel) = cached_channel_for_tls_generation(addr, &cache_key).await {
-            debug!("Using cached bulk gRPC channel {} for: {}", idx, addr);
             return Ok(channel);
         }
         build_channel_until(addr, &cache_key, deadline, internode_rpc_timeout()).await
@@ -3388,8 +3481,8 @@ pub async fn get_channel_for_class(addr: &str, class: ChannelClass) -> Result<Ch
 /// - Connect timeout: `DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS` (3s)
 /// - TCP keepalive: `DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS` (10s)
 /// - HTTP/2 keepalive interval: `DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS` (5s)
-/// - HTTP/2 keepalive timeout: `DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS` (3s)
-/// - RPC timeout: `DEFAULT_INTERNODE_RPC_TIMEOUT_SECS` (10s)
+/// - HTTP/2 keepalive timeout: `DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS` (20s)
+/// - RPC timeout: `DEFAULT_INTERNODE_RPC_TIMEOUT_SECS` (30s)
 pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
     // The control channel is cached under the bare address, preserving the legacy key.
     build_channel(addr, addr).await
@@ -3479,10 +3572,27 @@ async fn dial_channel(dial_addr: &str, cache_key: &str, rpc_timeout: Duration) -
     let tcp_keepalive = internode_tcp_keepalive();
     let http2_keepalive_interval = internode_http2_keep_alive_interval();
     let http2_keepalive_timeout = internode_http2_keep_alive_timeout();
+    let is_bulk = cache_key
+        .strip_prefix(dial_addr)
+        .is_some_and(|suffix| suffix.starts_with("\0bulk\0"));
+    let mut stream_window = internode_rpc_http2_stream_window();
+    let mut conn_window = internode_rpc_http2_conn_window();
+    let concurrency_limit = if is_bulk {
+        stream_window = bulk_window_override(rustfs_config::ENV_INTERNODE_BULK_HTTP2_STREAM_WINDOW_SIZE, stream_window);
+        conn_window = bulk_window_override(rustfs_config::ENV_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE, conn_window);
+        checked_rpc_capacity(rustfs_utils::get_env_usize(
+            rustfs_config::ENV_INTERNODE_BULK_CONCURRENCY_LIMIT,
+            rustfs_config::DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT,
+        ))?
+    } else {
+        INTERNODE_RPC_CONCURRENCY_LIMIT
+    };
+    let stream_window = checked_rpc_window(stream_window)?;
+    let conn_window = checked_rpc_window(conn_window)?;
     let mut connector = Endpoint::from_shared(dial_addr.to_string())?
         // Fast connection timeout for dead peer detection
         .connect_timeout(connect_timeout)
-        .concurrency_limit(INTERNODE_RPC_CONCURRENCY_LIMIT)
+        .concurrency_limit(concurrency_limit)
         // TCP-level keepalive - OS will probe connection
         .tcp_keepalive(Some(tcp_keepalive))
         // Disable Nagle so latency-sensitive control-plane RPCs (locks/health) are not batched
@@ -3496,13 +3606,18 @@ async fn dial_channel(dial_addr: &str, cache_key: &str, rpc_timeout: Duration) -
         // Overall timeout for any RPC - fail fast on unresponsive peers
         .timeout(rpc_timeout);
 
-    // Raise HTTP/2 flow-control windows above the 64KiB library default so larger unary
-    // responses (ReadMultiple/BatchReadVersion) are not throttled by the BDP. Mirrors the
-    // server-side window tuning in `rustfs/src/server/http.rs`.
-    if let Some(stream_window) = internode_rpc_http2_stream_window() {
+    // Apply the selected receive-credit policy. A zero setting delegates to the linked
+    // library's defaults; isolated bulk channels can override the common RPC settings.
+    if is_bulk {
+        connector = connector.buffer_size(checked_rpc_capacity(rustfs_utils::get_env_usize(
+            rustfs_config::ENV_INTERNODE_BULK_BUFFER_SIZE,
+            rustfs_config::DEFAULT_INTERNODE_BULK_BUFFER_SIZE,
+        ))?);
+    }
+    if let Some(stream_window) = stream_window {
         connector = connector.initial_stream_window_size(stream_window);
     }
-    if let Some(conn_window) = internode_rpc_http2_conn_window() {
+    if let Some(conn_window) = conn_window {
         connector = connector.initial_connection_window_size(conn_window);
     }
 
@@ -3512,26 +3627,13 @@ async fn dial_channel(dial_addr: &str, cache_key: &str, rpc_timeout: Duration) -
         generation = Some(outbound_tls.generation.0);
         if let Some(cert_pem) = outbound_tls.root_ca_pem.as_ref() {
             let ca = Certificate::from_pem(cert_pem);
-            // Derive the hostname from the HTTPS URL for TLS hostname verification.
-            let domain = dial_addr
-                .trim_start_matches(RUSTFS_HTTPS_PREFIX)
-                .split('/')
-                .next()
-                .unwrap_or("")
-                .split(':')
-                .next()
-                .unwrap_or("");
-            let tls = if !domain.is_empty() {
-                let mut cfg = ClientTlsConfig::new().ca_certificate(ca).domain_name(domain);
-                if let Some(id) = outbound_tls.mtls_identity.as_ref() {
-                    let identity = tonic::transport::Identity::from_pem(id.cert_pem.clone(), id.key_pem.clone());
-                    cfg = cfg.identity(identity);
-                }
-                cfg
-            } else {
-                // Fallback: configure TLS without explicit domain if parsing fails.
-                ClientTlsConfig::new().ca_certificate(ca)
-            };
+            // Tonic derives certificate verification and SNI from the endpoint URI,
+            // including bracketed IPv6 hosts.
+            let mut tls = ClientTlsConfig::new().ca_certificate(ca);
+            if let Some(id) = outbound_tls.mtls_identity.as_ref() {
+                let identity = tonic::transport::Identity::from_pem(id.cert_pem.clone(), id.key_pem.clone());
+                tls = tls.identity(identity);
+            }
             connector = connector.tls_config(tls)?;
             debug!("Configured TLS with custom root certificate for: {}", dial_addr);
         } else {
@@ -3566,6 +3668,14 @@ pub async fn evict_failed_connection(addr: &str) {
     evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Warn).await;
 }
 
+async fn evict_cached_channel(cache_key: &str, log_level: ConnectionEvictionLogLevel) {
+    // Match publication's lock order. Holding the generation guard across the short in-memory
+    // cache mutation prevents an older eviction from deleting a replacement's generation.
+    let mut generations = TLS_GENERATION_CACHE.write().await;
+    evict_connection_with_log_level(cache_key, log_level).await;
+    generations.remove(cache_key);
+}
+
 /// Evict a connection from the cache after a failure with an explicit log level.
 pub async fn evict_failed_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) {
     match log_level {
@@ -3593,8 +3703,7 @@ pub async fn evict_failed_connection_with_log_level(addr: &str, log_level: Conne
         ConnectionEvictionLogLevel::Warn => ConnectionEvictionLogLevel::Info,
         level => level,
     };
-    evict_connection_with_log_level(addr, cache_log_level).await;
-    TLS_GENERATION_CACHE.lock().await.remove(addr);
+    evict_cached_channel(addr, cache_log_level).await;
 
     // An RPC-triggered eviction is a peer-failure signal; feed it into the online/offline state so
     // the peer flips offline after enough consecutive failures (grpc-optimization P3).
@@ -3606,8 +3715,7 @@ pub async fn evict_failed_connection_with_log_level(addr: &str, log_level: Conne
     if channel_isolation_enabled() {
         for idx in 0..bulk_channel_pool_size() {
             let cache_key = bulk_cache_key(addr, idx);
-            evict_connection_with_log_level(&cache_key, cache_log_level).await;
-            TLS_GENERATION_CACHE.lock().await.remove(&cache_key);
+            evict_cached_channel(&cache_key, cache_log_level).await;
         }
     }
 }
@@ -3751,7 +3859,7 @@ mod tests {
         let stale_generation = generation.checked_add(1).unwrap_or(0);
         for key in [addr.to_string(), bulk_cache_key(addr, 0)] {
             cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await;
-            TLS_GENERATION_CACHE.lock().await.insert(key.clone(), stale_generation);
+            TLS_GENERATION_CACHE.write().await.insert(key.clone(), stale_generation);
             assert!(
                 cached_channel_for_tls_generation(addr, &key).await.is_none(),
                 "TLS02_INVARIANT stale HTTPS cache entry cannot be returned"
@@ -3769,7 +3877,7 @@ mod tests {
         let addr = "https://tls02-missing-generation.invalid:9000";
         for key in [addr.to_string(), bulk_cache_key(addr, 0)] {
             cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await;
-            TLS_GENERATION_CACHE.lock().await.remove(&key);
+            TLS_GENERATION_CACHE.write().await.remove(&key);
             assert!(
                 cached_channel_for_tls_generation(addr, &key).await.is_none(),
                 "TLS02_INVARIANT HTTPS channel without a generation cannot be returned"
@@ -3793,16 +3901,16 @@ mod tests {
         for cached_generation in [None, Some(generation.checked_add(1).unwrap_or(0))] {
             cache_connection(cache_key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await;
             if let Some(generation) = cached_generation {
-                TLS_GENERATION_CACHE.lock().await.insert(cache_key.clone(), generation);
+                TLS_GENERATION_CACHE.write().await.insert(cache_key.clone(), generation);
             } else {
-                TLS_GENERATION_CACHE.lock().await.remove(&cache_key);
+                TLS_GENERATION_CACHE.write().await.remove(&cache_key);
             }
             assert!(
                 create_channel_with_timeout(&addr, rpc_timeout).await.is_err(),
                 "recovery must dial the unavailable HTTPS peer instead of returning an unbound or stale cached channel"
             );
             assert!(!rustfs_common::has_cached_connection(&cache_key).await);
-            assert!(!TLS_GENERATION_CACHE.lock().await.contains_key(&cache_key));
+            assert!(!TLS_GENERATION_CACHE.write().await.contains_key(&cache_key));
         }
     }
 
@@ -3813,14 +3921,14 @@ mod tests {
         let generation = rustfs_tls_runtime::load_global_outbound_tls_generation().0;
         for key in [addr.to_string(), bulk_cache_key(addr, 0)] {
             cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await;
-            TLS_GENERATION_CACHE.lock().await.insert(key.clone(), generation);
+            TLS_GENERATION_CACHE.write().await.insert(key.clone(), generation);
             assert!(
                 cached_channel_for_tls_generation(addr, &key).await.is_some(),
                 "TLS02_INVARIANT HTTPS channel at the canonical generation is reused"
             );
             assert!(rustfs_common::has_cached_connection(&key).await);
             evict_connection_with_log_level(&key, ConnectionEvictionLogLevel::Debug).await;
-            TLS_GENERATION_CACHE.lock().await.remove(&key);
+            TLS_GENERATION_CACHE.write().await.remove(&key);
         }
     }
 
@@ -3832,14 +3940,14 @@ mod tests {
         for key in [addr.to_string(), bulk_cache_key(addr, 0)] {
             cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await;
             TLS_GENERATION_CACHE
-                .lock()
+                .write()
                 .await
                 .insert(key.clone(), generation.checked_add(1).unwrap_or(0));
             assert!(
                 cached_channel_for_tls_generation(addr, &key).await.is_some(),
                 "TLS02_INVARIANT HTTP cache is independent of TLS generations"
             );
-            TLS_GENERATION_CACHE.lock().await.remove(&key);
+            TLS_GENERATION_CACHE.write().await.remove(&key);
             assert!(
                 cached_channel_for_tls_generation(addr, &key).await.is_some(),
                 "TLS02_INVARIANT HTTP cache needs no TLS generation metadata"
@@ -3894,7 +4002,86 @@ mod tests {
             !rustfs_common::has_cached_connection(addr).await,
             "TLS02_INVARIANT late stale dial is never cached"
         );
-        assert!(!TLS_GENERATION_CACHE.lock().await.contains_key(addr));
+        assert!(!TLS_GENERATION_CACHE.write().await.contains_key(addr));
+    }
+
+    #[tokio::test]
+    async fn tls02_custom_ca_uses_ipv6_uri_and_rejects_wrong_certificate_names() {
+        use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration, publish_global_outbound_tls_state};
+        use tonic::codegen::tokio_stream::{StreamExt, wrappers::TcpListenerStream};
+        use tonic::transport::ServerTlsConfig;
+
+        let _test_guard = TLS02_TEST_LOCK.lock().await;
+        let previous = runtime_sources::outbound_tls_state().await;
+        let restore_generation = previous.generation.0.checked_add(3).expect("test generation capacity");
+        let mut observations = Vec::new();
+        for (offset, certificate_name) in [(1, "::1"), (2, "localhost")] {
+            let certificate =
+                rcgen::generate_simple_self_signed(vec![certificate_name.to_owned()]).expect("ephemeral TLS test certificate");
+            let certificate_pem = certificate.cert.pem();
+            let server_tls = ServerTlsConfig::new().identity(tonic::transport::Identity::from_pem(
+                &certificate_pem,
+                certificate.signing_key.serialize_pem(),
+            ));
+            let listener = tokio::net::TcpListener::bind("[::1]:0")
+                .await
+                .expect("IPv6 loopback TLS listener");
+            let addr = format!("https://{}", listener.local_addr().expect("IPv6 test server address"));
+            let accepted = Arc::new(AtomicUsize::new(0));
+            let counter = Arc::clone(&accepted);
+            let incoming = TcpListenerStream::new(listener).map(move |stream| {
+                if stream.is_ok() {
+                    counter.fetch_add(1, Ordering::SeqCst);
+                }
+                stream
+            });
+            let mut server = tonic::transport::Server::builder()
+                .tls_config(server_tls)
+                .expect("valid TLS server configuration");
+            let server = tokio::spawn(async move {
+                server
+                    .add_routes(tonic::service::Routes::default())
+                    .serve_with_incoming(incoming)
+                    .await
+            });
+            let material = OutboundTlsMaterial {
+                root_ca_pem: certificate_pem.into_bytes(),
+                mtls_identity: None,
+            };
+            publish_global_outbound_tls_state(TlsGeneration(previous.generation.0 + offset), &material).await;
+            let mut results = Vec::new();
+            for key in [addr.clone(), bulk_cache_key(&addr, 0)] {
+                results.push(
+                    build_channel(&addr, &key)
+                        .await
+                        .map(|_| ())
+                        .map_err(|error| error.to_string()),
+                );
+                evict_cached_channel(&key, ConnectionEvictionLogLevel::Debug).await;
+            }
+            server.abort();
+            let _ = server.await;
+            observations.push((results, accepted.load(Ordering::SeqCst)));
+        }
+        let restore = OutboundTlsMaterial {
+            root_ca_pem: previous.root_ca_pem.unwrap_or_default(),
+            mtls_identity: previous.mtls_identity,
+        };
+        publish_global_outbound_tls_state(TlsGeneration(restore_generation), &restore).await;
+
+        assert!(
+            observations[0].0.iter().all(Result::is_ok),
+            "IPv6 control and bulk channels must verify the URI host: {:?}",
+            observations[0].0
+        );
+        assert!(
+            observations[1].0.iter().all(Result::is_err),
+            "a trusted certificate for localhost must be rejected for ::1"
+        );
+        assert!(
+            observations.iter().all(|(_, accepted)| *accepted == 2),
+            "both certificate cases must exercise actual control and bulk TLS connections"
+        );
     }
 
     #[tokio::test]
@@ -4045,6 +4232,251 @@ mod tests {
         assert!(bulk_channel_pool_size() >= 1);
     }
 
+    #[test]
+    fn bulk_round_robin_is_independent_for_interleaved_peers() {
+        let cursors = BulkChannelCursors::default();
+        for round in 0..12 {
+            for peer in ["peer-a", "peer-b"] {
+                assert_eq!(cursors.for_peer(peer).next(2), round % 2, "each peer must use both slots");
+            }
+        }
+    }
+
+    #[test]
+    fn bulk_round_robin_balances_concurrent_acquisitions() {
+        let cursors = Arc::new(BulkChannelCursors::default());
+        let tasks = (0..8)
+            .map(|_| {
+                let cursors = Arc::clone(&cursors);
+                std::thread::spawn(move || {
+                    let mut counts = [[0; 4]; 2];
+                    for _ in 0..100 {
+                        for (peer, counts) in ["peer-a", "peer-b"].into_iter().zip(&mut counts) {
+                            counts[cursors.for_peer(peer).next(4)] += 1;
+                        }
+                    }
+                    counts
+                })
+            })
+            .collect::<Vec<_>>();
+        let mut total = [[0; 4]; 2];
+        for task in tasks {
+            for (total, counts) in total.iter_mut().zip(task.join().expect("selection worker")) {
+                for (total, count) in total.iter_mut().zip(counts) {
+                    *total += count;
+                }
+            }
+        }
+        assert_eq!(total, [[200; 4]; 2]);
+    }
+
+    #[test]
+    fn bulk_round_robin_bounds_retired_peer_bookkeeping() {
+        let cursors = BulkChannelCursors::default();
+        for index in 0..BULK_CHANNEL_CURSOR_CACHE_CAPACITY * 3 {
+            assert_eq!(cursors.for_peer(&format!("peer-{index}")).next(2), 0);
+        }
+        assert_eq!(cursors.peers.read().expect("cursor cache").len(), BULK_CHANNEL_CURSOR_CACHE_CAPACITY);
+        assert_eq!(cursors.for_peer("current-peer").next(2), 0);
+        assert_eq!(cursors.for_peer("current-peer").next(2), 1);
+    }
+
+    #[test]
+    fn bulk_pool_hits_borrow_stable_keys_and_large_pools_remain_bounded() {
+        let cursors = BulkChannelCursors::default();
+        let peer = cursors.for_peer("https://peer.example:9000");
+        let same_peer = cursors.for_peer("https://peer.example:9000");
+        assert!(Arc::ptr_eq(&peer, &same_peer));
+        for index in 0..BULK_CHANNEL_CACHED_KEYS {
+            let first = peer.cache_key("https://peer.example:9000", index);
+            let second = same_peer.cache_key("https://peer.example:9000", index);
+            assert!(matches!(first, Cow::Borrowed(_)));
+            assert_eq!(first, bulk_cache_key("https://peer.example:9000", index));
+            assert_eq!(first.as_ptr(), second.as_ptr(), "cache hits reuse the established key allocation");
+        }
+        let key = peer.cache_key("https://peer.example:9000", usize::MAX - 1);
+        assert!(matches!(key, Cow::Owned(_)));
+        assert_eq!(key, bulk_cache_key("https://peer.example:9000", usize::MAX - 1));
+        assert_eq!(peer.keys.len(), BULK_CHANNEL_CACHED_KEYS);
+        for (sequence, size) in [1, 2, 4, usize::MAX, 2].into_iter().enumerate() {
+            assert_eq!(peer.next(size), sequence % size, "pool changes preserve the peer's selection sequence");
+        }
+    }
+
+    #[test]
+    fn grpc_receive_windows_reject_overflow_and_preserve_library_default() {
+        assert_eq!(checked_rpc_window(None).expect("unset window"), None);
+        assert_eq!(checked_rpc_window(Some(0)).expect("default window"), None);
+        assert_eq!(checked_rpc_window(Some(1)).expect("minimum window"), Some(1));
+        assert_eq!(checked_rpc_window(Some(0x7fff_ffff)).expect("maximum window"), Some(0x7fff_ffff));
+        assert_eq!(
+            checked_rpc_window(Some(0x8000_0000)).expect_err("overflow window").kind(),
+            std::io::ErrorKind::InvalidInput
+        );
+    }
+
+    #[test]
+    fn grpc_bulk_capacity_rejects_overflow_and_normalizes_zero() {
+        assert_eq!(checked_rpc_capacity(0).expect("minimum capacity"), 1);
+        assert_eq!(checked_rpc_capacity(64).expect("default concurrency"), 64);
+        assert_eq!(checked_rpc_capacity(1024).expect("default buffer"), 1024);
+        assert_eq!(
+            checked_rpc_capacity(usize::MAX).expect_err("oversized capacity").kind(),
+            std::io::ErrorKind::InvalidInput
+        );
+    }
+
+    #[test]
+    fn grpc_bulk_windows_inherit_legacy_settings_and_allow_library_default() {
+        let name = rustfs_config::ENV_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE;
+        for (setting, expected) in [(None, Some(2097152)), (Some("0"), None), (Some("8388608"), Some(8388608))] {
+            temp_env::with_var(name, setting, || {
+                assert_eq!(
+                    checked_rpc_window(bulk_window_override(name, Some(2097152))).expect("valid bulk window"),
+                    expected
+                );
+            });
+        }
+    }
+
+    #[tokio::test]
+    async fn tls02_cache_hits_allow_concurrent_generation_readers() {
+        let _test_guard = TLS02_TEST_LOCK.lock().await;
+        let addr = "https://tls02-shared-cache.invalid:9000";
+        let generation = runtime_sources::outbound_tls_generation();
+        let channel = Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
+        assert!(cache_dialed_channel(addr, addr, &channel, Some(generation)).await);
+        let generations = TLS_GENERATION_CACHE.read().await;
+        let cached = tokio::time::timeout(Duration::from_secs(1), cached_channel_for_tls_generation(addr, addr))
+            .await
+            .expect("cache hit must not acquire an exclusive generation lock");
+        assert!(cached.is_some());
+        drop(generations);
+        evict_connection_with_log_level(addr, ConnectionEvictionLogLevel::Debug).await;
+        TLS_GENERATION_CACHE.write().await.remove(addr);
+    }
+
+    #[tokio::test]
+    async fn tls02_eviction_cannot_remove_a_replacement_generation() {
+        use std::future::Future;
+        use std::task::{Context, Waker};
+        let _test_guard = TLS02_TEST_LOCK.lock().await;
+        let addr = "https://tls02-replacement-race.invalid:9000";
+        let generation = runtime_sources::outbound_tls_generation();
+        let channel = Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
+        assert!(cache_dialed_channel(addr, addr, &channel, Some(generation)).await);
+
+        // Pause eviction at the common map, then attempt publication. Publishing a replacement
+        // before the earlier eviction finishes must never leave that replacement unbound.
+        let connections = rustfs_common::GLOBAL_CONN_MAP.read().await;
+        let mut eviction = Box::pin(evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Debug));
+        let mut replacement = Box::pin(cache_dialed_channel(addr, addr, &channel, Some(generation)));
+        let mut context = Context::from_waker(Waker::noop());
+        assert!(eviction.as_mut().poll(&mut context).is_pending());
+        assert!(replacement.as_mut().poll(&mut context).is_pending());
+        drop(connections);
+        let (_, published) = tokio::time::timeout(Duration::from_secs(1), async { tokio::join!(eviction, replacement) })
+            .await
+            .expect("eviction and replacement must make progress");
+        assert!(published);
+        assert!(
+            cached_channel_for_tls_generation(addr, addr).await.is_some(),
+            "replacement keeps its TLS generation"
+        );
+        evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Debug).await;
+    }
+
+    #[tokio::test]
+    async fn bulk_pool_interleaving_reuses_two_real_connections_per_peer() {
+        use tonic::codegen::tokio_stream::{StreamExt, wrappers::TcpListenerStream};
+        let _test_guard = TLS02_TEST_LOCK.lock().await;
+        temp_env::async_with_vars(
+            [
+                (rustfs_config::ENV_INTERNODE_CHANNEL_ISOLATION, Some("true")),
+                (rustfs_config::ENV_INTERNODE_BULK_CHANNELS, Some("2")),
+                (rustfs_config::ENV_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE, Some("8388608")),
+                (rustfs_config::ENV_INTERNODE_BULK_CONCURRENCY_LIMIT, Some("2")),
+                (rustfs_config::ENV_INTERNODE_BULK_BUFFER_SIZE, Some("4")),
+            ],
+            async {
+                let mut peers = Vec::new();
+                for _ in 0..2 {
+                    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind peer");
+                    let addr = format!("http://{}", listener.local_addr().expect("peer address"));
+                    let accepted = Arc::new(AtomicUsize::new(0));
+                    let changed = Arc::new(tokio::sync::Notify::new());
+                    let count = Arc::clone(&accepted);
+                    let notify = Arc::clone(&changed);
+                    let incoming = TcpListenerStream::new(listener).map(move |stream| {
+                        if stream.is_ok() {
+                            count.fetch_add(1, Ordering::SeqCst);
+                            notify.notify_one();
+                        }
+                        stream
+                    });
+                    let task = tokio::spawn(async move {
+                        tonic::transport::Server::builder()
+                            .add_routes(tonic::service::Routes::default())
+                            .serve_with_incoming(incoming)
+                            .await
+                    });
+                    peers.push((addr, accepted, changed, task));
+                }
+                for _ in 0..8 {
+                    for (addr, _, _, _) in &peers {
+                        get_channel_for_class(addr, ChannelClass::Bulk).await.expect("bulk channel");
+                    }
+                }
+                for (addr, accepted, changed, task) in peers {
+                    get_channel_for_class(&addr, ChannelClass::Control)
+                        .await
+                        .expect("control channel");
+                    tokio::time::timeout(Duration::from_secs(2), async {
+                        while accepted.load(Ordering::SeqCst) < 3 {
+                            changed.notified().await;
+                        }
+                    })
+                    .await
+                    .expect("server accepts all three pooled sockets");
+                    assert_eq!(accepted.load(Ordering::SeqCst), 3, "two bulk connections plus one control connection");
+                    for key in [addr.clone(), bulk_cache_key(&addr, 0), bulk_cache_key(&addr, 1)] {
+                        assert!(rustfs_common::has_cached_connection(&key).await);
+                        evict_connection_with_log_level(&key, ConnectionEvictionLogLevel::Debug).await;
+                    }
+                    task.abort();
+                    let _ = task.await;
+                }
+            },
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    async fn invalid_bulk_window_fails_before_dial_and_control_keeps_its_settings() {
+        let _test_guard = TLS02_TEST_LOCK.lock().await;
+        temp_env::async_with_vars(
+            [(rustfs_config::ENV_INTERNODE_BULK_HTTP2_STREAM_WINDOW_SIZE, Some("4294967295"))],
+            async {
+                let addr = "http://127.0.0.1:1";
+                let error = dial_channel(addr, &bulk_cache_key(addr, 0), Duration::from_secs(1))
+                    .await
+                    .expect_err("reject overflowing bulk window");
+                assert_eq!(
+                    error
+                        .downcast_ref::<std::io::Error>()
+                        .expect("typed configuration error")
+                        .kind(),
+                    std::io::ErrorKind::InvalidInput
+                );
+                assert_eq!(
+                    checked_rpc_window(internode_rpc_http2_stream_window()).expect("control setting"),
+                    Some(1024 * 1024)
+                );
+            },
+        )
+        .await;
+    }
+
     #[test]
     fn internode_rpc_msgpack_only_reuses_cached_env_until_reset() {
         let _guard = INTERNODE_RPC_MSGPACK_ONLY_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
@@ -4105,6 +4537,7 @@ mod tests {
 
     #[tokio::test]
     async fn get_channel_for_class_bulk_reuses_control_cache_when_isolation_disabled() {
+        let _test_guard = TLS02_TEST_LOCK.lock().await;
         // Isolation defaults off, so a Bulk request must reuse the control channel keyed by the
         // bare address and must NOT create a separate bulk-keyed entry (zero behavior change).
         let addr = "http://get-channel-isolation-off-test";

docs/operations/internode-grpc-benchmark-runbook.md

@@ -20,12 +20,25 @@ Every stage is env-gated, so "before" and "after" run the *same binary* with dif
 scripts/run_internode_grpc_ab_bench.sh --stage <p0|p1|p2|p3> --phase <before|after|request-only|canary|rollback> [--dry-run] [-- <bench args>]
 ```
 
-The driver writes the stage/phase server env to `<out-dir>/server-env.sh` and runs the underlying bench into `target/bench/internode-transport/<stage>-<phase>/`. `p0/p1/p3` accept `before|after`; `p2` also accepts `request-only|canary|rollback`. `--dry-run` prints the env and command only.
+The driver writes the stage/phase server env to `<out-dir>/server-env.sh` and runs the underlying bench into `target/bench/internode-transport/<stage>-<phase>/`. `p0/p1/p3` accept `before|after`; `p2` also accepts `request-only|canary|rollback`. Each phase first unsets its managed settings, then exports its overrides, so successive phases restore defaults correctly. Other environment variables retain their values. `--dry-run` writes the phase file and prints the command while skipping the workload. Apply the phase file through the deployment's service environment and read back the effective settings on every restarted node.
+
+For P3 single-variable experiments, select `--lifecycle-feature prewarm` or `--lifecycle-feature offline`. The default `both` preserves the combined experiment. Measure prewarm with cold first calls and delayed peer startup, and measure offline bypass with sustained peer loss and survivor-only client traffic. Record the selected feature with each result.
+
+For P0 single-parameter experiments, select `--transport-feature nodelay`, `stream-window`, `connection-window` or `max-message`. The default `all` retains the original combined preset and cannot attribute a result to one parameter. Selected P0 features write to `<out-root>/p0-<feature>-<phase>/`, keeping separate parameter artifacts from overwriting each other. Both legs clear the four P0-managed settings; the baseline overrides only the selected setting, while the candidate restores defaults. Apply any intended non-default production settings equally after the phase file, and read back effective settings, including bulk overrides that may take precedence. Zero receive windows select the linked library default, rather than a zero-byte or historical 64 KiB window; the phase names do not imply that the candidate is faster.
+
+```bash
+scripts/run_internode_grpc_ab_bench.sh --stage p0 --phase before --transport-feature connection-window --dry-run
+scripts/run_internode_grpc_ab_bench.sh --stage p0 --phase after --transport-feature connection-window --dry-run
+```
+
+Dry-run command display redacts access keys, secret keys and session tokens. Actual dispatch preserves the supplied arguments. Keep credentials out of published evidence and shell history; only use task-owned buckets because load tools may clear their selected bucket.
+
+Set `GRPC_BENCH_METRICS_URL` to the Collector Prometheus exporter endpoint. Generate each phase with `--dry-run`, apply its environment through the node service configuration, restart and verify all nodes, then execute the matching workload command.
 
 ```bash
 # P1 A/B: restart the cluster with each phase's server-env.sh between the two runs
-scripts/run_internode_grpc_ab_bench.sh --stage p1 --phase before -- --access-key AK --secret-key SK --metrics-url http://node1:9000/metrics
-scripts/run_internode_grpc_ab_bench.sh --stage p1 --phase after  -- --access-key AK --secret-key SK --metrics-url http://node1:9000/metrics
+scripts/run_internode_grpc_ab_bench.sh --stage p1 --phase before -- --access-key AK --secret-key SK --metrics-url "${GRPC_BENCH_METRICS_URL:?set the Collector Prometheus exporter URL}"
+scripts/run_internode_grpc_ab_bench.sh --stage p1 --phase after  -- --access-key AK --secret-key SK --metrics-url "${GRPC_BENCH_METRICS_URL:?set the Collector Prometheus exporter URL}"
 # P3 failover A/B (docker four-node)
 scripts/run_internode_grpc_ab_bench.sh --stage p3 --phase after
 # P2 env previews
@@ -45,7 +58,7 @@ Run **before** at the baseline column and **after** at the enabled column, every
 | P0 conn window | `RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE` | `0` | unset (`DEFAULT_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE`) |
 | P0 msg limit | `RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE` | `4194304` (tonic default) | unset (RustFS default, see `rustfs/src/server/http.rs`) |
 | P1 isolation | `RUSTFS_INTERNODE_CHANNEL_ISOLATION` | `false` (default) | `true` |
-| P1 bulk pool | `RUSTFS_INTERNODE_BULK_CHANNELS` | `1` | unset (`DEFAULT_INTERNODE_BULK_CHANNELS`) or higher |
+| P1 bulk pool | `RUSTFS_INTERNODE_BULK_CHANNELS` | unset | `2` |
 | P2 msgpack-only | `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` + `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` | both `false` (default) | per the convergence runbook |
 | P3 prewarm | `RUSTFS_INTERNODE_PREWARM` | `false` (default) | `true` |
 | P3 offline bypass | `RUSTFS_INTERNODE_OFFLINE_BYPASS` | `false` (default) | `true` |
@@ -53,12 +66,16 @@ Run **before** at the baseline column and **after** at the enabled column, every
 
 ## Metrics to capture
 
+The lifetime guards cover ReadAll, WriteAll, ReadMultiple and BatchReadVersion. Their active/completion metrics describe these measured operations and follow the general metrics runtime gate. Startup duration recording keeps the existing instrument, and disabled detailed metrics avoid the additional gauge/counter registrations. Bulk client unary duration covers each attempt from polling the tonic call through response-body decoding, including transport errors and cancellation by an outer deadline. Retries contribute separate samples. Channel acquisition and application serialization before the call require their own timing. The server handler duration covers handler entry through return or drop, including application payload decoding, disk work and response construction. Protobuf transport encoding and response transmission follow handler completion. Keep these two instruments separate when calculating quantiles.
+
 All names are defined in `crates/io-metrics/src/internode_metrics.rs`.
 
 | Metric | Stage signal |
 | --- | --- |
 | `rustfs_system_network_internode_operation_duration_ms{operation,backend}` | control-plane RTT (P0), lock/bulk latency (P1), first-op latency (P3) |
-| `rustfs_system_network_internode_operation_server_duration_ms{operation,backend}` | server-side `ReadMultiple` / `BatchReadVersion` handler lifetime, including decode, disk work, encode, errors, and cancellation; separate from client RTT |
+| `rustfs_system_network_internode_operation_server_duration_ms{operation,backend}` | server handler time for ReadAll, WriteAll, ReadMultiple and BatchReadVersion |
+| `rustfs_system_network_internode_rpc_active{operation,backend,side}` | live client attempts or server handlers, including pending unary response bodies on the client |
+| `rustfs_system_network_internode_rpc_completions_total{operation,backend,outcome}` | client attempt outcomes `success`, `error` and `cancelled`. An outer timeout cancels an attempt, while a returned transport timeout is an error |
 | `rustfs_system_network_internode_operation_payload_bytes` | payload size distribution (P0/P1 sizing) |
 | `rustfs_system_network_internode_operation_large_payloads_total` | large unary RPCs sharing a channel (P1 target) |
 | `rustfs_system_network_internode_dial_avg_time_nanos`, `rustfs_system_network_internode_dial_errors_total` | connect cost and failures (P3) |
@@ -66,6 +83,124 @@ All names are defined in `crates/io-metrics/src/internode_metrics.rs`.
 | `rustfs_cluster_servers_offline_total` | offline detection correctness (P3 bypass) |
 | lock p99 (lock metrics, needs `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true`) | P1 head-of-line-blocking win |
 
+Verify `RUSTFS_OBS_METRICS_EXPORT_ENABLED=true` and that the Collector exports the intended node/process identities. The client `success` outcome denotes an `Ok` tonic exchange. Interpret payload-level success using the returned operation result and operation error counters. RPC duration buckets use milliseconds and resolve sub-millisecond samples; object-lock duration buckets use seconds. The active-client metric counts the complete measured unary future, while tonic's transport concurrency limit ends when its HTTP response future completes. The two counts have different lifetimes. `BatchReadVersion` traffic requires the batch gate, such as `RUSTFS_GET_METADATA_READ_VERSION_COALESCE=on` on compatible peers; verify actual batch request counts and keep that setting identical across A/B legs.
+
+## Isolated bulk tuning
+
+These settings apply when `RUSTFS_INTERNODE_CHANNEL_ISOLATION=true` selects an isolated bulk cache key. Control and dedicated recovery channels use their existing transport settings. Defaults preserve the existing transport behavior.
+
+P1 phase files clear bulk-specific overrides so an isolation experiment inherits the same common RPC window policy on both legs. For a separate bulk-tuning experiment, apply the chosen override after the phase file and include it in the effective configuration record.
+
+| Environment variable | Default | Scope |
+| --- | --- | --- |
+| `RUSTFS_INTERNODE_BULK_HTTP2_STREAM_WINDOW_SIZE` | Inherit `RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE` | Bulk response receive credit per stream, bytes |
+| `RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE` | Inherit `RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE` | Bulk response receive credit per connection, bytes |
+| `RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT` | `64` | Active tonic transport response futures per bulk channel; minimum 1 |
+| `RUSTFS_INTERNODE_BULK_BUFFER_SIZE` | `1024` | Pending tonic service buffer capacity per bulk channel; minimum 1 |
+
+Zero receive-window values select the linked library default. In the locked Hyper 1.12 client this is 2 MiB per stream and 5 MiB per connection, while RustFS explicitly defaults to 1 MiB and 2 MiB. Record which policy the experiment actually selects. Values above the HTTP/2 31-bit limit fail channel acquisition with an invalid-input error. Change connection receive credit first, then stream credit only when measured bandwidth-delay product and resource budgets justify it. For example, compare the default 2 MiB connection window with `RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE=8388608`, holding other settings fixed. Request-upload flow control is configured by the receiving server. Keep server-wide window changes in a separate experiment.
+
+Bulk selection advances independently for each peer. Cursor bookkeeping caches up to 512 peer addresses. Each entry keeps four immutable slot keys so common bulk pools borrow their cache keys on hits. Larger pools construct the remaining slot keys on demand with the same cache identity. Replacing an entry restarts its selection sequence while established channels keep their existing cache keys. HTTPS cache hits share the TLS-generation read lock, with generation validation retained before handoff. Connection misses still coalesce under the existing per-key dial lock.
+
+Custom-CA channels use tonic's endpoint URI handling for certificate names and SNI, including IPv6 literals. The regression suite exercises real control and bulk TLS connections to IPv6 loopback and verifies that a trusted certificate for a different host is rejected.
+
+## Local transport experiment
+
+The [channel benchmark](../../crates/protos/examples/internode_channel_bench.rs) exercises the production channel acquisition path against two local tonic echo peers. It validates every returned byte and reports per-class request percentiles, bulk payload throughput and accepted connections per peer. An optional fourth argument selects control workers that continuously send 7-byte RPCs until the bulk workers complete. This is a closed-loop concurrency workload. Record achieved request rates and use fixed offered-load tests for the final control-plane SLO assessment. It bounds the requested workload and binds only to loopback. Its scope is the channel/HTTP2/TCP path. Distributed storage acceptance additionally requires the real four-node workload and fault matrix.
+
+```bash
+cargo build --release -p rustfs-protos --example internode_channel_bench
+RUSTFS_INTERNODE_CHANNEL_ISOLATION=false target/release/examples/internode_channel_bench 16 256 262144 2
+RUSTFS_INTERNODE_CHANNEL_ISOLATION=true RUSTFS_INTERNODE_BULK_CHANNELS=2 target/release/examples/internode_channel_bench 16 256 262144 2
+```
+
+Freeze the binary and environment, qualify repeated A baselines, then run A1/B1/B2/A2. Interpret payload throughput as one payload per validated round trip; the echo transfers that payload in both directions. Keep source revision, binary identity, workload arguments, connection counts, all errors and raw samples with the results. Stop attribution when baseline drift exceeds the declared gate. Use the effective four-node deployment configuration for final acceptance, with metadata, locks, multipart, TLS rotation and peer failure covered.
+
+For fleet-wide transport changes such as switching between plaintext and TLS, stop all test nodes and verify their processes have exited before starting any node with the new settings. A per-node stop/start loop can temporarily mix protocols and exhaust peer startup retries. Preserve real shutdown failures such as `http12_deadline_missed`; a successful `systemctl stop` command alone does not establish a clean application exit. Use independent collector endpoints and configure its own telemetry listener too, so a second collector does not collide with an existing instance's default internal metrics port.
+
+Object workload latency and RPC latency describe different boundaries. Verify that the intended bulk RPCs actually occurred before attributing isolation, pool or window effects; large S3 objects alone do not prove large unary gRPC payload coverage. Record the histogram boundaries when comparing code versions: a finer bucket layout changes quantile estimates without establishing a latency improvement. Keep the instrument name, units and client/server boundaries with the raw evidence.
+
+## Measurement identity and maintenance observations
+
+Freeze a fresh main revision and candidate before rebuilding both binaries in separate source and Cargo target directories. Record the actual deployed executable identity, configuration, workload and resource ownership. Earlier reports remain attached to their original revisions. Run fault recovery on separate test volumes from the steady-state campaign, or finish recovery and demonstrate stable maintenance observations before measuring. Keep healing enabled; bypassing recovery changes the correctness contract.
+
+`ReadVersion` uses the control channel and already exports `read_version_request_encode`, `read_version_rpc_roundtrip` and `read_version_response_decode` under `rustfs_system_network_internode_operation_stage_duration_ms`; its server stages include `read_version_disk_read`. Metrics export enables these GET stages without the PUT-specific stage flag. The roundtrip timer starts after Channel acquisition, so it does not measure TLS cache or connection acquisition. Ordinary small-object GET can predominantly exercise this control method. Confirm the actual bulk methods, payloads and achieved control request rate before changing bulk settings.
+
+Normal GET aggregation requires `RUSTFS_GET_METADATA_READ_VERSION_COALESCE=auto|on`, which defaults off. Setting only `RUSTFS_METADATA_BATCH_READ` does not assemble GET batches. The remote batch gate resolves `RUSTFS_GET_METADATA_READ_VERSION_COALESCE` before `RUSTFS_METADATA_BATCH_READ`, then the legacy `RUSTFS_BATCH_METADATA_RPC`; `auto` can fall back on UNIMPLEMENTED, while `on` cannot. Restart after changing these cached gates and verify `rustfs_get_metadata_read_version_coalescer_total{event="attempted_batch"}` and `rustfs_remote_disk_batch_read_version_gate_total{mode,decision}`. Treat enabling aggregation as its own experiment because it changes the method distribution and adds the coalescing delay.
+
+For each cell, record prepare, warmup, measure and cleanup separately. Request-level timestamps from the load driver define the measurement window; a Prometheus snapshot bracketing the entire Warp command includes preparation and cleanup. Do not divide such counter deltas by the timed workload duration or call object STAT/GET P99 a control-RPC P99. Record cumulative histogram bucket bounds and per-series timestamps; a bucket upper bound is not an exact quantile. Use external RPC samples for comparisons across changed bucket layouts.
+
+The offline [maintenance-window checker](../../scripts/check_internode_grpc_window.py) consumes recorded OTLP JSON, retaining resource identity and producer point timestamps that a collector's cached Prometheus exposition may conceal. Build one snapshot per observation from the most recent export of each current RustFS process. Do not concatenate multiple exports of the same series into a snapshot. Independently record each node's PID, `/proc/<pid>/stat` start-time ticks and `/proc/sys/kernel/random/boot_id` at every observation; process names can contain spaces and parentheses, so parse the fields after the final `)` before reading the start-time field. A PID alone does not detect reuse, and start-time ticks alone do not detect a reboot. Synchronize producer and capture clocks, and predeclare export-age and sample-gap budgets suitable for the configured export interval. Repeated cached points, stale/future points, resets, missing required gauges, process changes and phase mismatches invalidate evidence.
+
+For a continuously appended native JSONL file, establish and record a byte offset before the cell, discard only the remainder of an already partial record at that offset, and wait for complete new exports from every current process before releasing the load start marker. Replaying a large historical file can consume the probe's bounded startup budget before any load begins. Keep the original export file, preserve producer timestamps and full series identity, reject truncation and conflicting points, and retain the existing freshness and coverage checks; starting at a fresh offset does not justify filling absent gauges with zero or accepting stale points.
+
+The manifest is a bounded JSON object with this shape; replace the illustrative timestamps and identities with recorded values. Capture times at the first and last observations must match the driver-marked measurement boundaries. Every sample carries an independent process observation, even when the identity is unchanged.
+
+```json
+{
+  "scope": "measurement",
+  "start_ns": 100000000000,
+  "end_ns": 220000000000,
+  "max_sample_gap_ns": 2000000000,
+  "max_export_age_ns": 2000000000,
+  "processes": [{"node": "node1", "pid": 101, "start_ticks": 500, "boot_id": "11111111-1111-4111-8111-111111111111"}],
+  "samples": [
+    {"phase": "measurement", "captured_ns": 100000000000, "path": "measure-start.json", "processes": [{"node": "node1", "pid": 101, "start_ticks": 500, "boot_id": "11111111-1111-4111-8111-111111111111"}]},
+    {"phase": "measurement", "captured_ns": 102000000000, "path": "measure-002.json", "processes": [{"node": "node1", "pid": 101, "start_ticks": 500, "boot_id": "11111111-1111-4111-8111-111111111111"}]}
+  ]
+}
+```
+
+Include all intermediate observations and a final sample at `end_ns`; the abbreviated example is intentionally incomplete and does not pass. The `node` value matches OTLP `service.instance.id`, and `pid` matches `process.pid`. Supply every test node in each process inventory; `--expected-nodes` defaults to four and rejects a partial inventory. The checker ignores exports from other processes, requires current-process MRF, heal-activity and scanner-activity gauges on every node, checks optional running-task gauges when present, and detects heal-start counter increments even if the gauge is zero at both endpoints. Lazy counters may be absent while idle; a newly appearing nonzero counter is activity, not an assumed zero baseline.
+
+```bash
+python3 scripts/check_internode_grpc_window.py --manifest /path/to/cell/window.json --out /path/to/cell/maintenance-report.json
+```
+
+Exit 0 means the sampled maintenance observations were stable, 1 means activity was observed, and 2 means the evidence is invalid. The report binds the manifest and each snapshot's SHA-256. Use a fresh output path per invocation and retain the exit status; exit 2 cannot qualify even if an earlier report exists. This does not authorize performance attribution or establish that no activity occurred between samples: method coverage, external RPC timing, sufficient duration/sample count, errors, resource isolation and baseline/ABBA drift gates remain separate requirements. Do not relabel old whole-command snapshots as measurement-only evidence or fill missing points with zero. Collect native OTLP for new experiments; the checker deliberately does not turn old Prometheus snapshots into fresh producer observations.
+
+Use this zero-maintenance check for isolated experiments that predeclare maintenance-free observations. A busy verdict is not a production health failure: legitimate healing or scanning under mixed load can be part of the workload. Such experiments need a separately declared maintenance-rate/queue envelope and comparable system observations on both sides; do not disable required recovery or reuse the zero-maintenance verdict as their general acceptance criterion.
+
+The shared server currently uses fixed HTTP/2 receive windows. With locked Hyper 1.12.0, the fixed stream and connection window setters following `adaptive_window(true)` disable adaptation. Moving that call last would change flow-control behavior; it is a separate experiment affecting the shared HTTP/2 listener, not a no-op cleanup. Keep fixed-window and adaptive-policy changes separate from client receive-window experiments.
+
+## Shutdown diagnostics
+
+Enable the existing `RUSTFS_OBS_LOG_STDOUT_ENABLED=true` on the task-owned diagnostic deployment and verify the sink before testing; `RUST_LOG` only sets filtering. Select the server module at debug level for the low-frequency `http_connection_drain` lifecycle events, while keeping other filtering identical between compared cells. Record `graceful_retired`, `children_retired`, `tls_reloader_retired`, and timeout `phase` (`graceful`, `child_join`, `completion_observed`, or `tls_reloader`), elapsed time, budget available at phase entry and tracked future count. `completion_observed` means the result was observed after the original deadline, not that tasks remain running. A tracked-future count is a snapshot, not proof that the original JoinHandles have been observed. No task identity, request body or credential is emitted by these diagnostics.
+
+Module-wide debug filtering also enables request-path diagnostics. Keep that diagnostic deployment separate from performance cells; freeze and read back the same production-like filter, such as `RUST_LOG=warn`, for both performance images while retaining native metrics and raw RPC samples. For task-owned systemd units, use `Type=exec` before validating `/proc/<MainPID>/exe`: `Type=simple` can report a successful start before the service child has executed the intended image. Preserve executable ownership checks instead of accepting an unverified process during that interval.
+
+When a listener stops accepting connections, unfinished TLS handshakes are cancelled before HTTP graceful retirement. These peers have not dispatched an HTTP, S3 or RPC request; established TLS/HTTP connections still follow the original graceful drain and shared deadline. Cover both a TCP peer that never sends a ClientHello and an established TLS/H2 response held across shutdown. This distinction does not permit cancelling accepted writes or treating an unrelated graceful timeout as resolved without reproduction and recovery evidence.
+
+Reproduce empty/active RPC drains on baseline and candidate, plaintext and TLS, single-node and full-stop barriers, with independent test data. Preserve the first deadline failure even if every process eventually exits. The original shared deadline remains authoritative, and timeout still waits for the real child/reloader retirement. Never abort accepted writes, discard join evidence, clear unclean-shutdown markers or extend the deadline merely to make a benchmark green. Compare confirmed writes and recovery state before declaring a shutdown fix.
+
+## Authenticated remote RPC probe
+
+The [remote probe](../../crates/ecstore/examples/internode_rpc_bench.rs) uses the production remote-disk ReadAll path and signed Ping client, with closed-loop bulk workers and fixed offered-rate control arrivals. It performs no remote writes or service operations. It always constructs remote disks, including for loopback test URLs, and rejects targets outside the declared `grpc-bench-` test-disk namespace. Prepare fixtures only after the isolated cluster has formatted its test disks and become ready: each test disk needs `.rustfs.sys/tmp/<fixture_id>/grpc-bench-payload.bin`, containing the exact bytes of the local `fixture_file`. This is a task-owned temporary-file RPC probe, not an erasure-coded S3 throughput or durability test. Preserve the separate S3 integrity/fault matrix.
+
+Recreate these temporary RPC fixtures after every test-service restart, once readiness and the formatted disk UUIDs have been revalidated. Startup cleanup can remove `.rustfs.sys/tmp` content, so a missing fixture after restart is not an object-durability oracle. Preserve confirmed S3 objects and multipart uploads separately and verify their content across shutdown/recovery before creating the next temporary probe fixture.
+
+```bash
+cargo build --release --locked -p rustfs-ecstore --example internode_rpc_bench
+# Load only this task's private RPC secret and outbound settings before running.
+target/release/examples/internode_rpc_bench /path/to/cell/probe.json /path/to/cell/raw-rpc.json
+```
+
+The manifest supplies `namespace`, `targets` (full test-disk URLs), a non-nil UUID `fixture_id`, `fixture_file`, `duration_secs`, `bulk_workers`, `control_rps`, and `rpc_timeout_ms`. For example, use duration 120, 32 bulk workers, a 1 MiB fixture, 100 Ping arrivals per second and a 1000 ms per-call deadline. Bounds are enforced in the source; an exhausted raw-sample budget invalidates the cell rather than truncating samples silently. HTTPS requires `tls_ca`; mTLS and certificate rotation remain separate acceptance cases. `RUSTFS_RPC_SECRET` must be explicitly supplied by the test deployment. Paths in the manifest resolve relative to that manifest, and output creation refuses existing files.
+
+The bulk recorder holds at most four million samples across its workers. Record the probe source and binary identity with the server identities, including any instrumentation overlay. Calibrate the workload before qualification when a fast local Docker network can exhaust the recorder capacity. Keep the complete failed attempt and restart qualification with the same probe on both sides.
+
+ReadVersion failure counters retain their existing totals. The classified error counter identifies `grpc_status`, `remote_response`, and `response_decode` at the client failure boundary. A `remote_response` failure can include an absent system metadata file, so inspect the returned storage error before treating it as a connection failure. Classification labels use a fixed set of values and do not contain file paths or peer-provided error messages.
+
+An optional `disk_ids` array identifies the formatted test disks by UUID, in exactly the same order as `targets`. Supply every target with a non-nil ID or omit the entire mapping. The probe installs these references through the existing remote-disk client API before ReadAll preflight; it does not modify server metadata. This is needed when the server registers a local path while the client dials a URL, as in isolated SNSD checks. Obtain IDs only from the task-owned formatted disks after verifying their ownership and inode; do not borrow identifiers from the original service's volumes. Preflight errors expose the numeric disk-error classification without printing error payloads.
+
+For this standalone client, apply `RUSTFS_INTERNODE_CHANNEL_ISOLATION`, bulk-pool and client receive-window settings to the probe process. Keep server configuration fixed in a client-parameter experiment. Node-internal clients require their own node environment changes in the separate S3 experiment; changing a server's outgoing pool does not configure an external probe's connection pool. The probe does not export client telemetry, and its report states that limit. Collect server telemetry and load-generator CPU/RSS/FD/socket observations alongside the raw requests; do not generalize a client-recorder-free result to all production observability costs.
+
+For aligned observation windows, supply distinct, initially absent `start_marker` and `end_marker` paths. After the `ready` receipt, the controller captures its initial native OTLP/process snapshot and atomically writes `{"time_ns": <capture_unix_nanoseconds>}` to the start marker. The start must arrive within the preceding second; any initial dispatch delay remains in the control samples. After `load_finished`, capture final producer exports and process facts and write the end marker. Both waits are bounded. Receipts and raw JSON bind the common epoch and observation endpoint; the observation window can include request drain/export synchronization, while the declared offering duration remains separate. Without markers the probe runs autonomously and does not establish a synchronized maintenance window.
+
+Every offered Ping is recorded, including schedule drops, deadlines and errors. Compute offered-load latency from `completed_ns - scheduled_ns`, RPC/acquisition time from `completed_ns - dispatched_ns` when dispatched, and dispatch lag from their difference. These are monotonic offsets from `start_unix_ns`. Record the control count, `errors`, `bulk_coverage_verified` and exit status before calculating quantiles. Any failure, payload mismatch, schedule drop or absent validated bulk coverage prevents attribution; selecting only successful samples would hide overload. Bulk payload bytes count only validated replies completed within the offering interval, with one response payload per RPC, not TCP/TLS wire bytes. The production remote-disk path retains its existing idempotent-read retry policy, so preserve per-attempt server/error counters too; zero final method errors alone does not prove no underlying retry occurred.
+
+Use identical probe source and settings on the two frozen library revisions, in separate Cargo targets, and record the harness overlay and binary identity independently from the already-built clean server binaries. Qualify repeated A cells and then A1/B1/B2/A2 before expanding to coalescing, isolation, pool or window experiments. A successful local mock regression verifies the probe's calling and accounting boundaries; acceptance against the actual authenticated four-node services is still required.
+
 ## Acceptance gates
 
 Record the verdict (pass/fail plus measured delta) in the paired directory's summary.

rustfs/src/server/http.rs

@@ -93,7 +93,7 @@ use std::io::{Error, Result};
 use std::net::{IpAddr, SocketAddr};
 use std::pin::Pin;
 use std::sync::Arc;
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::task::{Context, Poll};
 use std::time::{Duration, Instant};
 use tokio::net::{TcpListener, TcpStream};
@@ -883,6 +883,8 @@ async fn retire_http12_children(
     unverified_tls_reloader: bool,
     active_connections: usize,
 ) -> Http12ShutdownOutcome {
+    let started = tokio::time::Instant::now();
+    let graceful_complete = AtomicBool::new(false);
     #[cfg(test)]
     if let Some(control) = tasks.test_control() {
         *control.deadline.lock() = Some(deadline);
@@ -890,6 +892,16 @@ async fn retire_http12_children(
     }
     let drain = async {
         graceful.await;
+        graceful_complete.store(true, Ordering::Relaxed);
+        debug!(
+            event = EVENT_HTTP_CONNECTION_DRAIN,
+            component = LOG_COMPONENT_SERVER,
+            subsystem = LOG_SUBSYSTEM_TRANSPORT,
+            state = "graceful_retired",
+            elapsed_ms = started.elapsed().as_secs_f64() * 1000.0,
+            tracked_futures = tasks.0.tracker.len(),
+            "HTTP graceful shutdown retired"
+        );
         #[cfg(test)]
         if let Some(control) = tasks.test_control() {
             control.graceful_finished.cancel();
@@ -905,6 +917,16 @@ async fn retire_http12_children(
             component = LOG_COMPONENT_SERVER,
             subsystem = LOG_SUBSYSTEM_TRANSPORT,
             state = "timeout",
+            phase = if observed.is_ok() {
+                "completion_observed"
+            } else if graceful_complete.load(Ordering::Relaxed) {
+                "child_join"
+            } else {
+                "graceful"
+            },
+            elapsed_ms = started.elapsed().as_secs_f64() * 1000.0,
+            budget_ms = deadline.saturating_duration_since(started).as_secs_f64() * 1000.0,
+            tracked_futures = tasks.0.tracker.len(),
             active_connections,
             timeout_secs = 10,
             "HTTP connection drain timed out"
@@ -914,6 +936,16 @@ async fn retire_http12_children(
         Ok(result) => result,
         Err(_) => drain.as_mut().await,
     };
+    debug!(
+        event = EVENT_HTTP_CONNECTION_DRAIN,
+        component = LOG_COMPONENT_SERVER,
+        subsystem = LOG_SUBSYSTEM_TRANSPORT,
+        state = "children_retired",
+        deadline_missed,
+        child_failures,
+        elapsed_ms = started.elapsed().as_secs_f64() * 1000.0,
+        "HTTP child tasks retired"
+    );
     Http12ShutdownOutcome {
         deadline_missed,
         first_child_error,
@@ -947,15 +979,38 @@ pub(super) async fn retire_tls_reloader(
     outcome: &mut Http12ShutdownOutcome,
     deadline: tokio::time::Instant,
 ) {
+    let started = tokio::time::Instant::now();
     let stop = task.wait_stopped();
     tokio::pin!(stop);
     let observed = tokio::time::timeout_at(deadline, stop.as_mut()).await;
-    outcome.deadline_missed |= observed.is_err() || tokio::time::Instant::now() >= deadline;
+    let deadline_missed = observed.is_err() || tokio::time::Instant::now() >= deadline;
+    outcome.deadline_missed |= deadline_missed;
+    if deadline_missed {
+        warn!(
+            event = EVENT_HTTP_CONNECTION_DRAIN,
+            component = LOG_COMPONENT_SERVER,
+            subsystem = LOG_SUBSYSTEM_TRANSPORT,
+            state = "timeout",
+            phase = "tls_reloader",
+            elapsed_ms = started.elapsed().as_secs_f64() * 1000.0,
+            budget_ms = deadline.saturating_duration_since(started).as_secs_f64() * 1000.0,
+            "TLS reloader retirement timed out"
+        );
+    }
     outcome.unverified_tls_reloader = true;
     outcome.tls_reloader = Some(match observed {
         Ok(receipt) => receipt,
         Err(_) => stop.as_mut().await,
     });
+    debug!(
+        event = EVENT_HTTP_CONNECTION_DRAIN,
+        component = LOG_COMPONENT_SERVER,
+        subsystem = LOG_SUBSYSTEM_TRANSPORT,
+        state = "tls_reloader_retired",
+        deadline_missed,
+        elapsed_ms = started.elapsed().as_secs_f64() * 1000.0,
+        "TLS reloader retired"
+    );
 }
 
 #[cfg(test)]
@@ -971,6 +1026,7 @@ pub(crate) async fn test_retire_http12_scope(
 #[cfg(test)]
 #[derive(Default)]
 struct HttpLeafTestControl {
+    tls_handshake_started: tokio_util::sync::CancellationToken,
     body_entered: tokio_util::sync::CancellationToken,
     body_release: tokio_util::sync::CancellationToken,
     drain_entered: tokio_util::sync::CancellationToken,
@@ -2627,6 +2683,7 @@ async fn start_http_server_inner(
 
         let http_server = Arc::new(conn_builder);
         let graceful = GracefulShutdown::new();
+        let tls_handshake_stop = tokio_util::sync::CancellationToken::new();
         debug!(
             event = EVENT_HTTP_ACCEPT_LOOP_STATE,
             component = LOG_COMPONENT_SERVER,
@@ -2795,6 +2852,7 @@ async fn start_http_server_inner(
                 rate_limit_layer: api_rate_limit_layer.clone(),
                 server_ctx: Arc::clone(&server_ctx),
                 tls_handshake_timeout: Duration::from_secs(http1_header_read_timeout),
+                tls_handshake_stop: tls_handshake_stop.clone(),
                 request_body_idle_timeout: http_request_body_read_timeout,
             };
 
@@ -2833,6 +2891,9 @@ async fn start_http_server_inner(
             );
         }
         drop(listener);
+        // Unestablished TLS peers cannot receive HTTP graceful shutdown yet, but
+        // already hold a watcher. Retire only that pre-request phase immediately.
+        tls_handshake_stop.cancel();
         drop(http_server);
         #[cfg(feature = "http3")]
         let unverified_h3 = http3_server.is_some();
@@ -2909,6 +2970,7 @@ struct ConnectionContext {
     /// the existing slow-client bound for the pre-request phase, and is pre-computed with the
     /// other transport parameters to avoid a per-connection env read.
     tls_handshake_timeout: Duration,
+    tls_handshake_stop: tokio_util::sync::CancellationToken,
     /// Inter-chunk timeout used while discarding a raw HTTP/1 request body
     /// after an S3 operation has already produced its response.
     request_body_idle_timeout: Duration,
@@ -3035,6 +3097,7 @@ fn process_connection(
             rate_limit_layer,
             server_ctx,
             tls_handshake_timeout,
+            tls_handshake_stop,
             request_body_idle_timeout,
         } = context;
 
@@ -3260,7 +3323,16 @@ fn process_connection(
                 .ok()
                 .map_or_else(|| "unknown".to_string(), |addr| addr.to_string());
             let acceptor = holder.get();
-            match accept_tls_with_deadline(&acceptor, socket, tls_handshake_timeout).await {
+            #[cfg(test)]
+            if let Some(control) = task_owner.test_control() {
+                control.tls_handshake_started.cancel();
+            }
+            let accepted = tokio::select! {
+                biased;
+                () = tls_handshake_stop.cancelled() => return,
+                accepted = accept_tls_with_deadline(&acceptor, socket, tls_handshake_timeout) => accepted,
+            };
+            match accepted {
                 Ok(tls_socket) => {
                     trace!("TLS handshake successful");
                     let stream = TokioIo::new(tls_socket);
@@ -4276,6 +4348,113 @@ mod tests {
             .expect("actual production HTTP listener")
     }
 
+    async fn http_tls_leaf_fixture(control: Arc<HttpLeafTestControl>) -> (ShutdownHandle, SocketAddr, tempfile::TempDir) {
+        if rustls::crypto::CryptoProvider::get_default().is_none() {
+            let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+        }
+        let directory = tempfile::TempDir::new().expect("TLS fixture directory");
+        let pair = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).expect("TLS fixture certificate");
+        std::fs::write(directory.path().join(rustfs_config::RUSTFS_TLS_CERT), pair.cert.pem())
+            .expect("TLS fixture certificate file");
+        std::fs::write(directory.path().join(rustfs_config::RUSTFS_TLS_KEY), pair.signing_key.serialize_pem())
+            .expect("TLS fixture key file");
+        let mut config = config::Config::new("127.0.0.1:0", Vec::new());
+        config.console_enable = false;
+        config.tls_path = Some(directory.path().to_str().expect("TLS fixture path").to_owned());
+        let readiness = Arc::new(GlobalReadiness::new());
+        readiness.mark_stage(rustfs_common::SystemStage::FullReady);
+        let (shutdown, address) = start_http_server_inner(&config, readiness, ServerContextSlot::new(), Some(control))
+            .await
+            .expect("actual production TLS listener");
+        (shutdown, address, directory)
+    }
+
+    #[tokio::test]
+    async fn http12_tls_stalled_handshake_retires_before_shutdown_deadline() {
+        let control = Arc::new(HttpLeafTestControl::default());
+        let (mut shutdown, address, _directory) = http_tls_leaf_fixture(control.clone()).await;
+        let socket = TcpStream::connect(address).await.expect("open TCP without a ClientHello");
+        let entered = tokio::time::timeout(Duration::from_secs(5), control.tls_handshake_started.cancelled()).await;
+        shutdown.signal();
+        let timely = tokio::time::timeout(Duration::from_secs(1), shutdown.wait_checked()).await;
+        // Always release the peer and join the original owner, including on a regression.
+        drop(socket);
+        let (timely, joined) = match timely {
+            Ok(joined) => (true, joined),
+            Err(_) => (false, shutdown.wait_checked().await),
+        };
+        entered.expect("accepted TLS handshake was actually polled");
+        joined.expect("original TLS owner remains joined");
+        assert!(timely, "an unestablished TLS peer must not hold the HTTP drain");
+        assert!(!control.body_entered.is_cancelled(), "no HTTP request reached a handler");
+    }
+
+    #[tokio::test]
+    async fn http12_tls_established_h2_response_keeps_graceful_drain() {
+        use rustls::pki_types::pem::PemObject;
+
+        let control = Arc::new(HttpLeafTestControl::default());
+        let (mut shutdown, address, directory) = http_tls_leaf_fixture(control.clone()).await;
+        let mut client = None;
+        let observations = async {
+            let mut roots = rustls::RootCertStore::empty();
+            let certificate =
+                rustls::pki_types::CertificateDer::from_pem_file(directory.path().join(rustfs_config::RUSTFS_TLS_CERT))
+                    .map_err(Error::other)?;
+            roots.add(certificate).map_err(Error::other)?;
+            let mut config = rustls::ClientConfig::builder()
+                .with_root_certificates(roots)
+                .with_no_client_auth();
+            config.alpn_protocols = vec![b"h2".to_vec()];
+            let connector = tokio_rustls::TlsConnector::from(Arc::new(config));
+            let stream = connector
+                .connect("localhost".try_into().map_err(Error::other)?, TcpStream::connect(address).await?)
+                .await?;
+            let (mut sender, connection) =
+                hyper::client::conn::http2::handshake::<_, _, Empty<Bytes>>(TokioExecutor::new(), TokioIo::new(stream))
+                    .await
+                    .map_err(Error::other)?;
+            client = Some(tokio::spawn(connection));
+            let request = HttpRequest::builder()
+                .uri("https://localhost/__http_leaf_hold")
+                .body(Empty::new())
+                .map_err(Error::other)?;
+            let response = tokio::time::timeout(Duration::from_secs(5), sender.send_request(request))
+                .await
+                .map_err(Error::other)?
+                .map_err(Error::other)?;
+            tokio::time::timeout(Duration::from_secs(5), control.body_entered.cancelled())
+                .await
+                .map_err(Error::other)?;
+            shutdown.signal();
+            let held = tokio::time::timeout(Duration::from_millis(100), shutdown.wait_checked())
+                .await
+                .is_err();
+            control.body_release.cancel();
+            tokio::time::timeout(Duration::from_secs(5), response.into_body().collect())
+                .await
+                .map_err(Error::other)?
+                .map_err(Error::other)?;
+            drop(sender);
+            Ok::<_, Error>(held)
+        }
+        .await;
+        control.body_release.cancel();
+        shutdown.signal();
+        let joined = shutdown.wait_checked().await;
+        if let Some(client) = client {
+            client
+                .await
+                .expect("original TLS client task joined")
+                .expect("TLS H2 completed");
+        }
+        assert!(
+            observations.expect("verified TLS H2 observations"),
+            "established response must retain its graceful owner"
+        );
+        joined.expect("TLS response completed before the original shutdown deadline");
+    }
+
     #[cfg(feature = "http3")]
     async fn assert_tls_startup_failure_joins_started_http3_outer(fail: bool) {
         use super::super::tls_material::TlsMaterialError;

rustfs/src/storage/rpc/node_service/disk.rs

@@ -32,6 +32,7 @@ use crate::storage::storage_api::{
 };
 use bytes::Bytes;
 use rustfs_filemeta::FileInfo;
+use rustfs_io_metrics::internode_metrics::InternodeRpcDurationGuard;
 use rustfs_io_metrics::internode_metrics::{
     INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_REQUEST,
     INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_COMPARE_AND_UPDATE_FILE,
@@ -389,32 +390,6 @@ fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Insta
     }
 }
 
-/// Records the complete server-side lifetime of a unary internode RPC, including request
-/// decoding, disk work, response encoding, error paths, and cancellation.
-struct InternodeOperationDurationGuard {
-    operation: &'static str,
-    started_at: Instant,
-}
-
-impl InternodeOperationDurationGuard {
-    fn new(operation: &'static str) -> Self {
-        Self {
-            operation,
-            started_at: Instant::now(),
-        }
-    }
-}
-
-impl Drop for InternodeOperationDurationGuard {
-    fn drop(&mut self) {
-        global_internode_metrics().record_server_duration_for_operation_and_backend(
-            self.operation,
-            INTERNODE_TRANSPORT_BACKEND_GRPC,
-            self.started_at.elapsed(),
-        );
-    }
-}
-
 fn encode_batch_read_version_response_payloads(
     batch_read_version_resps: &[BatchReadVersionResp],
     request_decoded_from_msgpack: bool,
@@ -762,7 +737,7 @@ impl NodeService {
         &self,
         request: Request<ReadMultipleRequest>,
     ) -> Result<Response<ReadMultipleResponse>, Status> {
-        let _duration = InternodeOperationDurationGuard::new(INTERNODE_OPERATION_GRPC_READ_MULTIPLE);
+        let _duration = InternodeRpcDurationGuard::handler(INTERNODE_OPERATION_GRPC_READ_MULTIPLE);
         let request = request.into_inner();
         if let Some(disk) = self.find_disk(&request.disk).await? {
             let read_multiple_req = match decode_msgpack_or_json::<ReadMultipleReq>(
@@ -823,7 +798,7 @@ impl NodeService {
         &self,
         request: Request<BatchReadVersionRequest>,
     ) -> Result<Response<BatchReadVersionResponse>, Status> {
-        let _duration = InternodeOperationDurationGuard::new(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION);
+        let _duration = InternodeRpcDurationGuard::handler(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION);
         let attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
         let request = request.into_inner();
         if attribution_enabled {
@@ -2733,6 +2708,7 @@ impl NodeService {
     }
 
     pub(super) async fn handle_write_all(&self, request: Request<WriteAllRequest>) -> Result<Response<WriteAllResponse>, Status> {
+        let _duration = InternodeRpcDurationGuard::handler(INTERNODE_OPERATION_GRPC_WRITE_ALL);
         verify_disk_mutation_digest(&request, rustfs_protos::canonical_write_all_request_body(request.get_ref()), "write_all")?;
         let request = request.into_inner();
         let data_len = request.data.len();
@@ -2837,6 +2813,7 @@ impl NodeService {
     }
 
     pub(super) async fn handle_read_all(&self, request: Request<ReadAllRequest>) -> Result<Response<ReadAllResponse>, Status> {
+        let _duration = InternodeRpcDurationGuard::handler(INTERNODE_OPERATION_GRPC_READ_ALL);
         debug!("read all");
 
         let request = request.into_inner();
@@ -2886,7 +2863,7 @@ impl NodeService {
 mod tests {
     use super::{
         DeleteOptions, INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
-        InternodeOperationDurationGuard, compat_response_json, decode_msgpack_or_json, decode_multipart_rename_request_file_info,
+        InternodeRpcDurationGuard, compat_response_json, decode_msgpack_or_json, decode_multipart_rename_request_file_info,
         decode_rename_data_request_file_info, encode_batch_read_version_response_payloads, encode_delete_versions_errors,
         encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, encode_read_multiple_response_payloads,
         encode_rename_data_response_payloads, rejects_legacy_delete_transaction,
@@ -3008,7 +2985,7 @@ mod tests {
                     super::INTERNODE_TRANSPORT_BACKEND_GRPC,
                     std::time::Duration::from_secs(1),
                 );
-                drop(InternodeOperationDurationGuard::new(operation));
+                drop(InternodeRpcDurationGuard::handler(operation));
             }
         });
 
@@ -3055,7 +3032,9 @@ mod tests {
     #[serial]
     fn handlers_record_duration_on_missing_disk_errors() {
         use metrics_util::debugging::{DebugValue, DebuggingRecorder};
-        use rustfs_protos::proto_gen::node_service::{BatchReadVersionRequest, ReadMultipleRequest};
+        use rustfs_protos::proto_gen::node_service::{
+            BatchReadVersionRequest, ReadAllRequest, ReadMultipleRequest, WriteAllRequest,
+        };
 
         let recorder = DebuggingRecorder::new();
         let snapshotter = recorder.snapshotter();
@@ -3087,6 +3066,25 @@ mod tests {
                     .expect("BatchReadVersion handler should return a response")
                     .into_inner();
                 assert!(!batch_read_version.success);
+                let read_all = server
+                    .handle_read_all(Request::new(ReadAllRequest {
+                        disk: "missing-disk".to_owned(),
+                        volume: "bucket".to_owned(),
+                        path: "key".to_owned(),
+                    }))
+                    .await
+                    .expect("ReadAll failure response")
+                    .into_inner();
+                assert!(!read_all.success);
+                let write_all = server
+                    .handle_write_all(Request::new(WriteAllRequest {
+                        disk: "missing-disk".to_owned(),
+                        volume: "bucket".to_owned(),
+                        path: "key".to_owned(),
+                        data: bytes::Bytes::new(),
+                    }))
+                    .await;
+                assert!(!write_all.is_ok_and(|response| response.into_inner().success));
             });
         });
 
@@ -3098,7 +3096,7 @@ mod tests {
                 composite.key().name() == "rustfs_system_network_internode_operation_server_duration_ms"
             })
             .collect::<Vec<_>>();
-        assert_eq!(entries.len(), 2);
+        assert_eq!(entries.len(), 4);
         for (composite, _, _, value) in entries {
             let operation = composite
                 .key()
@@ -3107,7 +3105,13 @@ mod tests {
                 .map(|label| label.value().to_string())
                 .expect("handler duration metric should carry an operation label");
             assert!(
-                operation == INTERNODE_OPERATION_GRPC_READ_MULTIPLE || operation == INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION
+                [
+                    INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+                    INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+                    rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_GRPC_READ_ALL,
+                    rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_GRPC_WRITE_ALL,
+                ]
+                .contains(&operation.as_str())
             );
             assert!(matches!(value, DebugValue::Histogram(samples) if !samples.is_empty()));
         }

scripts/check_internode_grpc_window.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Check recorded gRPC measurement windows for process and maintenance stability.
+
+Read-only: consume OTLP JSON snapshots and independently recorded process facts.
+This checks observations, not throughput, RPC coverage, or an application SLO.
+"""
+
+import argparse
+import hashlib
+import json
+import math
+from pathlib import Path
+import sys
+import uuid
+
+from scanner_otlp_window import attributes, counter_point, identity, load, require, uint, unique_object
+
+
+REQUIRED_GAUGES = frozenset({
+    "rustfs_heal_mrf_queue_depth",
+    "rustfs_memory_allocator_reclaim_heal_activity_current",
+    "rustfs_memory_allocator_reclaim_scanner_activity_current",
+})
+OPTIONAL_GAUGES = frozenset({"rustfs_heal_task_running"})
+COUNTERS = frozenset({"rustfs_heal_task_start_total"})
+METRICS = REQUIRED_GAUGES | OPTIONAL_GAUGES | COUNTERS
+
+
+def processes(rows):
+    require(isinstance(rows, list) and 0 < len(rows) <= 512, "invalid process inventory")
+    result = {}
+    for row in rows:
+        node = row.get("node")
+        require(isinstance(node, str) and node and node not in result, "duplicate or missing node identity")
+        pid = uint(row.get("pid"), "process pid")
+        start = uint(row.get("start_ticks"), "process start ticks")
+        boot = row.get("boot_id")
+        require(isinstance(boot, str) and uuid.UUID(boot).int > 0, "missing process boot identity")
+        require(pid > 0 and start > 0, "missing process identity")
+        result[node] = (pid, start, boot)
+    return result
+
+
+def resource_process(resource):
+    values = attributes(resource.get("attributes", []))
+    node = values.get("service.instance.id", {}).get("stringValue")
+    pid = values.get("process.pid", {})
+    if set(pid) == {"intValue"}:
+        pid = uint(pid["intValue"], "resource process pid")
+    elif set(pid) == {"stringValue"}:
+        pid = uint(pid["stringValue"], "resource process pid")
+    else:
+        pid = None
+    return node, pid
+
+
+def gauge_point(point):
+    require(uint(point.get("flags", 0), "point flags") == 0, "gauge has no recorded value or unknown flags")
+    require(set(point) & {"asInt", "asDouble"} in ({"asInt"}, {"asDouble"}), "ambiguous or missing gauge value")
+    value = uint(point["asInt"], "gauge value") if "asInt" in point else point["asDouble"]
+    require(type(value) in (int, float) and math.isfinite(value) and value >= 0, "invalid gauge value")
+    return value
+
+
+def observations(document, inventory, captured, max_age):
+    require(isinstance(document, dict), "snapshot must be an OTLP export object")
+    found = set()
+    result = {}
+    ignored = 0
+    for resource_metrics in document.get("resourceMetrics", []):
+        resource = resource_metrics.get("resource", {})
+        node, pid = resource_process(resource)
+        if node not in inventory or pid != inventory[node][0]:
+            ignored += 1
+            continue
+        for scope_metrics in resource_metrics.get("scopeMetrics", []):
+            scope = scope_metrics.get("scope", {})
+            for metric in scope_metrics.get("metrics", []):
+                name = metric.get("name")
+                if name not in METRICS:
+                    continue
+                kind = "sum" if name in COUNTERS else "gauge"
+                require(set(metric) & {"sum", "gauge", "histogram", "summary", "exponentialHistogram"} == {kind},
+                        "unexpected maintenance metric type")
+                data = metric[kind]
+                if kind == "sum":
+                    require(data.get("isMonotonic") is True and data.get("aggregationTemporality") in
+                            (2, "AGGREGATION_TEMPORALITY_CUMULATIVE"), "maintenance counter must be cumulative")
+                for point in data.get("dataPoints", []):
+                    require(uint(point.get("droppedAttributesCount", 0), "point dropped attributes") == 0,
+                            "maintenance point identity has dropped attributes")
+                    end = uint(point.get("timeUnixNano"), "metric timestamp")
+                    require(0 < end <= captured and captured - end <= max_age, "stale or future maintenance observation")
+                    descriptor = identity(resource, scope, metric, point,
+                                          resource_metrics.get("schemaUrl", ""), scope_metrics.get("schemaUrl", ""))
+                    key = json.dumps(descriptor, sort_keys=True, allow_nan=False)
+                    require(key not in result, "duplicate maintenance series")
+                    value = counter_point(point) if kind == "sum" else {"value": gauge_point(point), "end": end}
+                    result[key] = {**value, "node": node, "name": name, "kind": kind}
+                    found.add((node, name))
+    require(all((node, name) in found for node in inventory for name in REQUIRED_GAUGES),
+            "missing current-process maintenance observations")
+    return result, ignored
+
+
+def check(manifest, root, expected_nodes=4):
+    require(isinstance(manifest, dict) and manifest.get("scope") == "measurement",
+            "a measurement-only window is required; command/prepare/cleanup windows cannot qualify")
+    inventory = processes(manifest.get("processes"))
+    require(type(expected_nodes) is int and 0 < expected_nodes <= 512 and len(inventory) == expected_nodes,
+            "process inventory does not match the declared fleet size")
+    start = uint(manifest.get("start_ns"), "window start")
+    end = uint(manifest.get("end_ns"), "window end")
+    gap = uint(manifest.get("max_sample_gap_ns"), "maximum sample gap")
+    age = uint(manifest.get("max_export_age_ns"), "maximum export age")
+    require(0 < start < end and gap > 0 and age > 0, "invalid observation window or sampling budget")
+    samples = manifest.get("samples")
+    require(isinstance(samples, list) and 3 <= len(samples) <= 4096, "at least three bounded observations are required")
+    previous = None
+    previous_time = None
+    ignored = 0
+    violations = []
+    paths = set()
+    receipts = []
+    for index, sample in enumerate(samples):
+        require(sample.get("phase") == "measurement", "observation belongs to a different workload phase")
+        require(processes(sample.get("processes")) == inventory, "process restarted, changed, or was not observed")
+        captured = uint(sample.get("captured_ns"), "capture timestamp")
+        require(start <= captured <= end, "observation outside the measurement window")
+        if previous_time is not None:
+            require(0 < captured - previous_time <= gap, "duplicate, reversed, or missing observations")
+        else:
+            require(captured == start, "missing measurement-start observation")
+        require(isinstance(sample.get("path"), str) and sample["path"], "missing snapshot path")
+        path = (root / sample["path"]).resolve()
+        require(path not in paths, "snapshot file reused across observations")
+        paths.add(path)
+        document, digest = load(path, "json")
+        receipts.append({"path": sample["path"], "sha256": digest, "captured_ns": captured})
+        current, skipped = observations(document, inventory, captured, age)
+        ignored += skipped
+        for key, point in current.items():
+            if point["kind"] == "gauge":
+                if point["value"] != 0:
+                    violations.append({"sample": index, "node": point["node"], "metric": point["name"], "reason": "nonzero"})
+            elif previous is not None:
+                before = previous.get(key)
+                if before is None:
+                    if point["value"] != 0:
+                        violations.append({"sample": index, "node": point["node"], "metric": point["name"],
+                                           "reason": "new_nonzero_counter"})
+                else:
+                    require(before["start"] == point["start"] and point["value"] >= before["value"],
+                            "maintenance counter reset within measurement window")
+                    if point["value"] > before["value"]:
+                        violations.append({"sample": index, "node": point["node"], "metric": point["name"],
+                                           "reason": "counter_increased"})
+            if previous is not None and key in previous:
+                require(point["end"] > previous[key]["end"], "cached maintenance point repeated across captures")
+        if previous is not None:
+            require(previous.keys() <= current.keys(), "maintenance series disappeared within measurement window")
+        previous, previous_time = current, captured
+    require(previous_time == end, "missing measurement-end observation")
+    return {"observed_stable": not violations, "scope": "measurement", "start_ns": start, "end_ns": end,
+            "samples": len(samples), "processes": len(inventory), "max_sample_gap_ns": gap,
+            "max_export_age_ns": age, "ignored_resource_exports": ignored, "violations": violations,
+            "snapshots": receipts,
+            "limits": "Sampled maintenance observations only; RPC coverage, per-request timing, drift and SLO gates remain required."}, paths
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--expected-nodes", type=int, default=4)
+    args = parser.parse_args()
+    try:
+        require(args.manifest.stat().st_size <= 1024 * 1024, "oversized window manifest")
+        raw = args.manifest.read_bytes()
+        manifest = json.loads(raw, object_pairs_hook=unique_object)
+        result, inputs = check(manifest, args.manifest.resolve().parent, args.expected_nodes)
+        require(args.out.resolve() not in inputs | {args.manifest.resolve()}, "output would overwrite raw evidence")
+        if args.out.exists():
+            require(not any(args.out.samefile(path) for path in inputs | {args.manifest.resolve()}),
+                    "output aliases raw evidence")
+        result["manifest_sha256"] = hashlib.sha256(raw).hexdigest()
+        args.out.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
+        return 0 if result["observed_stable"] else 1
+    except (ValueError, OSError, TypeError, KeyError, AttributeError) as error:
+        print(f"Invalid gRPC measurement evidence: {error}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/run_internode_grpc_ab_bench.sh

@@ -24,6 +24,8 @@ STAGE=""
 PHASE=""
 OUT_ROOT="${PROJECT_ROOT}/target/bench/internode-transport"
 DRY_RUN=false
+LIFECYCLE_FEATURE="both"
+TRANSPORT_FEATURE="all"
 PASSTHROUGH=()
 
 usage() {
@@ -38,6 +40,10 @@ Required:
 Options:
   --out-root <dir>          Artifact root. Default: target/bench/internode-transport
   --dry-run                 Print the env + the command, do not run the bench.
+  --lifecycle-feature <prewarm|offline|both>
+                            P3 feature to enable. Default: both (legacy combined experiment).
+  --transport-feature <nodelay|stream-window|connection-window|max-message|all>
+                            P0 parameter to compare. Default: all (legacy combined experiment).
   -h, --help                Show this help.
 
 Everything after `--` is passed through to the underlying bench:
@@ -60,6 +66,10 @@ while [[ $# -gt 0 ]]; do
     --phase) PHASE="$2"; shift 2 ;;
     --out-root) OUT_ROOT="$2"; shift 2 ;;
     --dry-run) DRY_RUN=true; shift ;;
+    --lifecycle-feature) LIFECYCLE_FEATURE="$2"; shift 2 ;;
+    --transport-feature)
+      [[ $# -ge 2 ]] || { echo "ERROR: --transport-feature requires a value" >&2; exit 2; }
+      TRANSPORT_FEATURE="$2"; shift 2 ;;
     -h|--help) usage; exit 0 ;;
     --) shift; PASSTHROUGH=("$@"); break ;;
     *) echo "Unknown argument: $1" >&2; usage; exit 2 ;;
@@ -67,6 +77,16 @@ while [[ $# -gt 0 ]]; do
 done
 
 [[ -n "${STAGE}" && -n "${PHASE}" ]] || { echo "ERROR: --stage and --phase are required" >&2; usage; exit 2; }
+case "${LIFECYCLE_FEATURE}" in prewarm|offline|both) ;; *) echo "ERROR: invalid --lifecycle-feature" >&2; exit 2 ;; esac
+case "${TRANSPORT_FEATURE}" in nodelay|stream-window|connection-window|max-message|all) ;; *) echo "ERROR: invalid --transport-feature" >&2; exit 2 ;; esac
+if [[ "${STAGE}" != "p0" && "${TRANSPORT_FEATURE}" != "all" ]]; then
+  echo "ERROR: --transport-feature is only valid for --stage p0" >&2
+  exit 2
+fi
+if [[ "${STAGE}" != "p3" && "${LIFECYCLE_FEATURE}" != "both" ]]; then
+  echo "ERROR: --lifecycle-feature is only valid for --stage p3" >&2
+  exit 2
+fi
 case "${PHASE}" in before|after|request-only|canary|rollback) ;; *) echo "ERROR: --phase must be before, after, request-only, canary, or rollback" >&2; exit 2 ;; esac
 if [[ "${STAGE}" != "p2" ]]; then
   case "${PHASE}" in
@@ -82,13 +102,20 @@ stage_env() {
   case "${stage}" in
     p0)
       if [[ "${phase}" == "before" ]]; then
-        printf '%s\n' \
-          'RUSTFS_INTERNODE_RPC_TCP_NODELAY=false' \
-          'RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE=0' \
-          'RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE=0' \
-          'RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE=4194304'
+        if [[ "${TRANSPORT_FEATURE}" == "all" || "${TRANSPORT_FEATURE}" == "nodelay" ]]; then
+          printf '%s\n' 'RUSTFS_INTERNODE_RPC_TCP_NODELAY=false'
+        fi
+        if [[ "${TRANSPORT_FEATURE}" == "all" || "${TRANSPORT_FEATURE}" == "stream-window" ]]; then
+          printf '%s\n' 'RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE=0'
+        fi
+        if [[ "${TRANSPORT_FEATURE}" == "all" || "${TRANSPORT_FEATURE}" == "connection-window" ]]; then
+          printf '%s\n' 'RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE=0'
+        fi
+        if [[ "${TRANSPORT_FEATURE}" == "all" || "${TRANSPORT_FEATURE}" == "max-message" ]]; then
+          printf '%s\n' 'RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE=4194304'
+        fi
       fi
-      # after: P0 defaults are the enabled state -> no overrides needed.
+      # The reset list below clears the baseline overrides before restoring defaults.
       ;;
     p1)
       if [[ "${phase}" == "after" ]]; then
@@ -120,8 +147,8 @@ stage_env() {
     p3)
       if [[ "${phase}" == "after" ]]; then
         printf '%s\n' \
-          'RUSTFS_INTERNODE_PREWARM=true' \
-          'RUSTFS_INTERNODE_OFFLINE_BYPASS=true' \
+          "RUSTFS_INTERNODE_PREWARM=$([[ "${LIFECYCLE_FEATURE}" != "offline" ]] && echo true || echo false)" \
+          "RUSTFS_INTERNODE_OFFLINE_BYPASS=$([[ "${LIFECYCLE_FEATURE}" != "prewarm" ]] && echo true || echo false)" \
           'RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=5' \
           'RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD=3'
       else
@@ -135,17 +162,31 @@ stage_env() {
 }
 
 ENV_LINES="$(stage_env "${STAGE}" "${PHASE}")"
-OUT_DIR="${OUT_ROOT}/${STAGE}-${PHASE}"
+# A phase owns only these settings. Clear them both in the emitted file and this process so
+# sourcing before/after in one shell cannot silently carry baseline overrides into a candidate.
+case "${STAGE}" in
+  p0) RESET_VARS=(RUSTFS_INTERNODE_RPC_TCP_NODELAY RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE) ;;
+  p1) RESET_VARS=(RUSTFS_INTERNODE_CHANNEL_ISOLATION RUSTFS_INTERNODE_BULK_CHANNELS RUSTFS_INTERNODE_BULK_HTTP2_STREAM_WINDOW_SIZE RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT RUSTFS_INTERNODE_BULK_BUFFER_SIZE) ;;
+  p2) RESET_VARS=(RUSTFS_INTERNODE_RPC_MSGPACK_ONLY RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED) ;;
+  p3) RESET_VARS=(RUSTFS_INTERNODE_PREWARM RUSTFS_INTERNODE_OFFLINE_BYPASS RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD) ;;
+esac
+ARTIFACT_STAGE="${STAGE}"
+if [[ "${STAGE}" == "p0" && "${TRANSPORT_FEATURE}" != "all" ]]; then
+  ARTIFACT_STAGE="${STAGE}-${TRANSPORT_FEATURE}"
+fi
+OUT_DIR="${OUT_ROOT}/${ARTIFACT_STAGE}-${PHASE}"
 mkdir -p "${OUT_DIR}"
 ENV_FILE="${OUT_DIR}/server-env.sh"
 
 {
   echo "# RustFS server env for internode stage ${STAGE} / phase ${PHASE}."
+  if [[ "${STAGE}" == "p0" ]]; then echo "# Transport feature: ${TRANSPORT_FEATURE}"; fi
   if [[ "${STAGE}" == "p2" && "${PHASE}" == "canary" ]]; then
     echo "# Source this only on the selected canary RustFS node before starting that node; keep the rest on p2/before or p2/request-only env."
   else
     echo "# Source this on EVERY rustfs node before starting the server, then run the bench."
   fi
+  for variable in "${RESET_VARS[@]}"; do echo "unset ${variable}"; done
   if [[ -n "${ENV_LINES}" ]]; then
     while IFS= read -r line; do echo "export ${line}"; done <<<"${ENV_LINES}"
   else
@@ -159,13 +200,33 @@ sed 's/^/   /' "${ENV_FILE}"
 echo "-- artifacts: ${OUT_DIR}"
 
 # Export for the p3 docker path (best-effort: only takes effect if the compose forwards the env).
+for variable in "${RESET_VARS[@]}"; do unset "${variable}"; done
 if [[ -n "${ENV_LINES}" ]]; then
   while IFS= read -r line; do export "${line?}"; done <<<"${ENV_LINES}"
 fi
 
 run() {
   if [[ "${DRY_RUN}" == "true" ]]; then
-    echo "-- dry-run, would execute:"; printf '   %q ' "$@"; echo
+    local redact_next=false argument
+    echo "-- dry-run, would execute:"
+    for argument in "$@"; do
+      if [[ "${redact_next}" == "true" ]]; then
+        printf '   %q ' REDACTED
+        redact_next=false
+        continue
+      fi
+      case "${argument}" in
+        --access-key|--secret-key|--session-token)
+          printf '   %q ' "${argument}"
+          redact_next=true
+          ;;
+        --access-key=*|--secret-key=*|--session-token=*|-accessKey=*|-secretKey=*)
+          printf '   %q ' "${argument%%=*}=REDACTED"
+          ;;
+        *) printf '   %q ' "${argument}" ;;
+      esac
+    done
+    echo
     return 0
   fi
   "$@"

scripts/test_internode_grpc_ab_bench.sh

@@ -86,3 +86,128 @@ rg -qx 'export RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=5' "$P3_AFTER_ENV"
 rg -qx 'export RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD=3' "$P3_AFTER_ENV"
 
 echo "internode grpc A/B bench env tests passed"
+
+# Exercise the generated files as operators use them, including a polluted starting shell.
+(
+  export RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=17
+  # shellcheck source=/dev/null
+  source "$P0_BEFORE_ENV"
+  [[ "$RUSTFS_INTERNODE_RPC_TCP_NODELAY" == false ]]
+  # shellcheck source=/dev/null
+  source "$P0_AFTER_ENV"
+  for variable in RUSTFS_INTERNODE_RPC_TCP_NODELAY RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE; do
+    [[ -z "${!variable+x}" ]] || { echo "P0 candidate retained ${variable}" >&2; exit 1; }
+  done
+  [[ "$RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS" == 17 ]]
+  # shellcheck source=/dev/null
+  source "$P0_BEFORE_ENV"
+  [[ "$RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE" == 4194304 ]]
+  export RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE=33554432
+  export RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT=1
+  # shellcheck source=/dev/null
+  source "$P1_AFTER_ENV"
+  [[ "$RUSTFS_INTERNODE_CHANNEL_ISOLATION" == true ]]
+  [[ -z "${RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE+x}" && -z "${RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT+x}" ]]
+  # shellcheck source=/dev/null
+  source "$P1_BEFORE_ENV"
+  [[ "$RUSTFS_INTERNODE_CHANNEL_ISOLATION" == false && -z "${RUSTFS_INTERNODE_BULK_CHANNELS+x}" ]]
+  # shellcheck source=/dev/null
+  source "$P3_AFTER_ENV"
+  [[ "$RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS" == 5 ]]
+  # shellcheck source=/dev/null
+  source "$P3_BEFORE_ENV"
+  [[ "$RUSTFS_INTERNODE_PREWARM" == false && "$RUSTFS_INTERNODE_OFFLINE_BYPASS" == false ]]
+  [[ -z "${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS+x}" && -z "${RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD+x}" ]]
+)
+
+for feature in prewarm offline; do
+  "$RUNNER" --stage p3 --phase after --lifecycle-feature "$feature" --out-root "$TMP_DIR/$feature" --dry-run >/dev/null
+  (
+    # shellcheck source=/dev/null
+    source "$TMP_DIR/$feature/p3-after/server-env.sh"
+    if [[ "$feature" == prewarm ]]; then
+      [[ "$RUSTFS_INTERNODE_PREWARM" == true && "$RUSTFS_INTERNODE_OFFLINE_BYPASS" == false ]]
+    else
+      [[ "$RUSTFS_INTERNODE_PREWARM" == false && "$RUSTFS_INTERNODE_OFFLINE_BYPASS" == true ]]
+    fi
+  )
+done
+if "$RUNNER" --stage p1 --phase after --lifecycle-feature prewarm --out-root "$TMP_DIR" --dry-run >/dev/null 2>&1; then
+  echo "expected lifecycle selection to be P3-only" >&2
+  exit 1
+fi
+echo "internode grpc A/B phase switching tests passed"
+
+features=(nodelay stream-window connection-window max-message)
+settings=(RUSTFS_INTERNODE_RPC_TCP_NODELAY RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE)
+values=(false 0 0 4194304)
+for index in "${!features[@]}"; do
+  feature="${features[$index]}"
+  for phase in before after; do
+    "$RUNNER" --stage p0 --phase "$phase" --transport-feature "$feature" --out-root "$TMP_DIR" --dry-run >/dev/null
+  done
+  before="$TMP_DIR/p0-$feature-before/server-env.sh"
+  after="$TMP_DIR/p0-$feature-after/server-env.sh"
+  [[ "$(rg -c '^export ' "$before")" == 1 ]]
+  rg -Fqx "export ${settings[$index]}=${values[$index]}" "$before"
+  if rg -q '^export ' "$after"; then
+    echo "single-parameter candidate must restore defaults: $feature" >&2
+    exit 1
+  fi
+  (
+    for variable in "${settings[@]}"; do export "$variable=polluted"; done
+    # shellcheck source=/dev/null
+    source "$before"
+    for variable in "${settings[@]}"; do
+      if [[ "$variable" == "${settings[$index]}" ]]; then
+        [[ "${!variable}" == "${values[$index]}" ]]
+      else
+        [[ -z "${!variable+x}" ]]
+      fi
+    done
+    # shellcheck source=/dev/null
+    source "$after"
+    for variable in "${settings[@]}"; do [[ -z "${!variable+x}" ]]; done
+  )
+done
+for invalid in unknown ../nodelay; do
+  if "$RUNNER" --stage p0 --phase before --transport-feature "$invalid" --out-root "$TMP_DIR/invalid" --dry-run >/dev/null 2>&1; then
+    echo "invalid transport feature was accepted" >&2
+    exit 1
+  fi
+done
+if "$RUNNER" --stage p1 --phase after --transport-feature nodelay --out-root "$TMP_DIR/invalid" --dry-run >/dev/null 2>&1; then
+  echo "expected transport selection to be P0-only" >&2
+  exit 1
+fi
+if "$RUNNER" --stage p0 --phase before --out-root "$TMP_DIR/invalid" --transport-feature >/dev/null 2>&1; then
+  echo "missing transport selection was accepted" >&2
+  exit 1
+fi
+[[ ! -e "$TMP_DIR/invalid" ]]
+
+test_access='grpc-driver-test-access'
+test_secret='grpc-driver-test-secret with spaces'
+test_token='grpc-driver-test-token'
+dry_output="$("$RUNNER" --stage p0 --phase before --transport-feature nodelay --out-root "$TMP_DIR/redaction" --dry-run -- --access-key "$test_access" --secret-key "$test_secret" --session-token="$test_token")"
+for sensitive in "$test_access" grpc-driver-test-secret "$test_token"; do
+  if rg -Fq "$sensitive" <<<"$dry_output"; then
+    echo "dry-run exposed a credential argument" >&2
+    exit 1
+  fi
+done
+rg -Fq REDACTED <<<"$dry_output"
+
+# Exercise actual dispatch with a local stub: display redaction must not change argv.
+fixture="$TMP_DIR/dispatch/scripts"
+mkdir -p "$fixture"
+cp "$RUNNER" "$fixture/run_internode_grpc_ab_bench.sh"
+cat >"$fixture/run_internode_transport_baseline.sh" <<'STUB'
+#!/usr/bin/env bash
+printf '%s\n' "$@" >"${GRPC_DRIVER_TEST_CAPTURE:?}"
+STUB
+chmod +x "$fixture/run_internode_transport_baseline.sh"
+GRPC_DRIVER_TEST_CAPTURE="$TMP_DIR/dispatched-args" "$fixture/run_internode_grpc_ab_bench.sh" --stage p0 --phase before --transport-feature nodelay --out-root "$TMP_DIR/dispatch-output" -- --access-key "$test_access" --secret-key "$test_secret" >/dev/null
+rg -Fqx "$test_access" "$TMP_DIR/dispatched-args"
+rg -Fqx "$test_secret" "$TMP_DIR/dispatched-args"
+echo "internode grpc single-parameter and credential-display tests passed"

scripts/test_internode_grpc_window.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""Regression cases for the offline gRPC maintenance-window gate."""
+
+import copy
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+import check_internode_grpc_window as gate
+
+
+class WindowTests(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.addCleanup(self.directory.cleanup)
+        self.root = Path(self.directory.name)
+        self.inventory = [{"node": "node1", "pid": 101, "start_ticks": 500,
+                           "boot_id": "11111111-1111-4111-8111-111111111111"}]
+        self.manifest = {"scope": "measurement", "start_ns": 100, "end_ns": 120,
+                         "max_sample_gap_ns": 10, "max_export_age_ns": 5,
+                         "processes": self.inventory, "samples": []}
+        for time in (100, 110, 120):
+            path = self.root / f"{time}.json"
+            path.write_text(json.dumps(self.document(time)))
+            self.manifest["samples"].append({"phase": "measurement", "captured_ns": time,
+                                              "processes": copy.deepcopy(self.inventory), "path": path.name})
+
+    @staticmethod
+    def document(time, pid=101, counter=7):
+        metrics = [{"name": name, "gauge": {"dataPoints": [{"timeUnixNano": str(time), "asInt": "0"}]}}
+                   for name in ["rustfs_heal_mrf_queue_depth", "rustfs_memory_allocator_reclaim_heal_activity_current",
+                                "rustfs_memory_allocator_reclaim_scanner_activity_current"]]
+        metrics.append({"name": "rustfs_heal_task_start_total", "sum": {
+            "isMonotonic": True, "aggregationTemporality": 2, "dataPoints": [
+                {"startTimeUnixNano": "50", "timeUnixNano": str(time), "asInt": str(counter)}]}})
+        return {"resourceMetrics": [{"resource": {"attributes": [
+            {"key": "service.instance.id", "value": {"stringValue": "node1"}},
+            {"key": "process.pid", "value": {"intValue": str(pid)}}]},
+            "scopeMetrics": [{"scope": {"name": "test"}, "metrics": metrics}]}]}
+
+    def edit(self, index, callback):
+        path = self.root / self.manifest["samples"][index]["path"]
+        document = json.loads(path.read_text())
+        callback(document["resourceMetrics"][0]["scopeMetrics"][0]["metrics"])
+        path.write_text(json.dumps(document))
+
+    def check(self):
+        return gate.check(self.manifest, self.root, expected_nodes=1)[0]
+
+    def test_partial_fleet_inventory_cannot_qualify_four_node_window(self):
+        with self.assertRaisesRegex(ValueError, "declared fleet size"):
+            gate.check(self.manifest, self.root)
+
+    def test_all_declared_nodes_must_export_current_gauges(self):
+        self.manifest["processes"] = self.inventory + [{"node": "node2", "pid": 102, "start_ticks": 600,
+                                                        "boot_id": "22222222-2222-4222-8222-222222222222"}]
+        for sample in self.manifest["samples"]:
+            sample["processes"] = copy.deepcopy(self.manifest["processes"])
+        with self.assertRaisesRegex(ValueError, "missing current-process"):
+            gate.check(self.manifest, self.root, expected_nodes=2)
+
+    def test_unknown_flags_and_dropped_point_identity_fail(self):
+        for update in ({"flags": 1}, {"droppedAttributesCount": 1}):
+            with self.subTest(update=update):
+                path = self.root / self.manifest["samples"][1]["path"]
+                path.write_text(json.dumps(self.document(110)))
+                self.edit(1, lambda metrics: metrics[0]["gauge"]["dataPoints"][0].update(update))
+                with self.assertRaises(ValueError):
+                    self.check()
+
+    def test_valid_window_ignores_old_pid_with_background_activity(self):
+        for sample in self.manifest["samples"]:
+            path = self.root / sample["path"]
+            current = json.loads(path.read_text())
+            old = self.document(1, pid=99, counter=999)
+            old["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["gauge"]["dataPoints"][0]["asInt"] = "16"
+            current["resourceMetrics"].extend(old["resourceMetrics"])
+            path.write_text(json.dumps(current))
+        result = self.check()
+        self.assertTrue(result["observed_stable"])
+        self.assertEqual(result["ignored_resource_exports"], 3)
+
+    def test_old_pid_cannot_replace_missing_current_process(self):
+        path = self.root / self.manifest["samples"][1]["path"]
+        path.write_text(json.dumps(self.document(110, pid=99)))
+        with self.assertRaisesRegex(ValueError, "missing current-process"):
+            self.check()
+
+    def test_current_background_activity_fails_observation(self):
+        self.edit(1, lambda metrics: metrics[0]["gauge"]["dataPoints"][0].update(asInt="1"))
+        self.assertFalse(self.check()["observed_stable"])
+
+    def test_each_maintenance_gauge_is_required(self):
+        for name in ["rustfs_heal_mrf_queue_depth", "rustfs_memory_allocator_reclaim_heal_activity_current",
+                     "rustfs_memory_allocator_reclaim_scanner_activity_current"]:
+            with self.subTest(name=name):
+                path = self.root / self.manifest["samples"][1]["path"]
+                path.write_text(json.dumps(self.document(110)))
+                def remove_metric(metrics):
+                    metrics[:] = [metric for metric in metrics if metric["name"] != name]
+                self.edit(1, remove_metric)
+                with self.assertRaisesRegex(ValueError, "missing current-process"):
+                    self.check()
+
+    def test_double_gauge_values_are_validated_without_integer_default(self):
+        def replace(metrics, value):
+            point = metrics[0]["gauge"]["dataPoints"][0]
+            point.pop("asInt")
+            point["asDouble"] = value
+        self.edit(1, lambda metrics: replace(metrics, 0.0))
+        self.assertTrue(self.check()["observed_stable"])
+        for value in (float("nan"), float("inf"), -1.0, True):
+            with self.subTest(value=value):
+                path = self.root / self.manifest["samples"][1]["path"]
+                path.write_text(json.dumps(self.document(110)))
+                self.edit(1, lambda metrics: replace(metrics, value))
+                with self.assertRaises(ValueError):
+                    self.check()
+
+    def test_counter_increment_catches_activity_between_zero_gauges(self):
+        for index in (1, 2):
+            self.edit(index, lambda metrics: metrics[-1]["sum"]["dataPoints"][0].update(asInt="8"))
+        result = self.check()
+        self.assertFalse(result["observed_stable"])
+        self.assertEqual(result["violations"][0]["reason"], "counter_increased")
+
+    def test_new_nonzero_counter_is_not_assumed_to_start_at_zero(self):
+        self.edit(0, lambda metrics: metrics.pop())
+        self.assertEqual(self.check()["violations"][0]["reason"], "new_nonzero_counter")
+
+    def test_idle_process_without_lazy_counter_can_pass(self):
+        for index in range(3):
+            self.edit(index, lambda metrics: metrics.pop())
+        self.assertTrue(self.check()["observed_stable"])
+
+    def test_restart_or_pid_reuse_invalidates_inventory(self):
+        for field, replacement in (("pid", 102), ("start_ticks", 501),
+                                    ("boot_id", "22222222-2222-4222-8222-222222222222")):
+            with self.subTest(field=field):
+                original = copy.deepcopy(self.manifest["samples"][1]["processes"])
+                self.manifest["samples"][1]["processes"][0][field] = replacement
+                with self.assertRaisesRegex(ValueError, "process restarted"):
+                    self.check()
+                self.manifest["samples"][1]["processes"] = original
+
+    def test_duplicate_reversed_or_missing_scrapes_fail(self):
+        for time in (100, 99, 111):
+            with self.subTest(time=time):
+                self.manifest["samples"][1]["captured_ns"] = time
+                with self.assertRaises(ValueError):
+                    self.check()
+
+    def test_oversized_sample_gap_fails(self):
+        self.manifest["max_sample_gap_ns"] = 9
+        with self.assertRaisesRegex(ValueError, "missing observations"):
+            self.check()
+
+    def test_command_or_prepare_scope_cannot_qualify(self):
+        self.manifest["scope"] = "command"
+        with self.assertRaisesRegex(ValueError, "measurement-only"):
+            self.check()
+        self.manifest["scope"] = "measurement"
+        self.manifest["samples"][1]["phase"] = "prepare"
+        with self.assertRaisesRegex(ValueError, "different workload phase"):
+            self.check()
+
+    def test_incomplete_window_is_rejected(self):
+        self.manifest["end_ns"] = 121
+        with self.assertRaisesRegex(ValueError, "measurement-end"):
+            self.check()
+
+    def test_stale_future_repeated_or_unrecorded_points_fail(self):
+        for point in ({"timeUnixNano": "1"}, {"timeUnixNano": "111"},
+                      {"timeUnixNano": "100"}, {"flags": 1}, {"asDouble": float("nan")}):
+            with self.subTest(point=point):
+                path = self.root / self.manifest["samples"][1]["path"]
+                path.write_text(json.dumps(self.document(110)))
+                self.edit(1, lambda metrics: metrics[0]["gauge"]["dataPoints"][0].update(point))
+                with self.assertRaises(ValueError):
+                    self.check()
+
+    def test_counter_reset_or_start_epoch_change_fails(self):
+        for update in ({"asInt": "6"}, {"startTimeUnixNano": "60"}):
+            with self.subTest(update=update):
+                path = self.root / self.manifest["samples"][1]["path"]
+                path.write_text(json.dumps(self.document(110)))
+                self.edit(1, lambda metrics: metrics[-1]["sum"]["dataPoints"][0].update(update))
+                with self.assertRaisesRegex(ValueError, "counter reset"):
+                    self.check()
+
+    def test_missing_disappearing_or_duplicate_series_fail(self):
+        for action in (lambda metrics: metrics.pop(0), lambda metrics: metrics.pop(),
+                       lambda metrics: metrics.append(copy.deepcopy(metrics[0]))):
+            with self.subTest(action=action):
+                path = self.root / self.manifest["samples"][1]["path"]
+                path.write_text(json.dumps(self.document(110)))
+                self.edit(1, action)
+                with self.assertRaises(ValueError):
+                    self.check()
+
+    def run_cli(self, output):
+        manifest = self.root / "manifest.json"
+        manifest.write_text(json.dumps(self.manifest))
+        return subprocess.run([sys.executable, str(Path(gate.__file__)), "--manifest", str(manifest),
+                               "--out", str(output), "--expected-nodes", "1"], capture_output=True, text=True)
+
+    def test_actual_entry_point_distinguishes_stable_busy_and_invalid(self):
+        output = self.root / "report.json"
+        self.assertEqual(self.run_cli(output).returncode, 0)
+        result = json.loads(output.read_text())
+        self.assertEqual(len(result["manifest_sha256"]), 64)
+        self.assertEqual(len(result["snapshots"]), 3)
+        self.edit(1, lambda metrics: metrics[0]["gauge"]["dataPoints"][0].update(asInt="16"))
+        self.assertEqual(self.run_cli(output).returncode, 1)
+        self.assertFalse(json.loads(output.read_text())["observed_stable"])
+        self.manifest["scope"] = "command"
+        self.assertEqual(self.run_cli(output).returncode, 2)
+
+    def test_actual_entry_point_preserves_raw_evidence(self):
+        output = self.root / self.manifest["samples"][0]["path"]
+        before = output.read_bytes()
+        self.assertEqual(self.run_cli(output).returncode, 2)
+        self.assertEqual(output.read_bytes(), before)
+
+    def test_actual_entry_point_preserves_hard_linked_raw_evidence(self):
+        raw = self.root / self.manifest["samples"][0]["path"]
+        output = self.root / "alias.json"
+        os.link(raw, output)
+        before = raw.read_bytes()
+        self.assertEqual(self.run_cli(output).returncode, 2)
+        self.assertEqual(raw.read_bytes(), before)
+
+
+if __name__ == "__main__":
+    unittest.main()