mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-31 09:18:28 +00:00
feat(storage): harden internode data-path controls (#4224)
* fix(rio): propagate http writer shutdown errors * fix(ecstore): unify remote lock rpc deadlines * fix(storage): reject corrupt read multiple payloads * feat(rio): add internode http tuning profiles * feat(metrics): add internode baseline signals * feat(ecstore): observe shard locality topology * feat(ecstore): gate shard locality scheduling * feat(ecstore): gate batch read version rpc * feat(ecstore): observe batch processor adaptation * feat(ecstore): gate batch processor observation * docs: add get benchmark regression analysis * docs: add issue 797 execution plan status * fix(ecstore): require explicit batch rpc support * fix(ecstore): honor documented batch read gate * fix(ecstore): keep batch read gate stable per call * chore: update workspace dependencies * feat(ecstore): log batch read gate decisions * feat(ecstore): count batch read gate decisions * test(issue-797): add local internode A/B runner * test(rio): fix tuning profile spelling fixture * fix(protocols): adapt sftp channel open callbacks * fix(metrics): wrap batch processor observation args * chore(docs): keep issue notes local only * fix(storage): address internode review feedback * fix(storage): address internode data-path review findings - Run the BatchReadVersion auto-mode unary fallback outside the batch RPC deadline so each read_version keeps its own per-op timeout and health accounting instead of racing the whole batch against one drive timeout. - Cap adaptive batch-processor concurrency growth at a hard multiple of the configured baseline so sustained fast batches cannot ratchet past the configured limit. - Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE, and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading the environment on hot paths. - Skip shard read-cost collection in observe mode when stage metrics are disabled, and cache the local endpoint host list instead of rebuilding it on every read. - Allow --warp-extra-args values starting with -- and drop the unused warp_hosts_csv helper in the issue-797 A/B runner. * fix(storage): address internode data-path review findings - Run the BatchReadVersion auto-mode unary fallback outside the batch RPC deadline so each read_version keeps its own per-op timeout and health accounting instead of racing the whole batch against one drive timeout. - Cap adaptive batch-processor concurrency growth at a hard multiple of the configured baseline so sustained fast batches cannot ratchet past the configured limit. - Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE, and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading the environment on hot paths. - Skip shard read-cost collection in observe mode when stage metrics are disabled, and cache the local endpoint host list instead of rebuilding it on every read. - Allow --warp-extra-args values starting with -- and drop the unused warp_hosts_csv helper in the issue-797 A/B runner. Co-Authored-By: heihutu<heihutu@gmail.com> * fix(storage): align buffer clamp test with media cap * fix(ecstore): release optimized read locks before streaming --------- Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
houseme
@@ -33,6 +33,7 @@ const BACKEND_LABEL: &str = "backend";
|
||||
const CLASSIFICATION_LABEL: &str = "classification";
|
||||
const STAGE_LABEL: &str = "stage";
|
||||
const DOMINANT_ERROR_LABEL: &str = "dominant_error";
|
||||
const HTTP_VERSION_LABEL: &str = "http_version";
|
||||
const INTERNODE_OPERATION_SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_sent_bytes_total";
|
||||
const INTERNODE_OPERATION_RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_recv_bytes_total";
|
||||
const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_outgoing_total";
|
||||
@@ -42,6 +43,10 @@ const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_o
|
||||
const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total";
|
||||
const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total";
|
||||
const INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL: &str = "rustfs_system_network_internode_operation_retry_successes_total";
|
||||
const INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL: &str = "rustfs_system_network_internode_operation_http_versions_total";
|
||||
const INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL: &str = "rustfs_system_network_internode_operation_stall_timeouts_total";
|
||||
const INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL: &str =
|
||||
"rustfs_system_network_internode_operation_write_shutdown_errors_total";
|
||||
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -52,6 +57,7 @@ pub struct InternodeOperationMetricDescriptor {
|
||||
|
||||
const OPERATION_BACKEND_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL];
|
||||
const OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
|
||||
const OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL];
|
||||
const QUORUM_FAILURE_LABELS: &[&str] = &[STAGE_LABEL, DOMINANT_ERROR_LABEL];
|
||||
|
||||
pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &[
|
||||
@@ -91,6 +97,18 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
|
||||
name: INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL,
|
||||
labels: OPERATION_BACKEND_CLASSIFICATION_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL,
|
||||
labels: OPERATION_BACKEND_HTTP_VERSION_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL,
|
||||
labels: OPERATION_BACKEND_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL,
|
||||
labels: OPERATION_BACKEND_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
|
||||
labels: QUORUM_FAILURE_LABELS,
|
||||
@@ -107,6 +125,9 @@ pub struct InternodeMetricsSnapshot {
|
||||
pub dial_errors_total: u64,
|
||||
pub dial_avg_time_nanos: u64,
|
||||
pub last_dial_unix_millis: u64,
|
||||
pub operation_http_versions_total: u64,
|
||||
pub operation_stall_timeouts_total: u64,
|
||||
pub operation_write_shutdown_errors_total: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -120,6 +141,9 @@ pub struct InternodeMetrics {
|
||||
dial_total_time_nanos: AtomicU64,
|
||||
dial_samples_total: AtomicU64,
|
||||
last_dial_unix_millis: AtomicU64,
|
||||
operation_http_versions_total: AtomicU64,
|
||||
operation_stall_timeouts_total: AtomicU64,
|
||||
operation_write_shutdown_errors_total: AtomicU64,
|
||||
}
|
||||
|
||||
impl InternodeMetrics {
|
||||
@@ -264,6 +288,33 @@ impl InternodeMetrics {
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_http_version_for_operation_and_backend(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
http_version: &'static str,
|
||||
) {
|
||||
self.operation_http_versions_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(
|
||||
INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL,
|
||||
OPERATION_LABEL => operation,
|
||||
BACKEND_LABEL => backend,
|
||||
HTTP_VERSION_LABEL => http_version
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_stall_timeout_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
|
||||
self.operation_stall_timeouts_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1);
|
||||
}
|
||||
|
||||
pub fn record_write_shutdown_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
|
||||
self.operation_write_shutdown_errors_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
|
||||
counter!(
|
||||
ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
|
||||
@@ -307,6 +358,9 @@ impl InternodeMetrics {
|
||||
dial_errors_total: self.dial_errors_total.load(Ordering::Relaxed),
|
||||
dial_avg_time_nanos,
|
||||
last_dial_unix_millis: self.last_dial_unix_millis.load(Ordering::Relaxed),
|
||||
operation_http_versions_total: self.operation_http_versions_total.load(Ordering::Relaxed),
|
||||
operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed),
|
||||
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -321,6 +375,9 @@ impl InternodeMetrics {
|
||||
self.dial_total_time_nanos.store(0, Ordering::Relaxed);
|
||||
self.dial_samples_total.store(0, Ordering::Relaxed);
|
||||
self.last_dial_unix_millis.store(0, Ordering::Relaxed);
|
||||
self.operation_http_versions_total.store(0, Ordering::Relaxed);
|
||||
self.operation_stall_timeouts_total.store(0, Ordering::Relaxed);
|
||||
self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -393,14 +450,21 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn operation_metric_descriptors_include_backend_and_operation_labels() {
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 10);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 13);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[..6] {
|
||||
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
for metric in &INTERNODE_OPERATION_METRICS[6..9] {
|
||||
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]);
|
||||
}
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[9].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[9].labels,
|
||||
&[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL]
|
||||
);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[10..12] {
|
||||
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[12].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -433,6 +497,18 @@ mod tests {
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[9].name,
|
||||
"rustfs_system_network_internode_operation_http_versions_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[10].name,
|
||||
"rustfs_system_network_internode_operation_stall_timeouts_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[11].name,
|
||||
"rustfs_system_network_internode_operation_write_shutdown_errors_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[12].name,
|
||||
"rustfs_system_storage_erasure_write_quorum_failures_total"
|
||||
);
|
||||
}
|
||||
@@ -456,6 +532,19 @@ mod tests {
|
||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
"connection_reset",
|
||||
);
|
||||
metrics.record_http_version_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_PUT_FILE_STREAM,
|
||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
"http/1.1",
|
||||
);
|
||||
metrics.record_stall_timeout_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
);
|
||||
metrics.record_write_shutdown_error_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_PUT_FILE_STREAM,
|
||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
);
|
||||
metrics.record_erasure_write_quorum_failure("write", "connection_reset");
|
||||
|
||||
let snapshot = metrics.snapshot();
|
||||
@@ -463,5 +552,8 @@ mod tests {
|
||||
assert_eq!(snapshot.recv_bytes_total, 0);
|
||||
assert_eq!(snapshot.outgoing_requests_total, 0);
|
||||
assert_eq!(snapshot.incoming_requests_total, 0);
|
||||
assert_eq!(snapshot.operation_http_versions_total, 1);
|
||||
assert_eq!(snapshot.operation_stall_timeouts_total, 1);
|
||||
assert_eq!(snapshot.operation_write_shutdown_errors_total, 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1098,6 +1098,19 @@ pub fn record_get_object_shard_locality_policy_disabled(path: &'static str) {
|
||||
}
|
||||
counter!("rustfs_io_get_object_shard_locality_policy_disabled_total", "path" => path).increment(1);
|
||||
}
|
||||
|
||||
/// Record observe-only shard-locality potential while scheduling remains disabled.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_shard_locality_observe_only(path: &'static str, remote_scheduled: usize, remote_avoid_potential: usize) {
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
histogram!("rustfs_io_get_object_shard_remote_scheduled_observe_only", "path" => path)
|
||||
.record(shard_read_fanout_to_f64(remote_scheduled));
|
||||
histogram!("rustfs_io_get_object_shard_remote_avoid_potential", "path" => path)
|
||||
.record(shard_read_fanout_to_f64(remote_avoid_potential));
|
||||
}
|
||||
|
||||
/// Record per-stripe shard-read fanout shape for GetObject read-path attribution.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_shard_read_fanout(
|
||||
@@ -1116,6 +1129,70 @@ pub fn record_get_object_shard_read_fanout(
|
||||
histogram!("rustfs_io_get_object_shard_read_failed", "path" => path).record(shard_read_fanout_to_f64(failed));
|
||||
}
|
||||
|
||||
fn batch_processor_count_to_f64(value: usize) -> f64 {
|
||||
u64::try_from(value).unwrap_or(u64::MAX) as f64
|
||||
}
|
||||
|
||||
fn batch_processor_count_to_u64(value: usize) -> u64 {
|
||||
u64::try_from(value).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
/// Observe-only batch processor shape and adaptive-concurrency advice.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct BatchProcessorObservation {
|
||||
pub operation: &'static str,
|
||||
pub batch_size: usize,
|
||||
pub configured_concurrency: usize,
|
||||
pub max_queue_wait_secs: f64,
|
||||
pub execution_latency_secs: f64,
|
||||
pub successes: usize,
|
||||
pub errors: usize,
|
||||
pub timeouts: usize,
|
||||
pub suggested_concurrency: usize,
|
||||
pub suggestion_reason: &'static str,
|
||||
}
|
||||
|
||||
/// Record observe-only batch processor shape and adaptive-concurrency advice.
|
||||
#[inline(always)]
|
||||
pub fn record_batch_processor_observation(observation: BatchProcessorObservation) {
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
|
||||
histogram!("rustfs_ecstore_batch_processor_batch_size", "operation" => observation.operation)
|
||||
.record(batch_processor_count_to_f64(observation.batch_size));
|
||||
histogram!("rustfs_ecstore_batch_processor_configured_concurrency", "operation" => observation.operation)
|
||||
.record(batch_processor_count_to_f64(observation.configured_concurrency));
|
||||
histogram!("rustfs_ecstore_batch_processor_queue_wait_seconds", "operation" => observation.operation)
|
||||
.record(observation.max_queue_wait_secs);
|
||||
histogram!("rustfs_ecstore_batch_processor_execution_latency_seconds", "operation" => observation.operation)
|
||||
.record(observation.execution_latency_secs);
|
||||
counter!(
|
||||
"rustfs_ecstore_batch_processor_results_total",
|
||||
"operation" => observation.operation,
|
||||
"outcome" => "success"
|
||||
)
|
||||
.increment(batch_processor_count_to_u64(observation.successes));
|
||||
counter!(
|
||||
"rustfs_ecstore_batch_processor_results_total",
|
||||
"operation" => observation.operation,
|
||||
"outcome" => "error"
|
||||
)
|
||||
.increment(batch_processor_count_to_u64(observation.errors));
|
||||
counter!(
|
||||
"rustfs_ecstore_batch_processor_results_total",
|
||||
"operation" => observation.operation,
|
||||
"outcome" => "timeout"
|
||||
)
|
||||
.increment(batch_processor_count_to_u64(observation.timeouts));
|
||||
histogram!(
|
||||
"rustfs_ecstore_batch_processor_suggested_concurrency",
|
||||
"operation" => observation.operation,
|
||||
"reason" => observation.suggestion_reason
|
||||
)
|
||||
.record(batch_processor_count_to_f64(observation.suggested_concurrency));
|
||||
}
|
||||
|
||||
/// Record the bitrot reader setup scheduling strategy selected for a GET read.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_reader_setup_strategy(strategy: &'static str, mode: &'static str) {
|
||||
@@ -2033,6 +2110,26 @@ mod tests {
|
||||
record_bytespool_return("xlarge", "dropped");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_batch_processor_observation() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
set_get_stage_metrics_enabled(true);
|
||||
record_batch_processor_observation(BatchProcessorObservation {
|
||||
operation: "read",
|
||||
batch_size: 16,
|
||||
configured_concurrency: 8,
|
||||
max_queue_wait_secs: 0.001,
|
||||
execution_latency_secs: 0.025,
|
||||
successes: 15,
|
||||
errors: 1,
|
||||
timeouts: 0,
|
||||
suggested_concurrency: 10,
|
||||
suggestion_reason: "improving",
|
||||
});
|
||||
assert!(get_stage_metrics_enabled());
|
||||
set_get_stage_metrics_enabled(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_zero_copy_write() {
|
||||
record_zero_copy_write(1024, 10.5);
|
||||
@@ -2094,6 +2191,7 @@ mod tests {
|
||||
record_get_object_pipeline_failure_for_path("codec_streaming", "decode", "read_quorum");
|
||||
record_get_object_shard_read_observation("codec_streaming", 0, "data", "local", "success", "none", 1024, 0.004, 0.001);
|
||||
record_get_object_shard_read_cost_summary("codec_streaming", 3, 1, 2, 0, 4, 4, 4, true);
|
||||
record_get_object_shard_locality_observe_only("codec_streaming", 2, 1);
|
||||
record_get_object_reader_setup_strategy("data_blocks_first", "read_quorum");
|
||||
record_get_object_reader_setup_strategy_by_size(
|
||||
"codec_streaming",
|
||||
@@ -2115,10 +2213,32 @@ mod tests {
|
||||
0,
|
||||
2,
|
||||
);
|
||||
record_batch_processor_observation(BatchProcessorObservation {
|
||||
operation: "read",
|
||||
batch_size: 16,
|
||||
configured_concurrency: 8,
|
||||
max_queue_wait_secs: 0.001,
|
||||
execution_latency_secs: 0.025,
|
||||
successes: 15,
|
||||
errors: 1,
|
||||
timeouts: 0,
|
||||
suggested_concurrency: 10,
|
||||
suggestion_reason: "improving",
|
||||
});
|
||||
|
||||
assert!(0.005_f64.is_sign_positive());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_object_shard_locality_observe_only_metrics_smoke() {
|
||||
let remote_scheduled = 2;
|
||||
let remote_avoid_potential = 1;
|
||||
|
||||
record_get_object_shard_locality_observe_only("codec_streaming", remote_scheduled, remote_avoid_potential);
|
||||
|
||||
assert!(remote_scheduled >= remote_avoid_potential);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_get_object_fill_metrics() {
|
||||
record_get_object_fill_queued("codec_streaming", "single_inflight", 1);
|
||||
|
||||
Reference in New Issue
Block a user