feat: optimize small GET read paths (#4022)

2026-08-24 13:16:28 +00:00 · 2026-07-01 15:40:00 +08:00
parent b0f491549a
commit b16120dbcc
25 changed files with 5318 additions and 502 deletions
@@ -186,6 +186,25 @@ const SHARD_READ_COST_SAME_NODE: &str = "same_node";
 const SHARD_READ_COST_UNKNOWN: &str = "unknown";
 const LOW_COST_QUORUM_CANDIDATE_FALSE: &str = "false";
 const LOW_COST_QUORUM_CANDIDATE_TRUE: &str = "true";
+pub const GET_OBJECT_SIZE_BUCKET_LE_4_KIB: &str = "le_4kib";
+pub const GET_OBJECT_SIZE_BUCKET_LE_16_KIB: &str = "le_16kib";
+pub const GET_OBJECT_SIZE_BUCKET_LE_64_KIB: &str = "le_64kib";
+pub const GET_OBJECT_SIZE_BUCKET_LE_128_KIB: &str = "le_128kib";
+pub const GET_OBJECT_SIZE_BUCKET_LE_1_MIB: &str = "le_1mib";
+pub const GET_OBJECT_SIZE_BUCKET_GT_1_MIB: &str = "gt_1mib";
+
+/// Return the bounded size bucket used by small-object GET diagnostics.
+#[inline(always)]
+pub const fn get_object_size_bucket(size_bytes: i64) -> &'static str {
+    match size_bytes {
+        ..=4_096 => GET_OBJECT_SIZE_BUCKET_LE_4_KIB,
+        4_097..=16_384 => GET_OBJECT_SIZE_BUCKET_LE_16_KIB,
+        16_385..=65_536 => GET_OBJECT_SIZE_BUCKET_LE_64_KIB,
+        65_537..=131_072 => GET_OBJECT_SIZE_BUCKET_LE_128_KIB,
+        131_073..=1_048_576 => GET_OBJECT_SIZE_BUCKET_LE_1_MIB,
+        _ => GET_OBJECT_SIZE_BUCKET_GT_1_MIB,
+    }
+}

 fn saturating_sub_atomic(counter: &AtomicU64, bytes: u64) -> u64 {
    let mut current = counter.load(Ordering::Relaxed);
@@ -404,6 +423,39 @@ pub fn record_get_object_reader_stream_poll(
    .record(duration_secs);
 }

+/// Record a poll of the single-chunk in-memory GetObject handoff stream.
+#[inline(always)]
+pub fn record_get_object_memory_body_stream_poll(source: &'static str, outcome: &'static str, bytes: usize, duration_secs: f64) {
+    if !get_stage_metrics_enabled() {
+        return;
+    }
+    let bytes_counter = u64::try_from(bytes).unwrap_or(u64::MAX);
+    counter!(
+        "rustfs_io_get_object_memory_body_stream_poll_total",
+        "source" => source,
+        "outcome" => outcome
+    )
+    .increment(1);
+    counter!(
+        "rustfs_io_get_object_memory_body_stream_poll_bytes_total",
+        "source" => source,
+        "outcome" => outcome
+    )
+    .increment(bytes_counter);
+    histogram!(
+        "rustfs_io_get_object_memory_body_stream_poll_bytes",
+        "source" => source,
+        "outcome" => outcome
+    )
+    .record(usize_to_f64(bytes));
+    histogram!(
+        "rustfs_io_get_object_memory_body_stream_poll_duration_seconds",
+        "source" => source,
+        "outcome" => outcome
+    )
+    .record(duration_secs);
+}
+
 /// Record I/O queue congestion observation.
 #[inline(always)]
 pub fn record_io_queue_congestion() {
@@ -446,6 +498,28 @@ pub fn record_get_object_stage_duration(path: &'static str, stage: &'static str,
    histogram!("rustfs_io_get_object_stage_duration_seconds", "path" => path, "stage" => stage).record(duration_secs);
 }

+/// Record GetObject stage duration with bounded object class and size labels.
+#[inline(always)]
+pub fn record_get_object_stage_duration_by_size(
+    path: &'static str,
+    stage: &'static str,
+    object_class: &'static str,
+    size_bucket: &'static str,
+    duration_secs: f64,
+) {
+    if !get_stage_metrics_enabled() {
+        return;
+    }
+    histogram!(
+        "rustfs_io_get_object_stage_duration_seconds_by_size",
+        "path" => path,
+        "stage" => stage,
+        "object_class" => object_class,
+        "size_bucket" => size_bucket
+    )
+    .record(duration_secs);
+}
+
 /// Record GetObject metadata fanout duration.
 #[inline(always)]
 pub fn record_get_object_metadata_fanout_duration(path: &'static str, duration_secs: f64) {
@@ -485,6 +559,16 @@ pub fn record_get_object_metadata_response(path: &'static str, outcome: &'static
    counter!("rustfs_io_get_object_metadata_response_total", "path" => path, "outcome" => outcome).increment(1);
 }

+/// Record one bounded metadata cache decision.
+#[inline(always)]
+pub fn record_get_object_metadata_cache_decision(path: &'static str, decision: &'static str, reason: &'static str) {
+    if !get_stage_metrics_enabled() {
+        return;
+    }
+    counter!("rustfs_io_get_object_metadata_cache_total", "path" => path, "decision" => decision, "reason" => reason)
+        .increment(1);
+}
+
 /// Record aggregate metadata fanout shape for one GetObject metadata read.
 #[inline(always)]
 pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize, valid: usize, ignored: usize, errors: usize) {
@@ -603,6 +687,37 @@ pub fn record_get_object_reader_path(path: &'static str) {
    counter!("rustfs_io_get_object_reader_path_total", "path" => path).increment(1);
 }

+/// Record the selected GetObject reader path with bounded object class and size labels.
+#[inline(always)]
+pub fn record_get_object_reader_path_by_size(path: &'static str, object_class: &'static str, size_bucket: &'static str) {
+    if !get_stage_metrics_enabled() {
+        return;
+    }
+    counter!(
+        "rustfs_io_get_object_reader_path_by_size_total",
+        "path" => path,
+        "object_class" => object_class,
+        "size_bucket" => size_bucket
+    )
+    .increment(1);
+}
+
+/// Record the concrete subpath used by the direct-memory GetObject reader.
+#[inline(always)]
+pub fn record_get_object_direct_memory_subpath(subpath: &'static str, object_class: &'static str, size_bucket: &'static str) {
+    if !get_stage_metrics_enabled() {
+        return;
+    }
+    counter!("rustfs_io_get_object_direct_memory_subpath_total", "subpath" => subpath).increment(1);
+    counter!(
+        "rustfs_io_get_object_direct_memory_subpath_by_size_total",
+        "subpath" => subpath,
+        "object_class" => object_class,
+        "size_bucket" => size_bucket
+    )
+    .increment(1);
+}
+
 /// Record why the codec streaming reader was not selected.
 #[inline(always)]
 pub fn record_get_object_codec_streaming_fallback(reason: &'static str) {
@@ -627,6 +742,27 @@ pub fn record_get_object_codec_streaming_decision(outcome: &'static str, object_
    .increment(1);
 }

+/// Record the final codec-streaming rollout decision with bounded size attribution.
+#[inline(always)]
+pub fn record_get_object_codec_streaming_decision_by_size(
+    outcome: &'static str,
+    object_class: &'static str,
+    reason: &'static str,
+    size_bucket: &'static str,
+) {
+    if !get_stage_metrics_enabled() {
+        return;
+    }
+    counter!(
+        "rustfs_io_get_object_codec_streaming_decision_by_size_total",
+        "outcome" => outcome,
+        "object_class" => object_class,
+        "reason" => reason,
+        "size_bucket" => size_bucket
+    )
+    .increment(1);
+}
+
 /// Record one decoded reader stripe processed by a GetObject read path.
 #[inline(always)]
 pub fn record_get_object_reader_stripe(path: &'static str) {
@@ -974,6 +1110,66 @@ pub fn record_get_object_shard_read_fanout(
    histogram!("rustfs_io_get_object_shard_read_failed", "path" => path).record(shard_read_fanout_to_f64(failed));
 }

+/// Record the bitrot reader setup scheduling strategy selected for a GET read.
+#[inline(always)]
+pub fn record_get_object_reader_setup_strategy(strategy: &'static str, mode: &'static str) {
+    if !get_stage_metrics_enabled() {
+        return;
+    }
+    counter!(
+        "rustfs_io_get_object_reader_setup_strategy_total",
+        "strategy" => strategy,
+        "mode" => mode
+    )
+    .increment(1);
+}
+
+/// Record the final bitrot reader setup fanout shape for a GET read.
+#[inline(always)]
+pub fn record_get_object_reader_setup_fanout(
+    strategy: &'static str,
+    mode: &'static str,
+    scheduled: usize,
+    attempted: usize,
+    ready: usize,
+    failed: usize,
+    deferred: usize,
+) {
+    if !get_stage_metrics_enabled() {
+        return;
+    }
+    histogram!(
+        "rustfs_io_get_object_reader_setup_scheduled",
+        "strategy" => strategy,
+        "mode" => mode
+    )
+    .record(shard_read_fanout_to_f64(scheduled));
+    histogram!(
+        "rustfs_io_get_object_reader_setup_attempted",
+        "strategy" => strategy,
+        "mode" => mode
+    )
+    .record(shard_read_fanout_to_f64(attempted));
+    histogram!(
+        "rustfs_io_get_object_reader_setup_ready",
+        "strategy" => strategy,
+        "mode" => mode
+    )
+    .record(shard_read_fanout_to_f64(ready));
+    histogram!(
+        "rustfs_io_get_object_reader_setup_failed",
+        "strategy" => strategy,
+        "mode" => mode
+    )
+    .record(shard_read_fanout_to_f64(failed));
+    histogram!(
+        "rustfs_io_get_object_reader_setup_deferred",
+        "strategy" => strategy,
+        "mode" => mode
+    )
+    .record(shard_read_fanout_to_f64(deferred));
+}
+
 /// Record GetObject metadata resolution duration.
 #[inline(always)]
 pub fn record_get_object_metadata_phase_duration(duration_secs: f64) {
@@ -1694,10 +1890,13 @@ mod tests {
    #[test]
    fn test_record_get_object_stage_metrics() {
        record_get_object_stage_duration("s3_handler", "request_context", 0.001);
+        record_get_object_stage_duration_by_size("legacy_duplex", "metadata", "plain_single_part", "le_4kib", 0.001);
        record_get_object_reader_path("codec_streaming");
+        record_get_object_reader_path_by_size("codec_streaming", "plain_single_part", "le_1mib");
        record_get_object_codec_streaming_fallback("range");
        record_get_object_codec_streaming_decision("fallback", "range", "range");
        record_get_object_codec_streaming_decision("use", "plain_single_part", "none");
+        record_get_object_codec_streaming_decision_by_size("fallback", "plain_single_part", "below_min_size", "le_128kib");
        record_get_object_reader_stripe("codec_streaming");
        record_get_object_reader_bytes("codec_streaming", 1024);
        record_get_object_reader_buffer("codec_streaming", "output", 1024);
@@ -1735,6 +1934,8 @@ mod tests {
        record_get_object_pipeline_failure_for_path("codec_streaming", "decode", "read_quorum");
        record_get_object_shard_read_observation("codec_streaming", 0, "data", "local", "success", "none", 1024, 0.004, 0.001);
        record_get_object_shard_read_cost_summary("codec_streaming", 3, 1, 2, 0, 4, 4, 4, true);
+        record_get_object_reader_setup_strategy("data_blocks_first", "read_quorum");
+        record_get_object_reader_setup_fanout("data_blocks_first", "read_quorum", 3, 2, 2, 0, 2);

        assert!(0.005_f64.is_sign_positive());
    }
@@ -1750,6 +1951,7 @@ mod tests {
        record_get_object_reader_prefetch_bytes("codec_streaming", "single_inflight", 4096);
        record_get_object_reader_stream_buffer_size("standard", "selected", 131072);
        record_get_object_reader_stream_poll("standard", "selected", "ready_data", 8192, 4096, 0.0002);
+        record_get_object_memory_body_stream_poll("buffered_body", "ready_data", 4096, 0.0001);

        assert!(0.0003_f64.is_sign_positive());
    }
@@ -1794,8 +1996,11 @@ mod tests {
        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        set_get_stage_metrics_enabled(true);
        record_get_object_stage_duration("s3_handler", "request_context", 0.001);
+        record_get_object_stage_duration_by_size("legacy_duplex", "metadata", "plain_single_part", "le_4kib", 0.001);
        record_get_object_reader_path("codec_streaming");
+        record_get_object_reader_path_by_size("codec_streaming", "plain_single_part", "le_1mib");
        record_get_object_codec_streaming_fallback("range");
+        record_get_object_codec_streaming_decision_by_size("fallback", "plain_single_part", "below_min_size", "le_128kib");
        record_get_object_reader_stripe("codec_streaming");
        record_get_object_reader_bytes("codec_streaming", 1024);
        record_get_object_reader_buffer("codec_streaming", "output", 1024);
@@ -1838,8 +2043,11 @@ mod tests {
        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        set_get_stage_metrics_enabled(false);
        record_get_object_stage_duration("s3_handler", "request_context", 0.001);
+        record_get_object_stage_duration_by_size("legacy_duplex", "metadata", "plain_single_part", "le_4kib", 0.001);
        record_get_object_reader_path("codec_streaming");
+        record_get_object_reader_path_by_size("codec_streaming", "plain_single_part", "le_1mib");
        record_get_object_codec_streaming_fallback("range");
+        record_get_object_codec_streaming_decision_by_size("fallback", "plain_single_part", "below_min_size", "le_128kib");
        record_get_object_reader_stripe("codec_streaming");
        record_get_object_reader_bytes("codec_streaming", 1024);
        record_get_object_reader_buffer("codec_streaming", "output", 1024);
@@ -1879,6 +2087,22 @@ mod tests {
        assert!(!get_stage_metrics_enabled());
    }

+    #[test]
+    fn test_get_object_size_buckets_match_issue714_matrix() {
+        assert_eq!(get_object_size_bucket(0), GET_OBJECT_SIZE_BUCKET_LE_4_KIB);
+        assert_eq!(get_object_size_bucket(1024), GET_OBJECT_SIZE_BUCKET_LE_4_KIB);
+        assert_eq!(get_object_size_bucket(4096), GET_OBJECT_SIZE_BUCKET_LE_4_KIB);
+        assert_eq!(get_object_size_bucket(4097), GET_OBJECT_SIZE_BUCKET_LE_16_KIB);
+        assert_eq!(get_object_size_bucket(10 * 1024), GET_OBJECT_SIZE_BUCKET_LE_16_KIB);
+        assert_eq!(get_object_size_bucket(16 * 1024), GET_OBJECT_SIZE_BUCKET_LE_16_KIB);
+        assert_eq!(get_object_size_bucket((16 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_64_KIB);
+        assert_eq!(get_object_size_bucket(100 * 1024), GET_OBJECT_SIZE_BUCKET_LE_128_KIB);
+        assert_eq!(get_object_size_bucket(128 * 1024), GET_OBJECT_SIZE_BUCKET_LE_128_KIB);
+        assert_eq!(get_object_size_bucket((128 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_LE_1_MIB);
+        assert_eq!(get_object_size_bucket(1024 * 1024), GET_OBJECT_SIZE_BUCKET_LE_1_MIB);
+        assert_eq!(get_object_size_bucket((1024 * 1024) + 1), GET_OBJECT_SIZE_BUCKET_GT_1_MIB);
+    }
+
    #[test]
    fn test_record_stage_duration_generic() {
        // Generic stage duration should always record (no gating flag)