rustfs/crates/io-metrics/src/list_objects_metrics.rs

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use metrics::{counter, describe_counter, describe_histogram, histogram};
use std::sync::OnceLock;

use crate::get_stage_metrics_enabled;

pub const LIST_OBJECTS_SOURCE_WALKER: &str = "walker";
pub const LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED: &str = "limit_reached";
pub const LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED: &str = "input_closed";
pub const LIST_OBJECTS_MERGE_OUTCOME_STARTED: &str = "started";

const LIST_OBJECTS_GATHER_TOTAL: &str = "rustfs_s3_list_objects_gather_total";
const LIST_OBJECTS_GATHER_DURATION_MS: &str = "rustfs_s3_list_objects_gather_duration_ms";
const LIST_OBJECTS_GATHER_SCANNED_ENTRIES: &str = "rustfs_s3_list_objects_gather_scanned_entries";
const LIST_OBJECTS_GATHER_RETURNED_ENTRIES: &str = "rustfs_s3_list_objects_gather_returned_entries";
const LIST_OBJECTS_GATHER_FILTERED_ENTRIES: &str = "rustfs_s3_list_objects_gather_filtered_entries";
const LIST_OBJECTS_GATHER_SCAN_AMPLIFICATION: &str = "rustfs_s3_list_objects_gather_scan_amplification";
const LIST_OBJECTS_GATHER_LIMIT: &str = "rustfs_s3_list_objects_gather_limit";
const LIST_OBJECTS_MERGE_FAN_IN: &str = "rustfs_s3_list_objects_merge_fan_in";
const LIST_OBJECTS_MERGE_READ_QUORUM: &str = "rustfs_s3_list_objects_merge_read_quorum";
const LIST_OBJECTS_INDEX_ATTEMPT_TOTAL: &str = "rustfs_s3_list_objects_index_attempt_total";
const LIST_OBJECTS_INDEX_FALLBACK_TOTAL: &str = "rustfs_s3_list_objects_index_fallback_total";
const LIST_OBJECTS_INDEX_SERVED_TOTAL: &str = "rustfs_s3_list_objects_index_served_total";
const LIST_OBJECTS_INDEX_CANDIDATE_KEYS: &str = "rustfs_s3_list_objects_index_candidate_keys";
const LIST_OBJECTS_INDEX_LIVE_VERIFY_ATTEMPTS: &str = "rustfs_s3_list_objects_index_live_verify_attempts";
const LIST_OBJECTS_INDEX_LIVE_VERIFY_HITS: &str = "rustfs_s3_list_objects_index_live_verify_hits";
const LIST_OBJECTS_INDEX_LIVE_VERIFY_MISSES: &str = "rustfs_s3_list_objects_index_live_verify_misses";
const LIST_OBJECTS_INDEX_LIVE_VERIFY_FAILURE_TOTAL: &str = "rustfs_s3_list_objects_index_live_verify_failure_total";
const LIST_OBJECTS_INDEX_RETURNED_OBJECTS: &str = "rustfs_s3_list_objects_index_returned_objects";
const LIST_OBJECTS_INDEX_RETURNED_PREFIXES: &str = "rustfs_s3_list_objects_index_returned_prefixes";
const LIST_OBJECTS_INDEX_VERIFICATION_IO_AMPLIFICATION: &str = "rustfs_s3_list_objects_index_verification_io_amplification";
const LIST_OBJECTS_LOCAL_READ_DIR_TOTAL: &str = "rustfs_s3_list_objects_local_read_dir_total";
const LIST_OBJECTS_LOCAL_READ_DIR_DURATION_MS: &str = "rustfs_s3_list_objects_local_read_dir_duration_ms";
const LIST_OBJECTS_LOCAL_READ_DIR_ENTRIES: &str = "rustfs_s3_list_objects_local_read_dir_entries";
const LIST_OBJECTS_LOCAL_READ_DIR_LIMIT: &str = "rustfs_s3_list_objects_local_read_dir_limit";

pub const LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_OK: &str = "ok";
pub const LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_ERROR: &str = "error";

#[derive(Clone, Copy, Debug, PartialEq)]
pub struct ListObjectsGatherObservation {
    pub source: &'static str,
    pub outcome: &'static str,
    pub limit: i32,
    pub scanned_entries: usize,
    pub returned_entries: usize,
    pub duration_ms: f64,
    pub has_prefix: bool,
    pub has_delimiter: bool,
    pub has_marker: bool,
}

#[derive(Clone, Copy, Debug, PartialEq)]
pub struct ListObjectsIndexPageObservation {
    pub source: &'static str,
    pub provider: &'static str,
    pub candidate_keys: usize,
    pub live_verify_attempts: usize,
    pub live_verify_hits: usize,
    pub live_verify_misses: usize,
    pub returned_objects: usize,
    pub returned_prefixes: usize,
    pub is_truncated: bool,
}

#[derive(Clone, Copy, Debug, PartialEq)]
pub struct ListObjectsLocalReadDirObservation {
    pub outcome: &'static str,
    pub requested_count: i32,
    pub returned_entries: usize,
    pub duration_ms: f64,
    pub is_root: bool,
    pub has_filter_prefix: bool,
    pub has_forward: bool,
}

#[inline(always)]
fn bool_label(value: bool) -> &'static str {
    if value { "true" } else { "false" }
}

#[inline(always)]
fn count_as_f64(value: usize) -> f64 {
    value as f64
}

#[inline(always)]
fn limit_as_f64(value: i32) -> f64 {
    value.max(0) as f64
}

#[inline(always)]
fn count_mode_label(count: i32) -> &'static str {
    if count < 0 { "whole" } else { "bounded" }
}

#[inline(always)]
fn read_dir_count_as_f64(value: i32) -> f64 {
    f64::from(value)
}

pub fn init_list_objects_metrics() {
    static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new();
    METRICS_DESC_INIT.get_or_init(|| {
        describe_counter!(
            LIST_OBJECTS_GATHER_TOTAL,
            "Total number of ListObjects gather phases by source, outcome, and request shape."
        );
        describe_histogram!(LIST_OBJECTS_GATHER_DURATION_MS, "ListObjects gather phase duration in milliseconds.");
        describe_histogram!(
            LIST_OBJECTS_GATHER_SCANNED_ENTRIES,
            "Number of entries consumed by ListObjects gather before producing a page."
        );
        describe_histogram!(
            LIST_OBJECTS_GATHER_RETURNED_ENTRIES,
            "Number of entries returned by ListObjects gather before pagination trimming."
        );
        describe_histogram!(
            LIST_OBJECTS_GATHER_FILTERED_ENTRIES,
            "Number of entries filtered by ListObjects gather before producing a page."
        );
        describe_histogram!(
            LIST_OBJECTS_GATHER_SCAN_AMPLIFICATION,
            "Ratio of scanned entries to returned entries in ListObjects gather."
        );
        describe_histogram!(LIST_OBJECTS_GATHER_LIMIT, "Requested internal ListObjects gather page limit.");
        describe_histogram!(LIST_OBJECTS_MERGE_FAN_IN, "Number of input streams merged for ListObjects pages.");
        describe_histogram!(LIST_OBJECTS_MERGE_READ_QUORUM, "Read quorum used while merging ListObjects entries.");
        describe_counter!(
            LIST_OBJECTS_INDEX_ATTEMPT_TOTAL,
            "Total number of opt-in ListObjects index serving attempts by source, provider, and request shape."
        );
        describe_counter!(
            LIST_OBJECTS_INDEX_FALLBACK_TOTAL,
            "Total number of opt-in ListObjects index attempts that fell back to the live walker."
        );
        describe_counter!(
            LIST_OBJECTS_INDEX_SERVED_TOTAL,
            "Total number of ListObjects pages served by an opt-in index provider."
        );
        describe_histogram!(
            LIST_OBJECTS_INDEX_CANDIDATE_KEYS,
            "Number of candidate keys proposed by the opt-in ListObjects index provider per page."
        );
        describe_histogram!(
            LIST_OBJECTS_INDEX_LIVE_VERIFY_ATTEMPTS,
            "Number of live xl.meta verification attempts made by opt-in ListObjects index serving per page."
        );
        describe_histogram!(
            LIST_OBJECTS_INDEX_LIVE_VERIFY_HITS,
            "Number of opt-in ListObjects index candidates accepted by live xl.meta verification per page."
        );
        describe_histogram!(
            LIST_OBJECTS_INDEX_LIVE_VERIFY_MISSES,
            "Number of opt-in ListObjects index candidates rejected as stale or missing by live xl.meta verification per page."
        );
        describe_counter!(
            LIST_OBJECTS_INDEX_LIVE_VERIFY_FAILURE_TOTAL,
            "Total number of opt-in ListObjects index live xl.meta verification failures."
        );
        describe_histogram!(
            LIST_OBJECTS_INDEX_RETURNED_OBJECTS,
            "Number of objects returned by opt-in ListObjects index serving per page."
        );
        describe_histogram!(
            LIST_OBJECTS_INDEX_RETURNED_PREFIXES,
            "Number of common prefixes returned by opt-in ListObjects index serving per page."
        );
        describe_histogram!(
            LIST_OBJECTS_INDEX_VERIFICATION_IO_AMPLIFICATION,
            "Ratio of live verification attempts to returned objects for opt-in ListObjects index serving."
        );
        describe_counter!(
            LIST_OBJECTS_LOCAL_READ_DIR_TOTAL,
            "Total number of local read_dir calls made while serving live-walker ListObjects pages."
        );
        describe_histogram!(
            LIST_OBJECTS_LOCAL_READ_DIR_DURATION_MS,
            "Duration in milliseconds of local read_dir calls made while serving live-walker ListObjects pages."
        );
        describe_histogram!(
            LIST_OBJECTS_LOCAL_READ_DIR_ENTRIES,
            "Number of immediate directory entries returned by local read_dir while serving live-walker ListObjects pages."
        );
        describe_histogram!(
            LIST_OBJECTS_LOCAL_READ_DIR_LIMIT,
            "Requested local read_dir count used while serving live-walker ListObjects pages; negative counts mean whole-directory enumeration."
        );
    });
}

pub fn record_list_objects_gather(observation: ListObjectsGatherObservation) {
    if !get_stage_metrics_enabled() {
        return;
    }

    let filtered_entries = observation.scanned_entries.saturating_sub(observation.returned_entries);
    let scan_amplification = if observation.returned_entries == 0 {
        count_as_f64(observation.scanned_entries)
    } else {
        count_as_f64(observation.scanned_entries) / count_as_f64(observation.returned_entries)
    };

    counter!(
        LIST_OBJECTS_GATHER_TOTAL,
        "source" => observation.source,
        "outcome" => observation.outcome,
        "has_prefix" => bool_label(observation.has_prefix),
        "has_delimiter" => bool_label(observation.has_delimiter),
        "has_marker" => bool_label(observation.has_marker),
    )
    .increment(1);
    histogram!(
        LIST_OBJECTS_GATHER_DURATION_MS,
        "source" => observation.source,
        "outcome" => observation.outcome
    )
    .record(observation.duration_ms);
    histogram!(LIST_OBJECTS_GATHER_SCANNED_ENTRIES, "source" => observation.source)
        .record(count_as_f64(observation.scanned_entries));
    histogram!(LIST_OBJECTS_GATHER_RETURNED_ENTRIES, "source" => observation.source)
        .record(count_as_f64(observation.returned_entries));
    histogram!(LIST_OBJECTS_GATHER_FILTERED_ENTRIES, "source" => observation.source).record(count_as_f64(filtered_entries));
    histogram!(LIST_OBJECTS_GATHER_SCAN_AMPLIFICATION, "source" => observation.source).record(scan_amplification);
    histogram!(LIST_OBJECTS_GATHER_LIMIT, "source" => observation.source).record(limit_as_f64(observation.limit));
}

pub fn record_list_objects_merge(source: &'static str, input_channels: usize, read_quorum: usize) {
    if !get_stage_metrics_enabled() {
        return;
    }

    histogram!(
        LIST_OBJECTS_MERGE_FAN_IN,
        "source" => source,
        "outcome" => LIST_OBJECTS_MERGE_OUTCOME_STARTED
    )
    .record(count_as_f64(input_channels));
    histogram!(
        LIST_OBJECTS_MERGE_READ_QUORUM,
        "source" => source,
        "outcome" => LIST_OBJECTS_MERGE_OUTCOME_STARTED
    )
    .record(count_as_f64(read_quorum));
}

pub fn record_list_objects_index_fallback(source: &'static str, reason: &'static str) {
    if !get_stage_metrics_enabled() {
        return;
    }

    counter!(
        LIST_OBJECTS_INDEX_FALLBACK_TOTAL,
        "source" => source,
        "reason" => reason
    )
    .increment(1);
}

pub fn record_list_objects_index_attempt(
    source: &'static str,
    provider: &'static str,
    has_prefix: bool,
    has_delimiter: bool,
    has_marker: bool,
) {
    if !get_stage_metrics_enabled() {
        return;
    }

    counter!(
        LIST_OBJECTS_INDEX_ATTEMPT_TOTAL,
        "source" => source,
        "provider" => provider,
        "has_prefix" => bool_label(has_prefix),
        "has_delimiter" => bool_label(has_delimiter),
        "has_marker" => bool_label(has_marker),
    )
    .increment(1);
}

pub fn record_list_objects_index_live_verify_failure(source: &'static str, reason: &'static str) {
    if !get_stage_metrics_enabled() {
        return;
    }

    counter!(
        LIST_OBJECTS_INDEX_LIVE_VERIFY_FAILURE_TOTAL,
        "source" => source,
        "reason" => reason,
    )
    .increment(1);
}

pub fn record_list_objects_index_served(observation: ListObjectsIndexPageObservation) {
    if !get_stage_metrics_enabled() {
        return;
    }

    let returned_objects = count_as_f64(observation.returned_objects);
    let verification_io_amplification = if observation.returned_objects == 0 {
        count_as_f64(observation.live_verify_attempts)
    } else {
        count_as_f64(observation.live_verify_attempts) / returned_objects
    };

    counter!(
        LIST_OBJECTS_INDEX_SERVED_TOTAL,
        "source" => observation.source,
        "provider" => observation.provider,
        "is_truncated" => bool_label(observation.is_truncated),
    )
    .increment(1);
    histogram!(
        LIST_OBJECTS_INDEX_CANDIDATE_KEYS,
        "source" => observation.source,
        "provider" => observation.provider,
    )
    .record(count_as_f64(observation.candidate_keys));
    histogram!(
        LIST_OBJECTS_INDEX_LIVE_VERIFY_ATTEMPTS,
        "source" => observation.source,
        "provider" => observation.provider,
    )
    .record(count_as_f64(observation.live_verify_attempts));
    histogram!(
        LIST_OBJECTS_INDEX_LIVE_VERIFY_HITS,
        "source" => observation.source,
        "provider" => observation.provider,
    )
    .record(count_as_f64(observation.live_verify_hits));
    histogram!(
        LIST_OBJECTS_INDEX_LIVE_VERIFY_MISSES,
        "source" => observation.source,
        "provider" => observation.provider,
    )
    .record(count_as_f64(observation.live_verify_misses));
    histogram!(
        LIST_OBJECTS_INDEX_RETURNED_OBJECTS,
        "source" => observation.source,
        "provider" => observation.provider,
    )
    .record(returned_objects);
    histogram!(
        LIST_OBJECTS_INDEX_RETURNED_PREFIXES,
        "source" => observation.source,
        "provider" => observation.provider,
    )
    .record(count_as_f64(observation.returned_prefixes));
    histogram!(
        LIST_OBJECTS_INDEX_VERIFICATION_IO_AMPLIFICATION,
        "source" => observation.source,
        "provider" => observation.provider,
    )
    .record(verification_io_amplification);
}

pub fn record_list_objects_local_read_dir(observation: ListObjectsLocalReadDirObservation) {
    if !get_stage_metrics_enabled() {
        return;
    }

    let count_mode = count_mode_label(observation.requested_count);

    counter!(
        LIST_OBJECTS_LOCAL_READ_DIR_TOTAL,
        "outcome" => observation.outcome,
        "count_mode" => count_mode,
        "is_root" => bool_label(observation.is_root),
        "has_filter_prefix" => bool_label(observation.has_filter_prefix),
        "has_forward" => bool_label(observation.has_forward),
    )
    .increment(1);
    histogram!(
        LIST_OBJECTS_LOCAL_READ_DIR_DURATION_MS,
        "outcome" => observation.outcome,
        "count_mode" => count_mode,
    )
    .record(observation.duration_ms);
    histogram!(
        LIST_OBJECTS_LOCAL_READ_DIR_ENTRIES,
        "outcome" => observation.outcome,
        "count_mode" => count_mode,
    )
    .record(count_as_f64(observation.returned_entries));
    histogram!(LIST_OBJECTS_LOCAL_READ_DIR_LIMIT, "count_mode" => count_mode)
        .record(read_dir_count_as_f64(observation.requested_count));
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn record_gather_observation_handles_empty_page() {
        init_list_objects_metrics();
        record_list_objects_gather(ListObjectsGatherObservation {
            source: LIST_OBJECTS_SOURCE_WALKER,
            outcome: LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED,
            limit: 1001,
            scanned_entries: 42,
            returned_entries: 0,
            duration_ms: 3.5,
            has_prefix: true,
            has_delimiter: false,
            has_marker: true,
        });
    }

    #[test]
    fn record_merge_observation_accepts_zero_quorum() {
        init_list_objects_metrics();
        record_list_objects_merge(LIST_OBJECTS_SOURCE_WALKER, 4, 0);
    }

    #[test]
    fn record_index_fallback_observation_accepts_reason() {
        init_list_objects_metrics();
        record_list_objects_index_fallback("index_key_only", "unsupported_request");
    }

    #[test]
    fn record_index_attempt_and_served_observations_accept_counts() {
        init_list_objects_metrics();
        record_list_objects_index_attempt("index_key_only", "walker_key_only", true, true, false);
        record_list_objects_index_served(ListObjectsIndexPageObservation {
            source: "index_key_only",
            provider: "walker_key_only",
            candidate_keys: 1000,
            live_verify_attempts: 700,
            live_verify_hits: 650,
            live_verify_misses: 50,
            returned_objects: 600,
            returned_prefixes: 10,
            is_truncated: true,
        });
        record_list_objects_index_live_verify_failure("index_key_only", "read_error");
    }

    #[test]
    fn record_local_read_dir_observation_accepts_whole_directory_counts() {
        init_list_objects_metrics();
        record_list_objects_local_read_dir(ListObjectsLocalReadDirObservation {
            outcome: LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_OK,
            requested_count: -1,
            returned_entries: 4096,
            duration_ms: 12.5,
            is_root: true,
            has_filter_prefix: false,
            has_forward: false,
        });
        record_list_objects_local_read_dir(ListObjectsLocalReadDirObservation {
            outcome: LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_ERROR,
            requested_count: -1,
            returned_entries: 0,
            duration_ms: 5000.0,
            is_root: true,
            has_filter_prefix: false,
            has_forward: true,
        });
    }
}