feat(tier): report cluster tier stats and count tier requests (#7110)

`GET /v3/tier-stats` answered from whichever process received the
request, returning that node's rolling 24-hour transition counters as
if they were cluster totals, and the `TierRequestsSuccess` and
`TierRequestsFailure` metric names had no producer at all.

The body now separates the two quantities a tier carries. Stored
inventory comes from the persisted scanner usage snapshot, which is
already cluster-wide; rolling activity is summed over every member
through a new read-only `TierDailyStats` peer RPC. Rings are merged
rather than added, so an idle node's expired hours age out, and each
node counts only its own committed transitions, so a retry is counted
once. Coverage travels with the numbers: `activity.status` names the
reporting members and the ones that could not be asked, timed out, or
answered with a ring this build refuses to merge, and per-tier
inventory is absent rather than zero when the snapshot has no
accounting. The version 1 body stays reachable at `?format=legacy`.

Tier request counters are recorded at the two seams every remote
request passes through, so a new provider is counted by construction,
with a closed operation/outcome label set that can never grow a tier
name, endpoint or object key.

Closes rustfs/backlog#2207

Co-authored-by: cxymds <cxymds@gmail.com>

crates/obs/src/metrics/collectors/mod.rs

@@ -40,6 +40,7 @@ pub mod system_memory;
 pub mod system_network;
 pub mod system_network_host;
 pub mod system_process;
+pub mod tier;
 
 pub(crate) use audit::{AuditTargetRuntimeStats, collect_audit_runtime_metrics};
 pub use audit::{AuditTargetStats, collect_audit_metrics};
@@ -94,3 +95,4 @@ pub use system_process::{
     ProcessAttributeError, ProcessAttributes, ProcessStats, ProcessStatusType, collect_process_attributes,
     collect_process_metrics,
 };
+pub use tier::{TierRequestStats, collect_tier_request_metrics};

crates/obs/src/metrics/collectors/tier.rs

@@ -0,0 +1,96 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Remote tier request metrics collector.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::tier::*;
+
+/// One operation/outcome cell of the tier request counters.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TierRequestStats {
+    pub operation: &'static str,
+    pub outcome: &'static str,
+    pub count: u64,
+}
+
+/// Split the fixed operation/outcome cells over the success and failure
+/// counters.
+///
+/// The success counter carries no outcome label: `success` is the only outcome
+/// it can report, and repeating it would make the two counters look like they
+/// share a label set they do not.
+pub fn collect_tier_request_metrics(stats: &[TierRequestStats]) -> Vec<PrometheusMetric> {
+    stats
+        .iter()
+        .map(|stat| {
+            if stat.outcome == "success" {
+                PrometheusMetric::from_descriptor(&TIER_REQUESTS_SUCCESS_MD, stat.count as f64)
+                    .with_label(OPERATION_LABEL, stat.operation)
+            } else {
+                PrometheusMetric::from_descriptor(&TIER_REQUESTS_FAILURE_MD, stat.count as f64)
+                    .with_label(OPERATION_LABEL, stat.operation)
+                    .with_label(OUTCOME_LABEL, stat.outcome)
+            }
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn stats() -> Vec<TierRequestStats> {
+        vec![
+            TierRequestStats {
+                operation: "put",
+                outcome: "success",
+                count: 7,
+            },
+            TierRequestStats {
+                operation: "put",
+                outcome: "timeout",
+                count: 2,
+            },
+        ]
+    }
+
+    #[test]
+    fn success_and_failure_land_on_their_own_counters() {
+        let metrics = collect_tier_request_metrics(&stats());
+
+        let success = metrics
+            .iter()
+            .find(|metric| metric.name == TIER_REQUESTS_SUCCESS_MD.get_full_metric_name())
+            .expect("a success cell must produce the success counter");
+        assert_eq!(success.value, 7.0);
+        assert!(
+            success.labels.iter().all(|(name, _)| *name != OUTCOME_LABEL),
+            "the success counter must not carry an outcome label"
+        );
+
+        let failure = metrics
+            .iter()
+            .find(|metric| metric.name == TIER_REQUESTS_FAILURE_MD.get_full_metric_name())
+            .expect("a non-success cell must produce the failure counter");
+        assert_eq!(failure.value, 2.0);
+        assert!(
+            failure
+                .labels
+                .iter()
+                .any(|(name, value)| *name == OUTCOME_LABEL && value.as_ref() == "timeout"),
+            "the failure counter must keep the outcome that produced it"
+        );
+    }
+}

crates/obs/src/metrics/scheduler.rs

@@ -77,6 +77,7 @@ use crate::metrics::collectors::{
     collect_request_metrics,
     collect_resource_metrics,
     collect_scanner_runtime_metrics,
+    collect_tier_request_metrics,
 };
 use crate::metrics::config::{
     DEFAULT_AUDIT_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
@@ -151,7 +152,7 @@ use crate::metrics::stats_collector::{
     collect_disk_and_system_drive_runtime_stats, collect_erasure_set_stats, collect_host_network_stats, collect_iam_stats,
     collect_ilm_runtime_metric_stats, collect_internode_network_stats, collect_on_demand_migration_backfill_stats,
     collect_on_demand_migration_stats, collect_process_metric_bundle_with, collect_replication_stats,
-    collect_scanner_runtime_metric_stats, collect_system_cpu_and_memory_stats_with,
+    collect_scanner_runtime_metric_stats, collect_system_cpu_and_memory_stats_with, collect_tier_request_metric_stats,
 };
 use crate::node_identity::{SERVER_LABEL, current_local_node_identity};
 use crate::telemetry::retire_metric_series;
@@ -2397,6 +2398,8 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                                 metrics.extend(collect_ilm_runtime_metrics(&stats));
                             }
 
+                            metrics.extend(collect_tier_request_metrics(&collect_tier_request_metric_stats()));
+
                             let mut retire_scanner_cycle_bucket_drive_result_keys = Vec::new();
                             let mut retire_scanner_bucket_drive_result_keys = Vec::new();
                             let mut retire_scanner_active_bucket_drive_keys = Vec::new();

crates/obs/src/metrics/schema/entry/subsystem.rs

@@ -47,6 +47,7 @@ pub enum MetricSubsystem {
 
     // other service related subsystems
     Ilm,
+    Tier,
     Audit,
     Replication,
     Notification,
@@ -91,6 +92,7 @@ impl MetricSubsystem {
 
             // other service related subsystems
             Self::Ilm => "/ilm",
+            Self::Tier => "/tier",
             Self::Audit => "/audit",
             Self::Replication => "/replication",
             Self::Notification => "/notification",
@@ -140,6 +142,7 @@ impl MetricSubsystem {
 
             // Other service-related subsystems
             "/ilm" => Self::Ilm,
+            "/tier" => Self::Tier,
             "/audit" => Self::Audit,
             "/replication" => Self::Replication,
             "/notification" => Self::Notification,
@@ -202,6 +205,7 @@ pub mod subsystems {
     pub const CLUSTER_IAM: MetricSubsystem = MetricSubsystem::ClusterIam;
     pub const CLUSTER_CONFIG: MetricSubsystem = MetricSubsystem::ClusterConfig;
     pub const ILM: MetricSubsystem = MetricSubsystem::Ilm;
+    pub const TIER: MetricSubsystem = MetricSubsystem::Tier;
     pub const AUDIT: MetricSubsystem = MetricSubsystem::Audit;
     pub const REPLICATION: MetricSubsystem = MetricSubsystem::Replication;
     pub const NOTIFICATION: MetricSubsystem = MetricSubsystem::Notification;

crates/obs/src/metrics/schema/mod.rs

@@ -40,6 +40,7 @@ pub mod system_memory;
 pub mod system_network;
 pub mod system_network_host;
 pub mod system_process;
+pub mod tier;
 
 pub use entry::descriptor::MetricDescriptor;
 pub use entry::metric_name::MetricName;

crates/obs/src/metrics/schema/tier.rs

@@ -0,0 +1,43 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Remote tier request metric descriptors.
+//!
+//! The label set is fixed by the operation and outcome enums the recording
+//! site uses, so the series count is bounded by construction and cannot grow
+//! with tier names, endpoints or object keys.
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, subsystems};
+use std::sync::LazyLock;
+
+pub const OPERATION_LABEL: &str = "operation";
+pub const OUTCOME_LABEL: &str = "outcome";
+
+pub static TIER_REQUESTS_SUCCESS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::TierRequestsSuccess,
+        "Remote tier requests the backend acknowledged, by operation",
+        &[OPERATION_LABEL],
+        subsystems::TIER,
+    )
+});
+
+pub static TIER_REQUESTS_FAILURE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::TierRequestsFailure,
+        "Remote tier requests that did not complete, by operation and outcome",
+        &[OPERATION_LABEL, OUTCOME_LABEL],
+        subsystems::TIER,
+    )
+});

crates/obs/src/metrics/stats_collector.rs

@@ -27,7 +27,7 @@ use crate::metrics::collectors::{
     DriveDetailedStats, DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats,
     IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats,
     OdmBackfillRuntimeStats, OnDemandMigrationBucketStats, ProcessStats, ProcessStatusType, ReplicationMetricsSnapshot,
-    ResourceStats, ScannerRuntimeStats, ScannerStats,
+    ResourceStats, ScannerRuntimeStats, ScannerStats, TierRequestStats,
 };
 use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
 use crate::metrics::{
@@ -1396,6 +1396,23 @@ fn ilm_backpressure_stats(metrics: &ScannerMetricsReport) -> Vec<IlmBackpressure
     ]
 }
 
+/// Collect the remote tier request counters from the lifecycle runtime.
+///
+/// Every operation/outcome cell is reported, including zero ones, so the
+/// series set is stable from the first scrape rather than appearing one label
+/// combination at a time.
+pub fn collect_tier_request_metric_stats() -> Vec<TierRequestStats> {
+    global_metrics()
+        .tier_request_counts()
+        .into_iter()
+        .map(|count| TierRequestStats {
+            operation: count.operation.as_label(),
+            outcome: count.outcome.as_label(),
+            count: count.count,
+        })
+        .collect()
+}
+
 /// Collect ILM metrics from the current lifecycle runtime state.
 pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
     collect_ilm_runtime_metric_stats().await.map(|stats| stats.stats)