feat(tier): report cluster tier stats and count tier requests (#7110)

`GET /v3/tier-stats` answered from whichever process received the
request, returning that node's rolling 24-hour transition counters as
if they were cluster totals, and the `TierRequestsSuccess` and
`TierRequestsFailure` metric names had no producer at all.

The body now separates the two quantities a tier carries. Stored
inventory comes from the persisted scanner usage snapshot, which is
already cluster-wide; rolling activity is summed over every member
through a new read-only `TierDailyStats` peer RPC. Rings are merged
rather than added, so an idle node's expired hours age out, and each
node counts only its own committed transitions, so a retry is counted
once. Coverage travels with the numbers: `activity.status` names the
reporting members and the ones that could not be asked, timed out, or
answered with a ring this build refuses to merge, and per-tier
inventory is absent rather than zero when the snapshot has no
accounting. The version 1 body stays reachable at `?format=legacy`.

Tier request counters are recorded at the two seams every remote
request passes through, so a new provider is counted by construction,
with a closed operation/outcome label set that can never grow a tier
name, endpoint or object key.

Closes rustfs/backlog#2207

Co-authored-by: cxymds <cxymds@gmail.com>

crates/e2e_test/README.md

@@ -25,7 +25,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **policy** | [`src/policy/`](src/policy), `existing_object_tag_policy_test`, `bucket_policy_check_test`, `anonymous_access_test`, `security_boundary_test`, `multipart_auth_test` | IAM / bucket-policy / STS session policy, policy variables, anonymous access, DoS/SSRF boundaries. Own guide: [`src/policy/README.md`](src/policy/README.md) |
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
-| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
+| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 

crates/e2e_test/src/lib.rs

@@ -218,6 +218,10 @@ mod cluster_multidrive_pool_test;
 #[cfg(test)]
 mod inline_fast_path_cluster_test;
 
+// backlog#2207: two-node gate for the cluster-authoritative tier stats contract.
+#[cfg(test)]
+mod tier_stats_cluster_test;
+
 // PutObject / MultipartUpload with checksum (Content-MD5, x-amz-checksum-*)
 #[cfg(test)]
 mod checksum_upload_test;

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -904,6 +904,13 @@ impl NodeService for MinimalLockNodeService {
     ) -> Result<Response<rustfs_protos::proto_gen::node_service::LoadTransitionTierConfigResponse>, Status> {
         Err(Status::unimplemented("lock-only test server"))
     }
+
+    async fn tier_daily_stats(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::TierDailyStatsRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::TierDailyStatsResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
 }
 
 /// Spawn a gRPC lock server on a random port

crates/e2e_test/src/tier_stats_cluster_test.rs

@@ -0,0 +1,144 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Two-node gate for the tier stats wire contract (rustfs/backlog#2207).
+//!
+//! Before this contract, `GET /v3/tier-stats` answered from the process that
+//! happened to receive the request, so the same query returned different
+//! numbers depending on which node a client reached, with nothing in the body
+//! saying so. These tests pin the two properties that fix costs: the answer is
+//! node-independent, and it states how much of the cluster it covers.
+
+use crate::common::{RustFSTestClusterEnvironment, admin_request, init_logging};
+use http::Method;
+use http::StatusCode;
+use serde_json::Value;
+use std::time::Duration;
+use tokio::time::{Instant, sleep};
+
+type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
+
+const TIER_STATS_PATH: &str = "/rustfs/admin/v3/tier-stats";
+const PEER_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);
+
+async fn tier_stats(cluster: &RustFSTestClusterEnvironment, node: usize, query: &str) -> TestResult<(StatusCode, String)> {
+    admin_request(
+        &cluster.nodes[node].url,
+        Method::GET,
+        &format!("{TIER_STATS_PATH}{query}"),
+        None,
+        &cluster.access_key,
+        &cluster.secret_key,
+    )
+    .await
+}
+
+async fn tier_stats_json(cluster: &RustFSTestClusterEnvironment, node: usize) -> TestResult<Value> {
+    let (status, body) = tier_stats(cluster, node, "").await?;
+    assert_eq!(status, StatusCode::OK, "node {node} must answer tier-stats: {body}");
+    Ok(serde_json::from_str(&body)?)
+}
+
+/// Wait until every node reports the whole cluster.
+///
+/// Peer clients are established during startup, so a query issued in the first
+/// moments can legitimately see a peer as unavailable. Polling separates that
+/// startup window from the failure this test is about: an answer that stays
+/// partial because the peer never reports at all.
+async fn wait_for_complete_activity(cluster: &RustFSTestClusterEnvironment) -> TestResult<Vec<Value>> {
+    let deadline = Instant::now() + PEER_CONVERGENCE_TIMEOUT;
+    loop {
+        let mut bodies = Vec::with_capacity(cluster.nodes.len());
+        for node in 0..cluster.nodes.len() {
+            bodies.push(tier_stats_json(cluster, node).await?);
+        }
+
+        if bodies.iter().all(|body| body["activity"]["status"] == "complete") {
+            return Ok(bodies);
+        }
+        if Instant::now() >= deadline {
+            return Err(format!("tier-stats activity never became complete on every node: {bodies:?}").into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+#[tokio::test]
+async fn tier_stats_answers_the_same_cluster_result_from_either_node() -> TestResult {
+    init_logging();
+
+    let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
+    cluster.start().await?;
+
+    let bodies = wait_for_complete_activity(&cluster).await?;
+    let (first, second) = (&bodies[0], &bodies[1]);
+
+    for (node, body) in bodies.iter().enumerate() {
+        assert_eq!(body["contractVersion"], 2, "node {node} must name the current contract version");
+        assert_eq!(
+            body["activity"]["nodesExpected"], 2,
+            "node {node} must expect both cluster members, not only itself"
+        );
+        assert_eq!(
+            body["activity"]["nodesReporting"], 2,
+            "node {node} must include its peer's rolling window in the sum"
+        );
+        assert_eq!(
+            body["activity"]["unavailableNodes"],
+            serde_json::json!([]),
+            "node {node} reported a complete result while naming an unavailable member"
+        );
+    }
+
+    // A fixture cluster has no remote tier, so this pair is equal over an
+    // empty list; `nodesReporting` above is what proves the peer answered.
+    // The per-tier summing itself is pinned by the aggregator unit tests.
+    assert_eq!(
+        first["tiers"], second["tiers"],
+        "the same query must not depend on which node received it"
+    );
+    assert_eq!(
+        first["inventory"]["status"], second["inventory"]["status"],
+        "both nodes read the same persisted usage snapshot"
+    );
+
+    cluster.stop();
+    Ok(())
+}
+
+#[tokio::test]
+async fn tier_stats_keeps_the_legacy_body_reachable_and_rejects_unknown_formats() -> TestResult {
+    init_logging();
+
+    let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
+    cluster.start().await?;
+
+    let (status, body) = tier_stats(&cluster, 0, "?format=legacy").await?;
+    assert_eq!(status, StatusCode::OK, "the pinned legacy body must stay reachable: {body}");
+    let legacy: Value = serde_json::from_str(&body)?;
+    assert!(
+        legacy.is_object() && legacy.get("contractVersion").is_none(),
+        "the legacy body is the bare tier map, not the current envelope: {legacy}"
+    );
+
+    let (status, body) = tier_stats(&cluster, 0, "?format=v3").await?;
+    assert_eq!(
+        status,
+        StatusCode::BAD_REQUEST,
+        "an unknown format must be rejected rather than answered in another shape: {body}"
+    );
+
+    cluster.stop();
+    Ok(())
+}

crates/ecstore/src/api/mod.rs

@@ -118,7 +118,7 @@ pub mod bucket {
         }
 
         pub mod tier_last_day_stats {
-            pub use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
+            pub use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats, TierDailyStatsWire};
         }
 
         pub mod tier_sweeper {
@@ -472,7 +472,7 @@ pub mod notification {
     #[cfg(any(test, feature = "test-util"))]
     pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
     pub use crate::services::notification_sys::{
-        CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
+        ClusterTierDailyStats, CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
         acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
         new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
     };

crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs

@@ -19,6 +19,7 @@
 #![allow(clippy::all)]
 
 use rustfs_data_usage::TierStats;
+use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 use std::collections::HashMap;
 use std::ops::Sub;
@@ -27,7 +28,24 @@ use tracing::{error, warn};
 
 pub type DailyAllTierStats = HashMap<String, LastDayTierStats>;
 
-#[derive(Clone)]
+/// One bin per hour of the rolling day. The bin index is the UTC hour, so the
+/// array is a ring the writer ages forward rather than a queue.
+pub const TIER_DAILY_STATS_BINS: usize = 24;
+
+/// Interchange form of [`LastDayTierStats`] for the internode tier-stats RPC.
+///
+/// The in-memory type keeps its bins private because the ring is only
+/// meaningful together with `updated_at`; this type carries both across the
+/// wire and is validated back into the ring by [`LastDayTierStats::from_wire`].
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TierDailyStatsWire {
+    pub bins: Vec<TierStats>,
+    /// Seconds since the Unix epoch. Bins are hour-resolution, so a coarser
+    /// timestamp than the in-memory `OffsetDateTime` loses nothing.
+    pub updated_at_unix_secs: i64,
+}
+
+#[derive(Clone, Debug)]
 pub struct LastDayTierStats {
     bins: [TierStats; 24],
     updated_at: OffsetDateTime,
@@ -80,11 +98,57 @@ impl LastDayTierStats {
         }
     }
 
-    #[allow(
-        dead_code,
-        reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
-    )]
-    fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
+    /// The rolling ring as observed, without aging it forward.
+    ///
+    /// Only meaningful together with [`LastDayTierStats::updated_at`]: a bin
+    /// belongs to the hour of its index within the day that ends at
+    /// `updated_at`.
+    pub fn bins(&self) -> &[TierStats; TIER_DAILY_STATS_BINS] {
+        &self.bins
+    }
+
+    pub fn updated_at(&self) -> OffsetDateTime {
+        self.updated_at
+    }
+
+    pub fn to_wire(&self) -> TierDailyStatsWire {
+        TierDailyStatsWire {
+            bins: self.bins.to_vec(),
+            updated_at_unix_secs: self.updated_at.unix_timestamp(),
+        }
+    }
+
+    /// Rebuild the ring from a peer's response.
+    ///
+    /// A ring of the wrong width or an unrepresentable timestamp is corrupt
+    /// peer input, not a zero sample: it returns an error so the caller can
+    /// report the node as non-reporting instead of merging a plausible but
+    /// wrong day into a cluster total.
+    pub fn from_wire(wire: TierDailyStatsWire) -> Result<Self, std::io::Error> {
+        let bins: [TierStats; TIER_DAILY_STATS_BINS] = wire.bins.try_into().map_err(|bins: Vec<TierStats>| {
+            std::io::Error::new(
+                std::io::ErrorKind::InvalidData,
+                format!("tier daily stats must carry {TIER_DAILY_STATS_BINS} bins, got {}", bins.len()),
+            )
+        })?;
+        let updated_at = OffsetDateTime::from_unix_timestamp(wire.updated_at_unix_secs).map_err(|err| {
+            std::io::Error::new(
+                std::io::ErrorKind::InvalidData,
+                format!("tier daily stats carry an unrepresentable timestamp: {err}"),
+            )
+        })?;
+
+        Ok(Self { bins, updated_at })
+    }
+
+    /// Combine two independently observed rings.
+    ///
+    /// Each node counts only the transitions it completed itself, so summing
+    /// bins across nodes is a cluster total rather than a double count. The
+    /// older ring is aged forward to the newer one's clock first, so a node
+    /// that stopped transitioning hours ago contributes its still-current
+    /// bins and not its expired ones.
+    pub fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
         let mut cl = self.clone();
         let mut cm = m;
         let mut merged = LastDayTierStats::default();
@@ -108,6 +172,7 @@ impl LastDayTierStats {
 #[cfg(test)]
 mod test {
     use super::*;
+    use time::Duration;
 
     #[test]
     fn total_sums_all_recorded_stats() {
@@ -132,4 +197,83 @@ mod test {
             }
         );
     }
+
+    fn sample(total_size: u64) -> TierStats {
+        TierStats {
+            total_size,
+            num_versions: 1,
+            num_objects: 1,
+        }
+    }
+
+    #[test]
+    fn wire_round_trip_preserves_the_ring_and_its_clock() {
+        let mut stats = LastDayTierStats::default();
+        stats.add_stats(sample(10));
+
+        let restored = LastDayTierStats::from_wire(stats.to_wire()).expect("a ring this node produced must decode");
+
+        assert_eq!(restored.bins(), stats.bins(), "every bin must survive the wire");
+        assert_eq!(
+            restored.updated_at().unix_timestamp(),
+            stats.updated_at().unix_timestamp(),
+            "the ring's clock must survive the wire"
+        );
+    }
+
+    #[test]
+    fn a_ring_of_the_wrong_width_is_rejected() {
+        let mut wire = LastDayTierStats::default().to_wire();
+        wire.bins.pop();
+
+        let err = LastDayTierStats::from_wire(wire).expect_err("a short ring must not decode as a zero day");
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+    }
+
+    #[test]
+    fn an_unrepresentable_clock_is_rejected() {
+        let mut wire = LastDayTierStats::default().to_wire();
+        wire.updated_at_unix_secs = i64::MIN;
+
+        let err = LastDayTierStats::from_wire(wire).expect_err("an unrepresentable clock must not decode");
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+    }
+
+    #[test]
+    fn merge_sums_two_nodes_that_transitioned_in_the_same_hour() {
+        let mut left = LastDayTierStats::default();
+        left.add_stats(sample(10));
+        let mut right = LastDayTierStats::default();
+        right.add_stats(sample(20));
+
+        assert_eq!(
+            left.merge(right).total(),
+            TierStats {
+                total_size: 30,
+                num_versions: 2,
+                num_objects: 2,
+            },
+            "each node counts only its own completions, so a merge is a cluster total"
+        );
+    }
+
+    #[test]
+    fn merge_ages_out_a_peer_ring_older_than_a_day() {
+        let mut stale = LastDayTierStats::default();
+        stale.add_stats(sample(10));
+        stale.updated_at -= Duration::days(2);
+
+        let mut fresh = LastDayTierStats::default();
+        fresh.add_stats(sample(20));
+
+        assert_eq!(
+            fresh.merge(stale).total(),
+            TierStats {
+                total_size: 20,
+                num_versions: 1,
+                num_objects: 1,
+            },
+            "a node that stopped transitioning more than a day ago must not keep contributing"
+        );
+    }
 }

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -23,6 +23,7 @@ use crate::storage_api_contracts::internode::{
     SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
 };
 use crate::{
+    bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats, TierDailyStatsWire},
     bucket::replication::BucketStats,
     disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout},
     layout::endpoints::EndpointServerPools,
@@ -48,7 +49,7 @@ use rustfs_protos::proto_gen::node_service::{
     LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
     ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
     ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
-    StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
+    StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest,
     TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest,
     node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
@@ -986,6 +987,40 @@ impl PeerRestClient {
         Ok(cpus)
     }
 
+    /// This peer's own rolling-day transition counters, per remote tier.
+    ///
+    /// The response is untrusted peer input: a ring of the wrong width or an
+    /// unrepresentable clock is rejected here rather than merged, so a corrupt
+    /// answer makes the node non-reporting instead of silently shifting a
+    /// cluster total.
+    pub async fn tier_daily_stats(&self) -> Result<DailyAllTierStats> {
+        self.finalize_result(self.tier_daily_stats_inner().await).await
+    }
+
+    async fn tier_daily_stats_inner(&self) -> Result<DailyAllTierStats> {
+        let mut client = self.get_client().await?;
+        let request = Request::new(TierDailyStatsRequest {});
+
+        let response = client.tier_daily_stats(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::other(msg));
+            }
+            return Err(peer_failure_without_details("tier_daily_stats", None));
+        }
+
+        let mut buf = Deserializer::new(Cursor::new(response.tier_daily_stats));
+        let wire: HashMap<String, TierDailyStatsWire> = Deserialize::deserialize(&mut buf)?;
+
+        wire.into_iter()
+            .map(|(tier, stats)| {
+                LastDayTierStats::from_wire(stats)
+                    .map(|stats| (tier, stats))
+                    .map_err(Error::from)
+            })
+            .collect()
+    }
+
     pub async fn get_net_info(&self) -> Result<NetInfo> {
         self.finalize_result(self.get_net_info_inner().await).await
     }

crates/ecstore/src/services/notification_sys.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
 use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, ScannerPublicationLease, TierConfigReloadOutcome};
 use crate::diagnostics::admin_server_info::get_commit_id;
 use crate::disk::DiskAPI;
@@ -50,6 +51,7 @@ const LOG_SUBSYSTEM_NOTIFICATION: &str = "notification";
 const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation";
 const EVENT_NOTIFICATION_CAPABILITY_PROBE: &str = "notification_capability_probe";
 const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
+const TIER_DAILY_STATS_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
 const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
 const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
@@ -973,6 +975,120 @@ impl NotificationSys {
     }
 }
 
+/// Rolling tier activity summed over every cluster member that answered, with
+/// the reporting coverage behind the sum.
+///
+/// The coverage is part of the result rather than a log line: a sum over a
+/// subset of the cluster is not a cluster total, and a caller that renders it
+/// as one is the defect this type exists to prevent.
+pub struct ClusterTierDailyStats {
+    pub stats: DailyAllTierStats,
+    /// Members whose rolling day is included, always at least this node.
+    pub nodes_reporting: usize,
+    /// Members this deployment expects to hear from, including this node.
+    pub nodes_expected: usize,
+    /// Members that could not be asked, could not answer, or answered with a
+    /// ring this build refuses to merge. Sorted, and named by grid host.
+    pub unavailable_nodes: Vec<String>,
+}
+
+impl ClusterTierDailyStats {
+    pub fn is_complete(&self) -> bool {
+        self.unavailable_nodes.is_empty() && self.nodes_reporting == self.nodes_expected
+    }
+}
+
+/// Fold one member's rolling day into the running cluster total.
+///
+/// Merging (rather than adding totals) ages each member's ring to the newer
+/// clock first, so a member that stopped transitioning yesterday contributes
+/// only the hours still inside the rolling day.
+fn merge_tier_daily_stats(into: &mut DailyAllTierStats, from: DailyAllTierStats) {
+    for (tier, stats) in from {
+        match into.remove(&tier) {
+            Some(existing) => {
+                into.insert(tier, existing.merge(stats));
+            }
+            None => {
+                into.insert(tier, stats);
+            }
+        }
+    }
+}
+
+impl NotificationSys {
+    /// Sum this node's rolling tier activity with every reachable peer's.
+    ///
+    /// Each node records only the transitions it completed itself, so the sum
+    /// is a cluster total and a retried transition is counted once, by the
+    /// node that finally committed it. Peers are probed concurrently under a
+    /// per-peer deadline so one black-holed member cannot hold the admin
+    /// request open.
+    pub async fn tier_daily_stats(&self, local: DailyAllTierStats) -> ClusterTierDailyStats {
+        let mut stats = local;
+        let nodes_expected = self.peer_clients.len() + 1;
+        let mut nodes_reporting = 1;
+        let mut unavailable_nodes = Vec::new();
+
+        let mut probes = Vec::with_capacity(self.peer_clients.len());
+        for (idx, client) in self.peer_clients.iter().enumerate() {
+            let host = self.tier_daily_stats_peer_host(idx, client.as_ref());
+            probes.push(async move {
+                let Some(client) = client.as_ref() else {
+                    return (host, Err(Error::other("peer is not reachable")));
+                };
+                // The peer is already named by the caller's `peer` log field
+                // and by `unavailable_nodes`, so the deadline is reported as
+                // the typed variant rather than a formatted fragment.
+                let result = timeout(TIER_DAILY_STATS_PROBE_TIMEOUT, client.tier_daily_stats())
+                    .await
+                    .unwrap_or(Err(Error::Timeout));
+                (host, result)
+            });
+        }
+
+        for (host, result) in join_all(probes).await {
+            match result {
+                Ok(peer_stats) => {
+                    nodes_reporting += 1;
+                    merge_tier_daily_stats(&mut stats, peer_stats);
+                }
+                Err(err) => {
+                    warn!(
+                        event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                        peer = host,
+                        error = %err,
+                        "tier daily stats peer did not report"
+                    );
+                    unavailable_nodes.push(host);
+                }
+            }
+        }
+
+        unavailable_nodes.sort();
+        ClusterTierDailyStats {
+            stats,
+            nodes_reporting,
+            nodes_expected,
+            unavailable_nodes,
+        }
+    }
+
+    /// Name a peer slot even when no client was ever built for it, so an
+    /// unreachable member is reported by host instead of disappearing.
+    fn tier_daily_stats_peer_host(&self, idx: usize, client: Option<&PeerRestClient>) -> String {
+        if let Some(client) = client {
+            return client.grid_host.clone();
+        }
+        self.peer_topology_hosts
+            .get(idx)
+            .cloned()
+            .unwrap_or_else(|| format!("peer[{idx}]"))
+    }
+}
+
 pub struct NotificationPeerErr {
     pub host: String,
     pub err: Option<Error>,
@@ -2935,6 +3051,65 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
+    use rustfs_data_usage::TierStats;
+
+    fn ring(total_size: u64) -> LastDayTierStats {
+        let mut stats = LastDayTierStats::default();
+        stats.add_stats(TierStats {
+            total_size,
+            num_versions: 1,
+            num_objects: 1,
+        });
+        stats
+    }
+
+    #[test]
+    fn merging_peer_rings_sums_each_tier_without_double_counting() {
+        let mut cluster = DailyAllTierStats::from([("WARM".to_string(), ring(10))]);
+
+        merge_tier_daily_stats(
+            &mut cluster,
+            DailyAllTierStats::from([("WARM".to_string(), ring(20)), ("COLD".to_string(), ring(5))]),
+        );
+
+        assert_eq!(
+            cluster.get("WARM").expect("the shared tier must survive the merge").total(),
+            TierStats {
+                total_size: 30,
+                num_versions: 2,
+                num_objects: 2,
+            },
+            "both nodes' completions belong in the cluster total"
+        );
+        assert_eq!(
+            cluster.get("COLD").expect("a tier only one node saw must be kept").total(),
+            TierStats {
+                total_size: 5,
+                num_versions: 1,
+                num_objects: 1,
+            }
+        );
+    }
+
+    #[test]
+    fn a_single_member_result_is_complete_and_a_missing_peer_is_not() {
+        let complete = ClusterTierDailyStats {
+            stats: DailyAllTierStats::new(),
+            nodes_reporting: 1,
+            nodes_expected: 1,
+            unavailable_nodes: Vec::new(),
+        };
+        assert!(complete.is_complete(), "a single-member deployment reports its whole cluster");
+
+        let partial = ClusterTierDailyStats {
+            stats: DailyAllTierStats::new(),
+            nodes_reporting: 1,
+            nodes_expected: 2,
+            unavailable_nodes: vec!["node-b:9000".to_string()],
+        };
+        assert!(!partial.is_complete(), "a silent member must make the sum partial");
+    }
 
     #[test]
     fn cross_pool_policy_versions_authorize_only_their_supported_protocols() {

crates/ecstore/src/services/tier/tier_config.rs

@@ -38,7 +38,7 @@ const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
     ("eu-south-1", "https://s3.it-1.wasabisys.com"),
 ];
 
-#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[derive(Serialize, Deserialize, Default, Debug, Clone, PartialEq)]
 pub enum TierType {
     #[default]
     Unsupported,

crates/ecstore/src/services/tier/warm_backend.rs

@@ -43,6 +43,7 @@ use rustfs_s3_client::{
     api_put_object::{AdvancedPutOptions, PutObjectOptions},
     transition_api::{ReadCloser, ReaderImpl},
 };
+use rustfs_scanner_metrics::metrics::{TierRequestOperation, TierRequestOutcome, global_metrics};
 use rustfs_utils::egress::validate_outbound_url;
 use rustfs_utils::http::headers::{
     CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
@@ -346,6 +347,110 @@ pub(crate) fn optimal_part_size(object_size: i64, min_part_size: i64) -> Result<
     Ok(part_size)
 }
 
+/// Counts every remote-tier request exactly once, whatever backend performs it.
+///
+/// The counters are the only production update path for the `tier` request
+/// metrics, and they must not grow with tier names, endpoints or object keys,
+/// so the wrapper deliberately records nothing but the fixed operation and
+/// outcome labels. Wrapping here rather than inside each provider keeps a new
+/// backend counted by construction, and keeps one call per request: the
+/// overridden `remove_exact` delegates to the inner backend, so the inner
+/// default's own `remove` cannot count the same request a second time.
+struct MeteredWarmBackend {
+    inner: WarmBackendImpl,
+}
+
+impl MeteredWarmBackend {
+    fn record<T>(operation: TierRequestOperation, result: Result<T, std::io::Error>) -> Result<T, std::io::Error> {
+        let outcome = match &result {
+            Ok(_) => TierRequestOutcome::Success,
+            Err(err) => TierRequestOutcome::from_error(err),
+        };
+        global_metrics().record_tier_request(operation, outcome);
+        result
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for MeteredWarmBackend {
+    /// Delegated without a counter: only one backend issues a remote request
+    /// here, and every other one takes the trait default, so a `validate`
+    /// counter would mostly record requests that never happened.
+    async fn validate(&self) -> Result<(), std::io::Error> {
+        self.inner.validate().await
+    }
+
+    /// A local check of a value this process already holds; it issues no
+    /// remote request, so it is delegated without a counter.
+    fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
+        self.inner.validate_remote_version_id(remote_version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        Self::record(TierRequestOperation::Put, self.inner.put(object, r, length).await)
+    }
+
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        Self::record(TierRequestOperation::Put, self.inner.put_with_meta(object, r, length, meta).await)
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        Self::record(TierRequestOperation::Get, self.inner.get(object, rv, opts).await)
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        Self::record(TierRequestOperation::Remove, self.inner.remove(object, rv).await)
+    }
+
+    async fn remove_exact(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        Self::record(TierRequestOperation::Remove, self.inner.remove_exact(object, rv).await)
+    }
+
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let result = self.inner.probe_transition_candidate(object).await;
+        // `Unsupported` is the trait default: the backend issued no request,
+        // so counting it would inflate the probe counter on every backend that
+        // does not implement probing.
+        if matches!(result, Ok(TransitionCandidateProbe::Unsupported)) {
+            return result;
+        }
+        Self::record(TierRequestOperation::Probe, result)
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        Self::record(TierRequestOperation::InUse, self.inner.in_use().await)
+    }
+}
+
+/// The reconciler is a second remote probe path, reached from recovery rather
+/// than from the `WarmBackend` handle, so it needs its own counter: a `probe`
+/// counter that saw only one of the two paths would read as a complete count
+/// while missing every recovery probe.
+struct MeteredTransitionCandidateReconciler {
+    inner: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>,
+}
+
+#[async_trait::async_trait]
+impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let result = self.inner.probe_transition_candidate_for(object, identity).await;
+        if matches!(result, Ok(TransitionCandidateProbe::Unsupported)) {
+            return result;
+        }
+        MeteredWarmBackend::record(TierRequestOperation::Probe, result)
+    }
+}
+
 pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
     let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
     w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
@@ -593,6 +698,8 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
         status_code: StatusCode::BAD_REQUEST,
     })?;
 
+    let d: WarmBackendImpl = Box::new(MeteredWarmBackend { inner: d });
+
     if probe {
         d.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
     }
@@ -641,7 +748,7 @@ pub(crate) async fn new_transition_candidate_reconciler(
         ),
         _ => return Ok(None),
     };
-    Ok(Some(reconciler))
+    Ok(Some(Box::new(MeteredTransitionCandidateReconciler { inner: reconciler })))
 }
 
 #[cfg(test)]
@@ -655,6 +762,156 @@ mod tests {
 
     const PROBE_VERSION: &str = "remote-v2";
 
+    struct CountingBackend {
+        put_result: fn() -> Result<String, std::io::Error>,
+        removes: Arc<AtomicUsize>,
+    }
+
+    #[async_trait::async_trait]
+    impl WarmBackend for CountingBackend {
+        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
+            (self.put_result)()
+        }
+
+        async fn put_with_meta(
+            &self,
+            object: &str,
+            r: ReaderImpl,
+            length: i64,
+            _meta: HashMap<String, String>,
+        ) -> Result<String, std::io::Error> {
+            self.put(object, r, length).await
+        }
+
+        async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+            Err(std::io::Error::other("unused"))
+        }
+
+        async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
+            self.removes.fetch_add(1, Ordering::SeqCst);
+            Ok(())
+        }
+
+        async fn in_use(&self) -> Result<bool, std::io::Error> {
+            Ok(false)
+        }
+    }
+
+    fn tier_cell(operation: TierRequestOperation, outcome: TierRequestOutcome) -> u64 {
+        global_metrics()
+            .tier_request_counts()
+            .into_iter()
+            .find(|count| count.operation == operation && count.outcome == outcome)
+            .map(|count| count.count)
+            .expect("every operation/outcome cell must be reported")
+    }
+
+    #[tokio::test]
+    async fn a_metered_put_counts_its_outcome_once() {
+        let before_success = tier_cell(TierRequestOperation::Put, TierRequestOutcome::Success);
+        let before_failure = tier_cell(TierRequestOperation::Put, TierRequestOutcome::BackendError);
+        let backend = MeteredWarmBackend {
+            inner: Box::new(CountingBackend {
+                put_result: || Ok("remote-v1".to_string()),
+                removes: Arc::new(AtomicUsize::new(0)),
+            }),
+        };
+
+        backend
+            .put("object", ReaderImpl::Body(Bytes::from_static(b"payload")), 7)
+            .await
+            .expect("the fake backend accepts the put");
+
+        assert_eq!(
+            tier_cell(TierRequestOperation::Put, TierRequestOutcome::Success),
+            before_success + 1,
+            "an acknowledged put must be counted once"
+        );
+        assert_eq!(
+            tier_cell(TierRequestOperation::Put, TierRequestOutcome::BackendError),
+            before_failure,
+            "a success must not also increment a failure cell"
+        );
+    }
+
+    #[tokio::test]
+    async fn a_metered_put_failure_is_classified_by_error_kind() {
+        let before_timeout = tier_cell(TierRequestOperation::Put, TierRequestOutcome::Timeout);
+        let backend = MeteredWarmBackend {
+            inner: Box::new(CountingBackend {
+                put_result: || Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "endpoint stalled")),
+                removes: Arc::new(AtomicUsize::new(0)),
+            }),
+        };
+
+        backend
+            .put("object", ReaderImpl::Body(Bytes::from_static(b"payload")), 7)
+            .await
+            .expect_err("the fake backend rejects the put");
+
+        assert_eq!(
+            tier_cell(TierRequestOperation::Put, TierRequestOutcome::Timeout),
+            before_timeout + 1,
+            "a timed-out request must land in the timeout cell"
+        );
+    }
+
+    fn tier_probe_total() -> u64 {
+        global_metrics()
+            .tier_request_counts()
+            .into_iter()
+            .filter(|count| count.operation == TierRequestOperation::Probe)
+            .map(|count| count.count)
+            .sum()
+    }
+
+    #[tokio::test]
+    async fn an_unsupported_probe_is_not_counted_as_a_request() {
+        let before = tier_probe_total();
+        let backend = MeteredWarmBackend {
+            inner: Box::new(CountingBackend {
+                put_result: || Ok(String::new()),
+                removes: Arc::new(AtomicUsize::new(0)),
+            }),
+        };
+
+        let probe = backend
+            .probe_transition_candidate("object")
+            .await
+            .expect("the trait default answers without a remote request");
+
+        assert_eq!(probe, TransitionCandidateProbe::Unsupported);
+        assert_eq!(
+            tier_probe_total(),
+            before,
+            "a backend that issues no probe request must not appear in the probe counters"
+        );
+    }
+
+    #[tokio::test]
+    async fn an_exact_remove_is_counted_once_not_twice() {
+        let before = tier_cell(TierRequestOperation::Remove, TierRequestOutcome::Success);
+        let removes = Arc::new(AtomicUsize::new(0));
+        let backend = MeteredWarmBackend {
+            inner: Box::new(CountingBackend {
+                put_result: || Ok(String::new()),
+                removes: Arc::clone(&removes),
+            }),
+        };
+
+        backend
+            .remove_exact("object", "remote-v1")
+            .await
+            .expect("the fake backend accepts the remove");
+
+        assert_eq!(removes.load(Ordering::SeqCst), 1, "the inner backend performs one request");
+        assert_eq!(
+            tier_cell(TierRequestOperation::Remove, TierRequestOutcome::Success),
+            before + 1,
+            "the default remove_exact must not count its own inner remove a second time"
+        );
+    }
+
     struct RejectingValidationBackend {
         validations: Arc<AtomicUsize>,
         puts: Arc<AtomicUsize>,

crates/obs/src/metrics/collectors/mod.rs

@@ -40,6 +40,7 @@ pub mod system_memory;
 pub mod system_network;
 pub mod system_network_host;
 pub mod system_process;
+pub mod tier;
 
 pub(crate) use audit::{AuditTargetRuntimeStats, collect_audit_runtime_metrics};
 pub use audit::{AuditTargetStats, collect_audit_metrics};
@@ -94,3 +95,4 @@ pub use system_process::{
     ProcessAttributeError, ProcessAttributes, ProcessStats, ProcessStatusType, collect_process_attributes,
     collect_process_metrics,
 };
+pub use tier::{TierRequestStats, collect_tier_request_metrics};

crates/obs/src/metrics/collectors/tier.rs

@@ -0,0 +1,96 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Remote tier request metrics collector.
+
+use crate::metrics::report::PrometheusMetric;
+use crate::metrics::schema::tier::*;
+
+/// One operation/outcome cell of the tier request counters.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TierRequestStats {
+    pub operation: &'static str,
+    pub outcome: &'static str,
+    pub count: u64,
+}
+
+/// Split the fixed operation/outcome cells over the success and failure
+/// counters.
+///
+/// The success counter carries no outcome label: `success` is the only outcome
+/// it can report, and repeating it would make the two counters look like they
+/// share a label set they do not.
+pub fn collect_tier_request_metrics(stats: &[TierRequestStats]) -> Vec<PrometheusMetric> {
+    stats
+        .iter()
+        .map(|stat| {
+            if stat.outcome == "success" {
+                PrometheusMetric::from_descriptor(&TIER_REQUESTS_SUCCESS_MD, stat.count as f64)
+                    .with_label(OPERATION_LABEL, stat.operation)
+            } else {
+                PrometheusMetric::from_descriptor(&TIER_REQUESTS_FAILURE_MD, stat.count as f64)
+                    .with_label(OPERATION_LABEL, stat.operation)
+                    .with_label(OUTCOME_LABEL, stat.outcome)
+            }
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn stats() -> Vec<TierRequestStats> {
+        vec![
+            TierRequestStats {
+                operation: "put",
+                outcome: "success",
+                count: 7,
+            },
+            TierRequestStats {
+                operation: "put",
+                outcome: "timeout",
+                count: 2,
+            },
+        ]
+    }
+
+    #[test]
+    fn success_and_failure_land_on_their_own_counters() {
+        let metrics = collect_tier_request_metrics(&stats());
+
+        let success = metrics
+            .iter()
+            .find(|metric| metric.name == TIER_REQUESTS_SUCCESS_MD.get_full_metric_name())
+            .expect("a success cell must produce the success counter");
+        assert_eq!(success.value, 7.0);
+        assert!(
+            success.labels.iter().all(|(name, _)| *name != OUTCOME_LABEL),
+            "the success counter must not carry an outcome label"
+        );
+
+        let failure = metrics
+            .iter()
+            .find(|metric| metric.name == TIER_REQUESTS_FAILURE_MD.get_full_metric_name())
+            .expect("a non-success cell must produce the failure counter");
+        assert_eq!(failure.value, 2.0);
+        assert!(
+            failure
+                .labels
+                .iter()
+                .any(|(name, value)| *name == OUTCOME_LABEL && value.as_ref() == "timeout"),
+            "the failure counter must keep the outcome that produced it"
+        );
+    }
+}

crates/obs/src/metrics/scheduler.rs

@@ -77,6 +77,7 @@ use crate::metrics::collectors::{
     collect_request_metrics,
     collect_resource_metrics,
     collect_scanner_runtime_metrics,
+    collect_tier_request_metrics,
 };
 use crate::metrics::config::{
     DEFAULT_AUDIT_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
@@ -151,7 +152,7 @@ use crate::metrics::stats_collector::{
     collect_disk_and_system_drive_runtime_stats, collect_erasure_set_stats, collect_host_network_stats, collect_iam_stats,
     collect_ilm_runtime_metric_stats, collect_internode_network_stats, collect_on_demand_migration_backfill_stats,
     collect_on_demand_migration_stats, collect_process_metric_bundle_with, collect_replication_stats,
-    collect_scanner_runtime_metric_stats, collect_system_cpu_and_memory_stats_with,
+    collect_scanner_runtime_metric_stats, collect_system_cpu_and_memory_stats_with, collect_tier_request_metric_stats,
 };
 use crate::node_identity::{SERVER_LABEL, current_local_node_identity};
 use crate::telemetry::retire_metric_series;
@@ -2397,6 +2398,8 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                                 metrics.extend(collect_ilm_runtime_metrics(&stats));
                             }
 
+                            metrics.extend(collect_tier_request_metrics(&collect_tier_request_metric_stats()));
+
                             let mut retire_scanner_cycle_bucket_drive_result_keys = Vec::new();
                             let mut retire_scanner_bucket_drive_result_keys = Vec::new();
                             let mut retire_scanner_active_bucket_drive_keys = Vec::new();

crates/obs/src/metrics/schema/entry/subsystem.rs

@@ -47,6 +47,7 @@ pub enum MetricSubsystem {
 
     // other service related subsystems
     Ilm,
+    Tier,
     Audit,
     Replication,
     Notification,
@@ -91,6 +92,7 @@ impl MetricSubsystem {
 
             // other service related subsystems
             Self::Ilm => "/ilm",
+            Self::Tier => "/tier",
             Self::Audit => "/audit",
             Self::Replication => "/replication",
             Self::Notification => "/notification",
@@ -140,6 +142,7 @@ impl MetricSubsystem {
 
             // Other service-related subsystems
             "/ilm" => Self::Ilm,
+            "/tier" => Self::Tier,
             "/audit" => Self::Audit,
             "/replication" => Self::Replication,
             "/notification" => Self::Notification,
@@ -202,6 +205,7 @@ pub mod subsystems {
     pub const CLUSTER_IAM: MetricSubsystem = MetricSubsystem::ClusterIam;
     pub const CLUSTER_CONFIG: MetricSubsystem = MetricSubsystem::ClusterConfig;
     pub const ILM: MetricSubsystem = MetricSubsystem::Ilm;
+    pub const TIER: MetricSubsystem = MetricSubsystem::Tier;
     pub const AUDIT: MetricSubsystem = MetricSubsystem::Audit;
     pub const REPLICATION: MetricSubsystem = MetricSubsystem::Replication;
     pub const NOTIFICATION: MetricSubsystem = MetricSubsystem::Notification;

crates/obs/src/metrics/schema/mod.rs

@@ -40,6 +40,7 @@ pub mod system_memory;
 pub mod system_network;
 pub mod system_network_host;
 pub mod system_process;
+pub mod tier;
 
 pub use entry::descriptor::MetricDescriptor;
 pub use entry::metric_name::MetricName;

crates/obs/src/metrics/schema/tier.rs

@@ -0,0 +1,43 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Remote tier request metric descriptors.
+//!
+//! The label set is fixed by the operation and outcome enums the recording
+//! site uses, so the series count is bounded by construction and cannot grow
+//! with tier names, endpoints or object keys.
+
+use crate::{MetricDescriptor, MetricName, new_counter_md, subsystems};
+use std::sync::LazyLock;
+
+pub const OPERATION_LABEL: &str = "operation";
+pub const OUTCOME_LABEL: &str = "outcome";
+
+pub static TIER_REQUESTS_SUCCESS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::TierRequestsSuccess,
+        "Remote tier requests the backend acknowledged, by operation",
+        &[OPERATION_LABEL],
+        subsystems::TIER,
+    )
+});
+
+pub static TIER_REQUESTS_FAILURE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::TierRequestsFailure,
+        "Remote tier requests that did not complete, by operation and outcome",
+        &[OPERATION_LABEL, OUTCOME_LABEL],
+        subsystems::TIER,
+    )
+});

crates/obs/src/metrics/stats_collector.rs

@@ -27,7 +27,7 @@ use crate::metrics::collectors::{
     DriveDetailedStats, DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats,
     IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats,
     OdmBackfillRuntimeStats, OnDemandMigrationBucketStats, ProcessStats, ProcessStatusType, ReplicationMetricsSnapshot,
-    ResourceStats, ScannerRuntimeStats, ScannerStats,
+    ResourceStats, ScannerRuntimeStats, ScannerStats, TierRequestStats,
 };
 use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
 use crate::metrics::{
@@ -1396,6 +1396,23 @@ fn ilm_backpressure_stats(metrics: &ScannerMetricsReport) -> Vec<IlmBackpressure
     ]
 }
 
+/// Collect the remote tier request counters from the lifecycle runtime.
+///
+/// Every operation/outcome cell is reported, including zero ones, so the
+/// series set is stable from the first scrape rather than appearing one label
+/// combination at a time.
+pub fn collect_tier_request_metric_stats() -> Vec<TierRequestStats> {
+    global_metrics()
+        .tier_request_counts()
+        .into_iter()
+        .map(|count| TierRequestStats {
+            operation: count.operation.as_label(),
+            outcome: count.outcome.as_label(),
+            count: count.count,
+        })
+        .collect()
+}
+
 /// Collect ILM metrics from the current lifecycle runtime state.
 pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
     collect_ilm_runtime_metric_stats().await.map(|stats| stats.stats)

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1482,6 +1482,25 @@ pub struct LoadTransitionTierConfigResponse {
     #[prost(enumeration = "ControlPlaneErrorCode", optional, tag = "3")]
     pub error_code: ::core::option::Option<i32>,
 }
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct TierDailyStatsRequest {}
+/// One node's own rolling-day transition counters, per remote tier.
+///
+/// `tier_daily_stats` is a msgpack map of tier name to the responder's 24-bin
+/// ring plus the clock that ring was last aged to. A node counts only the
+/// transitions it completed itself, so a caller sums the rings of every node to
+/// obtain a cluster total. A peer that does not implement this RPC answers
+/// UNIMPLEMENTED, which the caller reports as a non-reporting node rather than
+/// as zero activity.
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct TierDailyStatsResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(bytes = "bytes", tag = "2")]
+    pub tier_daily_stats: ::prost::bytes::Bytes,
+    #[prost(string, optional, tag = "3")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct TierMutationPrepareRequest {
     #[prost(uint32, tag = "1")]
@@ -3075,6 +3094,21 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "LoadTransitionTierConfig"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn tier_daily_stats(
+            &mut self,
+            request: impl tonic::IntoRequest<super::TierDailyStatsRequest>,
+        ) -> std::result::Result<tonic::Response<super::TierDailyStatsResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/TierDailyStats");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "TierDailyStats"));
+            self.inner.unary(req, path, codec).await
+        }
         pub async fn get_live_events(
             &mut self,
             request: impl tonic::IntoRequest<super::GetLiveEventsRequest>,
@@ -3475,6 +3509,10 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::LoadTransitionTierConfigRequest>,
         ) -> std::result::Result<tonic::Response<super::LoadTransitionTierConfigResponse>, tonic::Status>;
+        async fn tier_daily_stats(
+            &self,
+            request: tonic::Request<super::TierDailyStatsRequest>,
+        ) -> std::result::Result<tonic::Response<super::TierDailyStatsResponse>, tonic::Status>;
         async fn get_live_events(
             &self,
             request: tonic::Request<super::GetLiveEventsRequest>,
@@ -6080,6 +6118,34 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
+                "/node_service.NodeService/TierDailyStats" => {
+                    #[allow(non_camel_case_types)]
+                    struct TierDailyStatsSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::TierDailyStatsRequest> for TierDailyStatsSvc<T> {
+                        type Response = super::TierDailyStatsResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::TierDailyStatsRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::tier_daily_stats(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = TierDailyStatsSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/node_service.NodeService/GetLiveEvents" => {
                     #[allow(non_camel_case_types)]
                     struct GetLiveEventsSvc<T: NodeService>(pub Arc<T>);

crates/protos/src/node.proto

@@ -1043,6 +1043,22 @@ message LoadTransitionTierConfigResponse {
   optional ControlPlaneErrorCode error_code = 3;
 }
 
+message TierDailyStatsRequest {}
+
+// One node's own rolling-day transition counters, per remote tier.
+//
+// `tier_daily_stats` is a msgpack map of tier name to the responder's 24-bin
+// ring plus the clock that ring was last aged to. A node counts only the
+// transitions it completed itself, so a caller sums the rings of every node to
+// obtain a cluster total. A peer that does not implement this RPC answers
+// UNIMPLEMENTED, which the caller reports as a non-reporting node rather than
+// as zero activity.
+message TierDailyStatsResponse {
+  bool success = 1;
+  bytes tier_daily_stats = 2;
+  optional string error_info = 3;
+}
+
 message TierMutationPrepareRequest {
   uint32 version = 1;
   string mutation_id = 2;
@@ -1203,6 +1219,7 @@ service NodeService {
   rpc CancelDecommission(CancelDecommissionRequest) returns (CancelDecommissionResponse) {}; // auth-policy: body-bound
   rpc ClearDecommission(ClearDecommissionRequest) returns (ClearDecommissionResponse) {}; // auth-policy: body-bound
   rpc LoadTransitionTierConfig(LoadTransitionTierConfigRequest) returns (LoadTransitionTierConfigResponse) {}; // auth-policy: body-bound
+  rpc TierDailyStats(TierDailyStatsRequest) returns (TierDailyStatsResponse) {}; // auth-policy: read-only
   rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only
 }
 

crates/scanner-metrics/src/metrics.rs

@@ -798,6 +798,88 @@ struct ScannerActiveBucketDriveValue {
 // Metrics
 // ---------------------------------------------------------------------------
 
+/// A remote-tier request a warm backend issues on the cluster's behalf.
+///
+/// The set is closed on purpose: these values become metric labels, so a new
+/// operation is a deliberate schema change rather than something a call site
+/// can invent.
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
+pub enum TierRequestOperation {
+    Put,
+    Get,
+    Remove,
+    Probe,
+    InUse,
+}
+
+impl TierRequestOperation {
+    pub const ALL: [Self; 5] = [Self::Put, Self::Get, Self::Remove, Self::Probe, Self::InUse];
+
+    pub const fn as_label(self) -> &'static str {
+        match self {
+            Self::Put => "put",
+            Self::Get => "get",
+            Self::Remove => "remove",
+            Self::Probe => "probe",
+            Self::InUse => "in_use",
+        }
+    }
+
+    const fn index(self) -> usize {
+        self as usize
+    }
+}
+
+/// How a remote-tier request ended.
+///
+/// This classifies the request only. A transition whose remote PUT succeeded
+/// and whose local commit then failed is a `Success` here: the remote service
+/// did perform the request, and reporting it as a tier failure would hide a
+/// leaked remote object behind an apparent backend outage.
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
+pub enum TierRequestOutcome {
+    Success,
+    BackendError,
+    Timeout,
+    Cancelled,
+}
+
+impl TierRequestOutcome {
+    pub const ALL: [Self; 4] = [Self::Success, Self::BackendError, Self::Timeout, Self::Cancelled];
+
+    pub const fn as_label(self) -> &'static str {
+        match self {
+            Self::Success => "success",
+            Self::BackendError => "backend_error",
+            Self::Timeout => "timeout",
+            Self::Cancelled => "cancelled",
+        }
+    }
+
+    const fn index(self) -> usize {
+        self as usize
+    }
+
+    /// Classify a warm backend error without letting its message into a label.
+    pub fn from_error(err: &std::io::Error) -> Self {
+        match err.kind() {
+            std::io::ErrorKind::TimedOut => Self::Timeout,
+            std::io::ErrorKind::Interrupted => Self::Cancelled,
+            _ => Self::BackendError,
+        }
+    }
+}
+
+/// One fixed operation/outcome cell of the tier request counters.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct TierRequestCount {
+    pub operation: TierRequestOperation,
+    pub outcome: TierRequestOutcome,
+    pub count: u64,
+}
+
+const TIER_REQUEST_COUNTER_SLOTS: usize = TierRequestOperation::ALL.len() * TierRequestOutcome::ALL.len();
+
 pub struct Metrics {
     operations: Vec<AtomicU64>,
     latency: Vec<LockedLastMinuteLatency>,
@@ -892,6 +974,10 @@ pub struct Metrics {
     scanner_transition_queued_total: AtomicU64,
     scanner_transition_missed_total: AtomicU64,
     scanner_transition_completed: AtomicU64,
+    /// Remote tier requests by operation and outcome. The width is fixed by
+    /// the two label enums, so the series count cannot grow with tier names,
+    /// endpoints or object keys.
+    tier_requests: [AtomicU64; TIER_REQUEST_COUNTER_SLOTS],
     scanner_transition_failed: AtomicU64,
     scanner_throttle_idle_mode_enabled: AtomicBool,
     scanner_throttle_sleep_factor_micros: AtomicU64,
@@ -1943,6 +2029,7 @@ impl Metrics {
             scanner_transition_queued_total: AtomicU64::new(0),
             scanner_transition_missed_total: AtomicU64::new(0),
             scanner_transition_completed: AtomicU64::new(0),
+            tier_requests: std::array::from_fn(|_| AtomicU64::new(0)),
             scanner_transition_failed: AtomicU64::new(0),
             scanner_throttle_idle_mode_enabled: AtomicBool::new(false),
             scanner_throttle_sleep_factor_micros: AtomicU64::new(0),
@@ -2214,6 +2301,33 @@ impl Metrics {
             .store(state.compensation_running, Ordering::Relaxed);
     }
 
+    /// Count one completed remote tier request.
+    ///
+    /// Called once per request at the single point where every warm backend
+    /// call returns, so a retried request is counted once per attempt and a
+    /// successful attempt is never also counted as a failure.
+    pub fn record_tier_request(&self, operation: TierRequestOperation, outcome: TierRequestOutcome) {
+        let slot = operation.index() * TierRequestOutcome::ALL.len() + outcome.index();
+        self.tier_requests[slot].fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Every operation/outcome cell, including the zero ones, so a scrape has
+    /// a stable series set from the first request onward.
+    pub fn tier_request_counts(&self) -> Vec<TierRequestCount> {
+        let mut counts = Vec::with_capacity(TIER_REQUEST_COUNTER_SLOTS);
+        for operation in TierRequestOperation::ALL {
+            for outcome in TierRequestOutcome::ALL {
+                let slot = operation.index() * TierRequestOutcome::ALL.len() + outcome.index();
+                counts.push(TierRequestCount {
+                    operation,
+                    outcome,
+                    count: self.tier_requests[slot].load(Ordering::Relaxed),
+                });
+            }
+        }
+        counts
+    }
+
     pub fn record_scanner_transition_completed(&self, count: u64) {
         self.scanner_transition_completed.fetch_add(count, Ordering::Relaxed);
     }
@@ -3549,6 +3663,76 @@ impl Drop for CloseDiskGuard {
 mod tests {
     use super::*;
 
+    fn tier_cell(metrics: &Metrics, operation: TierRequestOperation, outcome: TierRequestOutcome) -> u64 {
+        metrics
+            .tier_request_counts()
+            .into_iter()
+            .find(|count| count.operation == operation && count.outcome == outcome)
+            .map(|count| count.count)
+            .expect("every operation/outcome cell must be reported")
+    }
+
+    #[test]
+    fn a_tier_request_lands_in_exactly_one_operation_outcome_cell() {
+        let metrics = Metrics::default();
+        metrics.record_tier_request(TierRequestOperation::Put, TierRequestOutcome::Success);
+
+        assert_eq!(tier_cell(&metrics, TierRequestOperation::Put, TierRequestOutcome::Success), 1);
+        assert_eq!(
+            metrics
+                .tier_request_counts()
+                .into_iter()
+                .map(|count| count.count)
+                .sum::<u64>(),
+            1,
+            "one request must not increment a second cell"
+        );
+    }
+
+    #[test]
+    fn every_operation_outcome_pair_has_its_own_cell() {
+        let metrics = Metrics::default();
+        for operation in TierRequestOperation::ALL {
+            for outcome in TierRequestOutcome::ALL {
+                metrics.record_tier_request(operation, outcome);
+            }
+        }
+
+        let counts = metrics.tier_request_counts();
+        assert_eq!(
+            counts.len(),
+            TierRequestOperation::ALL.len() * TierRequestOutcome::ALL.len(),
+            "the series set is fixed by the two label enums"
+        );
+        assert!(
+            counts.iter().all(|count| count.count == 1),
+            "two different pairs must not share a counter slot"
+        );
+    }
+
+    #[test]
+    fn a_backend_error_is_classified_without_reading_its_message() {
+        assert_eq!(
+            TierRequestOutcome::from_error(&std::io::Error::new(std::io::ErrorKind::TimedOut, "s3.example.com:9000 timed out")),
+            TierRequestOutcome::Timeout
+        );
+        assert_eq!(
+            TierRequestOutcome::from_error(&std::io::Error::new(std::io::ErrorKind::Interrupted, "cancelled")),
+            TierRequestOutcome::Cancelled
+        );
+        assert_eq!(
+            TierRequestOutcome::from_error(&std::io::Error::other("AccessDenied")),
+            TierRequestOutcome::BackendError
+        );
+    }
+
+    #[test]
+    fn tier_request_labels_are_stable_identifiers() {
+        assert_eq!(TierRequestOperation::Put.as_label(), "put");
+        assert_eq!(TierRequestOperation::InUse.as_label(), "in_use");
+        assert_eq!(TierRequestOutcome::BackendError.as_label(), "backend_error");
+    }
+
     #[test]
     fn scanner_metrics_report_timestamps_serialize_as_rfc3339_utc() {
         let report = ScannerMetricsReport {