fix: rename admin metrics endpoint to realtime (#8046)

This commit is contained in:
cxymds
2026-09-21 19:23:58 +08:00
committed by GitHub
parent 077b961232
commit cd2fa50e50
13 changed files with 24 additions and 24 deletions
@@ -184,7 +184,7 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
let (status, metrics_body) = admin_request(
&node.url,
Method::GET,
"/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
"/rustfs/admin/v3/realtime?n=1&by-host=true&by-disk=true",
None,
&dist.cluster.access_key,
&dist.cluster.secret_key,
@@ -257,7 +257,7 @@ async fn node_admin_body(dist: &DistCluster, node: usize, path: &str) -> TestRes
}
async fn http_put_counts(dist: &DistCluster, node: usize) -> TestResult<[u64; 2]> {
let body = node_admin_body(dist, node, "/rustfs/admin/v3/metrics?types=512&by-host=true&n=1").await?;
let body = node_admin_body(dist, node, "/rustfs/admin/v3/realtime?types=512&by-host=true&n=1").await?;
let sample: RealtimeMetrics = serde_json::from_str(body.lines().next().ok_or("empty HTTP metrics stream")?)?;
assert!(sample.errors.is_empty(), "HTTP metrics returned errors: {:?}", sample.errors);
let http = sample.aggregated.http.ok_or("HTTP metrics missing at WARN log level")?;
@@ -41,7 +41,7 @@ Every route asserted in `rustfs/src/admin/route_registration_test.rs` is registe
| Pools decommission / cancel / clear | 部分兼容 | `rustfs/src/admin/handlers/pools.rs` returns `NotImplemented` when endpoints are not initialized (single-pool or uninitialized clusters). |
| `/v3/top/drives`, `/v3/top/net` | 缺失 | Only `/v3/top/locks` is registered (`rustfs/src/admin/handlers/diagnostics.rs`). |
| Bucket / site replication per-object diff | 缺失 | `/v3/replicationmetrics` and site-replication status exist; no diff endpoint. |
| MRF (most-recent-failures) replication metrics breakdown | 缺失 | Only the generic `/v3/metrics` stream and replication metrics wire (`rustfs/src/admin/replication_metrics_wire.rs`). |
| MRF (most-recent-failures) replication metrics breakdown | 缺失 | Only the generic `/v3/realtime` stream and replication metrics wire (`rustfs/src/admin/replication_metrics_wire.rs`). |
Formerly-missing families that are now registered and therefore not exceptions: `/v3/healthinfo`, `/v3/obdinfo`, `/v3/force-unlock`, `/v3/top/locks`, `/v3/speedtest*`, `/v3/log`, `/v3/trace`, `/v3/profile`, `/v3/profiling/*`, `/v3/idp/{ldap|openid}/*`, `/v3/idp-config/*`.
@@ -11,7 +11,7 @@ Every stage is env-gated, so "before" and "after" run the *same binary* with dif
| --- | --- |
| RPC secret | Internode RPC fails closed: remote endpoints with default credentials and no `RUSTFS_RPC_SECRET` abort startup with `store init aborted: endpoints include remote nodes but ...` (`crates/ecstore/src/store/init.rs`). Set a non-default `RUSTFS_RPC_SECRET`, identical on every node. |
| systemd start timeout | `deploy/build/rustfs.service` is `Type=notify` and ships `TimeoutStartSec=120s`; READY fires only after quorum. If freshly purged disks need longer, raise it in a drop-in rather than lowering it. |
| Metrics export | RustFS has no Prometheus pull endpoint (`/admin/v3/metrics` is NDJSON, see `rustfs/src/admin/handlers/metrics.rs`); it pushes OTLP. Run an otel-collector (OTLP receiver → Prometheus exporter) and set `RUSTFS_OBS_ENDPOINT`, `RUSTFS_OBS_METRICS_EXPORT_ENABLED=true`, `RUSTFS_OBS_METER_INTERVAL=5`. For lock p99 also set `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true` (default off). |
| Metrics export | RustFS has no Prometheus pull endpoint (`/admin/v3/realtime` is NDJSON, see `rustfs/src/admin/handlers/realtime.rs`); it pushes OTLP. Run an otel-collector (OTLP receiver → Prometheus exporter) and set `RUSTFS_OBS_ENDPOINT`, `RUSTFS_OBS_METRICS_EXPORT_ENABLED=true`, `RUSTFS_OBS_METER_INTERVAL=5`. For lock p99 also set `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true` (default off). |
| Server env | `RUSTFS_INTERNODE_*` are **server** env. For p0/p1/p2, source the emitted `server-env.sh` on every node and restart before the run; the driver cannot mutate a running server. |
## Driver
@@ -20,7 +20,7 @@ sum by (instance) (rate(rustfs_s3_http_requests_total{method="PUT",outcome=~"[1-
Inspect `service_error` and `cancelled` separately; neither implies a received HTTP status. A zero denominator or absent series means no observed traffic, not proof of health. Use `rate` or reset-aware deltas because counters restart with the process. The older `rustfs_s3_operations_total` counter measures handler entries and excludes pre-dispatch rejections; it is not this HTTP denominator.
The authenticated admin metrics endpoint exposes the same counters through the optional `http` field in `aggregated` and `by_host`. Request `/rustfs/admin/v3/metrics?types=512&by-host=true&n=1` on each node for HTTP-only data. The default type selection also includes HTTP outcomes. This endpoint remains an NDJSON stream and does not become a cluster-wide peer fanout. `http.requests` contains `method`, `operation`, `outcome`, and `total`; `http.collected` timestamps collection. Compare consecutive samples from the same host. Concurrent snapshots are not atomic across series.
The authenticated admin metrics endpoint exposes the same counters through the optional `http` field in `aggregated` and `by_host`. Request `/rustfs/admin/v3/realtime?types=512&by-host=true&n=1` on each node for HTTP-only data. The default type selection also includes HTTP outcomes. This endpoint remains an NDJSON stream and does not become a cluster-wide peer fanout. `http.requests` contains `method`, `operation`, `outcome`, and `total`; `http.collected` timestamps collection. Compare consecutive samples from the same host. Concurrent snapshots are not atomic across series.
A missing `http` field means an older or non-reporting node, not zero failures. Old map-encoded RPC readers ignore the additive field; new readers accept older snapshots. In mixed-version deployments, check reporting coverage before aggregating a fleet-wide ratio.
+1 -1
View File
@@ -317,7 +317,7 @@ Use the harness around a post-start bucket creation workload to cover the timing
1. Start RustFS from an empty data path.
2. Start the harness before creating buckets.
3. Create a bucket, upload objects, and keep the harness running until at least one usage save is observed.
4. Compare `scanner-summary.csv` with `/rustfs/admin/v3/metrics?types=1&n=1` bucket metrics.
4. Compare `scanner-summary.csv` with `/rustfs/admin/v3/realtime?types=1&n=1` bucket metrics.
Expected evidence: dirty usage is marked, `life_time_scan_cycle` or `life_time_scan_bucket_drive` advances, `life_time_scan_object` advances for object workloads, and `life_time_save_usage` plus `usage_last_save_result=success` appear before non-zero bucket usage metrics are accepted as fresh.
+2 -2
View File
@@ -481,7 +481,7 @@ Read this snapshot before changing scanner controls: `blocked_source` with `life
## Reading Distributed Metrics
`/rustfs/admin/v3/scanner/status` and `/rustfs/admin/v3/metrics` report the node that handles the HTTP request; the metrics endpoint does not fan out to peers. In distributed deployments, query every node explicitly and keep `by-host=true` so each response includes that node's host view:
`/rustfs/admin/v3/scanner/status` and `/rustfs/admin/v3/realtime` report the node that handles the HTTP request; the metrics endpoint does not fan out to peers. In distributed deployments, query every node explicitly and keep `by-host=true` so each response includes that node's host view:
```bash
for endpoint in http://node-a:9000 http://node-b:9000 http://node-c:9000; do
@@ -493,7 +493,7 @@ for endpoint in http://node-a:9000 http://node-b:9000 http://node-c:9000; do
--access_key "$RUSTFS_ACCESS_KEY" \
--secret_key "$RUSTFS_SECRET_KEY" \
--request GET \
"${endpoint}/rustfs/admin/v3/metrics?types=1&by-host=true&n=1" \
"${endpoint}/rustfs/admin/v3/realtime?types=1&by-host=true&n=1" \
> "artifacts/scanner-metrics.${node}.$(date -u +%Y%m%dT%H%M%SZ).ndjson"
done
```
+2 -2
View File
@@ -43,7 +43,6 @@ pub mod kms_key_metadata;
pub mod kms_keys;
pub mod kms_management;
pub mod kms_rekey;
pub mod metrics;
pub mod mfa;
pub mod module_switch;
mod notify_runtime_access;
@@ -58,6 +57,7 @@ pub mod pools;
pub mod profile;
pub mod profile_admin;
pub mod quota;
pub mod realtime;
pub mod rebalance;
pub mod replication;
pub mod scanner;
@@ -145,7 +145,7 @@ mod tests {
let _inspect_data_handler = system::InspectDataHandler {};
let _storage_info_handler = system::StorageInfoHandler {};
let _data_usage_handler = system::DataUsageInfoHandler {};
let _metrics_handler = metrics::MetricsHandler {};
let _metrics_handler = realtime::MetricsHandler {};
let _profile_handler = profile_admin::ProfileHandler {};
let _profile_status_handler = profile_admin::ProfileStatusHandler {};
let _tls_status_handler = tls_debug::TlsStatusHandler {};
@@ -14,7 +14,7 @@
//! Console realtime metrics API.
//!
//! This preserves the console's fixed `/admin/v3/metrics` contract while
//! This preserves the console's fixed `/admin/v3/realtime` contract while
//! keeping the response format explicitly NDJSON. It is not a Prometheus text
//! exposition endpoint.
@@ -302,7 +302,7 @@ mod tests {
#[test]
fn metrics_params_default_to_single_sample() {
let uri: Uri = "/rustfs/admin/v3/metrics".parse().unwrap();
let uri: Uri = "/rustfs/admin/v3/realtime".parse().unwrap();
let mp = extract_metrics_init_params(&uri);
assert_eq!(resolve_sample_count(&mp), DEFAULT_METRICS_SAMPLES);
@@ -310,7 +310,7 @@ mod tests {
#[test]
fn metrics_params_treat_zero_as_single_sample() {
let uri: Uri = "/rustfs/admin/v3/metrics?n=0".parse().unwrap();
let uri: Uri = "/rustfs/admin/v3/realtime?n=0".parse().unwrap();
let mp = extract_metrics_init_params(&uri);
assert_eq!(resolve_sample_count(&mp), DEFAULT_METRICS_SAMPLES);
@@ -318,7 +318,7 @@ mod tests {
#[test]
fn metrics_params_cap_samples_to_safety_limit() {
let uri: Uri = "/rustfs/admin/v3/metrics?n=9999".parse().unwrap();
let uri: Uri = "/rustfs/admin/v3/realtime?n=9999".parse().unwrap();
let mp = extract_metrics_init_params(&uri);
assert_eq!(resolve_sample_count(&mp), MAX_METRICS_SAMPLES);
@@ -332,7 +332,7 @@ mod tests {
#[tokio::test]
async fn metrics_handler_rejects_missing_credentials() {
let result = MetricsHandler {}
.call(build_metrics_request("/rustfs/admin/v3/metrics"), Params::new())
.call(build_metrics_request("/rustfs/admin/v3/realtime"), Params::new())
.await;
let err = match result {
Ok(_) => panic!("metrics handler must reject unauthenticated requests"),
+3 -3
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::{cluster_snapshot, metrics};
use super::{cluster_snapshot, realtime};
use crate::admin::auth::authorize_admin_request;
use crate::admin::handlers::account::{ACCOUNT_INFO_ROUTE, ACCOUNT_PASSWORD_ROUTE};
use crate::admin::handlers::mfa::{ACCOUNT_MFA_ROUTE, MFA_CHALLENGE_ROUTE, USER_MFA_ROUTE};
@@ -167,8 +167,8 @@ pub fn register_system_route(r: &mut S3Router<AdminOperation>) -> std::io::Resul
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v3/metrics").as_str(),
AdminOperation(&metrics::MetricsHandler {}),
format!("{}{}", ADMIN_PREFIX, "/v3/realtime").as_str(),
AdminOperation(&realtime::MetricsHandler {}),
)?;
r.insert(
+2 -2
View File
@@ -320,7 +320,7 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
),
admin(HttpMethod::Get, "/rustfs/admin/v3/info", SERVER_INFO, RouteRiskLevel::Sensitive),
admin(HttpMethod::Get, "/rustfs/admin/v3/storageinfo", STORAGE_INFO, RouteRiskLevel::Sensitive),
admin(HttpMethod::Get, "/rustfs/admin/v3/metrics", GET_METRICS, RouteRiskLevel::Sensitive),
admin(HttpMethod::Get, "/rustfs/admin/v3/realtime", GET_METRICS, RouteRiskLevel::Sensitive),
admin(
HttpMethod::Get,
"/rustfs/admin/v3/object-data-cache/stats",
@@ -2303,7 +2303,7 @@ mod tests {
#[test]
fn route_policy_maps_metrics_to_explicit_admin_action() {
assert_action(HttpMethod::Get, "/rustfs/admin/v3/metrics", GET_METRICS);
assert_action(HttpMethod::Get, "/rustfs/admin/v3/realtime", GET_METRICS);
}
#[test]
+2 -2
View File
@@ -190,7 +190,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
admin_route(Method::GET, "/v3/storageinfo"),
admin_route(Method::GET, "/v3/datausageinfo"),
admin_route_sample(Method::GET, "/v3/usage/{bucket}", "/v3/usage/test-bucket"),
admin_route(Method::GET, "/v3/metrics"),
admin_route(Method::GET, "/v3/realtime"),
admin_route(Method::GET, "/v3/object-data-cache/stats"),
admin_route(Method::POST, "/v3/object-data-cache/flush"),
admin_route(Method::GET, "/v3/pools/list"),
@@ -1333,7 +1333,7 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::POST, &admin_path("/v3/service"));
assert_route(&router, Method::GET, &admin_path("/v3/info"));
assert_route(&router, Method::GET, &admin_path("/v3/storageinfo"));
assert_route(&router, Method::GET, &admin_path("/v3/metrics"));
assert_route(&router, Method::GET, &admin_path("/v3/realtime"));
assert_route(&router, Method::GET, &admin_path("/v3/pools/list"));
assert_route(&router, Method::GET, &admin_path("/v3/decommission/status"));
+1 -1
View File
@@ -2538,7 +2538,7 @@ mod tests {
assert!(futures::poll!(cancelled.as_mut()).is_pending());
drop(cancelled);
for path in [
"/rustfs/admin/v3/metrics",
"/rustfs/admin/v3/realtime",
"/minio/admin/v3/storageinfo",
CONSOLE_PREFIX,
"/rustfs/rpc/test",
@@ -2277,7 +2277,7 @@ fi
rustfs/src/admin/router.rs \
rustfs/src/admin/console.rs \
rustfs/src/admin/handlers/heal.rs \
rustfs/src/admin/handlers/metrics.rs \
rustfs/src/admin/handlers/realtime.rs \
rustfs/src/admin/handlers/object_zip_download.rs || true
) >"$RUSTFS_APP_ADMIN_STORAGE_API_BYPASS_HITS_FILE"