diff --git a/crates/e2e_test/src/distributed/observability_test.rs b/crates/e2e_test/src/distributed/observability_test.rs index 280bb6a46..494f1ce09 100644 --- a/crates/e2e_test/src/distributed/observability_test.rs +++ b/crates/e2e_test/src/distributed/observability_test.rs @@ -184,7 +184,7 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() let (status, metrics_body) = admin_request( &node.url, Method::GET, - "/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true", + "/rustfs/admin/v3/realtime?n=1&by-host=true&by-disk=true", None, &dist.cluster.access_key, &dist.cluster.secret_key, @@ -257,7 +257,7 @@ async fn node_admin_body(dist: &DistCluster, node: usize, path: &str) -> TestRes } async fn http_put_counts(dist: &DistCluster, node: usize) -> TestResult<[u64; 2]> { - let body = node_admin_body(dist, node, "/rustfs/admin/v3/metrics?types=512&by-host=true&n=1").await?; + let body = node_admin_body(dist, node, "/rustfs/admin/v3/realtime?types=512&by-host=true&n=1").await?; let sample: RealtimeMetrics = serde_json::from_str(body.lines().next().ok_or("empty HTTP metrics stream")?)?; assert!(sample.errors.is_empty(), "HTTP metrics returned errors: {:?}", sample.errors); let http = sample.aggregated.http.ok_or("HTTP metrics missing at WARN log level")?; diff --git a/docs/architecture/minio-rustfs-router-compatibility.md b/docs/architecture/minio-rustfs-router-compatibility.md index 806ccc098..5677b78c8 100644 --- a/docs/architecture/minio-rustfs-router-compatibility.md +++ b/docs/architecture/minio-rustfs-router-compatibility.md @@ -41,7 +41,7 @@ Every route asserted in `rustfs/src/admin/route_registration_test.rs` is registe | Pools decommission / cancel / clear | 部分兼容 | `rustfs/src/admin/handlers/pools.rs` returns `NotImplemented` when endpoints are not initialized (single-pool or uninitialized clusters). | | `/v3/top/drives`, `/v3/top/net` | 缺失 | Only `/v3/top/locks` is registered (`rustfs/src/admin/handlers/diagnostics.rs`). | | Bucket / site replication per-object diff | 缺失 | `/v3/replicationmetrics` and site-replication status exist; no diff endpoint. | -| MRF (most-recent-failures) replication metrics breakdown | 缺失 | Only the generic `/v3/metrics` stream and replication metrics wire (`rustfs/src/admin/replication_metrics_wire.rs`). | +| MRF (most-recent-failures) replication metrics breakdown | 缺失 | Only the generic `/v3/realtime` stream and replication metrics wire (`rustfs/src/admin/replication_metrics_wire.rs`). | Formerly-missing families that are now registered and therefore not exceptions: `/v3/healthinfo`, `/v3/obdinfo`, `/v3/force-unlock`, `/v3/top/locks`, `/v3/speedtest*`, `/v3/log`, `/v3/trace`, `/v3/profile`, `/v3/profiling/*`, `/v3/idp/{ldap|openid}/*`, `/v3/idp-config/*`. diff --git a/docs/operations/internode-grpc-benchmark-runbook.md b/docs/operations/internode-grpc-benchmark-runbook.md index 2c2fb6385..ed7095009 100644 --- a/docs/operations/internode-grpc-benchmark-runbook.md +++ b/docs/operations/internode-grpc-benchmark-runbook.md @@ -11,7 +11,7 @@ Every stage is env-gated, so "before" and "after" run the *same binary* with dif | --- | --- | | RPC secret | Internode RPC fails closed: remote endpoints with default credentials and no `RUSTFS_RPC_SECRET` abort startup with `store init aborted: endpoints include remote nodes but ...` (`crates/ecstore/src/store/init.rs`). Set a non-default `RUSTFS_RPC_SECRET`, identical on every node. | | systemd start timeout | `deploy/build/rustfs.service` is `Type=notify` and ships `TimeoutStartSec=120s`; READY fires only after quorum. If freshly purged disks need longer, raise it in a drop-in rather than lowering it. | -| Metrics export | RustFS has no Prometheus pull endpoint (`/admin/v3/metrics` is NDJSON, see `rustfs/src/admin/handlers/metrics.rs`); it pushes OTLP. Run an otel-collector (OTLP receiver → Prometheus exporter) and set `RUSTFS_OBS_ENDPOINT`, `RUSTFS_OBS_METRICS_EXPORT_ENABLED=true`, `RUSTFS_OBS_METER_INTERVAL=5`. For lock p99 also set `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true` (default off). | +| Metrics export | RustFS has no Prometheus pull endpoint (`/admin/v3/realtime` is NDJSON, see `rustfs/src/admin/handlers/realtime.rs`); it pushes OTLP. Run an otel-collector (OTLP receiver → Prometheus exporter) and set `RUSTFS_OBS_ENDPOINT`, `RUSTFS_OBS_METRICS_EXPORT_ENABLED=true`, `RUSTFS_OBS_METER_INTERVAL=5`. For lock p99 also set `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true` (default off). | | Server env | `RUSTFS_INTERNODE_*` are **server** env. For p0/p1/p2, source the emitted `server-env.sh` on every node and restart before the run; the driver cannot mutate a running server. | ## Driver diff --git a/docs/operations/s3-write-failure-diagnostics.md b/docs/operations/s3-write-failure-diagnostics.md index 16d56e788..3e146d000 100644 --- a/docs/operations/s3-write-failure-diagnostics.md +++ b/docs/operations/s3-write-failure-diagnostics.md @@ -20,7 +20,7 @@ sum by (instance) (rate(rustfs_s3_http_requests_total{method="PUT",outcome=~"[1- Inspect `service_error` and `cancelled` separately; neither implies a received HTTP status. A zero denominator or absent series means no observed traffic, not proof of health. Use `rate` or reset-aware deltas because counters restart with the process. The older `rustfs_s3_operations_total` counter measures handler entries and excludes pre-dispatch rejections; it is not this HTTP denominator. -The authenticated admin metrics endpoint exposes the same counters through the optional `http` field in `aggregated` and `by_host`. Request `/rustfs/admin/v3/metrics?types=512&by-host=true&n=1` on each node for HTTP-only data. The default type selection also includes HTTP outcomes. This endpoint remains an NDJSON stream and does not become a cluster-wide peer fanout. `http.requests` contains `method`, `operation`, `outcome`, and `total`; `http.collected` timestamps collection. Compare consecutive samples from the same host. Concurrent snapshots are not atomic across series. +The authenticated admin metrics endpoint exposes the same counters through the optional `http` field in `aggregated` and `by_host`. Request `/rustfs/admin/v3/realtime?types=512&by-host=true&n=1` on each node for HTTP-only data. The default type selection also includes HTTP outcomes. This endpoint remains an NDJSON stream and does not become a cluster-wide peer fanout. `http.requests` contains `method`, `operation`, `outcome`, and `total`; `http.collected` timestamps collection. Compare consecutive samples from the same host. Concurrent snapshots are not atomic across series. A missing `http` field means an older or non-reporting node, not zero failures. Old map-encoded RPC readers ignore the additive field; new readers accept older snapshots. In mixed-version deployments, check reporting coverage before aggregating a fleet-wide ratio. diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index 9d245a641..46684d268 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -317,7 +317,7 @@ Use the harness around a post-start bucket creation workload to cover the timing 1. Start RustFS from an empty data path. 2. Start the harness before creating buckets. 3. Create a bucket, upload objects, and keep the harness running until at least one usage save is observed. -4. Compare `scanner-summary.csv` with `/rustfs/admin/v3/metrics?types=1&n=1` bucket metrics. +4. Compare `scanner-summary.csv` with `/rustfs/admin/v3/realtime?types=1&n=1` bucket metrics. Expected evidence: dirty usage is marked, `life_time_scan_cycle` or `life_time_scan_bucket_drive` advances, `life_time_scan_object` advances for object workloads, and `life_time_save_usage` plus `usage_last_save_result=success` appear before non-zero bucket usage metrics are accepted as fresh. diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index c28288738..1c55d0097 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -481,7 +481,7 @@ Read this snapshot before changing scanner controls: `blocked_source` with `life ## Reading Distributed Metrics -`/rustfs/admin/v3/scanner/status` and `/rustfs/admin/v3/metrics` report the node that handles the HTTP request; the metrics endpoint does not fan out to peers. In distributed deployments, query every node explicitly and keep `by-host=true` so each response includes that node's host view: +`/rustfs/admin/v3/scanner/status` and `/rustfs/admin/v3/realtime` report the node that handles the HTTP request; the metrics endpoint does not fan out to peers. In distributed deployments, query every node explicitly and keep `by-host=true` so each response includes that node's host view: ```bash for endpoint in http://node-a:9000 http://node-b:9000 http://node-c:9000; do @@ -493,7 +493,7 @@ for endpoint in http://node-a:9000 http://node-b:9000 http://node-c:9000; do --access_key "$RUSTFS_ACCESS_KEY" \ --secret_key "$RUSTFS_SECRET_KEY" \ --request GET \ - "${endpoint}/rustfs/admin/v3/metrics?types=1&by-host=true&n=1" \ + "${endpoint}/rustfs/admin/v3/realtime?types=1&by-host=true&n=1" \ > "artifacts/scanner-metrics.${node}.$(date -u +%Y%m%dT%H%M%SZ).ndjson" done ``` diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 45b2f2d67..cf709f22e 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -43,7 +43,6 @@ pub mod kms_key_metadata; pub mod kms_keys; pub mod kms_management; pub mod kms_rekey; -pub mod metrics; pub mod mfa; pub mod module_switch; mod notify_runtime_access; @@ -58,6 +57,7 @@ pub mod pools; pub mod profile; pub mod profile_admin; pub mod quota; +pub mod realtime; pub mod rebalance; pub mod replication; pub mod scanner; @@ -145,7 +145,7 @@ mod tests { let _inspect_data_handler = system::InspectDataHandler {}; let _storage_info_handler = system::StorageInfoHandler {}; let _data_usage_handler = system::DataUsageInfoHandler {}; - let _metrics_handler = metrics::MetricsHandler {}; + let _metrics_handler = realtime::MetricsHandler {}; let _profile_handler = profile_admin::ProfileHandler {}; let _profile_status_handler = profile_admin::ProfileStatusHandler {}; let _tls_status_handler = tls_debug::TlsStatusHandler {}; diff --git a/rustfs/src/admin/handlers/metrics.rs b/rustfs/src/admin/handlers/realtime.rs similarity index 96% rename from rustfs/src/admin/handlers/metrics.rs rename to rustfs/src/admin/handlers/realtime.rs index 298338508..b13f10b8c 100644 --- a/rustfs/src/admin/handlers/metrics.rs +++ b/rustfs/src/admin/handlers/realtime.rs @@ -14,7 +14,7 @@ //! Console realtime metrics API. //! -//! This preserves the console's fixed `/admin/v3/metrics` contract while +//! This preserves the console's fixed `/admin/v3/realtime` contract while //! keeping the response format explicitly NDJSON. It is not a Prometheus text //! exposition endpoint. @@ -302,7 +302,7 @@ mod tests { #[test] fn metrics_params_default_to_single_sample() { - let uri: Uri = "/rustfs/admin/v3/metrics".parse().unwrap(); + let uri: Uri = "/rustfs/admin/v3/realtime".parse().unwrap(); let mp = extract_metrics_init_params(&uri); assert_eq!(resolve_sample_count(&mp), DEFAULT_METRICS_SAMPLES); @@ -310,7 +310,7 @@ mod tests { #[test] fn metrics_params_treat_zero_as_single_sample() { - let uri: Uri = "/rustfs/admin/v3/metrics?n=0".parse().unwrap(); + let uri: Uri = "/rustfs/admin/v3/realtime?n=0".parse().unwrap(); let mp = extract_metrics_init_params(&uri); assert_eq!(resolve_sample_count(&mp), DEFAULT_METRICS_SAMPLES); @@ -318,7 +318,7 @@ mod tests { #[test] fn metrics_params_cap_samples_to_safety_limit() { - let uri: Uri = "/rustfs/admin/v3/metrics?n=9999".parse().unwrap(); + let uri: Uri = "/rustfs/admin/v3/realtime?n=9999".parse().unwrap(); let mp = extract_metrics_init_params(&uri); assert_eq!(resolve_sample_count(&mp), MAX_METRICS_SAMPLES); @@ -332,7 +332,7 @@ mod tests { #[tokio::test] async fn metrics_handler_rejects_missing_credentials() { let result = MetricsHandler {} - .call(build_metrics_request("/rustfs/admin/v3/metrics"), Params::new()) + .call(build_metrics_request("/rustfs/admin/v3/realtime"), Params::new()) .await; let err = match result { Ok(_) => panic!("metrics handler must reject unauthenticated requests"), diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index 8b8ea09bd..81b1a2df8 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{cluster_snapshot, metrics}; +use super::{cluster_snapshot, realtime}; use crate::admin::auth::authorize_admin_request; use crate::admin::handlers::account::{ACCOUNT_INFO_ROUTE, ACCOUNT_PASSWORD_ROUTE}; use crate::admin::handlers::mfa::{ACCOUNT_MFA_ROUTE, MFA_CHALLENGE_ROUTE, USER_MFA_ROUTE}; @@ -167,8 +167,8 @@ pub fn register_system_route(r: &mut S3Router) -> std::io::Resul r.insert( Method::GET, - format!("{}{}", ADMIN_PREFIX, "/v3/metrics").as_str(), - AdminOperation(&metrics::MetricsHandler {}), + format!("{}{}", ADMIN_PREFIX, "/v3/realtime").as_str(), + AdminOperation(&realtime::MetricsHandler {}), )?; r.insert( diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 8080aa379..54cd46e4f 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -320,7 +320,7 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ ), admin(HttpMethod::Get, "/rustfs/admin/v3/info", SERVER_INFO, RouteRiskLevel::Sensitive), admin(HttpMethod::Get, "/rustfs/admin/v3/storageinfo", STORAGE_INFO, RouteRiskLevel::Sensitive), - admin(HttpMethod::Get, "/rustfs/admin/v3/metrics", GET_METRICS, RouteRiskLevel::Sensitive), + admin(HttpMethod::Get, "/rustfs/admin/v3/realtime", GET_METRICS, RouteRiskLevel::Sensitive), admin( HttpMethod::Get, "/rustfs/admin/v3/object-data-cache/stats", @@ -2303,7 +2303,7 @@ mod tests { #[test] fn route_policy_maps_metrics_to_explicit_admin_action() { - assert_action(HttpMethod::Get, "/rustfs/admin/v3/metrics", GET_METRICS); + assert_action(HttpMethod::Get, "/rustfs/admin/v3/realtime", GET_METRICS); } #[test] diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 496c93436..b4be4fc4b 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -190,7 +190,7 @@ fn expected_admin_route_matrix() -> Vec { admin_route(Method::GET, "/v3/storageinfo"), admin_route(Method::GET, "/v3/datausageinfo"), admin_route_sample(Method::GET, "/v3/usage/{bucket}", "/v3/usage/test-bucket"), - admin_route(Method::GET, "/v3/metrics"), + admin_route(Method::GET, "/v3/realtime"), admin_route(Method::GET, "/v3/object-data-cache/stats"), admin_route(Method::POST, "/v3/object-data-cache/flush"), admin_route(Method::GET, "/v3/pools/list"), @@ -1333,7 +1333,7 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::POST, &admin_path("/v3/service")); assert_route(&router, Method::GET, &admin_path("/v3/info")); assert_route(&router, Method::GET, &admin_path("/v3/storageinfo")); - assert_route(&router, Method::GET, &admin_path("/v3/metrics")); + assert_route(&router, Method::GET, &admin_path("/v3/realtime")); assert_route(&router, Method::GET, &admin_path("/v3/pools/list")); assert_route(&router, Method::GET, &admin_path("/v3/decommission/status")); diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index f0e9f484e..fc04368ed 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -2538,7 +2538,7 @@ mod tests { assert!(futures::poll!(cancelled.as_mut()).is_pending()); drop(cancelled); for path in [ - "/rustfs/admin/v3/metrics", + "/rustfs/admin/v3/realtime", "/minio/admin/v3/storageinfo", CONSOLE_PREFIX, "/rustfs/rpc/test", diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index d6f1a600f..7b6a2e3af 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -2277,7 +2277,7 @@ fi rustfs/src/admin/router.rs \ rustfs/src/admin/console.rs \ rustfs/src/admin/handlers/heal.rs \ - rustfs/src/admin/handlers/metrics.rs \ + rustfs/src/admin/handlers/realtime.rs \ rustfs/src/admin/handlers/object_zip_download.rs || true ) >"$RUSTFS_APP_ADMIN_STORAGE_API_BYPASS_HITS_FILE"