From fc0de983d83b66eaf85ec736dfc8ba795e47e2d8 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 6 Aug 2026 21:33:17 +0800 Subject: [PATCH] perf: add RPC auth profiling diagnostics (#5775) perf: add rpc auth profiling diagnostics Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/ecstore/src/api/mod.rs | 2 +- crates/ecstore/src/cluster/rpc/http_auth.rs | 65 +++++++++++++++++ crates/ecstore/src/cluster/rpc/mod.rs | 4 +- rustfs/src/server/http.rs | 19 +++++ rustfs/src/storage/storage_api.rs | 8 ++- rustfs/src/storage_api.rs | 2 +- scripts/run_samply_attach_window.sh | 79 +++++++++++++++++++++ scripts/test_run_samply_attach_window.sh | 71 ++++++++++++++++++ 8 files changed, 244 insertions(+), 6 deletions(-) create mode 100755 scripts/run_samply_attach_window.sh create mode 100755 scripts/test_run_samply_attach_window.sh diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e2f746a54..59bf363f9 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -439,7 +439,7 @@ pub mod rpc { gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, - tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response, + tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap, }; diff --git a/crates/ecstore/src/cluster/rpc/http_auth.rs b/crates/ecstore/src/cluster/rpc/http_auth.rs index b2e230b2c..c06033fd7 100644 --- a/crates/ecstore/src/cluster/rpc/http_auth.rs +++ b/crates/ecstore/src/cluster/rpc/http_auth.rs @@ -847,6 +847,46 @@ pub fn verify_tonic_rpc_signature_with_bootstrap( ) } +pub fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str { + match error.to_string().as_str() { + "Missing RPC audience" => "missing_audience", + "Invalid RPC request path" => "invalid_request_path", + "RPC replay-scoped authentication required" => "replay_scope_required", + "Missing RPC replay scope version" => "missing_replay_scope_version", + "Unsupported RPC replay scope version" => "unsupported_replay_scope_version", + "Missing RPC replay scope signature" => "missing_replay_scope_signature", + "Missing RPC replay scope nonce" => "missing_replay_scope_nonce", + "Invalid RPC replay scope nonce" => "invalid_replay_scope_nonce", + "Missing RPC boot epoch" => "missing_boot_epoch", + "Invalid RPC boot epoch" => "invalid_boot_epoch", + "Invalid RPC replay scope signature" => "invalid_replay_scope_signature", + "RPC boot epoch is stale" => "stale_boot_epoch", + "RPC request replay detected" => "replay_detected", + "RPC replay cache capacity exceeded" => "replay_cache_capacity", + "RPC replay cache unavailable" => "replay_cache_unavailable", + "RPC replay expiry overflow" => "replay_expiry_overflow", + "RPC request timestamp expired after clock regression" => "timestamp_expired_after_clock_regression", + "RPC v2 authentication required" => "v2_required", + "Missing RPC auth version" => "missing_v2_auth_version", + "Unsupported RPC auth version" => "unsupported_v2_auth_version", + "Missing RPC v2 signature" => "missing_v2_signature", + "Invalid RPC v2 signature" => "invalid_v2_signature", + "Missing timestamp header" => "missing_timestamp", + "Invalid timestamp format" => "invalid_timestamp", + "Request timestamp expired" => "timestamp_expired", + "Missing RPC nonce" => "missing_v2_nonce", + "Invalid RPC nonce" => "invalid_v2_nonce", + "Invalid unsigned RPC nonce" => "invalid_unsigned_v2_nonce", + "Missing RPC content SHA-256" => "missing_content_sha256", + "Invalid RPC content SHA-256" => "invalid_content_sha256", + "Missing signature header" => "missing_v1_signature", + "Invalid signature" => "invalid_v1_signature", + "Invalid RPC HMAC key" => "invalid_hmac_key", + message if message.contains(RPC_SECRET_REQUIRED_OPERATOR_MESSAGE) => "missing_rpc_secret", + _ => "unknown", + } +} + fn verify_tonic_rpc_signature_with_policy( audience: &str, path: &str, @@ -1699,6 +1739,31 @@ mod tests { assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err()); } + #[test] + fn tonic_rpc_auth_failure_reason_maps_security_relevant_errors() { + for (message, reason) in [ + ("Invalid RPC v2 signature", "invalid_v2_signature"), + ("RPC replay-scoped authentication required", "replay_scope_required"), + ("Missing RPC replay scope signature", "missing_replay_scope_signature"), + ("RPC boot epoch is stale", "stale_boot_epoch"), + ("RPC request replay detected", "replay_detected"), + ("Request timestamp expired", "timestamp_expired"), + ("Missing RPC content SHA-256", "missing_content_sha256"), + ("Invalid RPC content SHA-256", "invalid_content_sha256"), + ] { + assert_eq!( + tonic_rpc_auth_failure_reason(&std::io::Error::other(message)), + reason, + "message {message:?} should map to a stable low-cardinality reason" + ); + } + } + + #[test] + fn tonic_rpc_auth_failure_reason_falls_back_for_unclassified_errors() { + assert_eq!(tonic_rpc_auth_failure_reason(&std::io::Error::other("opaque failure")), "unknown"); + } + #[test] fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() { ensure_test_rpc_secret(); diff --git a/crates/ecstore/src/cluster/rpc/mod.rs b/crates/ecstore/src/cluster/rpc/mod.rs index 618083eb1..f13af6f09 100644 --- a/crates/ecstore/src/cluster/rpc/mod.rs +++ b/crates/ecstore/src/cluster/rpc/mod.rs @@ -34,8 +34,8 @@ pub use client::{ pub use http_auth::{ TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability, - sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability, - verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, + sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, + verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap, }; diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 0859d99bb..c832e3a92 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -1891,10 +1891,29 @@ fn check_auth(req: Request<()>) -> std::result::Result, Status> { allow_replay_scope_bootstrap, ) .map_err(|e| { + let rpc_path = target.uri.path(); + let rpc_service = rpc_path + .strip_prefix('/') + .and_then(|path| path.split_once('/')) + .map(|(service, _)| service) + .unwrap_or("unknown"); + let peer_addr = req + .extensions() + .get::() + .map(|addr| addr.0.to_string()) + .unwrap_or_else(|| "unknown".to_string()); + let failure_reason = storage::tonic_rpc_auth_failure_reason(&e); error!( event = EVENT_RPC_SIGNATURE_VERIFICATION_FAILED, component = LOG_COMPONENT_SERVER, subsystem = LOG_SUBSYSTEM_HTTP, + failure_reason, + rpc_path, + rpc_service, + rpc_method, + expected_audience = %audience, + peer_addr = %peer_addr, + replay_scope_bootstrap_allowed = allow_replay_scope_bootstrap, error = %e, "RPC signature verification failed" ); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 4b40fe6ca..bbd0c3796 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -498,8 +498,8 @@ pub(crate) mod ecstore_rpc { KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, - tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_canonical_body_digest, - verify_tonic_mutation_body_digest, verify_tonic_rpc_signature_with_bootstrap, + tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_rpc_signature, + verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_signature_with_bootstrap, }; #[cfg(test)] pub(crate) use rustfs_ecstore::api::rpc::{ @@ -1667,6 +1667,10 @@ pub(crate) fn verify_tonic_rpc_signature_with_bootstrap( ecstore_rpc::verify_tonic_rpc_signature_with_bootstrap(audience, path, headers, allow_replay_scope_bootstrap) } +pub(crate) fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str { + ecstore_rpc::tonic_rpc_auth_failure_reason(error) +} + pub(crate) fn tonic_boot_epoch_challenge(headers: &http::HeaderMap) -> std::io::Result> { ecstore_rpc::tonic_boot_epoch_challenge(headers) } diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 6850bc36f..c7b78db98 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -116,7 +116,7 @@ pub(crate) mod server { pub(crate) mod http { pub(crate) use crate::storage::storage_api::{ ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge, - tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap, + tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_tonic_rpc_signature_with_bootstrap, }; pub(crate) fn try_current_local_node_name() -> Option { diff --git a/scripts/run_samply_attach_window.sh b/scripts/run_samply_attach_window.sh new file mode 100755 index 000000000..e281e3fbb --- /dev/null +++ b/scripts/run_samply_attach_window.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -euo pipefail + +PID="" +DURATION_SECS="" +OUTPUT="" +RATE="999" +PRESYMBOLICATE="true" + +usage() { + cat <<'USAGE' +Usage: scripts/run_samply_attach_window.sh --pid --duration-secs --output [options] + +Attach samply to an already-running process for a bounded window and force a +Ctrl+C-style shutdown so samply writes the profile artifact. + +Options: + --pid Existing process id to profile. + --duration-secs Sampling window in seconds. + --output Profile output path. + --rate Sampling rate. Default: 999. + --no-presymbolicate Do not request samply's .syms.json sidecar. + -h, --help Show this help. +USAGE +} + +die() { + echo "error: $*" >&2 + exit 2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --pid) PID="${2:-}"; shift 2 ;; + --duration-secs) DURATION_SECS="${2:-}"; shift 2 ;; + --output) OUTPUT="${2:-}"; shift 2 ;; + --rate) RATE="${2:-}"; shift 2 ;; + --no-presymbolicate) PRESYMBOLICATE="false"; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +[[ "$PID" =~ ^[0-9]+$ ]] || die "--pid must be a positive integer" +[[ "$DURATION_SECS" =~ ^[0-9]+$ && "$DURATION_SECS" -gt 0 ]] || die "--duration-secs must be a positive integer" +[[ "$RATE" =~ ^[0-9]+$ && "$RATE" -gt 0 ]] || die "--rate must be a positive integer" +[[ -n "$OUTPUT" ]] || die "--output is required" +kill -0 "$PID" 2>/dev/null || die "process $PID is not running" +command -v timeout >/dev/null 2>&1 || die "timeout is required" +command -v samply >/dev/null 2>&1 || die "samply is required" + +mkdir -p "$(dirname "$OUTPUT")" + +cmd=(samply record -p "$PID" -r "$RATE" --save-only) +if [[ "$PRESYMBOLICATE" == "true" ]]; then + cmd+=(--unstable-presymbolicate) +fi +cmd+=(-o "$OUTPUT") + +set +e +timeout -s INT --kill-after=10s "${DURATION_SECS}s" "${cmd[@]}" +status=$? +set -e + +case "$status" in + 0|124|130) ;; + *) exit "$status" ;; +esac + +[[ -s "$OUTPUT" ]] || die "samply did not write profile output: $OUTPUT" +if [[ "$PRESYMBOLICATE" == "true" ]]; then + syms_output="${OUTPUT%.gz}.syms.json" + [[ -s "$syms_output" ]] || die "samply did not write symbol sidecar: $syms_output" +fi + +echo "profile=$OUTPUT" +if [[ "$PRESYMBOLICATE" == "true" ]]; then + echo "symbols=${OUTPUT%.gz}.syms.json" +fi diff --git a/scripts/test_run_samply_attach_window.sh b/scripts/test_run_samply_attach_window.sh new file mode 100755 index 000000000..0c5c27011 --- /dev/null +++ b/scripts/test_run_samply_attach_window.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="${SCRIPT_DIR}/run_samply_attach_window.sh" +TMP_DIR="$(mktemp -d)" +cleanup() { + if [[ -n "${TARGET_PID:-}" ]]; then + kill "$TARGET_PID" 2>/dev/null || true + wait "$TARGET_PID" 2>/dev/null || true + fi + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +mkdir -p "$TMP_DIR/bin" +cat >"$TMP_DIR/bin/samply" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +out="" +while [[ $# -gt 0 ]]; do + case "$1" in + -o) out="$2"; shift 2 ;; + *) shift ;; + esac +done +[[ -n "$out" ]] || exit 2 +write_outputs() { + printf '{"profile":"ok"}\n' >"$out" + printf '{"symbols":"ok"}\n' >"${out%.gz}.syms.json" + exit 0 +} +trap write_outputs INT TERM +while true; do sleep 1; done +MOCK +chmod +x "$TMP_DIR/bin/samply" +cat >"$TMP_DIR/bin/timeout" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +while [[ $# -gt 0 ]]; do + case "$1" in + -s) shift 2 ;; + --kill-after=*) shift ;; + *) break ;; + esac +done +shift +out="" +while [[ $# -gt 0 ]]; do + case "$1" in + -o) out="$2"; shift 2 ;; + *) shift ;; + esac +done +[[ -n "$out" ]] || exit 2 +printf '{"profile":"ok"}\n' >"$out" +printf '{"symbols":"ok"}\n' >"${out%.gz}.syms.json" +exit 124 +MOCK +chmod +x "$TMP_DIR/bin/timeout" + +sleep 30 & +TARGET_PID=$! + +OUTPUT="$TMP_DIR/profile.json.gz" +PATH="$TMP_DIR/bin:$PATH" "$RUNNER" --pid "$TARGET_PID" --duration-secs 1 --output "$OUTPUT" --rate 99 >"$TMP_DIR/run.out" + +test -s "$OUTPUT" +test -s "$TMP_DIR/profile.json.syms.json" +grep -qx "profile=$OUTPUT" "$TMP_DIR/run.out" +grep -qx "symbols=$TMP_DIR/profile.json.syms.json" "$TMP_DIR/run.out"