gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-07 05:43:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: add RPC auth profiling diagnostics (#5775)

Browse Source 
perf: add rpc auth profiling diagnostics

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-08-06 21:33:17 +08:00
committed by GitHub
parent e26b869259
commit fc0de983d8
8 changed files with 244 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/api/mod.rs
+1 -1
View File
@@ -439,7 +439,7 @@ pub mod rpc {
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response,
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_rpc_signature, verify_tonic_boot_epoch_response,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
};
crates/ecstore/src/cluster/rpc/http_auth.rs
+65
View File
@@ -847,6 +847,46 @@ pub fn verify_tonic_rpc_signature_with_bootstrap(
)
}
pub fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str {
match error.to_string().as_str() {
"Missing RPC audience" => "missing_audience",
"Invalid RPC request path" => "invalid_request_path",
"RPC replay-scoped authentication required" => "replay_scope_required",
"Missing RPC replay scope version" => "missing_replay_scope_version",
"Unsupported RPC replay scope version" => "unsupported_replay_scope_version",
"Missing RPC replay scope signature" => "missing_replay_scope_signature",
"Missing RPC replay scope nonce" => "missing_replay_scope_nonce",
"Invalid RPC replay scope nonce" => "invalid_replay_scope_nonce",
"Missing RPC boot epoch" => "missing_boot_epoch",
"Invalid RPC boot epoch" => "invalid_boot_epoch",
"Invalid RPC replay scope signature" => "invalid_replay_scope_signature",
"RPC boot epoch is stale" => "stale_boot_epoch",
"RPC request replay detected" => "replay_detected",
"RPC replay cache capacity exceeded" => "replay_cache_capacity",
"RPC replay cache unavailable" => "replay_cache_unavailable",
"RPC replay expiry overflow" => "replay_expiry_overflow",
"RPC request timestamp expired after clock regression" => "timestamp_expired_after_clock_regression",
"RPC v2 authentication required" => "v2_required",
"Missing RPC auth version" => "missing_v2_auth_version",
"Unsupported RPC auth version" => "unsupported_v2_auth_version",
"Missing RPC v2 signature" => "missing_v2_signature",
"Invalid RPC v2 signature" => "invalid_v2_signature",
"Missing timestamp header" => "missing_timestamp",
"Invalid timestamp format" => "invalid_timestamp",
"Request timestamp expired" => "timestamp_expired",
"Missing RPC nonce" => "missing_v2_nonce",
"Invalid RPC nonce" => "invalid_v2_nonce",
"Invalid unsigned RPC nonce" => "invalid_unsigned_v2_nonce",
"Missing RPC content SHA-256" => "missing_content_sha256",
"Invalid RPC content SHA-256" => "invalid_content_sha256",
"Missing signature header" => "missing_v1_signature",
"Invalid signature" => "invalid_v1_signature",
"Invalid RPC HMAC key" => "invalid_hmac_key",
message if message.contains(RPC_SECRET_REQUIRED_OPERATOR_MESSAGE) => "missing_rpc_secret",
_ => "unknown",
}
}
fn verify_tonic_rpc_signature_with_policy(
audience: &str,
path: &str,
@@ -1699,6 +1739,31 @@ mod tests {
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
}
#[test]
fn tonic_rpc_auth_failure_reason_maps_security_relevant_errors() {
for (message, reason) in [
("Invalid RPC v2 signature", "invalid_v2_signature"),
("RPC replay-scoped authentication required", "replay_scope_required"),
("Missing RPC replay scope signature", "missing_replay_scope_signature"),
("RPC boot epoch is stale", "stale_boot_epoch"),
("RPC request replay detected", "replay_detected"),
("Request timestamp expired", "timestamp_expired"),
("Missing RPC content SHA-256", "missing_content_sha256"),
("Invalid RPC content SHA-256", "invalid_content_sha256"),
] {
assert_eq!(
tonic_rpc_auth_failure_reason(&std::io::Error::other(message)),
reason,
"message {message:?} should map to a stable low-cardinality reason"
);
}
}
#[test]
fn tonic_rpc_auth_failure_reason_falls_back_for_unclassified_errors() {
assert_eq!(tonic_rpc_auth_failure_reason(&std::io::Error::other("opaque failure")), "unknown");
}
#[test]
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
ensure_test_rpc_secret();
crates/ecstore/src/cluster/rpc/mod.rs
+2 -2
View File
@@ -34,8 +34,8 @@ pub use client::{
pub use http_auth::{
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
};
rustfs/src/server/http.rs
+19
View File
@@ -1891,10 +1891,29 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
allow_replay_scope_bootstrap,
)
.map_err(|e| {
let rpc_path = target.uri.path();
let rpc_service = rpc_path
.strip_prefix('/')
.and_then(|path| path.split_once('/'))
.map(|(service, _)| service)
.unwrap_or("unknown");
let peer_addr = req
.extensions()
.get::<RemoteAddr>()
.map(|addr| addr.0.to_string())
.unwrap_or_else(|| "unknown".to_string());
let failure_reason = storage::tonic_rpc_auth_failure_reason(&e);
error!(
event = EVENT_RPC_SIGNATURE_VERIFICATION_FAILED,
component = LOG_COMPONENT_SERVER,
subsystem = LOG_SUBSYSTEM_HTTP,
failure_reason,
rpc_path,
rpc_service,
rpc_method,
expected_audience = %audience,
peer_addr = %peer_addr,
replay_scope_bootstrap_allowed = allow_replay_scope_bootstrap,
error = %e,
"RPC signature verification failed"
);
rustfs/src/storage/storage_api.rs
+6 -2
View File
@@ -498,8 +498,8 @@ pub(crate) mod ecstore_rpc {
KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX,
normalize_tonic_rpc_audience, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_signature_with_bootstrap,
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_rpc_signature,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_signature_with_bootstrap,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{
@@ -1667,6 +1667,10 @@ pub(crate) fn verify_tonic_rpc_signature_with_bootstrap(
ecstore_rpc::verify_tonic_rpc_signature_with_bootstrap(audience, path, headers, allow_replay_scope_bootstrap)
}
pub(crate) fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str {
ecstore_rpc::tonic_rpc_auth_failure_reason(error)
}
pub(crate) fn tonic_boot_epoch_challenge(headers: &http::HeaderMap) -> std::io::Result<Option<uuid::Uuid>> {
ecstore_rpc::tonic_boot_epoch_challenge(headers)
}
rustfs/src/storage_api.rs
+1 -1
View File
@@ -116,7 +116,7 @@ pub(crate) mod server {
pub(crate) mod http {
pub(crate) use crate::storage::storage_api::{
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge,
tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap,
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_tonic_rpc_signature_with_bootstrap,
};
pub(crate) fn try_current_local_node_name() -> Option<String> {
scripts/run_samply_attach_window.sh
Executable
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env bash
set -euo pipefail
PID=""
DURATION_SECS=""
OUTPUT=""
RATE="999"
PRESYMBOLICATE="true"
usage() {
cat <<'USAGE'
Usage: scripts/run_samply_attach_window.sh --pid <pid> --duration-secs <n> --output <profile.json.gz> [options]
Attach samply to an already-running process for a bounded window and force a
Ctrl+C-style shutdown so samply writes the profile artifact.
Options:
--pid <pid> Existing process id to profile.
--duration-secs <n> Sampling window in seconds.
--output <path> Profile output path.
--rate <hz> Sampling rate. Default: 999.
--no-presymbolicate Do not request samply's .syms.json sidecar.
-h, --help Show this help.
USAGE
}
die() {
echo "error: $*" >&2
exit 2
}
while [[ $# -gt 0 ]]; do
case "$1" in
--pid) PID="${2:-}"; shift 2 ;;
--duration-secs) DURATION_SECS="${2:-}"; shift 2 ;;
--output) OUTPUT="${2:-}"; shift 2 ;;
--rate) RATE="${2:-}"; shift 2 ;;
--no-presymbolicate) PRESYMBOLICATE="false"; shift ;;
-h|--help) usage; exit 0 ;;
*) die "unknown argument: $1" ;;
esac
done
[[ "$PID" =~ ^[0-9]+$ ]] || die "--pid must be a positive integer"
[[ "$DURATION_SECS" =~ ^[0-9]+$ && "$DURATION_SECS" -gt 0 ]] || die "--duration-secs must be a positive integer"
[[ "$RATE" =~ ^[0-9]+$ && "$RATE" -gt 0 ]] || die "--rate must be a positive integer"
[[ -n "$OUTPUT" ]] || die "--output is required"
kill -0 "$PID" 2>/dev/null || die "process $PID is not running"
command -v timeout >/dev/null 2>&1 || die "timeout is required"
command -v samply >/dev/null 2>&1 || die "samply is required"
mkdir -p "$(dirname "$OUTPUT")"
cmd=(samply record -p "$PID" -r "$RATE" --save-only)
if [[ "$PRESYMBOLICATE" == "true" ]]; then
cmd+=(--unstable-presymbolicate)
fi
cmd+=(-o "$OUTPUT")
set +e
timeout -s INT --kill-after=10s "${DURATION_SECS}s" "${cmd[@]}"
status=$?
set -e
case "$status" in
0|124|130) ;;
*) exit "$status" ;;
esac
[[ -s "$OUTPUT" ]] || die "samply did not write profile output: $OUTPUT"
if [[ "$PRESYMBOLICATE" == "true" ]]; then
syms_output="${OUTPUT%.gz}.syms.json"
[[ -s "$syms_output" ]] || die "samply did not write symbol sidecar: $syms_output"
fi
echo "profile=$OUTPUT"
if [[ "$PRESYMBOLICATE" == "true" ]]; then
echo "symbols=${OUTPUT%.gz}.syms.json"
fi
scripts/test_run_samply_attach_window.sh
Executable
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RUNNER="${SCRIPT_DIR}/run_samply_attach_window.sh"
TMP_DIR="$(mktemp -d)"
cleanup() {
if [[ -n "${TARGET_PID:-}" ]]; then
kill "$TARGET_PID" 2>/dev/null || true
wait "$TARGET_PID" 2>/dev/null || true
fi
rm -rf "$TMP_DIR"
}
trap cleanup EXIT
mkdir -p "$TMP_DIR/bin"
cat >"$TMP_DIR/bin/samply" <<'MOCK'
#!/usr/bin/env bash
set -euo pipefail
out=""
while [[ $# -gt 0 ]]; do
case "$1" in
-o) out="$2"; shift 2 ;;
*) shift ;;
esac
done
[[ -n "$out" ]] || exit 2
write_outputs() {
printf '{"profile":"ok"}\n' >"$out"
printf '{"symbols":"ok"}\n' >"${out%.gz}.syms.json"
exit 0
}
trap write_outputs INT TERM
while true; do sleep 1; done
MOCK
chmod +x "$TMP_DIR/bin/samply"
cat >"$TMP_DIR/bin/timeout" <<'MOCK'
#!/usr/bin/env bash
set -euo pipefail
while [[ $# -gt 0 ]]; do
case "$1" in
-s) shift 2 ;;
--kill-after=*) shift ;;
*) break ;;
esac
done
shift
out=""
while [[ $# -gt 0 ]]; do
case "$1" in
-o) out="$2"; shift 2 ;;
*) shift ;;
esac
done
[[ -n "$out" ]] || exit 2
printf '{"profile":"ok"}\n' >"$out"
printf '{"symbols":"ok"}\n' >"${out%.gz}.syms.json"
exit 124
MOCK
chmod +x "$TMP_DIR/bin/timeout"
sleep 30 &
TARGET_PID=$!
OUTPUT="$TMP_DIR/profile.json.gz"
PATH="$TMP_DIR/bin:$PATH" "$RUNNER" --pid "$TARGET_PID" --duration-secs 1 --output "$OUTPUT" --rate 99 >"$TMP_DIR/run.out"
test -s "$OUTPUT"
test -s "$TMP_DIR/profile.json.syms.json"
grep -qx "profile=$OUTPUT" "$TMP_DIR/run.out"
grep -qx "symbols=$TMP_DIR/profile.json.syms.json" "$TMP_DIR/run.out"