mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 23:47:28 +00:00
feat(rpc): add replay-scoped internode authentication (#5455)
This commit is contained in:
@@ -158,17 +158,33 @@ pub const DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT: bool = false;
|
|||||||
// rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
|
// rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
|
||||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
|
const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
|
||||||
|
|
||||||
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
/// Require the replay-scoped internode RPC signature after the fleet has converged on it.
|
||||||
/// one-time consumption of body-bound v2 signatures.
|
|
||||||
///
|
///
|
||||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope, so the steady
|
/// The default keeps v1/v2 peers available during a rolling upgrade. Operators may set this only
|
||||||
/// state holds roughly `mutating RPS x 601s` entries; the default sustains ~1,700 body-bound
|
/// after `rustfs_system_network_internode_replay_scope_fallback_total` remains zero for a full
|
||||||
/// mutating RPCs per second (about 120 MiB worst case, allocated only under sustained load).
|
/// release window. The node still accepts a v2-authenticated `Ping` carrying an epoch challenge:
|
||||||
/// Overflow fails closed — legitimate signed traffic is the only thing that can fill the cache
|
/// that narrowly scoped bootstrap lets an upgraded client learn the receiving process epoch and
|
||||||
/// (replays are rejected before insertion, and an attacker cannot mint valid nonces without the
|
/// immediately retry with the replay-scoped signature after a peer restart.
|
||||||
/// shared secret) — and increments
|
pub const ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT: &str = "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT";
|
||||||
|
pub const DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT: bool = false;
|
||||||
|
|
||||||
|
// Compile-time invariant: mixed-version clusters must remain available until operators make the
|
||||||
|
// observed fallback counter an explicit strictness decision.
|
||||||
|
const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
|
||||||
|
|
||||||
|
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
||||||
|
/// one-time consumption of authenticated RPC signatures.
|
||||||
|
///
|
||||||
|
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
|
||||||
|
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
|
||||||
|
/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
|
||||||
|
/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
|
||||||
|
/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
|
||||||
|
/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
||||||
|
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
|
||||||
|
/// the shared secret) — and increments
|
||||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
||||||
/// counter means this capacity is undersized for the node's peak mutation rate.
|
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
|
||||||
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
||||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
||||||
|
|
||||||
@@ -354,6 +370,12 @@ mod tests {
|
|||||||
assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
|
assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn internode_replay_scope_strict_env_name_is_stable() {
|
||||||
|
// The fail-open default invariant is asserted at compile time next to the definition.
|
||||||
|
assert_eq!(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn internode_replay_cache_capacity_defaults_and_env_name() {
|
fn internode_replay_cache_capacity_defaults_and_env_name() {
|
||||||
assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");
|
assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");
|
||||||
|
|||||||
@@ -13,8 +13,9 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
//! Cross-process replay / tamper acceptance for the internode NodeService v2 RPC
|
//! Cross-process replay / tamper acceptance for internode NodeService RPC
|
||||||
//! signature (<https://github.com/rustfs/backlog/issues/1327>).
|
//! signatures (<https://github.com/rustfs/backlog/issues/1327>,
|
||||||
|
//! <https://github.com/rustfs/backlog/issues/1542>).
|
||||||
//!
|
//!
|
||||||
//! # Why this exists on top of the in-process tests
|
//! # Why this exists on top of the in-process tests
|
||||||
//!
|
//!
|
||||||
@@ -78,6 +79,8 @@
|
|||||||
//! | mixed version: legacy-only still served, not blocked | [`legacy_only_signature_is_accepted_in_default_posture`] |
|
//! | mixed version: legacy-only still served, not blocked | [`legacy_only_signature_is_accepted_in_default_posture`] |
|
||||||
//! | strict flip closes the signature downgrade | [`signature_strict_rejects_legacy_only_downgrade`] |
|
//! | strict flip closes the signature downgrade | [`signature_strict_rejects_legacy_only_downgrade`] |
|
||||||
//! | strict flip closes the body-digest downgrade, incl. v1 | [`body_digest_strict_rejects_digestless_mutation`] |
|
//! | strict flip closes the body-digest downgrade, incl. v1 | [`body_digest_strict_rejects_digestless_mutation`] |
|
||||||
|
//! | replay scope binds every RPC and rejects restart replay | [`replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e`] |
|
||||||
|
//! | strict replay scope allows only Ping bootstrap before v3 | [`replay_scope_strict_requires_v3_after_ping_bootstrap_e2e`] |
|
||||||
//!
|
//!
|
||||||
//! Two acceptance items are deliberately left to the in-process tests. A stale
|
//! Two acceptance items are deliberately left to the in-process tests. A stale
|
||||||
//! timestamp cannot be forged from outside — it is inside the HMAC — so
|
//! timestamp cannot be forged from outside — it is inside the HMAC — so
|
||||||
@@ -88,18 +91,20 @@
|
|||||||
|
|
||||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||||
use crate::storage_api::internode_rpc_signature::{
|
use crate::storage_api::internode_rpc_signature::{
|
||||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
|
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||||
|
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
|
||||||
};
|
};
|
||||||
use http::{HeaderMap, Method};
|
use http::{HeaderMap, Method};
|
||||||
use rustfs_config::{
|
use rustfs_config::{
|
||||||
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_SIGNATURE_STRICT,
|
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||||
|
ENV_INTERNODE_RPC_SIGNATURE_STRICT,
|
||||||
};
|
};
|
||||||
use rustfs_protos::canonical_make_volume_request_body;
|
use rustfs_protos::canonical_make_volume_request_body;
|
||||||
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse};
|
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse};
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use tonic::{Code, Request, Status};
|
use tonic::{Code, Request, Response, Status};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||||
@@ -115,12 +120,14 @@ const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
|
|||||||
/// clears authentication stops harmlessly at `find_disk`.
|
/// clears authentication stops harmlessly at `find_disk`.
|
||||||
const ABSENT_DISK: &str = "/nonexistent/rustfs-signature-e2e-disk";
|
const ABSENT_DISK: &str = "/nonexistent/rustfs-signature-e2e-disk";
|
||||||
|
|
||||||
/// Wire names of the two v2 headers these tests edit. They are `pub(crate)` in
|
/// Wire names of the v2 and replay-scope headers these black-box tests edit. They are
|
||||||
/// ecstore, so they are repeated here rather than imported — [`overwrite_header`]
|
/// `pub(crate)` in ecstore, so they are repeated here rather than imported.
|
||||||
/// asserts the header it replaces was actually present, which turns a rename
|
/// [`overwrite_header`] asserts the header it replaces was actually present, which turns a
|
||||||
/// into a loud failure instead of silently reducing an attack to a no-op.
|
/// rename into a loud failure instead of silently reducing an attack to a no-op.
|
||||||
const CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
const CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
||||||
const NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
const NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
||||||
|
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||||
|
const BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
|
||||||
|
|
||||||
/// gRPC service name carried in the signed scope, i.e. `TONIC_RPC_PREFIX`
|
/// gRPC service name carried in the signed scope, i.e. `TONIC_RPC_PREFIX`
|
||||||
/// without its leading `/`.
|
/// without its leading `/`.
|
||||||
@@ -155,19 +162,28 @@ fn align_rpc_secret_with_server() {
|
|||||||
///
|
///
|
||||||
/// Uses the no-cleanup spawn so a `pkill` pattern cannot reap servers belonging
|
/// Uses the no-cleanup spawn so a `pkill` pattern cannot reap servers belonging
|
||||||
/// to other tests running in the same binary.
|
/// to other tests running in the same binary.
|
||||||
async fn start_server(extra_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
fn server_env(extra_env: &[(&'static str, &'static str)]) -> Vec<(&'static str, &'static str)> {
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
|
||||||
let mut child_env = vec![
|
let mut child_env = vec![
|
||||||
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
|
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
|
||||||
(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "false"),
|
(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "false"),
|
||||||
(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "false"),
|
(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "false"),
|
||||||
|
(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "false"),
|
||||||
(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "1048576"),
|
(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "1048576"),
|
||||||
];
|
];
|
||||||
child_env.extend_from_slice(extra_env);
|
child_env.extend_from_slice(extra_env);
|
||||||
env.start_rustfs_server_without_cleanup_with_env(&child_env).await?;
|
child_env
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn start_server_with_env(child_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||||
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
|
env.start_rustfs_server_without_cleanup_with_env(child_env).await?;
|
||||||
Ok(env)
|
Ok(env)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn start_server(extra_env: &[(&'static str, &'static str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||||
|
start_server_with_env(&server_env(extra_env)).await
|
||||||
|
}
|
||||||
|
|
||||||
/// Stop the child and drop the cached gRPC channel for its address.
|
/// Stop the child and drop the cached gRPC channel for its address.
|
||||||
///
|
///
|
||||||
/// `node_service_time_out_client_no_auth` memoises channels in a process-global
|
/// `node_service_time_out_client_no_auth` memoises channels in a process-global
|
||||||
@@ -238,12 +254,83 @@ fn overwrite_header(headers: &mut HeaderMap, name: &'static str, value: &str) {
|
|||||||
/// and nothing else — no interceptor adds or rewrites auth metadata, so the
|
/// and nothing else — no interceptor adds or rewrites auth metadata, so the
|
||||||
/// bytes on the wire are the ones the test chose.
|
/// bytes on the wire are the ones the test chose.
|
||||||
async fn call_make_volume(url: &str, request: MakeVolumeRequest, headers: HeaderMap) -> Result<MakeVolumeResponse, Status> {
|
async fn call_make_volume(url: &str, request: MakeVolumeRequest, headers: HeaderMap) -> Result<MakeVolumeResponse, Status> {
|
||||||
|
call_make_volume_response(url, request, headers)
|
||||||
|
.await
|
||||||
|
.map(Response::into_inner)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn call_make_volume_response(
|
||||||
|
url: &str,
|
||||||
|
request: MakeVolumeRequest,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> Result<Response<MakeVolumeResponse>, Status> {
|
||||||
let mut client = node_service_time_out_client_no_auth(&url.to_string())
|
let mut client = node_service_time_out_client_no_auth(&url.to_string())
|
||||||
.await
|
.await
|
||||||
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
|
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
|
||||||
let mut rpc_request = Request::new(request);
|
let mut rpc_request = Request::new(request);
|
||||||
rpc_request.metadata_mut().as_mut().extend(headers);
|
rpc_request.metadata_mut().as_mut().extend(headers);
|
||||||
client.make_volume(rpc_request).await.map(|response| response.into_inner())
|
client.make_volume(rpc_request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn call_ping_response(url: &str, headers: HeaderMap) -> Result<Response<PingResponse>, Status> {
|
||||||
|
let mut client = node_service_time_out_client_no_auth(&url.to_string())
|
||||||
|
.await
|
||||||
|
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
|
||||||
|
let mut rpc_request = Request::new(PingRequest {
|
||||||
|
version: 1,
|
||||||
|
body: bytes::Bytes::new(),
|
||||||
|
});
|
||||||
|
rpc_request.metadata_mut().as_mut().extend(headers);
|
||||||
|
client.ping(rpc_request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn attach_boot_epoch_challenge(headers: &mut HeaderMap) -> Uuid {
|
||||||
|
let challenge = Uuid::new_v4();
|
||||||
|
headers.insert(
|
||||||
|
BOOT_EPOCH_CHALLENGE_HEADER,
|
||||||
|
challenge.to_string().parse().expect("UUID must be a valid header value"),
|
||||||
|
);
|
||||||
|
challenge
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mint_replay_scope_headers(audience: &str, path: &str, content_sha256: &str, boot_epoch: Uuid) -> HeaderMap {
|
||||||
|
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(content_sha256));
|
||||||
|
let timestamp = headers
|
||||||
|
.get(TIMESTAMP_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.expect("v2 headers must carry a timestamp")
|
||||||
|
.to_string();
|
||||||
|
headers.extend(
|
||||||
|
gen_tonic_replay_scope_headers(audience, path, ×tamp, content_sha256, boot_epoch)
|
||||||
|
.expect("replay-scope headers must mint with the aligned RPC secret"),
|
||||||
|
);
|
||||||
|
headers
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn learn_boot_epoch_from_make_volume(url: &str, audience: &str) -> Uuid {
|
||||||
|
let request = make_volume_request("signature-e2e-epoch-bootstrap");
|
||||||
|
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(&canonical_digest(&request)));
|
||||||
|
let challenge = attach_boot_epoch_challenge(&mut headers);
|
||||||
|
let response = call_make_volume_response(url, request, headers)
|
||||||
|
.await
|
||||||
|
.expect("v2 request with epoch challenge must clear default authentication");
|
||||||
|
let boot_epoch = verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
|
||||||
|
.expect("server must HMAC-authenticate the advertised boot epoch");
|
||||||
|
assert_authenticated(
|
||||||
|
Ok(response.into_inner()),
|
||||||
|
"a v2 epoch-challenge request in the default replay-scope posture",
|
||||||
|
);
|
||||||
|
boot_epoch
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn learn_boot_epoch_from_ping(url: &str, audience: &str) -> Uuid {
|
||||||
|
let mut headers = mint_v2_headers(audience, "Ping", None);
|
||||||
|
let challenge = attach_boot_epoch_challenge(&mut headers);
|
||||||
|
let response = call_ping_response(url, headers)
|
||||||
|
.await
|
||||||
|
.expect("v2 Ping with an epoch challenge must bootstrap strict replay scope");
|
||||||
|
verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
|
||||||
|
.expect("strict replay-scope Ping must return a valid boot epoch proof")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Assert a call cleared authentication.
|
/// Assert a call cleared authentication.
|
||||||
@@ -332,6 +419,122 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A replay-scoped signature is usable exactly once against the exact gRPC path and the server
|
||||||
|
/// process epoch that minted it. This crosses the child-process boundary twice: the HMAC-protected
|
||||||
|
/// epoch is learned from a real response, then the same server is restarted in place to prove its
|
||||||
|
/// replacement epoch rejects the captured request even though the nonce cache is necessarily new.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult {
|
||||||
|
init_logging();
|
||||||
|
align_rpc_secret_with_server();
|
||||||
|
let child_env = server_env(&[]);
|
||||||
|
let mut env = start_server_with_env(&child_env).await?;
|
||||||
|
let url = env.url.clone();
|
||||||
|
let audience = audience_of(&env);
|
||||||
|
let boot_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
|
||||||
|
|
||||||
|
let request = make_volume_request("replay-scope-e2e-once");
|
||||||
|
let captured = mint_replay_scope_headers(
|
||||||
|
&audience,
|
||||||
|
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||||
|
&canonical_digest(&request),
|
||||||
|
boot_epoch,
|
||||||
|
);
|
||||||
|
assert_authenticated(
|
||||||
|
call_make_volume(&url, request.clone(), captured.clone()).await,
|
||||||
|
"the first replay-scoped mutation delivery",
|
||||||
|
);
|
||||||
|
assert_rejected(
|
||||||
|
call_make_volume(&url, request.clone(), captured).await,
|
||||||
|
Code::Unauthenticated,
|
||||||
|
None,
|
||||||
|
"the same replay-scoped mutation delivered twice",
|
||||||
|
);
|
||||||
|
|
||||||
|
let transplanted =
|
||||||
|
mint_replay_scope_headers(&audience, &format!("{TONIC_RPC_PREFIX}/Ping"), &canonical_digest(&request), boot_epoch);
|
||||||
|
assert_rejected(
|
||||||
|
call_make_volume(&url, request.clone(), transplanted).await,
|
||||||
|
Code::Unauthenticated,
|
||||||
|
None,
|
||||||
|
"a replay-scoped Ping signature transplanted onto MakeVolume",
|
||||||
|
);
|
||||||
|
|
||||||
|
let stale_epoch = mint_replay_scope_headers(
|
||||||
|
&audience,
|
||||||
|
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||||
|
&canonical_digest(&request),
|
||||||
|
boot_epoch,
|
||||||
|
);
|
||||||
|
env.restart_server_preserving_data(Vec::new(), &child_env).await?;
|
||||||
|
rustfs_protos::evict_failed_connection(&url).await;
|
||||||
|
assert_rejected(
|
||||||
|
call_make_volume(&url, request.clone(), stale_epoch).await,
|
||||||
|
Code::Unauthenticated,
|
||||||
|
None,
|
||||||
|
"a replay-scoped signature captured before the receiving process restart",
|
||||||
|
);
|
||||||
|
|
||||||
|
let restarted_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
|
||||||
|
assert_ne!(boot_epoch, restarted_epoch, "a restarted child process must advertise a new boot epoch");
|
||||||
|
let fresh_epoch = mint_replay_scope_headers(
|
||||||
|
&audience,
|
||||||
|
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||||
|
&canonical_digest(&request),
|
||||||
|
restarted_epoch,
|
||||||
|
);
|
||||||
|
assert_authenticated(
|
||||||
|
call_make_volume(&url, request, fresh_epoch).await,
|
||||||
|
"a replay-scoped mutation signed with the replacement process epoch",
|
||||||
|
);
|
||||||
|
|
||||||
|
stop_server(env, &url).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Strict replay scope leaves one authenticated v2 bootstrap: `Ping` carrying a fresh challenge.
|
||||||
|
/// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3
|
||||||
|
/// mutation succeeds. This protects a server restart without reopening a general downgrade path.
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult {
|
||||||
|
init_logging();
|
||||||
|
align_rpc_secret_with_server();
|
||||||
|
let env = start_server(&[(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "true")]).await?;
|
||||||
|
let url = env.url.clone();
|
||||||
|
let audience = audience_of(&env);
|
||||||
|
|
||||||
|
let v2_request = make_volume_request("replay-scope-e2e-strict-v2");
|
||||||
|
assert_rejected(
|
||||||
|
call_make_volume(
|
||||||
|
&url,
|
||||||
|
v2_request.clone(),
|
||||||
|
mint_v2_headers(&audience, "MakeVolume", Some(&canonical_digest(&v2_request))),
|
||||||
|
)
|
||||||
|
.await,
|
||||||
|
Code::Unauthenticated,
|
||||||
|
None,
|
||||||
|
"a v2 mutation after replay-scope strictness is enabled",
|
||||||
|
);
|
||||||
|
|
||||||
|
let boot_epoch = learn_boot_epoch_from_ping(&url, &audience).await;
|
||||||
|
let request = make_volume_request("replay-scope-e2e-strict-v3");
|
||||||
|
let replay_scoped = mint_replay_scope_headers(
|
||||||
|
&audience,
|
||||||
|
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||||
|
&canonical_digest(&request),
|
||||||
|
boot_epoch,
|
||||||
|
);
|
||||||
|
assert_authenticated(
|
||||||
|
call_make_volume(&url, request, replay_scoped).await,
|
||||||
|
"a replay-scoped mutation after Ping bootstrap under strict replay scope",
|
||||||
|
);
|
||||||
|
|
||||||
|
stop_server(env, &url).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Baseline: correctly signed mutations are accepted, both with and without a
|
/// Baseline: correctly signed mutations are accepted, both with and without a
|
||||||
/// body digest.
|
/// body digest.
|
||||||
///
|
///
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall
|
|||||||
use tonic::Request;
|
use tonic::Request;
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
use crate::storage_api::grpc_lock::{TonicInterceptor, node_service_time_out_client_no_auth};
|
use crate::storage_api::grpc_lock::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||||
|
|
||||||
/// gRPC lock client without authentication for testing
|
/// gRPC lock client without authentication for testing
|
||||||
/// Similar to RemoteClient but uses no_auth client
|
/// Similar to RemoteClient but uses no_auth client
|
||||||
@@ -42,7 +42,7 @@ impl GrpcLockClient {
|
|||||||
&self,
|
&self,
|
||||||
) -> Result<
|
) -> Result<
|
||||||
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
|
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
|
||||||
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, TonicInterceptor>,
|
tonic::service::interceptor::InterceptedService<AuthenticatedChannel, TonicInterceptor>,
|
||||||
>,
|
>,
|
||||||
> {
|
> {
|
||||||
node_service_time_out_client_no_auth(&self.addr)
|
node_service_time_out_client_no_auth(&self.addr)
|
||||||
|
|||||||
@@ -16,9 +16,12 @@
|
|||||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
|
pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
|
||||||
|
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::api::rpc::{TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers};
|
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||||
pub(crate) use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth};
|
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||||
|
verify_tonic_boot_epoch_response,
|
||||||
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
|
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||||
|
|
||||||
@@ -30,7 +33,7 @@ pub(crate) mod node_interact {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) mod grpc_lock {
|
pub(crate) mod grpc_lock {
|
||||||
pub(crate) use super::{TonicInterceptor, node_service_time_out_client_no_auth};
|
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Signing/transport surface used by the cross-process internode RPC signature
|
/// Signing/transport surface used by the cross-process internode RPC signature
|
||||||
@@ -40,7 +43,8 @@ pub(crate) mod grpc_lock {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) mod internode_rpc_signature {
|
pub(crate) mod internode_rpc_signature {
|
||||||
pub(crate) use super::{
|
pub(crate) use super::{
|
||||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
|
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||||
|
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -416,13 +416,15 @@ pub mod rio {
|
|||||||
|
|
||||||
pub mod rpc {
|
pub mod rpc {
|
||||||
pub use crate::cluster::rpc::{
|
pub use crate::cluster::rpc::{
|
||||||
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
|
AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
|
||||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
|
PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
|
||||||
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
|
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||||
verify_tonic_rpc_signature,
|
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||||
|
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||||
|
verify_tonic_rpc_signature_with_bootstrap,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,11 +12,20 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
|
#[cfg(test)]
|
||||||
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
|
||||||
|
use crate::cluster::rpc::http_auth::{
|
||||||
|
RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
|
||||||
|
RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
|
||||||
|
};
|
||||||
|
use crate::cluster::rpc::{
|
||||||
|
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
|
||||||
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
|
||||||
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
|
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
|
||||||
use crate::runtime::sources as runtime_sources;
|
use crate::runtime::sources as runtime_sources;
|
||||||
use http::Uri;
|
use http::{Request as HttpRequest, Response as HttpResponse, Uri};
|
||||||
use rustfs_protos::{
|
use rustfs_protos::{
|
||||||
ChannelClass, create_new_channel, get_channel_for_class,
|
ChannelClass, create_new_channel, get_channel_for_class,
|
||||||
proto_gen::node_service::{
|
proto_gen::node_service::{
|
||||||
@@ -24,9 +33,19 @@ use rustfs_protos::{
|
|||||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
use std::{error::Error, io::ErrorKind};
|
use std::{
|
||||||
|
collections::HashMap,
|
||||||
|
error::Error,
|
||||||
|
future::Future,
|
||||||
|
io::ErrorKind,
|
||||||
|
pin::Pin,
|
||||||
|
sync::{LazyLock, Mutex},
|
||||||
|
task::{Context, Poll},
|
||||||
|
};
|
||||||
use tonic::{service::interceptor::InterceptedService, transport::Channel};
|
use tonic::{service::interceptor::InterceptedService, transport::Channel};
|
||||||
|
use tower::Service;
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
|
use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
|
||||||
|
|
||||||
@@ -35,7 +54,7 @@ use super::context_propagation::{inject_request_id_into_metadata, inject_trace_c
|
|||||||
pub async fn node_service_time_out_client(
|
pub async fn node_service_time_out_client(
|
||||||
addr: &String,
|
addr: &String,
|
||||||
interceptor: TonicInterceptor,
|
interceptor: TonicInterceptor,
|
||||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||||
// Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the
|
// Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the
|
||||||
// `_for_class` variant below (grpc-optimization P1).
|
// `_for_class` variant below (grpc-optimization P1).
|
||||||
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
|
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
|
||||||
@@ -44,13 +63,14 @@ pub async fn node_service_time_out_client(
|
|||||||
pub async fn heal_control_time_out_client(
|
pub async fn heal_control_time_out_client(
|
||||||
addr: &str,
|
addr: &str,
|
||||||
interceptor: TonicInterceptor,
|
interceptor: TonicInterceptor,
|
||||||
) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
) -> Result<HealControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||||
Some(channel) => channel,
|
Some(channel) => channel,
|
||||||
None => create_new_channel(addr).await?,
|
None => create_new_channel(addr).await?,
|
||||||
};
|
};
|
||||||
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
|
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
|
||||||
|
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||||
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
|
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
|
||||||
.max_decoding_message_size(max_message_size)
|
.max_decoding_message_size(max_message_size)
|
||||||
.max_encoding_message_size(max_message_size))
|
.max_encoding_message_size(max_message_size))
|
||||||
@@ -59,13 +79,14 @@ pub async fn heal_control_time_out_client(
|
|||||||
pub async fn tier_mutation_control_time_out_client(
|
pub async fn tier_mutation_control_time_out_client(
|
||||||
addr: &str,
|
addr: &str,
|
||||||
interceptor: TonicInterceptor,
|
interceptor: TonicInterceptor,
|
||||||
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||||
Some(channel) => channel,
|
Some(channel) => channel,
|
||||||
None => create_new_channel(addr).await?,
|
None => create_new_channel(addr).await?,
|
||||||
};
|
};
|
||||||
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
|
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
|
||||||
|
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||||
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
|
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
|
||||||
.max_decoding_message_size(max_message_size)
|
.max_decoding_message_size(max_message_size)
|
||||||
.max_encoding_message_size(max_message_size))
|
.max_encoding_message_size(max_message_size))
|
||||||
@@ -81,7 +102,7 @@ pub async fn node_service_time_out_client_for_class(
|
|||||||
addr: &String,
|
addr: &String,
|
||||||
interceptor: TonicInterceptor,
|
interceptor: TonicInterceptor,
|
||||||
class: ChannelClass,
|
class: ChannelClass,
|
||||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||||
let channel = match class {
|
let channel = match class {
|
||||||
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
|
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
|
||||||
@@ -96,6 +117,7 @@ pub async fn node_service_time_out_client_for_class(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
|
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
|
||||||
|
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||||
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
|
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
|
||||||
.max_decoding_message_size(max_message_size)
|
.max_decoding_message_size(max_message_size)
|
||||||
.max_encoding_message_size(max_message_size))
|
.max_encoding_message_size(max_message_size))
|
||||||
@@ -103,7 +125,7 @@ pub async fn node_service_time_out_client_for_class(
|
|||||||
|
|
||||||
pub async fn node_service_time_out_client_no_auth(
|
pub async fn node_service_time_out_client_no_auth(
|
||||||
addr: &String,
|
addr: &String,
|
||||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||||
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
|
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -199,6 +221,104 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The transport service that learns an authenticated peer boot epoch and adds the replay-scoped
|
||||||
|
/// signature only after one has been observed. The v1/v2 interceptor stays inside this wrapper so
|
||||||
|
/// old servers continue receiving precisely the metadata they understand.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct ReplayScopeChannel<S> {
|
||||||
|
inner: S,
|
||||||
|
audience: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The channel type used by internode clients after v2 authentication and replay-scope handling.
|
||||||
|
pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
|
||||||
|
|
||||||
|
static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||||
|
|
||||||
|
impl<S> ReplayScopeChannel<S> {
|
||||||
|
fn new(inner: S, audience: Option<String>) -> Self {
|
||||||
|
Self { inner, audience }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
|
||||||
|
PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
|
||||||
|
if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
|
||||||
|
epochs.insert(audience, epoch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
|
||||||
|
where
|
||||||
|
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
|
||||||
|
S::Error: Send + 'static,
|
||||||
|
S::Future: Send + 'static,
|
||||||
|
ReqBody: Send + 'static,
|
||||||
|
ResBody: Send + 'static,
|
||||||
|
{
|
||||||
|
type Response = HttpResponse<ResBody>;
|
||||||
|
type Error = S::Error;
|
||||||
|
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||||
|
|
||||||
|
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||||
|
self.inner.poll_ready(cx)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
|
||||||
|
let authenticated = self.audience.as_ref().is_some_and(|_| {
|
||||||
|
request
|
||||||
|
.headers()
|
||||||
|
.get(RPC_AUTH_VERSION_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
== Some(RPC_AUTH_VERSION_V2)
|
||||||
|
});
|
||||||
|
let challenge = authenticated.then(Uuid::new_v4);
|
||||||
|
if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
|
||||||
|
// The challenge is independently HMAC-authenticated by the response proof. It is not
|
||||||
|
// part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
|
||||||
|
request.headers_mut().insert(
|
||||||
|
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||||
|
challenge.to_string().parse().expect("UUID must be a valid header value"),
|
||||||
|
);
|
||||||
|
if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
|
||||||
|
cached_peer_boot_epoch(audience),
|
||||||
|
request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
|
||||||
|
request
|
||||||
|
.headers()
|
||||||
|
.get(RPC_CONTENT_SHA256_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
) {
|
||||||
|
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
|
||||||
|
Ok(headers) => request.headers_mut().extend(headers),
|
||||||
|
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let audience = self.audience.clone();
|
||||||
|
let future = self.inner.call(request);
|
||||||
|
Box::pin(async move {
|
||||||
|
let response = future.await?;
|
||||||
|
if let (Some(audience), Some(challenge)) = (audience, challenge) {
|
||||||
|
match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
|
||||||
|
Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
|
||||||
|
Err(error)
|
||||||
|
if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
|
||||||
|
|| response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
|
||||||
|
{
|
||||||
|
debug!(error = %error, "peer boot epoch response proof was rejected")
|
||||||
|
}
|
||||||
|
Err(_) => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(response)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub struct TonicSignatureInterceptor {
|
pub struct TonicSignatureInterceptor {
|
||||||
audience: Option<String>,
|
audience: Option<String>,
|
||||||
}
|
}
|
||||||
@@ -257,6 +377,13 @@ impl TonicInterceptor {
|
|||||||
}
|
}
|
||||||
Ok(self)
|
Ok(self)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn replay_scope_audience(&self) -> Option<String> {
|
||||||
|
match self {
|
||||||
|
Self::Signature(interceptor) => interceptor.audience.clone(),
|
||||||
|
Self::NoOp(_) => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl tonic::service::Interceptor for TonicInterceptor {
|
impl tonic::service::Interceptor for TonicInterceptor {
|
||||||
@@ -279,6 +406,38 @@ mod tests {
|
|||||||
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
||||||
use tracing_subscriber::{Registry, layer::SubscriberExt};
|
use tracing_subscriber::{Registry, layer::SubscriberExt};
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct EpochProofService {
|
||||||
|
audience: String,
|
||||||
|
seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Service<HttpRequest<()>> for EpochProofService {
|
||||||
|
type Response = HttpResponse<()>;
|
||||||
|
type Error = std::convert::Infallible;
|
||||||
|
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
|
||||||
|
|
||||||
|
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||||
|
Poll::Ready(Ok(()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn call(&mut self, request: HttpRequest<()>) -> Self::Future {
|
||||||
|
self.seen_headers
|
||||||
|
.lock()
|
||||||
|
.expect("test header capture lock must not be poisoned")
|
||||||
|
.push(request.headers().clone());
|
||||||
|
let challenge = tonic_boot_epoch_challenge(request.headers())
|
||||||
|
.expect("client challenge must be syntactically valid")
|
||||||
|
.expect("authenticated client request must carry a boot epoch challenge");
|
||||||
|
let mut response = HttpResponse::new(());
|
||||||
|
response.headers_mut().extend(
|
||||||
|
tonic_boot_epoch_response_headers(&self.audience, challenge)
|
||||||
|
.expect("test server must be able to sign an epoch proof"),
|
||||||
|
);
|
||||||
|
std::future::ready(Ok(response))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn ensure_test_rpc_secret() {
|
fn ensure_test_rpc_secret() {
|
||||||
runtime_sources::ensure_test_rpc_secret();
|
runtime_sources::ensure_test_rpc_secret();
|
||||||
}
|
}
|
||||||
@@ -420,6 +579,52 @@ mod tests {
|
|||||||
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
|
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
|
||||||
|
ensure_test_rpc_secret();
|
||||||
|
let audience = "replay-scope-client-test:9000";
|
||||||
|
PEER_BOOT_EPOCHS
|
||||||
|
.lock()
|
||||||
|
.expect("peer epoch cache lock must not be poisoned")
|
||||||
|
.remove(audience);
|
||||||
|
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||||
|
let service = EpochProofService {
|
||||||
|
audience: audience.to_string(),
|
||||||
|
seen_headers: seen_headers.clone(),
|
||||||
|
};
|
||||||
|
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
|
||||||
|
let make_request = || {
|
||||||
|
let mut request = HttpRequest::builder()
|
||||||
|
.uri("/node_service.NodeService/Ping")
|
||||||
|
.body(())
|
||||||
|
.expect("test RPC request must build");
|
||||||
|
request.headers_mut().extend(
|
||||||
|
gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
|
||||||
|
.expect("v2 test headers must mint"),
|
||||||
|
);
|
||||||
|
request
|
||||||
|
};
|
||||||
|
|
||||||
|
futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
|
||||||
|
futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
|
||||||
|
|
||||||
|
let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
|
||||||
|
assert_eq!(headers.len(), 2);
|
||||||
|
assert!(headers[0].contains_key(RPC_BOOT_EPOCH_CHALLENGE_HEADER));
|
||||||
|
assert!(
|
||||||
|
!headers[0].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
|
||||||
|
"the first request must remain v2-compatible until the peer proves its epoch"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
|
||||||
|
"the second request must carry the replay-scoped v3 signature"
|
||||||
|
);
|
||||||
|
PEER_BOOT_EPOCHS
|
||||||
|
.lock()
|
||||||
|
.expect("peer epoch cache lock must not be poisoned")
|
||||||
|
.remove(audience);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_signature_interceptor_requires_generated_method_metadata() {
|
fn test_signature_interceptor_requires_generated_method_metadata() {
|
||||||
ensure_test_rpc_secret();
|
ensure_test_rpc_secret();
|
||||||
|
|||||||
@@ -20,8 +20,8 @@
|
|||||||
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
|
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
|
||||||
//! below, plus the broader negative-signature suite. The advisory class is: a
|
//! below, plus the broader negative-signature suite. The advisory class is: a
|
||||||
//! node must never accept an RPC whose auth is missing, malformed, or signed
|
//! node must never accept an RPC whose auth is missing, malformed, or signed
|
||||||
//! with the default/empty shared secret. Body-bound v2 requests additionally
|
//! with the default/empty shared secret. Body-bound v2 requests and all replay-scoped v3
|
||||||
//! receive process-local replay protection. See
|
//! requests additionally receive process-local replay protection. See
|
||||||
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
|
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
|
||||||
//!
|
//!
|
||||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
|
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
|
||||||
@@ -50,13 +50,22 @@ use uuid::Uuid;
|
|||||||
type HmacSha256 = Hmac<Sha256>;
|
type HmacSha256 = Hmac<Sha256>;
|
||||||
|
|
||||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||||
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||||
const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||||
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
|
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
|
||||||
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
||||||
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
||||||
const RPC_AUTH_VERSION_V2: &str = "2";
|
pub(crate) const RPC_AUTH_VERSION_V2: &str = "2";
|
||||||
|
pub const RPC_REPLAY_SCOPE_VERSION_HEADER: &str = "x-rustfs-rpc-replay-scope-version";
|
||||||
|
pub const RPC_REPLAY_SCOPE_SIGNATURE_HEADER: &str = "x-rustfs-rpc-signature-v3";
|
||||||
|
pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
|
||||||
|
pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
|
||||||
|
pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
|
||||||
|
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
|
||||||
|
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||||
|
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||||
|
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||||
@@ -75,9 +84,15 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
|||||||
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
|
static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||||
// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
|
get_env_bool(
|
||||||
// cannot disable replay protection by rejecting every body-bound request.
|
rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||||
|
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
|
||||||
|
// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
|
||||||
|
// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
|
||||||
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||||
rustfs_utils::get_env_usize(
|
rustfs_utils::get_env_usize(
|
||||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||||
@@ -86,6 +101,7 @@ static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
|||||||
.max(1)
|
.max(1)
|
||||||
});
|
});
|
||||||
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
||||||
|
static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct RpcNonceCache {
|
struct RpcNonceCache {
|
||||||
@@ -313,6 +329,189 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
|
|||||||
mac.verify_slice(&signature).is_ok()
|
mac.verify_slice(&signature).is_ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct ReplayScope<'a> {
|
||||||
|
audience: &'a str,
|
||||||
|
path: &'a str,
|
||||||
|
timestamp: &'a str,
|
||||||
|
nonce: Uuid,
|
||||||
|
content_sha256: &'a str,
|
||||||
|
boot_epoch: Uuid,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
|
||||||
|
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
|
||||||
|
for part in [
|
||||||
|
scope.audience.as_bytes(),
|
||||||
|
b"|",
|
||||||
|
scope.path.as_bytes(),
|
||||||
|
b"|POST|",
|
||||||
|
scope.timestamp.as_bytes(),
|
||||||
|
b"|",
|
||||||
|
scope.nonce.as_bytes(),
|
||||||
|
b"|",
|
||||||
|
scope.content_sha256.as_bytes(),
|
||||||
|
b"|",
|
||||||
|
scope.boot_epoch.as_bytes(),
|
||||||
|
] {
|
||||||
|
mac.update(part);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std::io::Result<String> {
|
||||||
|
let mut mac =
|
||||||
|
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||||
|
update_replay_scope(&mut mac, scope);
|
||||||
|
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
|
||||||
|
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
update_replay_scope(&mut mac, scope);
|
||||||
|
mac.verify_slice(&signature).is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update_boot_epoch_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
|
||||||
|
mac.update(RPC_BOOT_EPOCH_PROOF_DOMAIN);
|
||||||
|
mac.update(audience.as_bytes());
|
||||||
|
mac.update(b"|");
|
||||||
|
mac.update(challenge.as_bytes());
|
||||||
|
mac.update(boot_epoch.as_bytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid) -> std::io::Result<String> {
|
||||||
|
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
|
||||||
|
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
|
||||||
|
}
|
||||||
|
let mut mac =
|
||||||
|
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||||
|
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
|
||||||
|
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
|
||||||
|
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
|
||||||
|
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
|
||||||
|
}
|
||||||
|
let proof = general_purpose::STANDARD
|
||||||
|
.decode(proof)
|
||||||
|
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
|
||||||
|
let mut mac =
|
||||||
|
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||||
|
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
|
||||||
|
mac.verify_slice(&proof)
|
||||||
|
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
|
||||||
|
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
|
||||||
|
(!value.is_nil())
|
||||||
|
.then_some(value)
|
||||||
|
.ok_or_else(|| std::io::Error::other(format!("Invalid {name}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_tonic_rpc_path(path: &str) -> std::io::Result<(&str, &str)> {
|
||||||
|
path.strip_prefix('/')
|
||||||
|
.and_then(|path| path.split_once('/'))
|
||||||
|
.filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
|
||||||
|
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The process-unique epoch included in every replay-scoped server verification.
|
||||||
|
///
|
||||||
|
/// A fresh process gets a fresh value, so a signature captured before a server restart cannot be
|
||||||
|
/// admitted even though the bounded in-memory nonce cache necessarily starts empty again.
|
||||||
|
pub fn tonic_rpc_boot_epoch() -> Uuid {
|
||||||
|
*RPC_BOOT_EPOCH
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the additive replay-scope headers for a request that already carries rolling-upgrade-safe
|
||||||
|
/// v1/v2 metadata. `timestamp` and `content_sha256` are deliberately reused from the v2 scope so
|
||||||
|
/// old servers can continue validating the same request unchanged.
|
||||||
|
pub fn gen_tonic_replay_scope_headers(
|
||||||
|
audience: &str,
|
||||||
|
path: &str,
|
||||||
|
timestamp: &str,
|
||||||
|
content_sha256: &str,
|
||||||
|
boot_epoch: Uuid,
|
||||||
|
) -> std::io::Result<HeaderMap> {
|
||||||
|
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
|
||||||
|
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
|
||||||
|
}
|
||||||
|
parse_tonic_rpc_path(path)?;
|
||||||
|
timestamp
|
||||||
|
.parse::<i64>()
|
||||||
|
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||||
|
|
||||||
|
let nonce = Uuid::new_v4();
|
||||||
|
let signature = generate_replay_scope_signature(
|
||||||
|
&get_shared_secret()?,
|
||||||
|
ReplayScope {
|
||||||
|
audience,
|
||||||
|
path,
|
||||||
|
timestamp,
|
||||||
|
nonce,
|
||||||
|
content_sha256,
|
||||||
|
boot_epoch,
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
|
||||||
|
headers.insert(
|
||||||
|
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||||
|
header_value(&signature, RPC_REPLAY_SCOPE_SIGNATURE_HEADER)?,
|
||||||
|
);
|
||||||
|
headers.insert(
|
||||||
|
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||||
|
header_value(&nonce.to_string(), RPC_REPLAY_SCOPE_NONCE_HEADER)?,
|
||||||
|
);
|
||||||
|
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
|
||||||
|
Ok(headers)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse the optional client challenge used to authenticate a server boot-epoch advertisement.
|
||||||
|
pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option<Uuid>> {
|
||||||
|
headers
|
||||||
|
.get(RPC_BOOT_EPOCH_CHALLENGE_HEADER)
|
||||||
|
.map(|value| {
|
||||||
|
value
|
||||||
|
.to_str()
|
||||||
|
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch challenge"))
|
||||||
|
.and_then(|value| non_nil_uuid(value, "RPC boot epoch challenge"))
|
||||||
|
})
|
||||||
|
.transpose()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the authenticated response headers for a client boot-epoch challenge.
|
||||||
|
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
|
||||||
|
let boot_epoch = tonic_rpc_boot_epoch();
|
||||||
|
let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
|
||||||
|
headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
|
||||||
|
Ok(headers)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verify the server boot-epoch response for a challenge generated by this client.
|
||||||
|
pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
|
||||||
|
let boot_epoch = headers
|
||||||
|
.get(RPC_BOOT_EPOCH_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
|
||||||
|
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
|
||||||
|
let proof = headers
|
||||||
|
.get(RPC_BOOT_EPOCH_PROOF_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
|
||||||
|
verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
|
||||||
|
Ok(boot_epoch)
|
||||||
|
}
|
||||||
|
|
||||||
fn valid_content_sha256(value: &str) -> bool {
|
fn valid_content_sha256(value: &str) -> bool {
|
||||||
value == UNSIGNED_PAYLOAD
|
value == UNSIGNED_PAYLOAD
|
||||||
|| (value.len() == 64
|
|| (value.len() == 64
|
||||||
@@ -531,6 +730,17 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
|
|||||||
.any(|name| headers.contains_key(*name))
|
.any(|name| headers.contains_key(*name))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn has_replay_scope_headers(headers: &HeaderMap) -> bool {
|
||||||
|
[
|
||||||
|
RPC_REPLAY_SCOPE_VERSION_HEADER,
|
||||||
|
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||||
|
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||||
|
RPC_BOOT_EPOCH_HEADER,
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|name| headers.contains_key(*name))
|
||||||
|
}
|
||||||
|
|
||||||
/// Whether the server requires target-bound v2 authentication on every internode gRPC request,
|
/// Whether the server requires target-bound v2 authentication on every internode gRPC request,
|
||||||
/// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
|
/// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
|
||||||
/// lever gated on the v1-fallback counter reading zero fleet-wide; see
|
/// lever gated on the v1-fallback counter reading zero fleet-wide; see
|
||||||
@@ -540,9 +750,127 @@ fn internode_rpc_signature_strict() -> bool {
|
|||||||
*INTERNODE_RPC_SIGNATURE_STRICT
|
*INTERNODE_RPC_SIGNATURE_STRICT
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn internode_rpc_replay_scope_strict() -> bool {
|
||||||
|
*INTERNODE_RPC_REPLAY_SCOPE_STRICT
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
||||||
|
if audience.is_empty() {
|
||||||
|
return Err(std::io::Error::other("Missing RPC audience"));
|
||||||
|
}
|
||||||
|
parse_tonic_rpc_path(path)?;
|
||||||
|
|
||||||
|
let version = headers
|
||||||
|
.get(RPC_REPLAY_SCOPE_VERSION_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope version"))?;
|
||||||
|
if version != RPC_REPLAY_SCOPE_VERSION_V3 {
|
||||||
|
return Err(std::io::Error::other("Unsupported RPC replay scope version"));
|
||||||
|
}
|
||||||
|
let signature = headers
|
||||||
|
.get(RPC_REPLAY_SCOPE_SIGNATURE_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope signature"))?;
|
||||||
|
let timestamp = headers
|
||||||
|
.get(TIMESTAMP_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
|
||||||
|
let signed_at = timestamp
|
||||||
|
.parse::<i64>()
|
||||||
|
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||||
|
check_timestamp(signed_at)?;
|
||||||
|
let nonce = headers
|
||||||
|
.get(RPC_REPLAY_SCOPE_NONCE_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope nonce"))
|
||||||
|
.and_then(|value| non_nil_uuid(value, "RPC replay scope nonce"))?;
|
||||||
|
let content_sha256 = headers
|
||||||
|
.get(RPC_CONTENT_SHA256_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
|
||||||
|
if !valid_content_sha256(content_sha256) {
|
||||||
|
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||||
|
}
|
||||||
|
let boot_epoch = headers
|
||||||
|
.get(RPC_BOOT_EPOCH_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
|
||||||
|
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
|
||||||
|
let secret = get_shared_secret()?;
|
||||||
|
if !verify_replay_scope_signature(
|
||||||
|
&secret,
|
||||||
|
ReplayScope {
|
||||||
|
audience,
|
||||||
|
path,
|
||||||
|
timestamp,
|
||||||
|
nonce,
|
||||||
|
content_sha256,
|
||||||
|
boot_epoch,
|
||||||
|
},
|
||||||
|
signature,
|
||||||
|
) {
|
||||||
|
return Err(std::io::Error::other("Invalid RPC replay scope signature"));
|
||||||
|
}
|
||||||
|
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||||
|
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||||
|
}
|
||||||
|
check_and_record_nonce(nonce, signed_at)
|
||||||
|
}
|
||||||
|
|
||||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||||
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
||||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict())
|
verify_tonic_rpc_signature_with_policy(
|
||||||
|
audience,
|
||||||
|
path,
|
||||||
|
headers,
|
||||||
|
internode_rpc_signature_strict(),
|
||||||
|
internode_rpc_replay_scope_strict(),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verify gRPC authentication while allowing the narrowly scoped v2 `Ping` bootstrap used to
|
||||||
|
/// obtain an authenticated server boot epoch when replay-scope strictness is enabled.
|
||||||
|
pub fn verify_tonic_rpc_signature_with_bootstrap(
|
||||||
|
audience: &str,
|
||||||
|
path: &str,
|
||||||
|
headers: &HeaderMap,
|
||||||
|
allow_replay_scope_bootstrap: bool,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
verify_tonic_rpc_signature_with_policy(
|
||||||
|
audience,
|
||||||
|
path,
|
||||||
|
headers,
|
||||||
|
internode_rpc_signature_strict(),
|
||||||
|
internode_rpc_replay_scope_strict(),
|
||||||
|
allow_replay_scope_bootstrap,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_tonic_rpc_signature_with_policy(
|
||||||
|
audience: &str,
|
||||||
|
path: &str,
|
||||||
|
headers: &HeaderMap,
|
||||||
|
signature_strict: bool,
|
||||||
|
replay_scope_strict: bool,
|
||||||
|
allow_replay_scope_bootstrap: bool,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
if has_replay_scope_headers(headers) {
|
||||||
|
return verify_tonic_replay_scope_signature(audience, path, headers);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only a method-bound v2 Ping with a syntactically valid challenge may bootstrap a strict
|
||||||
|
// client after its peer restarts. Legacy metadata never gets this exception.
|
||||||
|
let bootstrap = allow_replay_scope_bootstrap
|
||||||
|
&& has_v2_auth_headers(headers)
|
||||||
|
&& tonic_boot_epoch_challenge(headers).is_ok_and(|challenge| challenge.is_some());
|
||||||
|
if replay_scope_strict && !bootstrap {
|
||||||
|
return Err(std::io::Error::other("RPC replay-scoped authentication required"));
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_tonic_rpc_signature_with_strictness(audience, path, headers, signature_strict)?;
|
||||||
|
global_internode_metrics().record_replay_scope_fallback();
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
|
/// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
|
||||||
@@ -1273,6 +1601,104 @@ mod tests {
|
|||||||
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_scope_binds_path_epoch_and_random_nonce() {
|
||||||
|
ensure_test_rpc_secret();
|
||||||
|
let path = "/node_service.NodeService/Ping";
|
||||||
|
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||||
|
.expect("v2 compatibility headers should build");
|
||||||
|
let timestamp = headers
|
||||||
|
.get(TIMESTAMP_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.expect("v2 timestamp")
|
||||||
|
.to_string();
|
||||||
|
let content_sha256 = headers
|
||||||
|
.get(RPC_CONTENT_SHA256_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.expect("v2 content digest")
|
||||||
|
.to_string();
|
||||||
|
headers.extend(
|
||||||
|
gen_tonic_replay_scope_headers("node-a:9000", path, ×tamp, &content_sha256, tonic_rpc_boot_epoch())
|
||||||
|
.expect("replay-scope headers should build"),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false).is_ok(),
|
||||||
|
"the first replay-scoped request must be accepted"
|
||||||
|
);
|
||||||
|
let replay = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false)
|
||||||
|
.expect_err("the random replay-scope nonce must be single-use");
|
||||||
|
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||||
|
|
||||||
|
let path_error = verify_tonic_replay_scope_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
|
||||||
|
.expect_err("a replay-scoped signature must not move to another method");
|
||||||
|
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
|
||||||
|
ensure_test_rpc_secret();
|
||||||
|
let path = "/node_service.NodeService/Ping";
|
||||||
|
let mut partial = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||||
|
.expect("v2 compatibility headers should build");
|
||||||
|
partial.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
|
||||||
|
let error = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
|
||||||
|
.expect_err("partial replay-scope metadata must never downgrade to v2");
|
||||||
|
assert_eq!(error.to_string(), "Missing RPC replay scope signature");
|
||||||
|
|
||||||
|
let timestamp = partial
|
||||||
|
.get(TIMESTAMP_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.expect("v2 timestamp")
|
||||||
|
.to_string();
|
||||||
|
let content_sha256 = partial
|
||||||
|
.get(RPC_CONTENT_SHA256_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.expect("v2 content digest")
|
||||||
|
.to_string();
|
||||||
|
let stale_epoch = Uuid::new_v4();
|
||||||
|
partial.extend(
|
||||||
|
gen_tonic_replay_scope_headers("node-a:9000", path, ×tamp, &content_sha256, stale_epoch)
|
||||||
|
.expect("replay-scope headers should build"),
|
||||||
|
);
|
||||||
|
let stale = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
|
||||||
|
.expect_err("a signature from a prior server boot epoch must be rejected");
|
||||||
|
assert_eq!(stale.to_string(), "RPC boot epoch is stale");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_scope_strictness_allows_only_authenticated_ping_bootstrap() {
|
||||||
|
ensure_test_rpc_secret();
|
||||||
|
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||||
|
.expect("v2 compatibility headers should build");
|
||||||
|
let rejected =
|
||||||
|
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, false)
|
||||||
|
.expect_err("strict replay scope must reject stripped new metadata");
|
||||||
|
assert_eq!(rejected.to_string(), "RPC replay-scoped authentication required");
|
||||||
|
|
||||||
|
headers.insert(
|
||||||
|
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||||
|
HeaderValue::from_str(&Uuid::new_v4().to_string()).expect("UUID header"),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, true,)
|
||||||
|
.is_ok(),
|
||||||
|
"only the signed Ping bootstrap may obtain a new server epoch in strict mode"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn boot_epoch_response_proof_binds_audience_challenge_and_epoch() {
|
||||||
|
ensure_test_rpc_secret();
|
||||||
|
let challenge = Uuid::new_v4();
|
||||||
|
let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("proof headers should build");
|
||||||
|
let epoch =
|
||||||
|
verify_tonic_boot_epoch_response("node-a:9000", challenge, &headers).expect("matching proof headers should verify");
|
||||||
|
assert_eq!(epoch, tonic_rpc_boot_epoch());
|
||||||
|
assert!(verify_tonic_boot_epoch_response("node-b:9000", challenge, &headers).is_err());
|
||||||
|
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
|
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
|
||||||
ensure_test_rpc_secret();
|
ensure_test_rpc_secret();
|
||||||
|
|||||||
@@ -28,13 +28,18 @@ pub(crate) use background_monitor::pin_callsite_interest_for_test;
|
|||||||
pub use background_monitor::shutdown_background_monitors;
|
pub use background_monitor::shutdown_background_monitors;
|
||||||
pub(crate) use background_monitor::spawn_background_monitor;
|
pub(crate) use background_monitor::spawn_background_monitor;
|
||||||
pub use client::{
|
pub use client::{
|
||||||
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||||
|
node_service_time_out_client_no_auth,
|
||||||
};
|
};
|
||||||
|
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
||||||
|
#[allow(unused_imports)]
|
||||||
pub use http_auth::{
|
pub use http_auth::{
|
||||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
|
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||||
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof,
|
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
|
||||||
verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
|
||||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||||
|
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||||
|
verify_tonic_rpc_signature_with_bootstrap,
|
||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::cluster::rpc::client::{
|
use crate::cluster::rpc::client::{
|
||||||
TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
||||||
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
|
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||||
};
|
};
|
||||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
||||||
@@ -66,7 +66,6 @@ use std::{
|
|||||||
use tokio::{net::TcpStream, time::Duration};
|
use tokio::{net::TcpStream, time::Duration};
|
||||||
use tonic::Request;
|
use tonic::Request;
|
||||||
use tonic::service::interceptor::InterceptedService;
|
use tonic::service::interceptor::InterceptedService;
|
||||||
use tonic::transport::Channel;
|
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
@@ -412,7 +411,7 @@ impl PeerRestClient {
|
|||||||
(remote, all, remote_topology_hosts)
|
(remote, all, remote_topology_hosts)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||||
if self.offline.load(Ordering::Acquire) {
|
if self.offline.load(Ordering::Acquire) {
|
||||||
self.mark_offline_and_spawn_recovery();
|
self.mark_offline_and_spawn_recovery();
|
||||||
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
||||||
@@ -433,7 +432,7 @@ impl PeerRestClient {
|
|||||||
&self,
|
&self,
|
||||||
) -> Result<
|
) -> Result<
|
||||||
rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
|
rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
|
||||||
InterceptedService<Channel, TonicInterceptor>,
|
InterceptedService<AuthenticatedChannel, TonicInterceptor>,
|
||||||
>,
|
>,
|
||||||
> {
|
> {
|
||||||
if self.offline.load(Ordering::Acquire) {
|
if self.offline.load(Ordering::Acquire) {
|
||||||
@@ -454,7 +453,7 @@ impl PeerRestClient {
|
|||||||
|
|
||||||
async fn get_tier_mutation_control_client(
|
async fn get_tier_mutation_control_client(
|
||||||
&self,
|
&self,
|
||||||
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||||
if self.offline.load(Ordering::Acquire) {
|
if self.offline.load(Ordering::Acquire) {
|
||||||
self.mark_offline_and_spawn_recovery();
|
self.mark_offline_and_spawn_recovery();
|
||||||
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
||||||
|
|||||||
@@ -14,7 +14,8 @@
|
|||||||
|
|
||||||
use crate::bucket::metadata_sys;
|
use crate::bucket::metadata_sys;
|
||||||
use crate::cluster::rpc::client::{
|
use crate::cluster::rpc::client::{
|
||||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
|
||||||
|
node_service_time_out_client,
|
||||||
};
|
};
|
||||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||||
use crate::disk::error::DiskError;
|
use crate::disk::error::DiskError;
|
||||||
@@ -52,7 +53,6 @@ use tokio::{net::TcpStream, sync::RwLock, time};
|
|||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use tonic::Request;
|
use tonic::Request;
|
||||||
use tonic::service::interceptor::InterceptedService;
|
use tonic::service::interceptor::InterceptedService;
|
||||||
use tonic::transport::Channel;
|
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
type Client = Arc<Box<dyn PeerS3Client>>;
|
type Client = Arc<Box<dyn PeerS3Client>>;
|
||||||
@@ -832,7 +832,7 @@ impl RemotePeerS3Client {
|
|||||||
client
|
client
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||||
node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
|
node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
|
||||||
.await
|
.await
|
||||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))
|
.map_err(|err| Error::other(format!("can not get client, err: {err}")))
|
||||||
|
|||||||
@@ -13,8 +13,8 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::cluster::rpc::client::{
|
use crate::cluster::rpc::client::{
|
||||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
|
||||||
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
||||||
};
|
};
|
||||||
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
|
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
|
||||||
use crate::cluster::rpc::internode_data_transport::{
|
use crate::cluster::rpc::internode_data_transport::{
|
||||||
@@ -71,7 +71,7 @@ use tokio::{
|
|||||||
time::timeout,
|
time::timeout,
|
||||||
};
|
};
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
|
use tonic::{Code, Request, service::interceptor::InterceptedService};
|
||||||
use tracing::{debug, trace, warn};
|
use tracing::{debug, trace, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
@@ -1083,7 +1083,7 @@ impl RemoteDisk {
|
|||||||
internode_offline_bypass_reason(&self.addr).map(Error::other)
|
internode_offline_bypass_reason(&self.addr).map(Error::other)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||||
if let Some(err) = self.offline_bypass_error() {
|
if let Some(err) = self.offline_bypass_error() {
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
@@ -1096,7 +1096,7 @@ impl RemoteDisk {
|
|||||||
/// Routes onto the isolated bulk channel pool so large transfers cannot head-of-line block
|
/// Routes onto the isolated bulk channel pool so large transfers cannot head-of-line block
|
||||||
/// lock/health RPCs (grpc-optimization P1). Falls back to the control channel when isolation
|
/// lock/health RPCs (grpc-optimization P1). Falls back to the control channel when isolation
|
||||||
/// is disabled.
|
/// is disabled.
|
||||||
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||||
if let Some(err) = self.offline_bypass_error() {
|
if let Some(err) = self.offline_bypass_error() {
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,9 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
use crate::cluster::rpc::client::{
|
||||||
|
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||||
|
};
|
||||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
@@ -29,7 +31,6 @@ use std::time::Duration;
|
|||||||
use tokio::time::timeout;
|
use tokio::time::timeout;
|
||||||
use tonic::Request;
|
use tonic::Request;
|
||||||
use tonic::service::interceptor::InterceptedService;
|
use tonic::service::interceptor::InterceptedService;
|
||||||
use tonic::transport::Channel;
|
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
/// Remote lock client implementation
|
/// Remote lock client implementation
|
||||||
@@ -78,7 +79,7 @@ impl RemoteClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||||
// P3-2 offline bypass (now covering the lock path too): fast-fail a peer already marked
|
// P3-2 offline bypass (now covering the lock path too): fast-fail a peer already marked
|
||||||
// offline instead of paying the connect timeout, so dsync reaches quorum sooner. Does not
|
// offline instead of paying the connect timeout, so dsync reaches quorum sooner. Does not
|
||||||
// change quorum; the self-healing re-probe keeps the peer recoverable.
|
// change quorum; the self-healing re-probe keeps the peer recoverable.
|
||||||
|
|||||||
@@ -67,6 +67,7 @@ const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
|
|||||||
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||||
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
|
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
|
||||||
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
|
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
|
||||||
|
const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
|
||||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
||||||
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
|
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
|
||||||
|
|
||||||
@@ -159,6 +160,7 @@ pub struct InternodeMetricsSnapshot {
|
|||||||
pub operation_write_shutdown_errors_total: u64,
|
pub operation_write_shutdown_errors_total: u64,
|
||||||
pub signature_v1_fallback_total: u64,
|
pub signature_v1_fallback_total: u64,
|
||||||
pub body_digest_fallback_total: u64,
|
pub body_digest_fallback_total: u64,
|
||||||
|
pub replay_scope_fallback_total: u64,
|
||||||
pub replay_cache_overflow_total: u64,
|
pub replay_cache_overflow_total: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -180,6 +182,7 @@ pub struct InternodeMetrics {
|
|||||||
msgpack_json_decode_error_total: AtomicU64,
|
msgpack_json_decode_error_total: AtomicU64,
|
||||||
signature_v1_fallback_total: AtomicU64,
|
signature_v1_fallback_total: AtomicU64,
|
||||||
body_digest_fallback_total: AtomicU64,
|
body_digest_fallback_total: AtomicU64,
|
||||||
|
replay_scope_fallback_total: AtomicU64,
|
||||||
replay_cache_overflow_total: AtomicU64,
|
replay_cache_overflow_total: AtomicU64,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -431,6 +434,13 @@ impl InternodeMetrics {
|
|||||||
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
|
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Count an accepted v1/v2 request that does not carry the replay-scoped signature. This is
|
||||||
|
/// the convergence signal for `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT`.
|
||||||
|
pub fn record_replay_scope_fallback(&self) {
|
||||||
|
self.replay_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
|
||||||
|
counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL).increment(1);
|
||||||
|
}
|
||||||
|
|
||||||
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
|
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
|
||||||
/// full. Overflow fails closed, so a sustained non-zero rate means
|
/// full. Overflow fails closed, so a sustained non-zero rate means
|
||||||
/// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
|
/// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
|
||||||
@@ -488,6 +498,7 @@ impl InternodeMetrics {
|
|||||||
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
|
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
|
||||||
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
|
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
|
||||||
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
|
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
|
||||||
|
replay_scope_fallback_total: self.replay_scope_fallback_total.load(Ordering::Relaxed),
|
||||||
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
|
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -510,6 +521,7 @@ impl InternodeMetrics {
|
|||||||
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
|
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
|
||||||
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
|
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
|
||||||
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
|
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
|
||||||
|
self.replay_scope_fallback_total.store(0, Ordering::Relaxed);
|
||||||
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
|
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -887,6 +899,19 @@ mod tests {
|
|||||||
assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
|
assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_scope_fallback_counter_updates_snapshot_and_resets() {
|
||||||
|
let metrics = InternodeMetrics::default();
|
||||||
|
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
|
||||||
|
|
||||||
|
metrics.record_replay_scope_fallback();
|
||||||
|
metrics.record_replay_scope_fallback();
|
||||||
|
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 2);
|
||||||
|
|
||||||
|
metrics.reset_for_test();
|
||||||
|
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cluster_peer_flips_offline_after_threshold_and_back_online() {
|
fn cluster_peer_flips_offline_after_threshold_and_back_online() {
|
||||||
// Unique addr keeps this independent of the process-global registry / other tests.
|
// Unique addr keeps this independent of the process-global registry / other tests.
|
||||||
|
|||||||
@@ -37,7 +37,8 @@ use crate::storage_api::server::http as storage;
|
|||||||
use crate::storage_api::server::http::rpc::InternodeRpcService;
|
use crate::storage_api::server::http::rpc::InternodeRpcService;
|
||||||
use crate::storage_api::server::http::tonic_service::make_server;
|
use crate::storage_api::server::http::tonic_service::make_server;
|
||||||
use crate::storage_api::server::http::{
|
use crate::storage_api::server::http::{
|
||||||
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, verify_tonic_rpc_signature,
|
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge,
|
||||||
|
tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap,
|
||||||
};
|
};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use http::{HeaderMap, Method, Request as HttpRequest, Response, Uri};
|
use http::{HeaderMap, Method, Request as HttpRequest, Response, Uri};
|
||||||
@@ -152,13 +153,17 @@ impl<S> RpcRequestPathService<S> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<S, B> Service<HttpRequest<B>> for RpcRequestPathService<S>
|
impl<S, B, ResBody> Service<HttpRequest<B>> for RpcRequestPathService<S>
|
||||||
where
|
where
|
||||||
S: Service<HttpRequest<B>>,
|
S: Service<HttpRequest<B>, Response = Response<ResBody>>,
|
||||||
|
S::Error: Send + 'static,
|
||||||
|
S::Future: Send + 'static,
|
||||||
|
B: Send + 'static,
|
||||||
|
ResBody: Send + 'static,
|
||||||
{
|
{
|
||||||
type Response = S::Response;
|
type Response = Response<ResBody>;
|
||||||
type Error = S::Error;
|
type Error = S::Error;
|
||||||
type Future = S::Future;
|
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
|
||||||
|
|
||||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||||
self.inner.poll_ready(cx)
|
self.inner.poll_ready(cx)
|
||||||
@@ -170,7 +175,22 @@ where
|
|||||||
method: req.method().clone(),
|
method: req.method().clone(),
|
||||||
};
|
};
|
||||||
req.extensions_mut().insert(target);
|
req.extensions_mut().insert(target);
|
||||||
self.inner.call(req)
|
let response_headers = tonic_boot_epoch_challenge(req.headers())
|
||||||
|
.ok()
|
||||||
|
.flatten()
|
||||||
|
.and_then(|challenge| {
|
||||||
|
storage::try_current_local_node_name()
|
||||||
|
.and_then(|node| normalize_tonic_rpc_audience(&node).ok())
|
||||||
|
.and_then(|audience| tonic_boot_epoch_response_headers(&audience, challenge).ok())
|
||||||
|
});
|
||||||
|
let future = self.inner.call(req);
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut response = future.await?;
|
||||||
|
if let Some(headers) = response_headers {
|
||||||
|
response.headers_mut().extend(headers);
|
||||||
|
}
|
||||||
|
Ok(response)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1862,7 +1882,15 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
|
|||||||
.filter(|method| !method.is_empty() && !method.contains('/'))
|
.filter(|method| !method.is_empty() && !method.contains('/'))
|
||||||
.ok_or_else(|| Status::unauthenticated("Invalid RPC request path"))?;
|
.ok_or_else(|| Status::unauthenticated("Invalid RPC request path"))?;
|
||||||
debug_assert!(!rpc_method.is_empty());
|
debug_assert!(!rpc_method.is_empty());
|
||||||
verify_tonic_rpc_signature(&audience, target.uri.path(), req.metadata().as_ref()).map_err(|e| {
|
let allow_replay_scope_bootstrap = target.uri.path() == "/node_service.NodeService/Ping"
|
||||||
|
&& tonic_boot_epoch_challenge(req.metadata().as_ref()).is_ok_and(|challenge| challenge.is_some());
|
||||||
|
verify_tonic_rpc_signature_with_bootstrap(
|
||||||
|
&audience,
|
||||||
|
target.uri.path(),
|
||||||
|
req.metadata().as_ref(),
|
||||||
|
allow_replay_scope_bootstrap,
|
||||||
|
)
|
||||||
|
.map_err(|e| {
|
||||||
error!(
|
error!(
|
||||||
event = EVENT_RPC_SIGNATURE_VERIFICATION_FAILED,
|
event = EVENT_RPC_SIGNATURE_VERIFICATION_FAILED,
|
||||||
component = LOG_COMPONENT_SERVER,
|
component = LOG_COMPONENT_SERVER,
|
||||||
|
|||||||
@@ -495,8 +495,9 @@ pub(crate) mod ecstore_rpc {
|
|||||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||||
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
|
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
|
||||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
|
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
|
||||||
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest,
|
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_signature,
|
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
||||||
|
verify_tonic_rpc_signature_with_bootstrap,
|
||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||||
@@ -1605,8 +1606,21 @@ pub(crate) fn sign_ns_scanner_capability(challenge: uuid::Uuid, server_epoch: uu
|
|||||||
ecstore_rpc::sign_ns_scanner_capability(challenge, server_epoch)
|
ecstore_rpc::sign_ns_scanner_capability(challenge, server_epoch)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &http::HeaderMap) -> std::io::Result<()> {
|
pub(crate) fn verify_tonic_rpc_signature_with_bootstrap(
|
||||||
ecstore_rpc::verify_tonic_rpc_signature(audience, path, headers)
|
audience: &str,
|
||||||
|
path: &str,
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
allow_replay_scope_bootstrap: bool,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
ecstore_rpc::verify_tonic_rpc_signature_with_bootstrap(audience, path, headers, allow_replay_scope_bootstrap)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn tonic_boot_epoch_challenge(headers: &http::HeaderMap) -> std::io::Result<Option<uuid::Uuid>> {
|
||||||
|
ecstore_rpc::tonic_boot_epoch_challenge(headers)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn tonic_boot_epoch_response_headers(audience: &str, challenge: uuid::Uuid) -> std::io::Result<http::HeaderMap> {
|
||||||
|
ecstore_rpc::tonic_boot_epoch_response_headers(audience, challenge)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
pub(crate) fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||||
|
|||||||
@@ -97,7 +97,8 @@ pub(crate) mod server {
|
|||||||
|
|
||||||
pub(crate) mod http {
|
pub(crate) mod http {
|
||||||
pub(crate) use crate::storage::storage_api::{
|
pub(crate) use crate::storage::storage_api::{
|
||||||
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, verify_tonic_rpc_signature,
|
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge,
|
||||||
|
tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(crate) fn try_current_local_node_name() -> Option<String> {
|
pub(crate) fn try_current_local_node_name() -> Option<String> {
|
||||||
|
|||||||
Reference in New Issue
Block a user