mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-08 22:33:22 +00:00
feat(rpc): bind canonical body digest into internode mutating disk RPC signatures (#5234)
* feat(rpc): bind canonical body digest into internode mutating disk RPC signatures Binds a domain-separated, length-prefixed canonical request-body digest into the v2 HMAC signature scope for every mutating NodeService disk RPC, so an on-path attacker on the default-plaintext internode channel can no longer tamper with a mutation payload (or strip the msgpack `_bin` field to force the JSON fallback decode) without invalidating the signature. Covers 13 mutating disk RPCs: RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes. The digest covers both the msgpack `_bin` payloads and their JSON compatibility copies. Gated fail-open by default (RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT) with a convergence counter, so rolling upgrades are byte-for-byte unaffected; the replay-cache capacity is now configurable and overflow fails closed with a metric. Refs https://github.com/rustfs/backlog/issues/1327 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix(rpc): satisfy architecture-migration compat-marker guard Put the removal condition on the RUSTFS_COMPAT_TODO marker line itself, and stop backticking env-var/metric names in the cleanup-register entry so the guard's id extractor only sees the task-id. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Zhengchao An
@@ -136,6 +136,42 @@ pub const DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT: bool = false;
|
||||
// rolling upgrades until the fleet-wide v1-fallback counter reads zero.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT);
|
||||
|
||||
/// Require a signature-bound canonical body digest on every mutating internode disk RPC
|
||||
/// (RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete,
|
||||
/// DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes), rejecting requests
|
||||
/// that authenticate without one (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
///
|
||||
/// Defaults to `false` (fail-open): a mutating request without a body digest keeps authenticating
|
||||
/// through the method-bound v2 (or legacy) signature, so peers from releases that predate
|
||||
/// body-digest signing survive rolling upgrades unchanged. Requests that do carry a digest are
|
||||
/// always verified with no downgrade, strict or not — the digest value is part of the signed v2
|
||||
/// scope, so an on-path attacker cannot strip it without invalidating the signature. This is a
|
||||
/// rollout lever gated on the body-digest fallback counter
|
||||
/// (`rustfs_system_network_internode_body_digest_fallback_total`) reading zero across a release
|
||||
/// window fleet-wide. Single-env rollback. It is deliberately separate from
|
||||
/// [`ENV_INTERNODE_RPC_SIGNATURE_STRICT`]: the two enforcement flips converge on different
|
||||
/// counters and must not gate each other.
|
||||
pub const ENV_INTERNODE_RPC_BODY_DIGEST_STRICT: &str = "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT";
|
||||
pub const DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT: bool = false;
|
||||
|
||||
// Compile-time invariant: fail-open by default so digestless peers keep authenticating during
|
||||
// rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
|
||||
|
||||
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
||||
/// one-time consumption of body-bound v2 signatures.
|
||||
///
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope, so the steady
|
||||
/// state holds roughly `mutating RPS x 601s` entries; the default sustains ~1,700 body-bound
|
||||
/// mutating RPCs per second (about 120 MiB worst case, allocated only under sustained load).
|
||||
/// Overflow fails closed — legitimate signed traffic is the only thing that can fill the cache
|
||||
/// (replays are rejected before insertion, and an attacker cannot mint valid nonces without the
|
||||
/// shared secret) — and increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
||||
/// counter means this capacity is undersized for the node's peak mutation rate.
|
||||
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
||||
|
||||
/// Consecutive-failure threshold after which an internode peer is marked offline (grpc-optimization
|
||||
/// P3 observability).
|
||||
///
|
||||
@@ -312,6 +348,18 @@ mod tests {
|
||||
assert_eq!(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_body_digest_strict_env_name_is_stable() {
|
||||
// The fail-open default invariant is asserted at compile time next to the definition.
|
||||
assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_replay_cache_capacity_defaults_and_env_name() {
|
||||
assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");
|
||||
assert_eq!(DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, 1_048_576);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_offline_failure_threshold_defaults_and_env_name() {
|
||||
assert_eq!(DEFAULT_INTERNODE_OFFLINE_FAILURE_THRESHOLD, 3);
|
||||
|
||||
@@ -411,7 +411,8 @@ pub mod rpc {
|
||||
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -61,7 +61,6 @@ const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
|
||||
const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536;
|
||||
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
|
||||
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
|
||||
static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
@@ -70,6 +69,22 @@ static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT,
|
||||
)
|
||||
});
|
||||
static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
get_env_bool(
|
||||
rustfs_config::ENV_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
)
|
||||
});
|
||||
// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
|
||||
// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
|
||||
// cannot disable replay protection by rejecting every body-bound request.
|
||||
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
)
|
||||
.max(1)
|
||||
});
|
||||
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -110,6 +125,10 @@ impl RpcNonceCache {
|
||||
return Err(std::io::Error::other("RPC request replay detected"));
|
||||
}
|
||||
if self.nonces.len() >= capacity {
|
||||
// Fail closed and alert: only legitimately signed traffic can fill the cache, so a
|
||||
// sustained overflow means RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY is undersized
|
||||
// for this node's peak mutation rate and writes are being refused.
|
||||
global_internode_metrics().record_replay_cache_overflow();
|
||||
return Err(std::io::Error::other("RPC replay cache capacity exceeded"));
|
||||
}
|
||||
self.nonces.insert(nonce);
|
||||
@@ -334,7 +353,7 @@ fn check_and_record_nonce(nonce: Uuid, signed_at: i64) -> std::io::Result<()> {
|
||||
let expires_at = now
|
||||
.checked_add(REPLAY_CACHE_RETENTION)
|
||||
.ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
|
||||
cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, MAX_REPLAY_PROTECTED_NONCES)
|
||||
cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, *REPLAY_CACHE_CAPACITY)
|
||||
}
|
||||
|
||||
/// Build headers with authentication signature
|
||||
@@ -447,6 +466,50 @@ pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canoni
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify a mutating disk RPC's canonical body digest with a rolling-upgrade fallback.
|
||||
///
|
||||
/// When the request carries a real (non-`UNSIGNED-PAYLOAD`) content SHA-256 it is verified exactly
|
||||
/// like [`verify_tonic_canonical_body_digest`]. The digest value is a member of the signed v2
|
||||
/// scope, so within the v2 lane it cannot be stripped or altered without invalidating the signature
|
||||
/// `check_auth` already enforced. When the request carries no digest — a peer that predates
|
||||
/// body-digest signing, or an attacker who downgraded the request to the legacy signature by
|
||||
/// dropping every v2 header — the request is accepted and counted on the body-digest fallback
|
||||
/// counter unless `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT` is enabled. That switch is what actually
|
||||
/// closes on-path body tampering for covered handlers: it rejects every digestless mutation,
|
||||
/// including v1-downgraded ones. It converges independently of the signature-strict switch
|
||||
/// (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
pub fn verify_tonic_mutation_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, *INTERNODE_RPC_BODY_DIGEST_STRICT)
|
||||
}
|
||||
|
||||
/// [`verify_tonic_mutation_body_digest`] with the strict gate injected as a parameter, so both
|
||||
/// rollout postures are unit-testable without racing on process-global environment variables.
|
||||
fn verify_tonic_mutation_body_digest_with_strictness<T>(
|
||||
request: &tonic::Request<T>,
|
||||
canonical_body: &[u8],
|
||||
strict: bool,
|
||||
) -> std::io::Result<()> {
|
||||
let digest = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
match digest {
|
||||
Some(digest) if digest != UNSIGNED_PAYLOAD => verify_tonic_canonical_body_digest(request, canonical_body),
|
||||
_ => {
|
||||
// RUSTFS_COMPAT_TODO(disk-mutation-body-digest): accept digestless peers during rolling upgrades. Remove after the
|
||||
// minimum supported RustFS peer version body-binds every mutating disk RPC.
|
||||
if strict {
|
||||
return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
|
||||
}
|
||||
// Count only ACCEPTED digestless mutations: this counter is the convergence gate that
|
||||
// must read zero fleet-wide across a release window before
|
||||
// `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT` may be enabled.
|
||||
global_internode_metrics().record_body_digest_fallback();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
|
||||
[
|
||||
RPC_AUTH_VERSION_HEADER,
|
||||
@@ -1410,6 +1473,129 @@ mod tests {
|
||||
assert!(cache.nonces.contains(&nonce_b));
|
||||
}
|
||||
|
||||
// The `rpc_body_digest_fallback_counter` serial group covers every test that drives (or
|
||||
// asserts on) the process-global body-digest fallback counter, so exact-delta assertions
|
||||
// cannot race with each other.
|
||||
#[test]
|
||||
#[serial_test::serial(rpc_body_digest_fallback_counter)]
|
||||
fn digestless_mutation_is_accepted_and_counted_while_strict_gate_is_off() {
|
||||
let request = tonic::Request::new(());
|
||||
let before = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
|
||||
assert!(
|
||||
verify_tonic_mutation_body_digest_with_strictness(&request, b"canonical-mutation-body", false).is_ok(),
|
||||
"a digestless peer must keep mutating while the strict gate is off"
|
||||
);
|
||||
|
||||
let after = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
assert_eq!(
|
||||
after,
|
||||
before + 1,
|
||||
"an accepted digestless mutation must increment the body-digest fallback counter exactly once"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(rpc_body_digest_fallback_counter)]
|
||||
fn strict_mutation_gate_rejects_digestless_but_keeps_body_bound() {
|
||||
let before = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
|
||||
let digestless = tonic::Request::new(());
|
||||
let error = verify_tonic_mutation_body_digest_with_strictness(&digestless, b"body", true)
|
||||
.expect_err("strict mode must reject a mutation without a body digest");
|
||||
assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
|
||||
|
||||
let mut unsigned = tonic::Request::new(());
|
||||
unsigned
|
||||
.metadata_mut()
|
||||
.as_mut()
|
||||
.insert(RPC_CONTENT_SHA256_HEADER, HeaderValue::from_static(UNSIGNED_PAYLOAD));
|
||||
let error = verify_tonic_mutation_body_digest_with_strictness(&unsigned, b"body", true)
|
||||
.expect_err("strict mode must reject an explicitly unsigned mutation payload");
|
||||
assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
|
||||
|
||||
let mut bound = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut bound, b"body").expect("digest metadata should encode");
|
||||
bound
|
||||
.metadata_mut()
|
||||
.as_mut()
|
||||
.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
assert!(
|
||||
verify_tonic_mutation_body_digest_with_strictness(&bound, b"body", true).is_ok(),
|
||||
"strict mode must keep accepting body-bound mutations"
|
||||
);
|
||||
let tampered = verify_tonic_mutation_body_digest_with_strictness(&bound, b"tampered-body", true)
|
||||
.expect_err("a tampered canonical body must fail even in strict mode");
|
||||
assert_eq!(tampered.to_string(), "RPC content SHA-256 mismatch");
|
||||
|
||||
let after = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"neither strict rejections nor bound verifications are digestless fallbacks"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(rpc_body_digest_fallback_counter)]
|
||||
fn mutation_digest_default_posture_is_fail_open_digestless_accept() {
|
||||
// The public entry point resolves strictness from the environment, whose compile-time
|
||||
// default is pinned to false in `rustfs_config`. A digestless peer therefore keeps
|
||||
// mutating through the default build with no configuration at all.
|
||||
let request = tonic::Request::new(());
|
||||
assert!(
|
||||
verify_tonic_mutation_body_digest_with_strictness(
|
||||
&request,
|
||||
b"canonical-mutation-body",
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
)
|
||||
.is_ok(),
|
||||
"the default strict posture must accept digestless mutations"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_mutation_contract_binds_method_nonce_and_body() {
|
||||
ensure_test_rpc_secret();
|
||||
let message = rustfs_protos::proto_gen::node_service::RenameDataRequest {
|
||||
disk: "http://node-a:9000/data/rustfs0".to_string(),
|
||||
src_volume: ".rustfs.sys/multipart".to_string(),
|
||||
src_path: "uploads/object".to_string(),
|
||||
file_info: "{\"volume\":\"bucket\"}".to_string(),
|
||||
dst_volume: "bucket".to_string(),
|
||||
dst_path: "object".to_string(),
|
||||
file_info_bin: vec![0x81, 0xA1, 0x76, 0x01].into(),
|
||||
};
|
||||
let body = rustfs_protos::canonical_rename_data_request_body(&message).expect("small request should encode");
|
||||
let mut request = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("canonical body digest should be attached");
|
||||
let content_sha256 = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "RenameData", content_sha256)
|
||||
.expect("body-bound auth headers should build");
|
||||
request.metadata_mut().as_mut().extend(headers.clone());
|
||||
|
||||
assert!(
|
||||
verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/RenameData", &headers).is_ok(),
|
||||
"the rename_data signature must bind destination, method, nonce, and body digest"
|
||||
);
|
||||
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/RenameData", &headers)
|
||||
.expect_err("reusing a consumed rename_data nonce must fail");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
let transplant = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/DeleteVersion", &headers)
|
||||
.expect_err("a rename_data signature must not authenticate a different method");
|
||||
assert_eq!(transplant.to_string(), "Invalid RPC v2 signature");
|
||||
|
||||
assert!(verify_tonic_mutation_body_digest(&request, &body).is_ok());
|
||||
let mut tampered = message;
|
||||
tampered.file_info_bin = Vec::new().into();
|
||||
let tampered_body = rustfs_protos::canonical_rename_data_request_body(&tampered).expect("small request should encode");
|
||||
let stripped = verify_tonic_mutation_body_digest(&request, &tampered_body)
|
||||
.expect_err("stripping the msgpack payload to force the JSON fallback decode must fail");
|
||||
assert_eq!(stripped.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_rejects_replay_after_wall_clock_regression() {
|
||||
let now = Instant::now();
|
||||
|
||||
@@ -31,7 +31,8 @@ pub use client::{
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability,
|
||||
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::cluster::rpc::client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
||||
};
|
||||
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
|
||||
use crate::cluster::rpc::internode_data_transport::{
|
||||
InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
|
||||
WriteStreamRequest,
|
||||
@@ -99,6 +100,18 @@ const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
|
||||
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
|
||||
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
|
||||
|
||||
/// Bind a mutating disk RPC to its canonical body: the digest lands in the request metadata, and
|
||||
/// the signing interceptor folds it (plus a replay-protected nonce) into the v2 signature scope
|
||||
/// (backlog#1327).
|
||||
fn attach_mutation_body_digest<T>(
|
||||
request: &mut Request<T>,
|
||||
canonical_body: std::result::Result<Vec<u8>, std::num::TryFromIntError>,
|
||||
op: &'static str,
|
||||
) -> Result<()> {
|
||||
let canonical_body = canonical_body.map_err(|_| Error::other(format!("{op} request length cannot be represented")))?;
|
||||
set_tonic_canonical_body_digest(request, &canonical_body).map_err(Error::other)
|
||||
}
|
||||
|
||||
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
|
||||
volume_infos
|
||||
.into_iter()
|
||||
@@ -1339,10 +1352,12 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(MakeVolumeRequest {
|
||||
let mut request = Request::new(MakeVolumeRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_make_volume_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "make_volume")?;
|
||||
|
||||
let response = client.make_volume(request).await?.into_inner();
|
||||
|
||||
@@ -1376,10 +1391,12 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(MakeVolumesRequest {
|
||||
let mut request = Request::new(MakeVolumesRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volumes: volumes.iter().map(|s| (*s).to_string()).collect(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_make_volumes_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "make_volumes")?;
|
||||
|
||||
let response = client.make_volumes(request).await?.into_inner();
|
||||
|
||||
@@ -1489,11 +1506,13 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(DeleteVolumeRequest {
|
||||
let mut request = Request::new(DeleteVolumeRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
force: force_delete,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_volume_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete_volume")?;
|
||||
|
||||
let response = client.delete_volume(request).await?.into_inner();
|
||||
|
||||
@@ -1542,7 +1561,7 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(DeleteVersionRequest {
|
||||
let mut request = Request::new(DeleteVersionRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
@@ -1552,6 +1571,8 @@ impl DiskAPI for RemoteDisk {
|
||||
file_info_bin: file_info_bin.into(),
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_version_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete_version")?;
|
||||
|
||||
let response = client.delete_version(request).await?.into_inner();
|
||||
|
||||
@@ -1643,7 +1664,7 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
};
|
||||
|
||||
let request = Request::new(DeleteVersionsRequest {
|
||||
let mut request = Request::new(DeleteVersionsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
versions: versions_str,
|
||||
@@ -1651,6 +1672,14 @@ impl DiskAPI for RemoteDisk {
|
||||
versions_bin,
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_versions_request_body(request.get_ref());
|
||||
if let Err(err) = attach_mutation_body_digest(&mut request, canonical_body, "delete_versions") {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for _ in 0..versions.len() {
|
||||
errors.push(Some(err.clone()));
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
// TODO: use Error not string
|
||||
|
||||
@@ -1719,11 +1748,13 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(DeletePathsRequest {
|
||||
let mut request = Request::new(DeletePathsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
paths: paths.clone(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_paths_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete_paths")?;
|
||||
|
||||
let response = client.delete_paths(request).await?.into_inner();
|
||||
|
||||
@@ -1762,13 +1793,15 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(WriteMetadataRequest {
|
||||
let mut request = Request::new(WriteMetadataRequest {
|
||||
disk,
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
file_info,
|
||||
file_info_bin: file_info_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_write_metadata_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "write_metadata")?;
|
||||
|
||||
let response = client.write_metadata(request).await?.into_inner();
|
||||
|
||||
@@ -1840,7 +1873,7 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(UpdateMetadataRequest {
|
||||
let mut request = Request::new(UpdateMetadataRequest {
|
||||
disk,
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
@@ -1849,6 +1882,8 @@ impl DiskAPI for RemoteDisk {
|
||||
file_info_bin: file_info_bin.into(),
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_update_metadata_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "update_metadata")?;
|
||||
|
||||
let response = client.update_metadata(request).await?.into_inner();
|
||||
|
||||
@@ -2096,7 +2131,7 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(RenameDataRequest {
|
||||
let mut request = Request::new(RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
@@ -2105,6 +2140,8 @@ impl DiskAPI for RemoteDisk {
|
||||
dst_path: dst_path.to_string(),
|
||||
file_info_bin: file_info_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
|
||||
let response = client.rename_data(request).await?.into_inner();
|
||||
|
||||
@@ -2361,13 +2398,15 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(RenameFileRequest {
|
||||
let mut request = Request::new(RenameFileRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_file_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_file")?;
|
||||
|
||||
let response = client.rename_file(request).await?.into_inner();
|
||||
|
||||
@@ -2404,7 +2443,7 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(RenamePartRequest {
|
||||
let mut request = Request::new(RenamePartRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
@@ -2412,6 +2451,8 @@ impl DiskAPI for RemoteDisk {
|
||||
dst_path: dst_path.to_string(),
|
||||
meta,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_part_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_part")?;
|
||||
|
||||
let response = client.rename_part(request).await?.into_inner();
|
||||
|
||||
@@ -2449,12 +2490,14 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(DeleteRequest {
|
||||
let mut request = Request::new(DeleteRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
options,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
|
||||
|
||||
let response = client.delete(request).await?.into_inner();
|
||||
|
||||
@@ -2665,12 +2708,14 @@ impl DiskAPI for RemoteDisk {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
|
||||
Error::other(format!("can not get client, err: {err}"))
|
||||
})?;
|
||||
let request = Request::new(WriteAllRequest {
|
||||
let mut request = Request::new(WriteAllRequest {
|
||||
disk,
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
data,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_write_all_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
|
||||
let response = match client.write_all(request).await {
|
||||
|
||||
@@ -65,6 +65,8 @@ const INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL: &str = "rustfs_system_network_in
|
||||
const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
|
||||
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
|
||||
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
||||
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -155,6 +157,8 @@ pub struct InternodeMetricsSnapshot {
|
||||
pub operation_stall_timeouts_total: u64,
|
||||
pub operation_write_shutdown_errors_total: u64,
|
||||
pub signature_v1_fallback_total: u64,
|
||||
pub body_digest_fallback_total: u64,
|
||||
pub replay_cache_overflow_total: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -173,6 +177,8 @@ pub struct InternodeMetrics {
|
||||
operation_write_shutdown_errors_total: AtomicU64,
|
||||
msgpack_json_decode_error_total: AtomicU64,
|
||||
signature_v1_fallback_total: AtomicU64,
|
||||
body_digest_fallback_total: AtomicU64,
|
||||
replay_cache_overflow_total: AtomicU64,
|
||||
}
|
||||
|
||||
impl InternodeMetrics {
|
||||
@@ -396,6 +402,26 @@ impl InternodeMetrics {
|
||||
counter!(INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
/// Count a mutating internode disk RPC that was accepted without a signature-bound canonical
|
||||
/// body digest (rolling-upgrade fallback, see
|
||||
/// <https://github.com/rustfs/backlog/issues/1327>). Only accepted requests count. This counter
|
||||
/// must read zero across a release window fleet-wide before
|
||||
/// `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT` may be enabled; after the strict flip digestless
|
||||
/// mutations are rejected and the counter stays flat.
|
||||
pub fn record_body_digest_fallback(&self) {
|
||||
self.body_digest_fallback_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
|
||||
/// full. Overflow fails closed, so a sustained non-zero rate means
|
||||
/// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
|
||||
/// mutation rate and writes are being refused — alert on this counter.
|
||||
pub fn record_replay_cache_overflow(&self) {
|
||||
self.replay_cache_overflow_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
|
||||
counter!(
|
||||
ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
|
||||
@@ -443,6 +469,8 @@ impl InternodeMetrics {
|
||||
operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed),
|
||||
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
|
||||
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
|
||||
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
|
||||
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -462,6 +490,8 @@ impl InternodeMetrics {
|
||||
self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed);
|
||||
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
|
||||
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -423,6 +423,562 @@ pub fn canonical_scanner_activity_response_body(
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
/// Length-prefixed, domain-separated byte builder for the disk-mutation canonical bodies below.
|
||||
/// Every variable-length field is u64-length-prefixed and every list u64-count-prefixed, so
|
||||
/// distinct field values can never collide into the same canonical bytes.
|
||||
struct CanonicalBodyBuilder {
|
||||
body: Vec<u8>,
|
||||
}
|
||||
|
||||
impl CanonicalBodyBuilder {
|
||||
fn new(domain: &'static [u8]) -> Self {
|
||||
let mut body = Vec::with_capacity(domain.len() + 128);
|
||||
body.extend_from_slice(domain);
|
||||
Self { body }
|
||||
}
|
||||
|
||||
fn push_bytes(&mut self, field: &[u8]) -> Result<(), std::num::TryFromIntError> {
|
||||
self.body.extend_from_slice(&u64::try_from(field.len())?.to_be_bytes());
|
||||
self.body.extend_from_slice(field);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn push_str(&mut self, field: &str) -> Result<(), std::num::TryFromIntError> {
|
||||
self.push_bytes(field.as_bytes())
|
||||
}
|
||||
|
||||
fn push_bool(&mut self, field: bool) {
|
||||
self.body.push(u8::from(field));
|
||||
}
|
||||
|
||||
fn push_count(&mut self, count: usize) -> Result<(), std::num::TryFromIntError> {
|
||||
self.body.extend_from_slice(&u64::try_from(count)?.to_be_bytes());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(self) -> Vec<u8> {
|
||||
self.body
|
||||
}
|
||||
}
|
||||
|
||||
// Canonical request bodies for the mutating NodeService disk RPCs (backlog#1327 body-digest
|
||||
// binding). Each covers every semantic wire field — including both the msgpack `_bin` payload and
|
||||
// its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
|
||||
// the JSON fallback decode path, breaks the signature-bound digest.
|
||||
|
||||
pub fn canonical_rename_data_request_body(
|
||||
request: &proto_gen::node_service::RenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-rename-data-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.file_info)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
body.push_bytes(&request.file_info_bin)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_version_request_body(
|
||||
request: &proto_gen::node_service::DeleteVersionRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-delete-version-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_str(&request.file_info)?;
|
||||
body.push_bool(request.force_del_marker);
|
||||
body.push_str(&request.opts)?;
|
||||
body.push_bytes(&request.file_info_bin)?;
|
||||
body.push_bytes(&request.opts_bin)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_versions_request_body(
|
||||
request: &proto_gen::node_service::DeleteVersionsRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-delete-versions-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_count(request.versions.len())?;
|
||||
for version in &request.versions {
|
||||
body.push_str(version)?;
|
||||
}
|
||||
body.push_str(&request.opts)?;
|
||||
body.push_count(request.versions_bin.len())?;
|
||||
for version_bin in &request.versions_bin {
|
||||
body.push_bytes(version_bin)?;
|
||||
}
|
||||
body.push_bytes(&request.opts_bin)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_write_metadata_request_body(
|
||||
request: &proto_gen::node_service::WriteMetadataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-write-metadata-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_str(&request.file_info)?;
|
||||
body.push_bytes(&request.file_info_bin)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_update_metadata_request_body(
|
||||
request: &proto_gen::node_service::UpdateMetadataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-update-metadata-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_str(&request.file_info)?;
|
||||
body.push_str(&request.opts)?;
|
||||
body.push_bytes(&request.file_info_bin)?;
|
||||
body.push_bytes(&request.opts_bin)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_write_all_request_body(
|
||||
request: &proto_gen::node_service::WriteAllRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-write-all-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_bytes(&request.data)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_request_body(
|
||||
request: &proto_gen::node_service::DeleteRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-delete-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_str(&request.options)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_paths_request_body(
|
||||
request: &proto_gen::node_service::DeletePathsRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-delete-paths-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_count(request.paths.len())?;
|
||||
for path in &request.paths {
|
||||
body.push_str(path)?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_file_request_body(
|
||||
request: &proto_gen::node_service::RenameFileRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-rename-file-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_part_request_body(
|
||||
request: &proto_gen::node_service::RenamePartRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-rename-part-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
body.push_bytes(&request.meta)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_volume_request_body(
|
||||
request: &proto_gen::node_service::DeleteVolumeRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-delete-volume-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
// Binding `force` is the point: an unbound recursive-delete flag lets an on-path attacker
|
||||
// turn a refuse-if-nonempty delete into a recursive volume wipe (backlog#1327).
|
||||
body.push_bool(request.force);
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_make_volume_request_body(
|
||||
request: &proto_gen::node_service::MakeVolumeRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-make-volume-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_make_volumes_request_body(
|
||||
request: &proto_gen::node_service::MakeVolumesRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-make-volumes-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_count(request.volumes.len())?;
|
||||
for volume in &request.volumes {
|
||||
body.push_str(volume)?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod disk_mutation_canonical_tests {
|
||||
use super::proto_gen::node_service::{
|
||||
DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, UpdateMetadataRequest, WriteAllRequest,
|
||||
WriteMetadataRequest,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
fn assert_all_distinct(bodies: &[Vec<u8>]) {
|
||||
for (i, a) in bodies.iter().enumerate() {
|
||||
for (j, b) in bodies.iter().enumerate() {
|
||||
if i != j {
|
||||
assert_ne!(a, b, "canonical bodies {i} and {j} must differ");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_canonical_body_binds_every_field() {
|
||||
let baseline = RenameDataRequest {
|
||||
disk: "disk-a".into(),
|
||||
src_volume: "src-vol".into(),
|
||||
src_path: "src-path".into(),
|
||||
file_info: "{\"v\":1}".into(),
|
||||
dst_volume: "dst-vol".into(),
|
||||
dst_path: "dst-path".into(),
|
||||
file_info_bin: vec![0x81, 0x01].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_data_request_body(&baseline).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut RenameDataRequest| r.disk = "disk-b".into(),
|
||||
|r: &mut RenameDataRequest| r.src_volume = "src-vol2".into(),
|
||||
|r: &mut RenameDataRequest| r.src_path = "src-path2".into(),
|
||||
|r: &mut RenameDataRequest| r.file_info = "{\"v\":2}".into(),
|
||||
|r: &mut RenameDataRequest| r.dst_volume = "dst-vol2".into(),
|
||||
|r: &mut RenameDataRequest| r.dst_path = "dst-path2".into(),
|
||||
|r: &mut RenameDataRequest| r.file_info_bin = vec![0x81, 0x02].into(),
|
||||
|r: &mut RenameDataRequest| r.file_info_bin = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_rename_data_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_canonical_body_is_injective_across_field_boundaries() {
|
||||
// Length prefixes must prevent moving bytes between adjacent fields from colliding.
|
||||
let shifted_left = RenameDataRequest {
|
||||
src_volume: "ab".into(),
|
||||
src_path: "c".into(),
|
||||
..Default::default()
|
||||
};
|
||||
let shifted_right = RenameDataRequest {
|
||||
src_volume: "a".into(),
|
||||
src_path: "bc".into(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_rename_data_request_body(&shifted_left).unwrap(),
|
||||
canonical_rename_data_request_body(&shifted_right).unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_version_canonical_body_binds_every_field() {
|
||||
let baseline = DeleteVersionRequest {
|
||||
disk: "disk-a".into(),
|
||||
volume: "vol".into(),
|
||||
path: "path".into(),
|
||||
file_info: "{\"v\":1}".into(),
|
||||
force_del_marker: false,
|
||||
opts: "{}".into(),
|
||||
file_info_bin: vec![0x81].into(),
|
||||
opts_bin: vec![0x80].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_delete_version_request_body(&baseline).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut DeleteVersionRequest| r.disk = "disk-b".into(),
|
||||
|r: &mut DeleteVersionRequest| r.volume = "vol2".into(),
|
||||
|r: &mut DeleteVersionRequest| r.path = "path2".into(),
|
||||
|r: &mut DeleteVersionRequest| r.file_info = "{\"v\":2}".into(),
|
||||
|r: &mut DeleteVersionRequest| r.force_del_marker = true,
|
||||
|r: &mut DeleteVersionRequest| r.opts = "{\"o\":1}".into(),
|
||||
|r: &mut DeleteVersionRequest| r.file_info_bin = vec![0x82].into(),
|
||||
|r: &mut DeleteVersionRequest| r.opts_bin = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_delete_version_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_versions_canonical_body_binds_lists_and_options() {
|
||||
let baseline = DeleteVersionsRequest {
|
||||
disk: "disk-a".into(),
|
||||
volume: "vol".into(),
|
||||
versions: vec!["v1".into(), "v2".into()],
|
||||
opts: "{}".into(),
|
||||
versions_bin: vec![vec![0x01].into(), vec![0x02].into()],
|
||||
opts_bin: vec![0x80].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_delete_versions_request_body(&baseline).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut DeleteVersionsRequest| r.disk = "disk-b".into(),
|
||||
|r: &mut DeleteVersionsRequest| r.volume = "vol2".into(),
|
||||
|r: &mut DeleteVersionsRequest| r.versions = vec!["v1".into(), "v3".into()],
|
||||
|r: &mut DeleteVersionsRequest| r.versions = vec!["v1v2".into()],
|
||||
|r: &mut DeleteVersionsRequest| r.opts = "{\"o\":1}".into(),
|
||||
|r: &mut DeleteVersionsRequest| r.versions_bin = vec![vec![0x01].into(), vec![0x03].into()],
|
||||
|r: &mut DeleteVersionsRequest| r.versions_bin = vec![vec![0x01, 0x02].into()],
|
||||
|r: &mut DeleteVersionsRequest| r.versions_bin = Vec::new(),
|
||||
|r: &mut DeleteVersionsRequest| r.opts_bin = vec![0x81].into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_delete_versions_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remaining_disk_mutation_canonical_bodies_bind_every_field() {
|
||||
// Mutating each field in turn and asserting all bodies differ catches a dropped or
|
||||
// duplicated `push_*` in these hand-written builders — an unbound field is tamperable.
|
||||
let write_metadata = WriteMetadataRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
file_info: "{\"a\":1}".into(),
|
||||
file_info_bin: vec![0x81].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_write_metadata_request_body(&write_metadata).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut WriteMetadataRequest| r.disk = "d2".into(),
|
||||
|r: &mut WriteMetadataRequest| r.volume = "v2".into(),
|
||||
|r: &mut WriteMetadataRequest| r.path = "p2".into(),
|
||||
|r: &mut WriteMetadataRequest| r.file_info = "{\"a\":2}".into(),
|
||||
|r: &mut WriteMetadataRequest| r.file_info_bin = vec![0x82].into(),
|
||||
] {
|
||||
let mut request = write_metadata.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_write_metadata_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let update_metadata = UpdateMetadataRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
file_info: "{\"a\":1}".into(),
|
||||
opts: "{\"o\":1}".into(),
|
||||
file_info_bin: vec![0x81].into(),
|
||||
opts_bin: vec![0x80].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_update_metadata_request_body(&update_metadata).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut UpdateMetadataRequest| r.disk = "d2".into(),
|
||||
|r: &mut UpdateMetadataRequest| r.volume = "v2".into(),
|
||||
|r: &mut UpdateMetadataRequest| r.path = "p2".into(),
|
||||
|r: &mut UpdateMetadataRequest| r.file_info = "{\"a\":2}".into(),
|
||||
|r: &mut UpdateMetadataRequest| r.opts = "{\"o\":2}".into(),
|
||||
|r: &mut UpdateMetadataRequest| r.file_info_bin = vec![0x82].into(),
|
||||
|r: &mut UpdateMetadataRequest| r.opts_bin = vec![0x81].into(),
|
||||
] {
|
||||
let mut request = update_metadata.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_update_metadata_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let write_all = WriteAllRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
data: vec![0xAA, 0xBB].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_write_all_request_body(&write_all).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut WriteAllRequest| r.disk = "d2".into(),
|
||||
|r: &mut WriteAllRequest| r.volume = "v2".into(),
|
||||
|r: &mut WriteAllRequest| r.path = "p2".into(),
|
||||
|r: &mut WriteAllRequest| r.data = vec![0xAA, 0xBC].into(),
|
||||
] {
|
||||
let mut request = write_all.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_write_all_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let delete = DeleteRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
options: "{\"o\":1}".into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_delete_request_body(&delete).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut DeleteRequest| r.disk = "d2".into(),
|
||||
|r: &mut DeleteRequest| r.volume = "v2".into(),
|
||||
|r: &mut DeleteRequest| r.path = "p2".into(),
|
||||
|r: &mut DeleteRequest| r.options = "{\"recursive\":true}".into(),
|
||||
] {
|
||||
let mut request = delete.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_delete_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let delete_paths = DeletePathsRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
paths: vec!["a".into(), "b".into()],
|
||||
};
|
||||
let mut bodies = vec![canonical_delete_paths_request_body(&delete_paths).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut DeletePathsRequest| r.disk = "d2".into(),
|
||||
|r: &mut DeletePathsRequest| r.volume = "v2".into(),
|
||||
|r: &mut DeletePathsRequest| r.paths = vec!["a".into(), "c".into()],
|
||||
|r: &mut DeletePathsRequest| r.paths = vec!["ab".into()],
|
||||
] {
|
||||
let mut request = delete_paths.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_delete_paths_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let rename_file = RenameFileRequest {
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dv".into(),
|
||||
dst_path: "dp".into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_file_request_body(&rename_file).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut RenameFileRequest| r.disk = "d2".into(),
|
||||
|r: &mut RenameFileRequest| r.src_volume = "sv2".into(),
|
||||
|r: &mut RenameFileRequest| r.src_path = "sp2".into(),
|
||||
|r: &mut RenameFileRequest| r.dst_volume = "dv2".into(),
|
||||
|r: &mut RenameFileRequest| r.dst_path = "dp2".into(),
|
||||
] {
|
||||
let mut request = rename_file.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_rename_file_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let rename_part = RenamePartRequest {
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dv".into(),
|
||||
dst_path: "dp".into(),
|
||||
meta: vec![0x01].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_part_request_body(&rename_part).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut RenamePartRequest| r.disk = "d2".into(),
|
||||
|r: &mut RenamePartRequest| r.src_volume = "sv2".into(),
|
||||
|r: &mut RenamePartRequest| r.src_path = "sp2".into(),
|
||||
|r: &mut RenamePartRequest| r.dst_volume = "dv2".into(),
|
||||
|r: &mut RenamePartRequest| r.dst_path = "dp2".into(),
|
||||
|r: &mut RenamePartRequest| r.meta = vec![0x02].into(),
|
||||
] {
|
||||
let mut request = rename_part.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_rename_part_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn volume_mutation_canonical_bodies_bind_their_fields() {
|
||||
let delete_volume = DeleteVolumeRequest {
|
||||
disk: "http://node-a:9000/data/rustfs0".into(),
|
||||
volume: "bucket".into(),
|
||||
force: false,
|
||||
};
|
||||
// The force flag must be part of the signed body: false → true is a recursive wipe.
|
||||
let mut recursive = delete_volume.clone();
|
||||
recursive.force = true;
|
||||
assert_ne!(
|
||||
canonical_delete_volume_request_body(&delete_volume).unwrap(),
|
||||
canonical_delete_volume_request_body(&recursive).unwrap(),
|
||||
);
|
||||
let mut retargeted = delete_volume.clone();
|
||||
retargeted.volume = "victim".into();
|
||||
assert_ne!(
|
||||
canonical_delete_volume_request_body(&delete_volume).unwrap(),
|
||||
canonical_delete_volume_request_body(&retargeted).unwrap(),
|
||||
);
|
||||
|
||||
let make_volume = MakeVolumeRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
};
|
||||
let mut tampered = make_volume.clone();
|
||||
tampered.volume = "v2".into();
|
||||
assert_ne!(
|
||||
canonical_make_volume_request_body(&make_volume).unwrap(),
|
||||
canonical_make_volume_request_body(&tampered).unwrap(),
|
||||
);
|
||||
|
||||
let make_volumes = MakeVolumesRequest {
|
||||
disk: "d".into(),
|
||||
volumes: vec!["a".into(), "b".into()],
|
||||
};
|
||||
let mut merged = make_volumes.clone();
|
||||
merged.volumes = vec!["ab".into()];
|
||||
assert_ne!(
|
||||
canonical_make_volumes_request_body(&make_volumes).unwrap(),
|
||||
canonical_make_volumes_request_body(&merged).unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_mutation_canonical_domains_are_distinct_per_message() {
|
||||
// The same field values must never authenticate one RPC's request as another's.
|
||||
let rename_file = RenameFileRequest {
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dv".into(),
|
||||
dst_path: "dp".into(),
|
||||
};
|
||||
let rename_part = RenamePartRequest {
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dv".into(),
|
||||
dst_path: "dp".into(),
|
||||
meta: Vec::new().into(),
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_rename_file_request_body(&rename_file).unwrap(),
|
||||
canonical_rename_part_request_body(&rename_part).unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod scanner_activity_tests {
|
||||
use super::{
|
||||
|
||||
Reference in New Issue
Block a user