mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
feat(rpc): bind canonical body digest into internode mutating disk RPC signatures (#5234)
* feat(rpc): bind canonical body digest into internode mutating disk RPC signatures Binds a domain-separated, length-prefixed canonical request-body digest into the v2 HMAC signature scope for every mutating NodeService disk RPC, so an on-path attacker on the default-plaintext internode channel can no longer tamper with a mutation payload (or strip the msgpack `_bin` field to force the JSON fallback decode) without invalidating the signature. Covers 13 mutating disk RPCs: RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes. The digest covers both the msgpack `_bin` payloads and their JSON compatibility copies. Gated fail-open by default (RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT) with a convergence counter, so rolling upgrades are byte-for-byte unaffected; the replay-cache capacity is now configurable and overflow fails closed with a metric. Refs https://github.com/rustfs/backlog/issues/1327 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix(rpc): satisfy architecture-migration compat-marker guard Put the removal condition on the RUSTFS_COMPAT_TODO marker line itself, and stop backticking env-var/metric names in the cleanup-register entry so the guard's id extractor only sees the task-id. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -136,6 +136,42 @@ pub const DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT: bool = false;
|
||||
// rolling upgrades until the fleet-wide v1-fallback counter reads zero.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT);
|
||||
|
||||
/// Require a signature-bound canonical body digest on every mutating internode disk RPC
|
||||
/// (RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete,
|
||||
/// DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes), rejecting requests
|
||||
/// that authenticate without one (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
///
|
||||
/// Defaults to `false` (fail-open): a mutating request without a body digest keeps authenticating
|
||||
/// through the method-bound v2 (or legacy) signature, so peers from releases that predate
|
||||
/// body-digest signing survive rolling upgrades unchanged. Requests that do carry a digest are
|
||||
/// always verified with no downgrade, strict or not — the digest value is part of the signed v2
|
||||
/// scope, so an on-path attacker cannot strip it without invalidating the signature. This is a
|
||||
/// rollout lever gated on the body-digest fallback counter
|
||||
/// (`rustfs_system_network_internode_body_digest_fallback_total`) reading zero across a release
|
||||
/// window fleet-wide. Single-env rollback. It is deliberately separate from
|
||||
/// [`ENV_INTERNODE_RPC_SIGNATURE_STRICT`]: the two enforcement flips converge on different
|
||||
/// counters and must not gate each other.
|
||||
pub const ENV_INTERNODE_RPC_BODY_DIGEST_STRICT: &str = "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT";
|
||||
pub const DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT: bool = false;
|
||||
|
||||
// Compile-time invariant: fail-open by default so digestless peers keep authenticating during
|
||||
// rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
|
||||
|
||||
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
||||
/// one-time consumption of body-bound v2 signatures.
|
||||
///
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope, so the steady
|
||||
/// state holds roughly `mutating RPS x 601s` entries; the default sustains ~1,700 body-bound
|
||||
/// mutating RPCs per second (about 120 MiB worst case, allocated only under sustained load).
|
||||
/// Overflow fails closed — legitimate signed traffic is the only thing that can fill the cache
|
||||
/// (replays are rejected before insertion, and an attacker cannot mint valid nonces without the
|
||||
/// shared secret) — and increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
||||
/// counter means this capacity is undersized for the node's peak mutation rate.
|
||||
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
||||
|
||||
/// Consecutive-failure threshold after which an internode peer is marked offline (grpc-optimization
|
||||
/// P3 observability).
|
||||
///
|
||||
@@ -312,6 +348,18 @@ mod tests {
|
||||
assert_eq!(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_body_digest_strict_env_name_is_stable() {
|
||||
// The fail-open default invariant is asserted at compile time next to the definition.
|
||||
assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_replay_cache_capacity_defaults_and_env_name() {
|
||||
assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");
|
||||
assert_eq!(DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, 1_048_576);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_offline_failure_threshold_defaults_and_env_name() {
|
||||
assert_eq!(DEFAULT_INTERNODE_OFFLINE_FAILURE_THRESHOLD, 3);
|
||||
|
||||
@@ -411,7 +411,8 @@ pub mod rpc {
|
||||
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -61,7 +61,6 @@ const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
|
||||
const MAX_REPLAY_PROTECTED_NONCES: usize = 65_536;
|
||||
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
|
||||
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
|
||||
static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
@@ -70,6 +69,22 @@ static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_SIGNATURE_STRICT,
|
||||
)
|
||||
});
|
||||
static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
get_env_bool(
|
||||
rustfs_config::ENV_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
)
|
||||
});
|
||||
// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
|
||||
// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
|
||||
// cannot disable replay protection by rejecting every body-bound request.
|
||||
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
)
|
||||
.max(1)
|
||||
});
|
||||
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -110,6 +125,10 @@ impl RpcNonceCache {
|
||||
return Err(std::io::Error::other("RPC request replay detected"));
|
||||
}
|
||||
if self.nonces.len() >= capacity {
|
||||
// Fail closed and alert: only legitimately signed traffic can fill the cache, so a
|
||||
// sustained overflow means RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY is undersized
|
||||
// for this node's peak mutation rate and writes are being refused.
|
||||
global_internode_metrics().record_replay_cache_overflow();
|
||||
return Err(std::io::Error::other("RPC replay cache capacity exceeded"));
|
||||
}
|
||||
self.nonces.insert(nonce);
|
||||
@@ -334,7 +353,7 @@ fn check_and_record_nonce(nonce: Uuid, signed_at: i64) -> std::io::Result<()> {
|
||||
let expires_at = now
|
||||
.checked_add(REPLAY_CACHE_RETENTION)
|
||||
.ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
|
||||
cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, MAX_REPLAY_PROTECTED_NONCES)
|
||||
cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, *REPLAY_CACHE_CAPACITY)
|
||||
}
|
||||
|
||||
/// Build headers with authentication signature
|
||||
@@ -447,6 +466,50 @@ pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canoni
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify a mutating disk RPC's canonical body digest with a rolling-upgrade fallback.
|
||||
///
|
||||
/// When the request carries a real (non-`UNSIGNED-PAYLOAD`) content SHA-256 it is verified exactly
|
||||
/// like [`verify_tonic_canonical_body_digest`]. The digest value is a member of the signed v2
|
||||
/// scope, so within the v2 lane it cannot be stripped or altered without invalidating the signature
|
||||
/// `check_auth` already enforced. When the request carries no digest — a peer that predates
|
||||
/// body-digest signing, or an attacker who downgraded the request to the legacy signature by
|
||||
/// dropping every v2 header — the request is accepted and counted on the body-digest fallback
|
||||
/// counter unless `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT` is enabled. That switch is what actually
|
||||
/// closes on-path body tampering for covered handlers: it rejects every digestless mutation,
|
||||
/// including v1-downgraded ones. It converges independently of the signature-strict switch
|
||||
/// (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
pub fn verify_tonic_mutation_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, *INTERNODE_RPC_BODY_DIGEST_STRICT)
|
||||
}
|
||||
|
||||
/// [`verify_tonic_mutation_body_digest`] with the strict gate injected as a parameter, so both
|
||||
/// rollout postures are unit-testable without racing on process-global environment variables.
|
||||
fn verify_tonic_mutation_body_digest_with_strictness<T>(
|
||||
request: &tonic::Request<T>,
|
||||
canonical_body: &[u8],
|
||||
strict: bool,
|
||||
) -> std::io::Result<()> {
|
||||
let digest = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
match digest {
|
||||
Some(digest) if digest != UNSIGNED_PAYLOAD => verify_tonic_canonical_body_digest(request, canonical_body),
|
||||
_ => {
|
||||
// RUSTFS_COMPAT_TODO(disk-mutation-body-digest): accept digestless peers during rolling upgrades. Remove after the
|
||||
// minimum supported RustFS peer version body-binds every mutating disk RPC.
|
||||
if strict {
|
||||
return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
|
||||
}
|
||||
// Count only ACCEPTED digestless mutations: this counter is the convergence gate that
|
||||
// must read zero fleet-wide across a release window before
|
||||
// `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT` may be enabled.
|
||||
global_internode_metrics().record_body_digest_fallback();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
|
||||
[
|
||||
RPC_AUTH_VERSION_HEADER,
|
||||
@@ -1410,6 +1473,129 @@ mod tests {
|
||||
assert!(cache.nonces.contains(&nonce_b));
|
||||
}
|
||||
|
||||
// The `rpc_body_digest_fallback_counter` serial group covers every test that drives (or
|
||||
// asserts on) the process-global body-digest fallback counter, so exact-delta assertions
|
||||
// cannot race with each other.
|
||||
#[test]
|
||||
#[serial_test::serial(rpc_body_digest_fallback_counter)]
|
||||
fn digestless_mutation_is_accepted_and_counted_while_strict_gate_is_off() {
|
||||
let request = tonic::Request::new(());
|
||||
let before = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
|
||||
assert!(
|
||||
verify_tonic_mutation_body_digest_with_strictness(&request, b"canonical-mutation-body", false).is_ok(),
|
||||
"a digestless peer must keep mutating while the strict gate is off"
|
||||
);
|
||||
|
||||
let after = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
assert_eq!(
|
||||
after,
|
||||
before + 1,
|
||||
"an accepted digestless mutation must increment the body-digest fallback counter exactly once"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(rpc_body_digest_fallback_counter)]
|
||||
fn strict_mutation_gate_rejects_digestless_but_keeps_body_bound() {
|
||||
let before = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
|
||||
let digestless = tonic::Request::new(());
|
||||
let error = verify_tonic_mutation_body_digest_with_strictness(&digestless, b"body", true)
|
||||
.expect_err("strict mode must reject a mutation without a body digest");
|
||||
assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
|
||||
|
||||
let mut unsigned = tonic::Request::new(());
|
||||
unsigned
|
||||
.metadata_mut()
|
||||
.as_mut()
|
||||
.insert(RPC_CONTENT_SHA256_HEADER, HeaderValue::from_static(UNSIGNED_PAYLOAD));
|
||||
let error = verify_tonic_mutation_body_digest_with_strictness(&unsigned, b"body", true)
|
||||
.expect_err("strict mode must reject an explicitly unsigned mutation payload");
|
||||
assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
|
||||
|
||||
let mut bound = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut bound, b"body").expect("digest metadata should encode");
|
||||
bound
|
||||
.metadata_mut()
|
||||
.as_mut()
|
||||
.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
assert!(
|
||||
verify_tonic_mutation_body_digest_with_strictness(&bound, b"body", true).is_ok(),
|
||||
"strict mode must keep accepting body-bound mutations"
|
||||
);
|
||||
let tampered = verify_tonic_mutation_body_digest_with_strictness(&bound, b"tampered-body", true)
|
||||
.expect_err("a tampered canonical body must fail even in strict mode");
|
||||
assert_eq!(tampered.to_string(), "RPC content SHA-256 mismatch");
|
||||
|
||||
let after = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"neither strict rejections nor bound verifications are digestless fallbacks"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(rpc_body_digest_fallback_counter)]
|
||||
fn mutation_digest_default_posture_is_fail_open_digestless_accept() {
|
||||
// The public entry point resolves strictness from the environment, whose compile-time
|
||||
// default is pinned to false in `rustfs_config`. A digestless peer therefore keeps
|
||||
// mutating through the default build with no configuration at all.
|
||||
let request = tonic::Request::new(());
|
||||
assert!(
|
||||
verify_tonic_mutation_body_digest_with_strictness(
|
||||
&request,
|
||||
b"canonical-mutation-body",
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
)
|
||||
.is_ok(),
|
||||
"the default strict posture must accept digestless mutations"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_mutation_contract_binds_method_nonce_and_body() {
|
||||
ensure_test_rpc_secret();
|
||||
let message = rustfs_protos::proto_gen::node_service::RenameDataRequest {
|
||||
disk: "http://node-a:9000/data/rustfs0".to_string(),
|
||||
src_volume: ".rustfs.sys/multipart".to_string(),
|
||||
src_path: "uploads/object".to_string(),
|
||||
file_info: "{\"volume\":\"bucket\"}".to_string(),
|
||||
dst_volume: "bucket".to_string(),
|
||||
dst_path: "object".to_string(),
|
||||
file_info_bin: vec![0x81, 0xA1, 0x76, 0x01].into(),
|
||||
};
|
||||
let body = rustfs_protos::canonical_rename_data_request_body(&message).expect("small request should encode");
|
||||
let mut request = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("canonical body digest should be attached");
|
||||
let content_sha256 = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "RenameData", content_sha256)
|
||||
.expect("body-bound auth headers should build");
|
||||
request.metadata_mut().as_mut().extend(headers.clone());
|
||||
|
||||
assert!(
|
||||
verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/RenameData", &headers).is_ok(),
|
||||
"the rename_data signature must bind destination, method, nonce, and body digest"
|
||||
);
|
||||
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/RenameData", &headers)
|
||||
.expect_err("reusing a consumed rename_data nonce must fail");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
let transplant = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/DeleteVersion", &headers)
|
||||
.expect_err("a rename_data signature must not authenticate a different method");
|
||||
assert_eq!(transplant.to_string(), "Invalid RPC v2 signature");
|
||||
|
||||
assert!(verify_tonic_mutation_body_digest(&request, &body).is_ok());
|
||||
let mut tampered = message;
|
||||
tampered.file_info_bin = Vec::new().into();
|
||||
let tampered_body = rustfs_protos::canonical_rename_data_request_body(&tampered).expect("small request should encode");
|
||||
let stripped = verify_tonic_mutation_body_digest(&request, &tampered_body)
|
||||
.expect_err("stripping the msgpack payload to force the JSON fallback decode must fail");
|
||||
assert_eq!(stripped.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_rejects_replay_after_wall_clock_regression() {
|
||||
let now = Instant::now();
|
||||
|
||||
@@ -31,7 +31,8 @@ pub use client::{
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability,
|
||||
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::cluster::rpc::client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
||||
};
|
||||
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
|
||||
use crate::cluster::rpc::internode_data_transport::{
|
||||
InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
|
||||
WriteStreamRequest,
|
||||
@@ -99,6 +100,18 @@ const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
|
||||
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
|
||||
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
|
||||
|
||||
/// Bind a mutating disk RPC to its canonical body: the digest lands in the request metadata, and
|
||||
/// the signing interceptor folds it (plus a replay-protected nonce) into the v2 signature scope
|
||||
/// (backlog#1327).
|
||||
fn attach_mutation_body_digest<T>(
|
||||
request: &mut Request<T>,
|
||||
canonical_body: std::result::Result<Vec<u8>, std::num::TryFromIntError>,
|
||||
op: &'static str,
|
||||
) -> Result<()> {
|
||||
let canonical_body = canonical_body.map_err(|_| Error::other(format!("{op} request length cannot be represented")))?;
|
||||
set_tonic_canonical_body_digest(request, &canonical_body).map_err(Error::other)
|
||||
}
|
||||
|
||||
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
|
||||
volume_infos
|
||||
.into_iter()
|
||||
@@ -1339,10 +1352,12 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(MakeVolumeRequest {
|
||||
let mut request = Request::new(MakeVolumeRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_make_volume_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "make_volume")?;
|
||||
|
||||
let response = client.make_volume(request).await?.into_inner();
|
||||
|
||||
@@ -1376,10 +1391,12 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(MakeVolumesRequest {
|
||||
let mut request = Request::new(MakeVolumesRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volumes: volumes.iter().map(|s| (*s).to_string()).collect(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_make_volumes_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "make_volumes")?;
|
||||
|
||||
let response = client.make_volumes(request).await?.into_inner();
|
||||
|
||||
@@ -1489,11 +1506,13 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(DeleteVolumeRequest {
|
||||
let mut request = Request::new(DeleteVolumeRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
force: force_delete,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_volume_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete_volume")?;
|
||||
|
||||
let response = client.delete_volume(request).await?.into_inner();
|
||||
|
||||
@@ -1542,7 +1561,7 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(DeleteVersionRequest {
|
||||
let mut request = Request::new(DeleteVersionRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
@@ -1552,6 +1571,8 @@ impl DiskAPI for RemoteDisk {
|
||||
file_info_bin: file_info_bin.into(),
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_version_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete_version")?;
|
||||
|
||||
let response = client.delete_version(request).await?.into_inner();
|
||||
|
||||
@@ -1643,7 +1664,7 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
};
|
||||
|
||||
let request = Request::new(DeleteVersionsRequest {
|
||||
let mut request = Request::new(DeleteVersionsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
versions: versions_str,
|
||||
@@ -1651,6 +1672,14 @@ impl DiskAPI for RemoteDisk {
|
||||
versions_bin,
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_versions_request_body(request.get_ref());
|
||||
if let Err(err) = attach_mutation_body_digest(&mut request, canonical_body, "delete_versions") {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
for _ in 0..versions.len() {
|
||||
errors.push(Some(err.clone()));
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
// TODO: use Error not string
|
||||
|
||||
@@ -1719,11 +1748,13 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(DeletePathsRequest {
|
||||
let mut request = Request::new(DeletePathsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
paths: paths.clone(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_paths_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete_paths")?;
|
||||
|
||||
let response = client.delete_paths(request).await?.into_inner();
|
||||
|
||||
@@ -1762,13 +1793,15 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(WriteMetadataRequest {
|
||||
let mut request = Request::new(WriteMetadataRequest {
|
||||
disk,
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
file_info,
|
||||
file_info_bin: file_info_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_write_metadata_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "write_metadata")?;
|
||||
|
||||
let response = client.write_metadata(request).await?.into_inner();
|
||||
|
||||
@@ -1840,7 +1873,7 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(UpdateMetadataRequest {
|
||||
let mut request = Request::new(UpdateMetadataRequest {
|
||||
disk,
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
@@ -1849,6 +1882,8 @@ impl DiskAPI for RemoteDisk {
|
||||
file_info_bin: file_info_bin.into(),
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_update_metadata_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "update_metadata")?;
|
||||
|
||||
let response = client.update_metadata(request).await?.into_inner();
|
||||
|
||||
@@ -2096,7 +2131,7 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(RenameDataRequest {
|
||||
let mut request = Request::new(RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
@@ -2105,6 +2140,8 @@ impl DiskAPI for RemoteDisk {
|
||||
dst_path: dst_path.to_string(),
|
||||
file_info_bin: file_info_bin.into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
|
||||
let response = client.rename_data(request).await?.into_inner();
|
||||
|
||||
@@ -2361,13 +2398,15 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(RenameFileRequest {
|
||||
let mut request = Request::new(RenameFileRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_file_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_file")?;
|
||||
|
||||
let response = client.rename_file(request).await?.into_inner();
|
||||
|
||||
@@ -2404,7 +2443,7 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(RenamePartRequest {
|
||||
let mut request = Request::new(RenamePartRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
@@ -2412,6 +2451,8 @@ impl DiskAPI for RemoteDisk {
|
||||
dst_path: dst_path.to_string(),
|
||||
meta,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_part_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_part")?;
|
||||
|
||||
let response = client.rename_part(request).await?.into_inner();
|
||||
|
||||
@@ -2449,12 +2490,14 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request = Request::new(DeleteRequest {
|
||||
let mut request = Request::new(DeleteRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
options,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_delete_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "delete")?;
|
||||
|
||||
let response = client.delete(request).await?.into_inner();
|
||||
|
||||
@@ -2665,12 +2708,14 @@ impl DiskAPI for RemoteDisk {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
|
||||
Error::other(format!("can not get client, err: {err}"))
|
||||
})?;
|
||||
let request = Request::new(WriteAllRequest {
|
||||
let mut request = Request::new(WriteAllRequest {
|
||||
disk,
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
data,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_write_all_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
|
||||
let response = match client.write_all(request).await {
|
||||
|
||||
@@ -65,6 +65,8 @@ const INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL: &str = "rustfs_system_network_in
|
||||
const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
|
||||
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
|
||||
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
||||
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -155,6 +157,8 @@ pub struct InternodeMetricsSnapshot {
|
||||
pub operation_stall_timeouts_total: u64,
|
||||
pub operation_write_shutdown_errors_total: u64,
|
||||
pub signature_v1_fallback_total: u64,
|
||||
pub body_digest_fallback_total: u64,
|
||||
pub replay_cache_overflow_total: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -173,6 +177,8 @@ pub struct InternodeMetrics {
|
||||
operation_write_shutdown_errors_total: AtomicU64,
|
||||
msgpack_json_decode_error_total: AtomicU64,
|
||||
signature_v1_fallback_total: AtomicU64,
|
||||
body_digest_fallback_total: AtomicU64,
|
||||
replay_cache_overflow_total: AtomicU64,
|
||||
}
|
||||
|
||||
impl InternodeMetrics {
|
||||
@@ -396,6 +402,26 @@ impl InternodeMetrics {
|
||||
counter!(INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
/// Count a mutating internode disk RPC that was accepted without a signature-bound canonical
|
||||
/// body digest (rolling-upgrade fallback, see
|
||||
/// <https://github.com/rustfs/backlog/issues/1327>). Only accepted requests count. This counter
|
||||
/// must read zero across a release window fleet-wide before
|
||||
/// `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT` may be enabled; after the strict flip digestless
|
||||
/// mutations are rejected and the counter stays flat.
|
||||
pub fn record_body_digest_fallback(&self) {
|
||||
self.body_digest_fallback_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
|
||||
/// full. Overflow fails closed, so a sustained non-zero rate means
|
||||
/// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
|
||||
/// mutation rate and writes are being refused — alert on this counter.
|
||||
pub fn record_replay_cache_overflow(&self) {
|
||||
self.replay_cache_overflow_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
|
||||
counter!(
|
||||
ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
|
||||
@@ -443,6 +469,8 @@ impl InternodeMetrics {
|
||||
operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed),
|
||||
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
|
||||
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
|
||||
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
|
||||
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -462,6 +490,8 @@ impl InternodeMetrics {
|
||||
self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed);
|
||||
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
|
||||
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -423,6 +423,562 @@ pub fn canonical_scanner_activity_response_body(
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
/// Length-prefixed, domain-separated byte builder for the disk-mutation canonical bodies below.
|
||||
/// Every variable-length field is u64-length-prefixed and every list u64-count-prefixed, so
|
||||
/// distinct field values can never collide into the same canonical bytes.
|
||||
struct CanonicalBodyBuilder {
|
||||
body: Vec<u8>,
|
||||
}
|
||||
|
||||
impl CanonicalBodyBuilder {
|
||||
fn new(domain: &'static [u8]) -> Self {
|
||||
let mut body = Vec::with_capacity(domain.len() + 128);
|
||||
body.extend_from_slice(domain);
|
||||
Self { body }
|
||||
}
|
||||
|
||||
fn push_bytes(&mut self, field: &[u8]) -> Result<(), std::num::TryFromIntError> {
|
||||
self.body.extend_from_slice(&u64::try_from(field.len())?.to_be_bytes());
|
||||
self.body.extend_from_slice(field);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn push_str(&mut self, field: &str) -> Result<(), std::num::TryFromIntError> {
|
||||
self.push_bytes(field.as_bytes())
|
||||
}
|
||||
|
||||
fn push_bool(&mut self, field: bool) {
|
||||
self.body.push(u8::from(field));
|
||||
}
|
||||
|
||||
fn push_count(&mut self, count: usize) -> Result<(), std::num::TryFromIntError> {
|
||||
self.body.extend_from_slice(&u64::try_from(count)?.to_be_bytes());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(self) -> Vec<u8> {
|
||||
self.body
|
||||
}
|
||||
}
|
||||
|
||||
// Canonical request bodies for the mutating NodeService disk RPCs (backlog#1327 body-digest
|
||||
// binding). Each covers every semantic wire field — including both the msgpack `_bin` payload and
|
||||
// its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
|
||||
// the JSON fallback decode path, breaks the signature-bound digest.
|
||||
|
||||
pub fn canonical_rename_data_request_body(
|
||||
request: &proto_gen::node_service::RenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-rename-data-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.file_info)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
body.push_bytes(&request.file_info_bin)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_version_request_body(
|
||||
request: &proto_gen::node_service::DeleteVersionRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-delete-version-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_str(&request.file_info)?;
|
||||
body.push_bool(request.force_del_marker);
|
||||
body.push_str(&request.opts)?;
|
||||
body.push_bytes(&request.file_info_bin)?;
|
||||
body.push_bytes(&request.opts_bin)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_versions_request_body(
|
||||
request: &proto_gen::node_service::DeleteVersionsRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-delete-versions-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_count(request.versions.len())?;
|
||||
for version in &request.versions {
|
||||
body.push_str(version)?;
|
||||
}
|
||||
body.push_str(&request.opts)?;
|
||||
body.push_count(request.versions_bin.len())?;
|
||||
for version_bin in &request.versions_bin {
|
||||
body.push_bytes(version_bin)?;
|
||||
}
|
||||
body.push_bytes(&request.opts_bin)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_write_metadata_request_body(
|
||||
request: &proto_gen::node_service::WriteMetadataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-write-metadata-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_str(&request.file_info)?;
|
||||
body.push_bytes(&request.file_info_bin)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_update_metadata_request_body(
|
||||
request: &proto_gen::node_service::UpdateMetadataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-update-metadata-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_str(&request.file_info)?;
|
||||
body.push_str(&request.opts)?;
|
||||
body.push_bytes(&request.file_info_bin)?;
|
||||
body.push_bytes(&request.opts_bin)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_write_all_request_body(
|
||||
request: &proto_gen::node_service::WriteAllRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-write-all-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_bytes(&request.data)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_request_body(
|
||||
request: &proto_gen::node_service::DeleteRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-delete-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_str(&request.options)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_paths_request_body(
|
||||
request: &proto_gen::node_service::DeletePathsRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-delete-paths-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
body.push_count(request.paths.len())?;
|
||||
for path in &request.paths {
|
||||
body.push_str(path)?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_file_request_body(
|
||||
request: &proto_gen::node_service::RenameFileRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-rename-file-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_part_request_body(
|
||||
request: &proto_gen::node_service::RenamePartRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-rename-part-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
body.push_bytes(&request.meta)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_delete_volume_request_body(
|
||||
request: &proto_gen::node_service::DeleteVolumeRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-delete-volume-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
// Binding `force` is the point: an unbound recursive-delete flag lets an on-path attacker
|
||||
// turn a refuse-if-nonempty delete into a recursive volume wipe (backlog#1327).
|
||||
body.push_bool(request.force);
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_make_volume_request_body(
|
||||
request: &proto_gen::node_service::MakeVolumeRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-make-volume-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.volume)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_make_volumes_request_body(
|
||||
request: &proto_gen::node_service::MakeVolumesRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-make-volumes-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_count(request.volumes.len())?;
|
||||
for volume in &request.volumes {
|
||||
body.push_str(volume)?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod disk_mutation_canonical_tests {
|
||||
use super::proto_gen::node_service::{
|
||||
DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, UpdateMetadataRequest, WriteAllRequest,
|
||||
WriteMetadataRequest,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
fn assert_all_distinct(bodies: &[Vec<u8>]) {
|
||||
for (i, a) in bodies.iter().enumerate() {
|
||||
for (j, b) in bodies.iter().enumerate() {
|
||||
if i != j {
|
||||
assert_ne!(a, b, "canonical bodies {i} and {j} must differ");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_canonical_body_binds_every_field() {
|
||||
let baseline = RenameDataRequest {
|
||||
disk: "disk-a".into(),
|
||||
src_volume: "src-vol".into(),
|
||||
src_path: "src-path".into(),
|
||||
file_info: "{\"v\":1}".into(),
|
||||
dst_volume: "dst-vol".into(),
|
||||
dst_path: "dst-path".into(),
|
||||
file_info_bin: vec![0x81, 0x01].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_data_request_body(&baseline).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut RenameDataRequest| r.disk = "disk-b".into(),
|
||||
|r: &mut RenameDataRequest| r.src_volume = "src-vol2".into(),
|
||||
|r: &mut RenameDataRequest| r.src_path = "src-path2".into(),
|
||||
|r: &mut RenameDataRequest| r.file_info = "{\"v\":2}".into(),
|
||||
|r: &mut RenameDataRequest| r.dst_volume = "dst-vol2".into(),
|
||||
|r: &mut RenameDataRequest| r.dst_path = "dst-path2".into(),
|
||||
|r: &mut RenameDataRequest| r.file_info_bin = vec![0x81, 0x02].into(),
|
||||
|r: &mut RenameDataRequest| r.file_info_bin = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_rename_data_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_canonical_body_is_injective_across_field_boundaries() {
|
||||
// Length prefixes must prevent moving bytes between adjacent fields from colliding.
|
||||
let shifted_left = RenameDataRequest {
|
||||
src_volume: "ab".into(),
|
||||
src_path: "c".into(),
|
||||
..Default::default()
|
||||
};
|
||||
let shifted_right = RenameDataRequest {
|
||||
src_volume: "a".into(),
|
||||
src_path: "bc".into(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_rename_data_request_body(&shifted_left).unwrap(),
|
||||
canonical_rename_data_request_body(&shifted_right).unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_version_canonical_body_binds_every_field() {
|
||||
let baseline = DeleteVersionRequest {
|
||||
disk: "disk-a".into(),
|
||||
volume: "vol".into(),
|
||||
path: "path".into(),
|
||||
file_info: "{\"v\":1}".into(),
|
||||
force_del_marker: false,
|
||||
opts: "{}".into(),
|
||||
file_info_bin: vec![0x81].into(),
|
||||
opts_bin: vec![0x80].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_delete_version_request_body(&baseline).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut DeleteVersionRequest| r.disk = "disk-b".into(),
|
||||
|r: &mut DeleteVersionRequest| r.volume = "vol2".into(),
|
||||
|r: &mut DeleteVersionRequest| r.path = "path2".into(),
|
||||
|r: &mut DeleteVersionRequest| r.file_info = "{\"v\":2}".into(),
|
||||
|r: &mut DeleteVersionRequest| r.force_del_marker = true,
|
||||
|r: &mut DeleteVersionRequest| r.opts = "{\"o\":1}".into(),
|
||||
|r: &mut DeleteVersionRequest| r.file_info_bin = vec![0x82].into(),
|
||||
|r: &mut DeleteVersionRequest| r.opts_bin = Vec::new().into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_delete_version_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_versions_canonical_body_binds_lists_and_options() {
|
||||
let baseline = DeleteVersionsRequest {
|
||||
disk: "disk-a".into(),
|
||||
volume: "vol".into(),
|
||||
versions: vec!["v1".into(), "v2".into()],
|
||||
opts: "{}".into(),
|
||||
versions_bin: vec![vec![0x01].into(), vec![0x02].into()],
|
||||
opts_bin: vec![0x80].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_delete_versions_request_body(&baseline).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut DeleteVersionsRequest| r.disk = "disk-b".into(),
|
||||
|r: &mut DeleteVersionsRequest| r.volume = "vol2".into(),
|
||||
|r: &mut DeleteVersionsRequest| r.versions = vec!["v1".into(), "v3".into()],
|
||||
|r: &mut DeleteVersionsRequest| r.versions = vec!["v1v2".into()],
|
||||
|r: &mut DeleteVersionsRequest| r.opts = "{\"o\":1}".into(),
|
||||
|r: &mut DeleteVersionsRequest| r.versions_bin = vec![vec![0x01].into(), vec![0x03].into()],
|
||||
|r: &mut DeleteVersionsRequest| r.versions_bin = vec![vec![0x01, 0x02].into()],
|
||||
|r: &mut DeleteVersionsRequest| r.versions_bin = Vec::new(),
|
||||
|r: &mut DeleteVersionsRequest| r.opts_bin = vec![0x81].into(),
|
||||
] {
|
||||
let mut request = baseline.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_delete_versions_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remaining_disk_mutation_canonical_bodies_bind_every_field() {
|
||||
// Mutating each field in turn and asserting all bodies differ catches a dropped or
|
||||
// duplicated `push_*` in these hand-written builders — an unbound field is tamperable.
|
||||
let write_metadata = WriteMetadataRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
file_info: "{\"a\":1}".into(),
|
||||
file_info_bin: vec![0x81].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_write_metadata_request_body(&write_metadata).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut WriteMetadataRequest| r.disk = "d2".into(),
|
||||
|r: &mut WriteMetadataRequest| r.volume = "v2".into(),
|
||||
|r: &mut WriteMetadataRequest| r.path = "p2".into(),
|
||||
|r: &mut WriteMetadataRequest| r.file_info = "{\"a\":2}".into(),
|
||||
|r: &mut WriteMetadataRequest| r.file_info_bin = vec![0x82].into(),
|
||||
] {
|
||||
let mut request = write_metadata.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_write_metadata_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let update_metadata = UpdateMetadataRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
file_info: "{\"a\":1}".into(),
|
||||
opts: "{\"o\":1}".into(),
|
||||
file_info_bin: vec![0x81].into(),
|
||||
opts_bin: vec![0x80].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_update_metadata_request_body(&update_metadata).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut UpdateMetadataRequest| r.disk = "d2".into(),
|
||||
|r: &mut UpdateMetadataRequest| r.volume = "v2".into(),
|
||||
|r: &mut UpdateMetadataRequest| r.path = "p2".into(),
|
||||
|r: &mut UpdateMetadataRequest| r.file_info = "{\"a\":2}".into(),
|
||||
|r: &mut UpdateMetadataRequest| r.opts = "{\"o\":2}".into(),
|
||||
|r: &mut UpdateMetadataRequest| r.file_info_bin = vec![0x82].into(),
|
||||
|r: &mut UpdateMetadataRequest| r.opts_bin = vec![0x81].into(),
|
||||
] {
|
||||
let mut request = update_metadata.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_update_metadata_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let write_all = WriteAllRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
data: vec![0xAA, 0xBB].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_write_all_request_body(&write_all).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut WriteAllRequest| r.disk = "d2".into(),
|
||||
|r: &mut WriteAllRequest| r.volume = "v2".into(),
|
||||
|r: &mut WriteAllRequest| r.path = "p2".into(),
|
||||
|r: &mut WriteAllRequest| r.data = vec![0xAA, 0xBC].into(),
|
||||
] {
|
||||
let mut request = write_all.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_write_all_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let delete = DeleteRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
options: "{\"o\":1}".into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_delete_request_body(&delete).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut DeleteRequest| r.disk = "d2".into(),
|
||||
|r: &mut DeleteRequest| r.volume = "v2".into(),
|
||||
|r: &mut DeleteRequest| r.path = "p2".into(),
|
||||
|r: &mut DeleteRequest| r.options = "{\"recursive\":true}".into(),
|
||||
] {
|
||||
let mut request = delete.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_delete_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let delete_paths = DeletePathsRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
paths: vec!["a".into(), "b".into()],
|
||||
};
|
||||
let mut bodies = vec![canonical_delete_paths_request_body(&delete_paths).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut DeletePathsRequest| r.disk = "d2".into(),
|
||||
|r: &mut DeletePathsRequest| r.volume = "v2".into(),
|
||||
|r: &mut DeletePathsRequest| r.paths = vec!["a".into(), "c".into()],
|
||||
|r: &mut DeletePathsRequest| r.paths = vec!["ab".into()],
|
||||
] {
|
||||
let mut request = delete_paths.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_delete_paths_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let rename_file = RenameFileRequest {
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dv".into(),
|
||||
dst_path: "dp".into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_file_request_body(&rename_file).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut RenameFileRequest| r.disk = "d2".into(),
|
||||
|r: &mut RenameFileRequest| r.src_volume = "sv2".into(),
|
||||
|r: &mut RenameFileRequest| r.src_path = "sp2".into(),
|
||||
|r: &mut RenameFileRequest| r.dst_volume = "dv2".into(),
|
||||
|r: &mut RenameFileRequest| r.dst_path = "dp2".into(),
|
||||
] {
|
||||
let mut request = rename_file.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_rename_file_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
|
||||
let rename_part = RenamePartRequest {
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dv".into(),
|
||||
dst_path: "dp".into(),
|
||||
meta: vec![0x01].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_part_request_body(&rename_part).unwrap()];
|
||||
for mutate in [
|
||||
|r: &mut RenamePartRequest| r.disk = "d2".into(),
|
||||
|r: &mut RenamePartRequest| r.src_volume = "sv2".into(),
|
||||
|r: &mut RenamePartRequest| r.src_path = "sp2".into(),
|
||||
|r: &mut RenamePartRequest| r.dst_volume = "dv2".into(),
|
||||
|r: &mut RenamePartRequest| r.dst_path = "dp2".into(),
|
||||
|r: &mut RenamePartRequest| r.meta = vec![0x02].into(),
|
||||
] {
|
||||
let mut request = rename_part.clone();
|
||||
mutate(&mut request);
|
||||
bodies.push(canonical_rename_part_request_body(&request).unwrap());
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn volume_mutation_canonical_bodies_bind_their_fields() {
|
||||
let delete_volume = DeleteVolumeRequest {
|
||||
disk: "http://node-a:9000/data/rustfs0".into(),
|
||||
volume: "bucket".into(),
|
||||
force: false,
|
||||
};
|
||||
// The force flag must be part of the signed body: false → true is a recursive wipe.
|
||||
let mut recursive = delete_volume.clone();
|
||||
recursive.force = true;
|
||||
assert_ne!(
|
||||
canonical_delete_volume_request_body(&delete_volume).unwrap(),
|
||||
canonical_delete_volume_request_body(&recursive).unwrap(),
|
||||
);
|
||||
let mut retargeted = delete_volume.clone();
|
||||
retargeted.volume = "victim".into();
|
||||
assert_ne!(
|
||||
canonical_delete_volume_request_body(&delete_volume).unwrap(),
|
||||
canonical_delete_volume_request_body(&retargeted).unwrap(),
|
||||
);
|
||||
|
||||
let make_volume = MakeVolumeRequest {
|
||||
disk: "d".into(),
|
||||
volume: "v".into(),
|
||||
};
|
||||
let mut tampered = make_volume.clone();
|
||||
tampered.volume = "v2".into();
|
||||
assert_ne!(
|
||||
canonical_make_volume_request_body(&make_volume).unwrap(),
|
||||
canonical_make_volume_request_body(&tampered).unwrap(),
|
||||
);
|
||||
|
||||
let make_volumes = MakeVolumesRequest {
|
||||
disk: "d".into(),
|
||||
volumes: vec!["a".into(), "b".into()],
|
||||
};
|
||||
let mut merged = make_volumes.clone();
|
||||
merged.volumes = vec!["ab".into()];
|
||||
assert_ne!(
|
||||
canonical_make_volumes_request_body(&make_volumes).unwrap(),
|
||||
canonical_make_volumes_request_body(&merged).unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_mutation_canonical_domains_are_distinct_per_message() {
|
||||
// The same field values must never authenticate one RPC's request as another's.
|
||||
let rename_file = RenameFileRequest {
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dv".into(),
|
||||
dst_path: "dp".into(),
|
||||
};
|
||||
let rename_part = RenamePartRequest {
|
||||
disk: "d".into(),
|
||||
src_volume: "sv".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dv".into(),
|
||||
dst_path: "dp".into(),
|
||||
meta: Vec::new().into(),
|
||||
};
|
||||
assert_ne!(
|
||||
canonical_rename_file_request_body(&rename_file).unwrap(),
|
||||
canonical_rename_part_request_body(&rename_part).unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod scanner_activity_tests {
|
||||
use super::{
|
||||
|
||||
@@ -16,6 +16,7 @@ for later deletion.
|
||||
- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Current protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while the distributed scanner publishes usage only after every peer returns authenticated protocol v5 state. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, and remove the protocol-v4 activity codec after every supported peer implements protocol v5; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
|
||||
- `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
|
||||
- `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
|
||||
- `disk-mutation-body-digest` internode mutating disk RPCs: servers temporarily accept mutating disk RPCs (RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes) that carry no signature-bound canonical body digest, so peers from releases that predate body-digest signing remain available during rolling upgrades. Accepted digestless mutations increment the internode body-digest fallback counter; that counter must read zero fleet-wide across a release window before RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT is enabled. Because body-bound requests now consume replay-cache nonces on the receiver, deploy the raised RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY default fleet-wide before enabling strict mode, and watch the internode replay-cache overflow counter for undersized capacity during the rollout. Remove the digestless fallback after the minimum supported RustFS peer version body-binds every mutating disk RPC.
|
||||
- `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus.
|
||||
- `backlog-1316` legacy encrypted multipart range seek: the feature remains opt-in until every server that can initiate, write, or complete multipart uploads supports the candidate-to-final marker protocol and uploadId commit lock, and pre-upgrade multipart uploads have drained. Remove the RUSTFS_ENCRYPTED_RANGE_SEEK switch after the minimum supported release does so; keep the quorum marker and malformed-layout full-read guards permanently.
|
||||
|
||||
|
||||
@@ -2393,6 +2393,238 @@ mod tests {
|
||||
request
|
||||
}
|
||||
|
||||
fn delete_request_message(options: &str) -> DeleteRequest {
|
||||
DeleteRequest {
|
||||
disk: "http://node-a:9000/data/rustfs0".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object".to_string(),
|
||||
options: options.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_mutation_body_digest_gate_runs_before_disk_lookup() {
|
||||
let service = make_server();
|
||||
|
||||
// A digestless mutation stays accepted through the default fail-open gate (rolling
|
||||
// upgrade posture) and proceeds to the disk lookup.
|
||||
let digestless = service
|
||||
.delete(Request::new(delete_request_message("{}")))
|
||||
.await
|
||||
.expect("a digestless mutation must stay accepted while the strict gate is off");
|
||||
assert!(!digestless.into_inner().success, "the unknown test disk cannot resolve");
|
||||
|
||||
// A digest bound to different request contents must be rejected before any disk work.
|
||||
let mut tampered = Request::new(delete_request_message("{}"));
|
||||
let other_body = rustfs_protos::canonical_delete_request_body(&delete_request_message("{\"recursive\":true}"))
|
||||
.expect("small request should encode");
|
||||
set_tonic_canonical_body_digest(&mut tampered, &other_body).expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut tampered);
|
||||
let tampered = service
|
||||
.delete(tampered)
|
||||
.await
|
||||
.expect_err("a tampered mutation must fail closed");
|
||||
assert_eq!(tampered.code(), tonic::Code::PermissionDenied);
|
||||
|
||||
// A digest matching the received wire fields authenticates and proceeds to the disk lookup.
|
||||
let mut signed = Request::new(delete_request_message("{}"));
|
||||
let body = rustfs_protos::canonical_delete_request_body(signed.get_ref()).expect("small request should encode");
|
||||
set_tonic_canonical_body_digest(&mut signed, &body).expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut signed);
|
||||
let signed = service
|
||||
.delete(signed)
|
||||
.await
|
||||
.expect("a correctly body-bound mutation must pass the digest gate");
|
||||
assert!(!signed.into_inner().success, "the unknown test disk cannot resolve");
|
||||
}
|
||||
|
||||
/// Per-handler wiring check for every mutating disk RPC. A mismatched digest must be rejected
|
||||
/// (catches a handler that omits its `verify_disk_mutation_digest` gate) and a correctly
|
||||
/// body-bound digest must pass the gate (catches a handler wired to the wrong
|
||||
/// `canonical_*_request_body`, which would reject legitimate traffic). Both failure modes are
|
||||
/// realistic across these copy-pasted call sites and are otherwise invisible to the digestless
|
||||
/// fail-open tests.
|
||||
#[tokio::test]
|
||||
async fn every_mutating_handler_enforces_its_body_digest() {
|
||||
let service = make_server();
|
||||
let disk = "http://node-a:9000/data/rustfs0".to_string();
|
||||
|
||||
macro_rules! assert_gated {
|
||||
($method:ident, $msg:expr, $canonical:path) => {{
|
||||
let msg = $msg;
|
||||
|
||||
// Correct digest: the gate passes and the handler proceeds to the (unknown) disk
|
||||
// lookup, so it must NOT fail with PermissionDenied.
|
||||
let mut ok = Request::new(msg.clone());
|
||||
let body = $canonical(ok.get_ref()).expect("canonical body should encode");
|
||||
set_tonic_canonical_body_digest(&mut ok, &body).expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut ok);
|
||||
if let Err(status) = service.$method(ok).await {
|
||||
assert_ne!(
|
||||
status.code(),
|
||||
tonic::Code::PermissionDenied,
|
||||
concat!(stringify!($method), ": a correctly body-bound request must pass the digest gate"),
|
||||
);
|
||||
}
|
||||
|
||||
// Mismatched digest: the gate must reject before any disk work.
|
||||
let mut bad = Request::new(msg);
|
||||
set_tonic_canonical_body_digest(&mut bad, b"unrelated-canonical-body").expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut bad);
|
||||
let err = service
|
||||
.$method(bad)
|
||||
.await
|
||||
.expect_err(concat!(stringify!($method), ": a tampered body must be rejected"));
|
||||
assert_eq!(
|
||||
err.code(),
|
||||
tonic::Code::PermissionDenied,
|
||||
concat!(stringify!($method), " must fail closed on a body-digest mismatch"),
|
||||
);
|
||||
}};
|
||||
}
|
||||
|
||||
assert_gated!(
|
||||
rename_data,
|
||||
RenameDataRequest {
|
||||
disk: disk.clone(),
|
||||
src_volume: "src".into(),
|
||||
src_path: "sp".into(),
|
||||
file_info: "{}".into(),
|
||||
dst_volume: "dst".into(),
|
||||
dst_path: "dp".into(),
|
||||
file_info_bin: vec![0x80].into(),
|
||||
},
|
||||
rustfs_protos::canonical_rename_data_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
delete_version,
|
||||
DeleteVersionRequest {
|
||||
disk: disk.clone(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
file_info: "{}".into(),
|
||||
force_del_marker: false,
|
||||
opts: "{}".into(),
|
||||
file_info_bin: vec![0x80].into(),
|
||||
opts_bin: vec![0x80].into(),
|
||||
},
|
||||
rustfs_protos::canonical_delete_version_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
delete_versions,
|
||||
DeleteVersionsRequest {
|
||||
disk: disk.clone(),
|
||||
volume: "v".into(),
|
||||
versions: vec!["a".into()],
|
||||
opts: "{}".into(),
|
||||
versions_bin: vec![vec![0x80].into()],
|
||||
opts_bin: vec![0x80].into(),
|
||||
},
|
||||
rustfs_protos::canonical_delete_versions_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
write_metadata,
|
||||
WriteMetadataRequest {
|
||||
disk: disk.clone(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
file_info: "{}".into(),
|
||||
file_info_bin: vec![0x80].into(),
|
||||
},
|
||||
rustfs_protos::canonical_write_metadata_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
update_metadata,
|
||||
UpdateMetadataRequest {
|
||||
disk: disk.clone(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
file_info: "{}".into(),
|
||||
opts: "{}".into(),
|
||||
file_info_bin: vec![0x80].into(),
|
||||
opts_bin: vec![0x80].into(),
|
||||
},
|
||||
rustfs_protos::canonical_update_metadata_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
write_all,
|
||||
WriteAllRequest {
|
||||
disk: disk.clone(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
data: vec![0x01, 0x02].into(),
|
||||
},
|
||||
rustfs_protos::canonical_write_all_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
delete,
|
||||
DeleteRequest {
|
||||
disk: disk.clone(),
|
||||
volume: "v".into(),
|
||||
path: "p".into(),
|
||||
options: "{}".into(),
|
||||
},
|
||||
rustfs_protos::canonical_delete_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
delete_paths,
|
||||
DeletePathsRequest {
|
||||
disk: disk.clone(),
|
||||
volume: "v".into(),
|
||||
paths: vec!["a".into()],
|
||||
},
|
||||
rustfs_protos::canonical_delete_paths_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
rename_file,
|
||||
RenameFileRequest {
|
||||
disk: disk.clone(),
|
||||
src_volume: "src".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dst".into(),
|
||||
dst_path: "dp".into(),
|
||||
},
|
||||
rustfs_protos::canonical_rename_file_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
rename_part,
|
||||
RenamePartRequest {
|
||||
disk: disk.clone(),
|
||||
src_volume: "src".into(),
|
||||
src_path: "sp".into(),
|
||||
dst_volume: "dst".into(),
|
||||
dst_path: "dp".into(),
|
||||
meta: vec![0x03].into(),
|
||||
},
|
||||
rustfs_protos::canonical_rename_part_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
delete_volume,
|
||||
DeleteVolumeRequest {
|
||||
disk: disk.clone(),
|
||||
volume: "v".into(),
|
||||
force: true,
|
||||
},
|
||||
rustfs_protos::canonical_delete_volume_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
make_volume,
|
||||
MakeVolumeRequest {
|
||||
disk: disk.clone(),
|
||||
volume: "v".into(),
|
||||
},
|
||||
rustfs_protos::canonical_make_volume_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
make_volumes,
|
||||
MakeVolumesRequest {
|
||||
disk,
|
||||
volumes: vec!["v".into()],
|
||||
},
|
||||
rustfs_protos::canonical_make_volumes_request_body
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_control_requires_body_bound_auth_before_topology_validation() {
|
||||
let service = make_heal_control_server();
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::storage::storage_api::rpc_consumer::node_service::{
|
||||
ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count,
|
||||
};
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
use crate::storage::storage_api::verify_tonic_mutation_body_digest;
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
@@ -81,6 +82,24 @@ fn encode_msgpack_named<T: serde::Serialize>(value: &T, value_name: &str) -> std
|
||||
Ok(serializer.into_inner())
|
||||
}
|
||||
|
||||
/// Enforce the signature-bound canonical body digest on a mutating disk RPC (backlog#1327).
|
||||
///
|
||||
/// Digest-bearing requests are verified against the canonical bytes rebuilt from the received
|
||||
/// wire fields — which cover both the msgpack `_bin` payloads and their JSON compatibility
|
||||
/// copies, so tampering with either encoding (or stripping `_bin` to force the JSON fallback
|
||||
/// decode) is rejected. Digestless requests fall back per
|
||||
/// `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT` (default: accept + convergence counter).
|
||||
fn verify_disk_mutation_digest<T>(
|
||||
request: &Request<T>,
|
||||
canonical_body: std::result::Result<Vec<u8>, std::num::TryFromIntError>,
|
||||
op: &'static str,
|
||||
) -> std::result::Result<(), Status> {
|
||||
let canonical_body =
|
||||
canonical_body.map_err(|_| Status::invalid_argument(format!("{op} request length cannot be represented")))?;
|
||||
verify_tonic_mutation_body_digest(request, &canonical_body)
|
||||
.map_err(|err| Status::permission_denied(format!("{op} authentication failed: {err}")))
|
||||
}
|
||||
|
||||
/// JSON compatibility string for a dual-encoded response field. Returns an empty string only when
|
||||
/// msgpack-only mode and its explicit fleet confirmation guard are both enabled; otherwise the
|
||||
/// legacy JSON encoding is retained for old peers. The paired `_bin` field is always sent.
|
||||
@@ -171,6 +190,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<DeleteVolumeRequest>,
|
||||
) -> Result<Response<DeleteVolumeResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_delete_volume_request_body(request.get_ref()),
|
||||
"delete_volume",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
match disk.delete_volume(&request.volume, request.force).await {
|
||||
@@ -325,6 +349,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<DeleteVersionsRequest>,
|
||||
) -> Result<Response<DeleteVersionsResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_delete_versions_request_body(request.get_ref()),
|
||||
"delete_versions",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
let mut versions = Vec::with_capacity(request.versions.len());
|
||||
@@ -380,6 +409,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<DeleteVersionRequest>,
|
||||
) -> Result<Response<DeleteVersionResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_delete_version_request_body(request.get_ref()),
|
||||
"delete_version",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
let file_info = match decode_msgpack_or_json::<FileInfo>(&request.file_info_bin, &request.file_info, "FileInfo") {
|
||||
@@ -544,6 +578,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<WriteMetadataRequest>,
|
||||
) -> Result<Response<WriteMetadataResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_write_metadata_request_body(request.get_ref()),
|
||||
"write_metadata",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
let file_info = match decode_msgpack_or_json::<FileInfo>(&request.file_info_bin, &request.file_info, "FileInfo") {
|
||||
@@ -577,6 +616,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<UpdateMetadataRequest>,
|
||||
) -> Result<Response<UpdateMetadataResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_update_metadata_request_body(request.get_ref()),
|
||||
"update_metadata",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
let file_info = match decode_msgpack_or_json::<FileInfo>(&request.file_info_bin, &request.file_info, "FileInfo") {
|
||||
@@ -648,6 +692,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<DeletePathsRequest>,
|
||||
) -> Result<Response<DeletePathsResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_delete_paths_request_body(request.get_ref()),
|
||||
"delete_paths",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
match disk.delete_paths(&request.volume, &request.paths).await {
|
||||
@@ -750,6 +799,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<MakeVolumeRequest>,
|
||||
) -> Result<Response<MakeVolumeResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_make_volume_request_body(request.get_ref()),
|
||||
"make_volume",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
match disk.make_volume(&request.volume).await {
|
||||
@@ -774,6 +828,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<MakeVolumesRequest>,
|
||||
) -> Result<Response<MakeVolumesResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_make_volumes_request_body(request.get_ref()),
|
||||
"make_volumes",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
match disk.make_volumes(request.volumes.iter().map(|s| &**s).collect()).await {
|
||||
@@ -798,6 +857,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<RenameDataRequest>,
|
||||
) -> Result<Response<RenameDataResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_rename_data_request_body(request.get_ref()),
|
||||
"rename_data",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
let file_info = match decode_msgpack_or_json::<FileInfo>(&request.file_info_bin, &request.file_info, "FileInfo") {
|
||||
@@ -888,6 +952,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<RenameFileRequest>,
|
||||
) -> Result<Response<RenameFileResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_rename_file_request_body(request.get_ref()),
|
||||
"rename_file",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
match disk
|
||||
@@ -915,6 +984,11 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<RenamePartRequest>,
|
||||
) -> Result<Response<RenamePartResponse>, Status> {
|
||||
verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_rename_part_request_body(request.get_ref()),
|
||||
"rename_part",
|
||||
)?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
match disk
|
||||
@@ -1083,6 +1157,7 @@ impl NodeService {
|
||||
}
|
||||
|
||||
pub(super) async fn handle_delete(&self, request: Request<DeleteRequest>) -> Result<Response<DeleteResponse>, Status> {
|
||||
verify_disk_mutation_digest(&request, rustfs_protos::canonical_delete_request_body(request.get_ref()), "delete")?;
|
||||
let request = request.into_inner();
|
||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
||||
let options = match serde_json::from_str::<DeleteOptions>(&request.options) {
|
||||
@@ -1113,6 +1188,7 @@ impl NodeService {
|
||||
}
|
||||
|
||||
pub(super) async fn handle_write_all(&self, request: Request<WriteAllRequest>) -> Result<Response<WriteAllResponse>, Status> {
|
||||
verify_disk_mutation_digest(&request, rustfs_protos::canonical_write_all_request_body(request.get_ref()), "write_all")?;
|
||||
let request = request.into_inner();
|
||||
let data_len = request.data.len();
|
||||
let metrics = runtime_sources::current_internode_metrics();
|
||||
|
||||
@@ -496,7 +496,7 @@ pub(crate) mod ecstore_rpc {
|
||||
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
|
||||
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_rpc_signature,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
@@ -1567,6 +1567,10 @@ pub(crate) fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>,
|
||||
ecstore_rpc::verify_tonic_canonical_body_digest(request, canonical_body)
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_mutation_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
ecstore_rpc::verify_tonic_mutation_body_digest(request, canonical_body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_tonic_canonical_body_digest<T>(request: &mut tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
ecstore_rpc::set_tonic_canonical_body_digest(request, canonical_body)
|
||||
|
||||
Reference in New Issue
Block a user