mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 05:06:28 +00:00
fix(scanner): fence unknown tier accounting
This commit is contained in:
@@ -456,11 +456,13 @@ pub mod rpc {
|
||||
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
|
||||
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_put_file_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature,
|
||||
verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -94,6 +94,7 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
|
||||
const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
|
||||
const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
|
||||
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
|
||||
const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
|
||||
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
|
||||
static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
get_env_bool(
|
||||
@@ -636,40 +637,79 @@ pub fn verify_put_file_capability(challenge: Uuid, server_epoch: Uuid, version:
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid put_file capability proof"))
|
||||
}
|
||||
|
||||
fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
|
||||
fn update_ns_scanner_capability_mac(
|
||||
mac: &mut HmacSha256,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
) {
|
||||
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
|
||||
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
|
||||
mac.update(challenge.as_bytes());
|
||||
mac.update(server_epoch.as_bytes());
|
||||
if supports_tier_registry_generation {
|
||||
// The optional response capability is part of the authenticated
|
||||
// scope. A proxy cannot turn an old/unsupported peer into a worker
|
||||
// that receives generation-fenced scanner work.
|
||||
mac.update(NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
|
||||
fn generate_ns_scanner_capability_proof(
|
||||
secret: &str,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
if challenge.is_nil() || server_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
|
||||
}
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
|
||||
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
|
||||
Ok(mac.finalize().into_bytes().to_vec())
|
||||
}
|
||||
|
||||
fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
|
||||
fn verify_ns_scanner_capability_proof(
|
||||
secret: &str,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
proof: &[u8],
|
||||
supports_tier_registry_generation: bool,
|
||||
) -> std::io::Result<()> {
|
||||
if challenge.is_nil() || server_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
|
||||
}
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
|
||||
update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
|
||||
mac.verify_slice(proof)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
|
||||
}
|
||||
|
||||
pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
|
||||
generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch)
|
||||
sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false)
|
||||
}
|
||||
|
||||
pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
|
||||
verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof)
|
||||
verify_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, proof, false)
|
||||
}
|
||||
|
||||
pub fn sign_ns_scanner_capability_with_tier_registry_generation(
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, supports_tier_registry_generation)
|
||||
}
|
||||
|
||||
pub fn verify_ns_scanner_capability_with_tier_registry_generation(
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
proof: &[u8],
|
||||
supports_tier_registry_generation: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof, supports_tier_registry_generation)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -1709,13 +1749,28 @@ mod tests {
|
||||
let secret = "test-scanner-capability-secret";
|
||||
let challenge = Uuid::new_v4();
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let proof =
|
||||
generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated");
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
|
||||
.expect("capability proof should be generated");
|
||||
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_capability_proof_binds_tier_registry_generation_support() {
|
||||
let secret = "test-scanner-capability-secret";
|
||||
let challenge = Uuid::new_v4();
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true)
|
||||
.expect("generation capability proof should be generated");
|
||||
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_err());
|
||||
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
|
||||
.expect("legacy capability proof should be generated");
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true).is_err());
|
||||
}
|
||||
|
||||
/// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
|
||||
|
||||
@@ -13,17 +13,18 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::{
|
||||
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability, verify_put_file_capability,
|
||||
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_put_file_capability,
|
||||
};
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{FileReader, FileWriter};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
|
||||
NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
|
||||
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY,
|
||||
PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION,
|
||||
PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY,
|
||||
NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
|
||||
PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
|
||||
PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
|
||||
@@ -137,6 +138,12 @@ fn put_file_capability_status_is_legacy(status: u16) -> bool {
|
||||
status == 404
|
||||
}
|
||||
|
||||
fn ns_scanner_capability_error_allows_legacy(error: &Error) -> bool {
|
||||
[400, 404, 405, 426]
|
||||
.into_iter()
|
||||
.any(|status| error.is_internode_http_status(status))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -220,6 +227,7 @@ pub struct NsScannerStreamRequest {
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NsScannerCapabilityRequest {
|
||||
pub endpoint: String,
|
||||
pub supports_tier_registry_generation: bool,
|
||||
}
|
||||
|
||||
/// Data-plane stream opener used by `RemoteDisk`.
|
||||
@@ -252,6 +260,15 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
|
||||
async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
|
||||
Err(Error::MethodNotAllowed)
|
||||
}
|
||||
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
|
||||
let server_epoch = self.probe_ns_scanner(request).await?;
|
||||
Ok(NsScannerCapabilityResponse {
|
||||
version: NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch,
|
||||
proof: Vec::new(),
|
||||
supports_tier_registry_generation: None,
|
||||
})
|
||||
}
|
||||
// Interface facet nobody calls yet: every transport implements both, but no
|
||||
// caller negotiates on them. Kept for the internode transport split
|
||||
// (backlog#1350); deleting them would delete the seam and six impls.
|
||||
@@ -335,27 +352,44 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
}
|
||||
|
||||
async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
|
||||
let challenge = Uuid::new_v4();
|
||||
let url = build_ns_scanner_capability_url(&request, challenge);
|
||||
let mut headers = msgpack_headers();
|
||||
build_auth_headers(&url, &Method::GET, &mut headers)?;
|
||||
let reader = HttpReader::new(url, Method::GET, headers, None).await?;
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
|
||||
.read_to_end(&mut body)
|
||||
.await?;
|
||||
if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
|
||||
return Err(Error::other("invalid remote namespace scanner capability response size"));
|
||||
Ok(self.probe_ns_scanner_capability(request).await?.server_epoch)
|
||||
}
|
||||
|
||||
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
|
||||
if request.supports_tier_registry_generation {
|
||||
return match self.probe_ns_scanner_capability_once(&request).await {
|
||||
Ok(response) => Ok(response),
|
||||
Err(marked_error) if ns_scanner_capability_error_allows_legacy(&marked_error) => {
|
||||
// A v3 peer may reject the additive query marker, ignore
|
||||
// it, or return its legacy proof. Retry once without the
|
||||
// marker and only downgrade after that legacy response is
|
||||
// authenticated; an unverified epoch is never trusted.
|
||||
let legacy_request = NsScannerCapabilityRequest {
|
||||
endpoint: request.endpoint.clone(),
|
||||
supports_tier_registry_generation: false,
|
||||
};
|
||||
match self.probe_ns_scanner_capability_once(&legacy_request).await {
|
||||
Ok(mut response) => {
|
||||
response.supports_tier_registry_generation = None;
|
||||
Ok(response)
|
||||
}
|
||||
Err(legacy_error) if ns_scanner_capability_error_allows_legacy(&legacy_error) => {
|
||||
// Some old deployments expose only the legacy
|
||||
// protocol response (or advertise 426). Treat
|
||||
// the pair as an explicit unsupported result so
|
||||
// the scanner can use its coordinator fallback.
|
||||
Err(Error::MethodNotAllowed)
|
||||
}
|
||||
Err(_) => Err(marked_error),
|
||||
}
|
||||
}
|
||||
// A server failure, network failure, or authentication error
|
||||
// is not evidence of an old parser. Do not issue an
|
||||
// unauthenticated legacy probe or silently downgrade.
|
||||
Err(marked_error) => Err(marked_error),
|
||||
};
|
||||
}
|
||||
let response: NsScannerCapabilityResponse =
|
||||
rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
|
||||
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
|
||||
return Err(Error::other("incompatible remote namespace scanner capability response"));
|
||||
}
|
||||
verify_ns_scanner_capability(challenge, response.server_epoch, &response.proof)
|
||||
.map_err(|err| Error::other(format!("remote namespace scanner capability authentication failed: {err}")))?;
|
||||
Ok(response.server_epoch)
|
||||
self.probe_ns_scanner_capability_once(&request).await
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
@@ -368,6 +402,53 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
}
|
||||
|
||||
impl TcpHttpInternodeDataTransport {
|
||||
async fn probe_ns_scanner_capability_once(
|
||||
&self,
|
||||
request: &NsScannerCapabilityRequest,
|
||||
) -> Result<NsScannerCapabilityResponse> {
|
||||
let challenge = Uuid::new_v4();
|
||||
let url = build_ns_scanner_capability_url(request, challenge);
|
||||
let mut headers = msgpack_headers();
|
||||
build_auth_headers(&url, &Method::GET, &mut headers)?;
|
||||
let reader = HttpReader::new(url, Method::GET, headers, None).await?;
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
|
||||
.read_to_end(&mut body)
|
||||
.await?;
|
||||
if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
|
||||
return Err(Error::other("invalid remote namespace scanner capability response size"));
|
||||
}
|
||||
let mut response: NsScannerCapabilityResponse =
|
||||
rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
|
||||
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
|
||||
return Err(Error::other("incompatible remote namespace scanner capability response"));
|
||||
}
|
||||
if let Err(err) = verify_ns_scanner_capability_with_tier_registry_generation(
|
||||
challenge,
|
||||
response.server_epoch,
|
||||
&response.proof,
|
||||
request.supports_tier_registry_generation,
|
||||
) {
|
||||
// A permissive older peer can ignore the additive marker and
|
||||
// return a valid legacy-scope proof with HTTP 200. Accept that
|
||||
// response only after independently authenticating the legacy
|
||||
// scope; all other verification failures remain fail-closed.
|
||||
if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
|
||||
response.supports_tier_registry_generation = None;
|
||||
return Ok(response);
|
||||
}
|
||||
return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
|
||||
}
|
||||
// The proof authenticates the requested capability scope, not the
|
||||
// optional response field. Derive the client-facing bit from that
|
||||
// verified scope so an intermediary cannot strip or rewrite the field
|
||||
// and force a silent downgrade after a successful generation-bound
|
||||
// handshake.
|
||||
normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn put_file_auth_capability(&self, endpoint: &str) -> Result<Option<Uuid>> {
|
||||
resolve_put_file_auth_capability(endpoint, || async {
|
||||
tokio::time::timeout(PUT_FILE_CAPABILITY_PROBE_TIMEOUT, self.probe_put_file_auth(endpoint))
|
||||
@@ -649,6 +730,14 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
fn normalize_ns_scanner_capability_response(response: &mut NsScannerCapabilityResponse, requested_generation_support: bool) {
|
||||
response.supports_tier_registry_generation = requested_generation_support.then_some(true);
|
||||
}
|
||||
|
||||
fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
|
||||
verify_ns_scanner_capability_with_tier_registry_generation(challenge, response.server_epoch, &response.proof, false).is_ok()
|
||||
}
|
||||
|
||||
fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
|
||||
let body_sha256 = hex_simd::encode_to_string(Sha256::digest(&request.body), hex_simd::AsciiCase::Lower);
|
||||
format!(
|
||||
@@ -675,13 +764,18 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
|
||||
|
||||
fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
|
||||
format!(
|
||||
"{}{}?{}={}&{}={}",
|
||||
"{}{}?{}={}&{}={}{}",
|
||||
request.endpoint,
|
||||
NS_SCANNER_PATH,
|
||||
NS_SCANNER_PROTOCOL_VERSION_QUERY,
|
||||
NS_SCANNER_PROTOCOL_VERSION,
|
||||
NS_SCANNER_CAPABILITY_CHALLENGE_QUERY,
|
||||
challenge
|
||||
challenge,
|
||||
if request.supports_tier_registry_generation {
|
||||
format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
@@ -794,6 +888,7 @@ mod tests {
|
||||
let probe_err = transport
|
||||
.probe_ns_scanner(NsScannerCapabilityRequest {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
supports_tier_registry_generation: false,
|
||||
})
|
||||
.await
|
||||
.expect_err("legacy transport should report namespace scanner as unsupported");
|
||||
@@ -1387,6 +1482,7 @@ mod tests {
|
||||
let url = build_ns_scanner_capability_url(
|
||||
&NsScannerCapabilityRequest {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
supports_tier_registry_generation: false,
|
||||
},
|
||||
challenge,
|
||||
);
|
||||
@@ -1399,6 +1495,85 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ns_scanner_capability_url_marks_generation_support_only_when_requested() {
|
||||
let challenge = Uuid::new_v4();
|
||||
let url = build_ns_scanner_capability_url(
|
||||
&NsScannerCapabilityRequest {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
supports_tier_registry_generation: true,
|
||||
},
|
||||
challenge,
|
||||
);
|
||||
|
||||
assert!(url.contains(&format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ns_scanner_capability_legacy_fallback_requires_explicit_compatibility_status() {
|
||||
for status in [400, 404, 405, 426] {
|
||||
let error = Error::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::from_u16(status).expect("test status")),
|
||||
));
|
||||
assert!(
|
||||
ns_scanner_capability_error_allows_legacy(&error),
|
||||
"status {status} should permit legacy retry"
|
||||
);
|
||||
}
|
||||
|
||||
let marked_server_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR),
|
||||
));
|
||||
let network_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
rustfs_rio::InternodeHttpErrorKind::ConnectionRefused,
|
||||
));
|
||||
let authentication_error = Error::other("remote namespace scanner capability authentication failed");
|
||||
assert!(!ns_scanner_capability_error_allows_legacy(&marked_server_error));
|
||||
assert!(!ns_scanner_capability_error_allows_legacy(&network_error));
|
||||
assert!(!ns_scanner_capability_error_allows_legacy(&authentication_error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authenticated_ns_scanner_capability_ignores_unprotected_response_bit() {
|
||||
let mut response = NsScannerCapabilityResponse {
|
||||
version: NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch: Uuid::new_v4(),
|
||||
proof: Vec::new(),
|
||||
supports_tier_registry_generation: None,
|
||||
};
|
||||
|
||||
normalize_ns_scanner_capability_response(&mut response, true);
|
||||
assert_eq!(response.supports_tier_registry_generation, Some(true));
|
||||
|
||||
response.supports_tier_registry_generation = Some(false);
|
||||
normalize_ns_scanner_capability_response(&mut response, false);
|
||||
assert_eq!(response.supports_tier_registry_generation, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ns_scanner_capability_accepts_only_authenticated_legacy_scope_after_marker_mismatch() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
let challenge = Uuid::new_v4();
|
||||
let response = NsScannerCapabilityResponse {
|
||||
version: NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch: Uuid::new_v4(),
|
||||
proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, Uuid::new_v4())
|
||||
.expect("placeholder proof should be generated"),
|
||||
supports_tier_registry_generation: None,
|
||||
};
|
||||
// A proof bound to a different challenge cannot authorize the legacy
|
||||
// fallback, even though the response has the expected shape.
|
||||
assert!(!ns_scanner_capability_legacy_proof_is_valid(challenge, &response));
|
||||
|
||||
let server_epoch = response.server_epoch;
|
||||
let valid_response = NsScannerCapabilityResponse {
|
||||
proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, server_epoch)
|
||||
.expect("legacy proof should be generated"),
|
||||
..response
|
||||
};
|
||||
assert!(ns_scanner_capability_legacy_proof_is_valid(challenge, &valid_response));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transport_config_defaults_to_tcp_http() {
|
||||
let transport = build_internode_data_transport(None).unwrap();
|
||||
|
||||
@@ -35,8 +35,9 @@ pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
|
||||
sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer,
|
||||
sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
|
||||
@@ -778,14 +778,16 @@ impl RemoteDisk {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
let probe = self.data_transport.probe_ns_scanner(NsScannerCapabilityRequest {
|
||||
let probe = self.data_transport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
|
||||
endpoint: self.endpoint.grid_host(),
|
||||
supports_tier_registry_generation: true,
|
||||
});
|
||||
let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
|
||||
.await
|
||||
.map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?;
|
||||
match result {
|
||||
Ok(server_epoch) => Ok(Some(server_epoch)),
|
||||
Ok(response) if response.supports_tier_registry_generation == Some(true) => Ok(Some(response.server_epoch)),
|
||||
Ok(_) => Ok(None),
|
||||
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
|
||||
Err(DiskError::MethodNotAllowed) => Ok(None),
|
||||
Err(err)
|
||||
@@ -3974,10 +3976,21 @@ mod tests {
|
||||
NsScannerProbe(NsScannerCapabilityRequest),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
#[derive(Debug, Clone)]
|
||||
struct RecordingInternodeDataTransport {
|
||||
calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
|
||||
ns_scanner_probe_status: Arc<StdMutex<Option<u16>>>,
|
||||
ns_scanner_generation_support: Arc<StdMutex<Option<bool>>>,
|
||||
}
|
||||
|
||||
impl Default for RecordingInternodeDataTransport {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
calls: Arc::default(),
|
||||
ns_scanner_probe_status: Arc::default(),
|
||||
ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -4197,6 +4210,15 @@ mod tests {
|
||||
Self {
|
||||
calls: Arc::default(),
|
||||
ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))),
|
||||
ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_ns_scanner_generation_support(support: Option<bool>) -> Self {
|
||||
Self {
|
||||
calls: Arc::default(),
|
||||
ns_scanner_probe_status: Arc::default(),
|
||||
ns_scanner_generation_support: Arc::new(StdMutex::new(support)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4862,6 +4884,23 @@ mod tests {
|
||||
Ok(Uuid::from_u128(1))
|
||||
}
|
||||
|
||||
async fn probe_ns_scanner_capability(
|
||||
&self,
|
||||
request: NsScannerCapabilityRequest,
|
||||
) -> Result<crate::storage_api_contracts::internode::NsScannerCapabilityResponse> {
|
||||
let server_epoch = self.probe_ns_scanner(request).await?;
|
||||
let supports_tier_registry_generation = *self
|
||||
.ns_scanner_generation_support
|
||||
.lock()
|
||||
.expect("namespace scanner generation support lock poisoned");
|
||||
Ok(crate::storage_api_contracts::internode::NsScannerCapabilityResponse {
|
||||
version: crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch,
|
||||
proof: Vec::new(),
|
||||
supports_tier_registry_generation,
|
||||
})
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"recording"
|
||||
}
|
||||
@@ -6674,6 +6713,22 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_disk_namespace_scanner_capability_falls_back_without_generation_support() {
|
||||
for support in [None, Some(false)] {
|
||||
let transport = RecordingInternodeDataTransport::with_ns_scanner_generation_support(support);
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await;
|
||||
|
||||
assert_eq!(
|
||||
remote_disk
|
||||
.ns_scanner_server_epoch()
|
||||
.await
|
||||
.expect("missing generation support should be classified as unsupported"),
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_disk_namespace_scanner_capability_rejects_legacy_transport() {
|
||||
let remote_disk = new_remote_disk_with_transport(Arc::new(RetryingOpenReadInternodeDataTransport::default())).await;
|
||||
|
||||
@@ -27,12 +27,12 @@ pub(crate) mod internode {
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
|
||||
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
|
||||
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY,
|
||||
NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN,
|
||||
PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN, PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1,
|
||||
PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY,
|
||||
PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse,
|
||||
PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
|
||||
PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user