mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 20:19:14 +00:00
feat(scanner): add bounded incarnation-scoped ACK receiver (#7182)
* chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * feat(scanner): add bounded incarnation-scoped ACK receiver Refs rustfs/backlog#2265 and rustfs/backlog#2240. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
@@ -208,6 +208,7 @@ const EVENT_PEER_ADDR_UNAVAILABLE: &str = "peer_addr_unavailable";
|
||||
const EVENT_RPC_SIGNATURE_VERIFICATION_FAILED: &str = "rpc_signature_verification_failed";
|
||||
const EVENT_GRPC_TRACE_CONTEXT_PROPAGATION_FAILED: &str = "grpc_trace_context_propagation_failed";
|
||||
const HEAL_CONTROL_TONIC_RPC_PATH: &str = "/node_service.HealControlService/HealControl";
|
||||
const SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH: &str = "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck";
|
||||
const TIER_MUTATION_PREPARE_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/PrepareTierMutation";
|
||||
const TIER_MUTATION_COMMIT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/CommitTierMutation";
|
||||
const TIER_MUTATION_ABORT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/AbortTierMutation";
|
||||
@@ -1856,6 +1857,7 @@ fn process_connection(
|
||||
);
|
||||
let rpc_service = RpcRequestPathService::new(
|
||||
Routes::new(node_service)
|
||||
.add_service(InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth))
|
||||
.add_service(heal_control_service)
|
||||
.add_service(tier_mutation_control_service)
|
||||
.prepare(),
|
||||
@@ -2259,6 +2261,7 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
|
||||
.strip_prefix(TONIC_RPC_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.or_else(|| (target.uri.path() == HEAL_CONTROL_TONIC_RPC_PATH).then_some("HealControl"))
|
||||
.or_else(|| (target.uri.path() == SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH).then_some("ScannerScopedDirtyUsageAck"))
|
||||
.or_else(|| (target.uri.path() == TIER_MUTATION_PREPARE_TONIC_RPC_PATH).then_some("PrepareTierMutation"))
|
||||
.or_else(|| (target.uri.path() == TIER_MUTATION_COMMIT_TONIC_RPC_PATH).then_some("CommitTierMutation"))
|
||||
.or_else(|| (target.uri.path() == TIER_MUTATION_ABORT_TONIC_RPC_PATH).then_some("AbortTierMutation"))
|
||||
@@ -3427,6 +3430,50 @@ mod tests {
|
||||
rustfs_common::set_global_local_node_name(&previous_node_name).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn scoped_dirty_usage_peer_probe_reaches_handler_through_production_auth() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string());
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind scoped ACK auth test");
|
||||
let addr = listener.local_addr().expect("listener address");
|
||||
let previous_node_name = rustfs_common::get_global_local_node_name().await;
|
||||
rustfs_common::set_global_local_node_name(&addr.to_string()).await;
|
||||
let node = InterceptedService::new(NodeServiceServer::new(make_server()), check_auth);
|
||||
let scanner = InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth);
|
||||
let service = RpcRequestPathService::new(Routes::new(node).add_service(scanner).prepare());
|
||||
let server = tokio::spawn(async move {
|
||||
let (socket, _) = listener.accept().await.expect("accept test connection");
|
||||
ConnBuilder::new(TokioExecutor::new())
|
||||
.serve_connection(TokioIo::new(socket), TowerToHyperService::new(service))
|
||||
.await
|
||||
.expect("serve scoped ACK auth test");
|
||||
});
|
||||
let host = rustfs_utils::XHost::try_from(addr.to_string()).expect("peer address");
|
||||
let client = storage::PeerRestClient::new(host, format!("http://{addr}"));
|
||||
let result = client
|
||||
.scanner_scoped_dirty_usage_capability(
|
||||
"11111111-1111-1111-1111-111111111111".to_string(),
|
||||
"a".repeat(32),
|
||||
vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
|
||||
bucket: "photos".into(),
|
||||
bucket_incarnation: vec![1; 16].into(),
|
||||
generation: 8,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
client.evict_connection().await;
|
||||
server.abort();
|
||||
let _ = server.await;
|
||||
rustfs_common::set_global_local_node_name(&previous_node_name).await;
|
||||
let error = result
|
||||
.expect_err("probe must fail closed without the requested storage owner")
|
||||
.to_string();
|
||||
assert!(
|
||||
error.contains("storage layer is not initialized") || error.contains("scoped dirty usage peer or process changed"),
|
||||
"signed probe must pass production path authentication and reach owner validation: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() {
|
||||
|
||||
@@ -493,6 +493,13 @@ impl std::fmt::Debug for NodeService {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn make_scanner_control_server() -> scanner_control_service_server::ScannerControlServiceServer<NodeService> {
|
||||
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
|
||||
scanner_control_service_server::ScannerControlServiceServer::new(make_server())
|
||||
.max_decoding_message_size(limit)
|
||||
.max_encoding_message_size(limit)
|
||||
}
|
||||
|
||||
pub fn make_server() -> NodeService {
|
||||
let context = runtime_sources::current_app_context();
|
||||
make_server_for_context(context)
|
||||
@@ -1087,6 +1094,74 @@ impl NodeService {
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl scanner_control_service_server::ScannerControlService for NodeService {
|
||||
async fn scanner_scoped_dirty_usage_ack(
|
||||
&self,
|
||||
request: Request<ScannerScopedDirtyUsageAckRequest>,
|
||||
) -> Result<Response<ScannerScopedDirtyUsageAckResponse>, Status> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
static ADMISSION: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4);
|
||||
|
||||
let canonical =
|
||||
canonical_scoped_dirty_usage_request(request.get_ref()).map_err(|err| Status::invalid_argument(err.to_string()))?;
|
||||
verify_tonic_canonical_body_digest(&request, &canonical)
|
||||
.map_err(|_| Status::permission_denied("scoped dirty usage authentication failed"))?;
|
||||
let _admission = ADMISSION
|
||||
.try_acquire()
|
||||
.map_err(|_| Status::resource_exhausted("scoped dirty usage receiver is busy"))?;
|
||||
let request = request.into_inner();
|
||||
let store = self
|
||||
.resolve_object_store()
|
||||
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
|
||||
if store.id.is_nil()
|
||||
|| request.owner_id != store.id.to_string()
|
||||
|| request.instance_id != rustfs_scanner::scanner_activity_epoch()
|
||||
{
|
||||
return Err(Status::failed_precondition("scoped dirty usage peer or process changed"));
|
||||
}
|
||||
let cleared = timeout(Duration::from_secs(30), async {
|
||||
// Strict bucket order is validated before admission. Acquire every
|
||||
// lifecycle/metadata fence before clearing any dirty record.
|
||||
let mut guards = Vec::with_capacity(request.entries.len());
|
||||
for entry in &request.entries {
|
||||
let incarnation = Uuid::from_slice(entry.bucket_incarnation.as_ref())
|
||||
.map_err(|_| Status::invalid_argument("invalid bucket incarnation"))?;
|
||||
let guard =
|
||||
crate::storage::storage_api::acquire_scanner_bucket_incarnation_fence(&entry.bucket, incarnation, store.id)
|
||||
.await
|
||||
.map_err(|_| Status::failed_precondition("trusted bucket incarnation is unavailable"))?;
|
||||
guards.push(guard);
|
||||
}
|
||||
let entries = guards
|
||||
.iter()
|
||||
.zip(&request.entries)
|
||||
.map(|(guard, entry)| (guard, entry.generation))
|
||||
.collect::<Vec<_>>();
|
||||
rustfs_scanner::acknowledge_scoped_dirty_usage(&request.instance_id, &entries, request.probe_only)
|
||||
.map_err(|err| Status::failed_precondition(err.to_string()))
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Status::deadline_exceeded("scoped dirty usage incarnation validation timed out"))??;
|
||||
let mut response = ScannerScopedDirtyUsageAckResponse {
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id: request.owner_id,
|
||||
instance_id: request.instance_id,
|
||||
supported: true,
|
||||
max_entries: SCOPED_DIRTY_USAGE_MAX_ENTRIES,
|
||||
max_request_bytes: SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES,
|
||||
cleared,
|
||||
response_proof: Bytes::new(),
|
||||
};
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Status::internal("scoped dirty usage response is too large"))?;
|
||||
response.response_proof = sign_tonic_rpc_response_proof(&body)
|
||||
.map_err(|_| Status::unavailable("scoped dirty usage response authentication is unavailable"))?
|
||||
.into();
|
||||
Ok(Response::new(response))
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl Node for NodeService {
|
||||
async fn ping(&self, request: Request<PingRequest>) -> Result<Response<PingResponse>, Status> {
|
||||
@@ -2623,6 +2698,7 @@ mod tests {
|
||||
use rustfs_kms::KmsServiceManager;
|
||||
use rustfs_protos::CanonicalMutationBody as _;
|
||||
use rustfs_protos::models::PingBodyBuilder;
|
||||
use rustfs_protos::proto_gen::node_service::scanner_control_service_server::ScannerControlService as _;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BackgroundHealStatusRequest, BatchGenerallyLockRequest, CancelDecommissionRequest, CheckPartsRequest,
|
||||
ClearDecommissionRequest, ControlPlaneErrorCode, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest,
|
||||
@@ -5990,6 +6066,74 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
fn scoped_dirty_usage_request() -> rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
|
||||
rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: 1,
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".into(),
|
||||
instance_id: "a".repeat(32),
|
||||
scope: 1,
|
||||
probe_only: false,
|
||||
entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
|
||||
bucket: "photos".into(),
|
||||
bucket_incarnation: vec![1; 16].into(),
|
||||
generation: 8,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_dirty_usage_authenticates_before_storage_and_rejects_tampering() {
|
||||
use rustfs_protos::scoped_dirty_usage::canonical_scoped_dirty_usage_request;
|
||||
let service = create_test_node_service();
|
||||
let unsigned = service
|
||||
.scanner_scoped_dirty_usage_ack(Request::new(scoped_dirty_usage_request()))
|
||||
.await
|
||||
.expect_err("unsigned ACK must not access storage");
|
||||
assert_eq!(unsigned.code(), tonic::Code::PermissionDenied);
|
||||
for field in 0..9 {
|
||||
let mut signed = Request::new(scoped_dirty_usage_request());
|
||||
let canonical = canonical_scoped_dirty_usage_request(signed.get_ref()).expect("canonical request");
|
||||
set_tonic_canonical_body_digest(&mut signed, &canonical).expect("digest");
|
||||
mark_v2_authenticated(&mut signed);
|
||||
match field {
|
||||
0 => signed.get_mut().challenge = vec![3; 16].into(),
|
||||
1 => signed.get_mut().owner_id = "22222222-2222-2222-2222-222222222222".into(),
|
||||
2 => signed.get_mut().instance_id = "b".repeat(32),
|
||||
3 => signed.get_mut().probe_only = true,
|
||||
4 => signed.get_mut().entries[0].bucket = "videos".into(),
|
||||
5 => signed.get_mut().entries[0].bucket_incarnation = vec![2; 16].into(),
|
||||
6 => signed.get_mut().entries[0].generation += 1,
|
||||
7 => signed.get_mut().scope += 1,
|
||||
_ => signed.get_mut().protocol_version += 1,
|
||||
}
|
||||
let error = service
|
||||
.scanner_scoped_dirty_usage_ack(signed)
|
||||
.await
|
||||
.expect_err("tampered ACK must fail");
|
||||
assert_eq!(
|
||||
error.code(),
|
||||
if field < 7 {
|
||||
tonic::Code::PermissionDenied
|
||||
} else {
|
||||
tonic::Code::InvalidArgument
|
||||
}
|
||||
);
|
||||
}
|
||||
let mut signed = Request::new(scoped_dirty_usage_request());
|
||||
let canonical = canonical_scoped_dirty_usage_request(signed.get_ref()).expect("canonical request");
|
||||
set_tonic_canonical_body_digest(&mut signed, &canonical).expect("digest");
|
||||
mark_v2_authenticated(&mut signed);
|
||||
assert_eq!(
|
||||
service
|
||||
.scanner_scoped_dirty_usage_ack(signed)
|
||||
.await
|
||||
.expect_err("missing owner cannot advertise capability")
|
||||
.code(),
|
||||
tonic::Code::Unavailable
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() {
|
||||
let service = create_test_node_service();
|
||||
@@ -6485,6 +6629,62 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_dirty_usage_transport_rejects_oversized_unknown_and_duplicate_fields() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind scoped ACK transport test");
|
||||
let addr = listener.local_addr().expect("test listener address");
|
||||
let (shutdown, stopped) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
tonic::transport::Server::builder()
|
||||
.add_service(super::make_scanner_control_server())
|
||||
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
|
||||
let _ = stopped.await;
|
||||
})
|
||||
.await
|
||||
.expect("scoped ACK transport server");
|
||||
});
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.http2_prior_knowledge()
|
||||
.build()
|
||||
.expect("HTTP/2 client");
|
||||
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
|
||||
for tag in [0x78, 0x0a] {
|
||||
// Unknown varint field 15, or repeated empty singular challenge:
|
||||
// both decode to a tiny default struct despite the large wire body.
|
||||
for oversized in [false, true] {
|
||||
let mut payload = [tag, 0].repeat(if oversized { (limit - 4) / 2 } else { limit / 2 });
|
||||
if oversized {
|
||||
// Unknown fixed32 field 15 makes a valid cap+1 protobuf.
|
||||
payload.extend_from_slice(&[0x7d, 0, 0, 0, 0]);
|
||||
}
|
||||
assert_eq!(payload.len(), limit + usize::from(oversized));
|
||||
let mut frame = vec![0];
|
||||
frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test payload").to_be_bytes());
|
||||
frame.extend_from_slice(&payload);
|
||||
let response = client
|
||||
.post(format!("http://{addr}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"))
|
||||
.header("content-type", "application/grpc")
|
||||
.header("te", "trailers")
|
||||
.body(frame)
|
||||
.send()
|
||||
.await
|
||||
.expect("send raw protobuf frame");
|
||||
let status = response.headers().get("grpc-status").expect("gRPC failure status");
|
||||
assert_eq!(
|
||||
status.to_str().expect("status text"),
|
||||
if oversized { "11" } else { "3" },
|
||||
"cap+1 must fail in the codec, while cap bytes reach request validation"
|
||||
);
|
||||
}
|
||||
}
|
||||
drop(client);
|
||||
shutdown.send(()).expect("stop test server");
|
||||
server.await.expect("join test server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_control_transport_enforces_codec_limit_and_fails_closed() {
|
||||
let Some(mut client) = connect_test_heal_control_client().await else {
|
||||
|
||||
@@ -379,7 +379,7 @@ pub(crate) mod tonic_service_consumer {
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::super::tonic_service::{heal_topology_fingerprint, make_heal_control_server_for_source};
|
||||
pub(crate) use super::super::tonic_service::{
|
||||
make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server,
|
||||
make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1704,6 +1704,14 @@ pub(crate) async fn acquire_bucket_metadata_transaction_lock(
|
||||
ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_scanner_bucket_incarnation_fence(
|
||||
bucket: &str,
|
||||
incarnation: uuid::Uuid,
|
||||
owner_id: uuid::Uuid,
|
||||
) -> Result<ecstore_bucket::metadata_sys::BucketMetadataMutationGuard> {
|
||||
ecstore_bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence(bucket, incarnation, owner_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn update_bucket_targets_under_transaction_lock(
|
||||
guard: &ecstore_bucket::metadata_sys::BucketMetadataMutationGuard,
|
||||
bucket: &str,
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_cache;
|
||||
pub(crate) use crate::storage::rpc::node_service::make_scanner_control_server;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source};
|
||||
pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server};
|
||||
|
||||
@@ -176,7 +176,7 @@ pub(crate) mod server {
|
||||
heal_topology_fingerprint, make_heal_control_server_for_source,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::tonic_service_consumer::{
|
||||
make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server,
|
||||
make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user