mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 20:06:37 +00:00
fix(tier): lock tier config mutations (#5080)
* fix(tier): lock tier config mutations Co-Authored-By: heihutu <heihutu@gmail.com> * fix(tier): add mutation RPC auth contract (#5082) Co-authored-by: heihutu <heihutu@gmail.com> * fix(tier): add peer mutation handler core (#5084) Co-authored-by: heihutu <heihutu@gmail.com> * fix(tier): add mutation control rpc service (#5087) Co-authored-by: heihutu <heihutu@gmail.com> * fix(tier): recover prepared mutation drains (#5093) Recover prepared tier mutation intent records into the local tier runtime so a restarted peer fails closed before issuing new remote-tier operation leases or conflicting admin publishes. Reconcile the recovered block map on each scan so committed, aborted, or removed intents clear stale local blocks instead of wedging the peer until process restart. Co-authored-by: heihutu <heihutu@gmail.com> * fix(tier): prove zero references before tier removal (#5092) Signed-off-by: houseme <housemecn@gmail.com> Co-authored-by: heihutu <heihutu@gmail.com> * fix(tier): clear peer mutation runtime blocks (#5094) Install prepared mutation runtime blocks when peer prepare requests are applied or replayed so followers fail closed immediately before restart recovery. Clear the in-memory block once peer commit or abort reaches a durable terminal state, including delayed duplicate prepare requests that observe a committed or aborted record. Co-authored-by: heihutu <heihutu@gmail.com> --------- Signed-off-by: houseme <housemecn@gmail.com> Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
@@ -1223,6 +1223,46 @@ pub struct LoadTransitionTierConfigResponse {
|
||||
#[prost(string, optional, tag = "2")]
|
||||
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TierMutationPrepareRequest {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub version: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub mutation_id: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub canonical_payload: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TierMutationCommitRequest {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub version: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub mutation_id: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub canonical_payload: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TierMutationAbortRequest {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub version: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub mutation_id: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub canonical_payload: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TierMutationControlResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(enumeration = "TierMutationPeerState", tag = "2")]
|
||||
pub state: i32,
|
||||
#[prost(bool, tag = "3")]
|
||||
pub applied: bool,
|
||||
#[prost(string, optional, tag = "4")]
|
||||
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(bytes = "bytes", tag = "5")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetLiveEventsRequest {
|
||||
#[prost(uint64, tag = "1")]
|
||||
@@ -1243,6 +1283,38 @@ pub struct GetLiveEventsResponse {
|
||||
#[prost(string, optional, tag = "5")]
|
||||
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
|
||||
#[repr(i32)]
|
||||
pub enum TierMutationPeerState {
|
||||
Unspecified = 0,
|
||||
Prepared = 1,
|
||||
Committed = 2,
|
||||
Aborted = 3,
|
||||
}
|
||||
impl TierMutationPeerState {
|
||||
/// String value of the enum field names used in the ProtoBuf definition.
|
||||
///
|
||||
/// The values are not transformed in any way and thus are considered stable
|
||||
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
|
||||
pub fn as_str_name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Unspecified => "TIER_MUTATION_PEER_STATE_UNSPECIFIED",
|
||||
Self::Prepared => "TIER_MUTATION_PEER_STATE_PREPARED",
|
||||
Self::Committed => "TIER_MUTATION_PEER_STATE_COMMITTED",
|
||||
Self::Aborted => "TIER_MUTATION_PEER_STATE_ABORTED",
|
||||
}
|
||||
}
|
||||
/// Creates an enum from field names used in the ProtoBuf definition.
|
||||
pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
|
||||
match value {
|
||||
"TIER_MUTATION_PEER_STATE_UNSPECIFIED" => Some(Self::Unspecified),
|
||||
"TIER_MUTATION_PEER_STATE_PREPARED" => Some(Self::Prepared),
|
||||
"TIER_MUTATION_PEER_STATE_COMMITTED" => Some(Self::Committed),
|
||||
"TIER_MUTATION_PEER_STATE_ABORTED" => Some(Self::Aborted),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod node_service_client {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
@@ -5590,3 +5662,334 @@ pub mod heal_control_service_server {
|
||||
const NAME: &'static str = SERVICE_NAME;
|
||||
}
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod tier_mutation_control_service_client {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
use tonic::codegen::http::Uri;
|
||||
use tonic::codegen::*;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TierMutationControlServiceClient<T> {
|
||||
inner: tonic::client::Grpc<T>,
|
||||
}
|
||||
impl TierMutationControlServiceClient<tonic::transport::Channel> {
|
||||
/// Attempt to create a new client by connecting to a given endpoint.
|
||||
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
|
||||
where
|
||||
D: TryInto<tonic::transport::Endpoint>,
|
||||
D::Error: Into<StdError>,
|
||||
{
|
||||
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
|
||||
Ok(Self::new(conn))
|
||||
}
|
||||
}
|
||||
impl<T> TierMutationControlServiceClient<T>
|
||||
where
|
||||
T: tonic::client::GrpcService<tonic::body::Body>,
|
||||
T::Error: Into<StdError>,
|
||||
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
|
||||
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
|
||||
{
|
||||
pub fn new(inner: T) -> Self {
|
||||
let inner = tonic::client::Grpc::new(inner);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_origin(inner: T, origin: Uri) -> Self {
|
||||
let inner = tonic::client::Grpc::with_origin(inner, origin);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_interceptor<F>(inner: T, interceptor: F) -> TierMutationControlServiceClient<InterceptedService<T, F>>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
T::ResponseBody: Default,
|
||||
T: tonic::codegen::Service<
|
||||
http::Request<tonic::body::Body>,
|
||||
Response = http::Response<<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody>,
|
||||
>,
|
||||
<T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
|
||||
Into<StdError> + std::marker::Send + std::marker::Sync,
|
||||
{
|
||||
TierMutationControlServiceClient::new(InterceptedService::new(inner, interceptor))
|
||||
}
|
||||
/// Compress requests with the given encoding.
|
||||
///
|
||||
/// This requires the server to support it otherwise it might respond with an
|
||||
/// error.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.send_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Enable decompressing responses.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.accept_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_decoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_encoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
pub async fn prepare_tier_mutation(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::TierMutationPrepareRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.TierMutationControlService/PrepareTierMutation");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.TierMutationControlService", "PrepareTierMutation"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn commit_tier_mutation(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::TierMutationCommitRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.TierMutationControlService/CommitTierMutation");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.TierMutationControlService", "CommitTierMutation"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn abort_tier_mutation(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::TierMutationAbortRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.TierMutationControlService/AbortTierMutation");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.TierMutationControlService", "AbortTierMutation"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated server implementations.
|
||||
pub mod tier_mutation_control_service_server {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
use tonic::codegen::*;
|
||||
/// Generated trait containing gRPC methods that should be implemented for use with TierMutationControlServiceServer.
|
||||
#[async_trait]
|
||||
pub trait TierMutationControlService: std::marker::Send + std::marker::Sync + 'static {
|
||||
async fn prepare_tier_mutation(
|
||||
&self,
|
||||
request: tonic::Request<super::TierMutationPrepareRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status>;
|
||||
async fn commit_tier_mutation(
|
||||
&self,
|
||||
request: tonic::Request<super::TierMutationCommitRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status>;
|
||||
async fn abort_tier_mutation(
|
||||
&self,
|
||||
request: tonic::Request<super::TierMutationAbortRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status>;
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct TierMutationControlServiceServer<T> {
|
||||
inner: Arc<T>,
|
||||
accept_compression_encodings: EnabledCompressionEncodings,
|
||||
send_compression_encodings: EnabledCompressionEncodings,
|
||||
max_decoding_message_size: Option<usize>,
|
||||
max_encoding_message_size: Option<usize>,
|
||||
}
|
||||
impl<T> TierMutationControlServiceServer<T> {
|
||||
pub fn new(inner: T) -> Self {
|
||||
Self::from_arc(Arc::new(inner))
|
||||
}
|
||||
pub fn from_arc(inner: Arc<T>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
accept_compression_encodings: Default::default(),
|
||||
send_compression_encodings: Default::default(),
|
||||
max_decoding_message_size: None,
|
||||
max_encoding_message_size: None,
|
||||
}
|
||||
}
|
||||
pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
{
|
||||
InterceptedService::new(Self::new(inner), interceptor)
|
||||
}
|
||||
/// Enable decompressing requests with the given encoding.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.accept_compression_encodings.enable(encoding);
|
||||
self
|
||||
}
|
||||
/// Compress responses with the given encoding, if the client supports it.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.send_compression_encodings.enable(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.max_decoding_message_size = Some(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.max_encoding_message_size = Some(limit);
|
||||
self
|
||||
}
|
||||
}
|
||||
impl<T, B> tonic::codegen::Service<http::Request<B>> for TierMutationControlServiceServer<T>
|
||||
where
|
||||
T: TierMutationControlService,
|
||||
B: Body + std::marker::Send + 'static,
|
||||
B::Error: Into<StdError> + std::marker::Send + 'static,
|
||||
{
|
||||
type Response = http::Response<tonic::body::Body>;
|
||||
type Error = std::convert::Infallible;
|
||||
type Future = BoxFuture<Self::Response, Self::Error>;
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
fn call(&mut self, req: http::Request<B>) -> Self::Future {
|
||||
match req.uri().path() {
|
||||
"/node_service.TierMutationControlService/PrepareTierMutation" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct PrepareTierMutationSvc<T: TierMutationControlService>(pub Arc<T>);
|
||||
impl<T: TierMutationControlService> tonic::server::UnaryService<super::TierMutationPrepareRequest> for PrepareTierMutationSvc<T> {
|
||||
type Response = super::TierMutationControlResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::TierMutationPrepareRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut =
|
||||
async move { <T as TierMutationControlService>::prepare_tier_mutation(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = PrepareTierMutationSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.TierMutationControlService/CommitTierMutation" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct CommitTierMutationSvc<T: TierMutationControlService>(pub Arc<T>);
|
||||
impl<T: TierMutationControlService> tonic::server::UnaryService<super::TierMutationCommitRequest> for CommitTierMutationSvc<T> {
|
||||
type Response = super::TierMutationControlResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::TierMutationCommitRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut =
|
||||
async move { <T as TierMutationControlService>::commit_tier_mutation(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = CommitTierMutationSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.TierMutationControlService/AbortTierMutation" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct AbortTierMutationSvc<T: TierMutationControlService>(pub Arc<T>);
|
||||
impl<T: TierMutationControlService> tonic::server::UnaryService<super::TierMutationAbortRequest> for AbortTierMutationSvc<T> {
|
||||
type Response = super::TierMutationControlResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::TierMutationAbortRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut =
|
||||
async move { <T as TierMutationControlService>::abort_tier_mutation(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = AbortTierMutationSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
_ => Box::pin(async move {
|
||||
let mut response = http::Response::new(tonic::body::Body::default());
|
||||
let headers = response.headers_mut();
|
||||
headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
|
||||
headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
|
||||
Ok(response)
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<T> Clone for TierMutationControlServiceServer<T> {
|
||||
fn clone(&self) -> Self {
|
||||
let inner = self.inner.clone();
|
||||
Self {
|
||||
inner,
|
||||
accept_compression_encodings: self.accept_compression_encodings,
|
||||
send_compression_encodings: self.send_compression_encodings,
|
||||
max_decoding_message_size: self.max_decoding_message_size,
|
||||
max_encoding_message_size: self.max_encoding_message_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated gRPC service name
|
||||
pub const SERVICE_NAME: &str = "node_service.TierMutationControlService";
|
||||
impl<T> tonic::server::NamedService for TierMutationControlServiceServer<T> {
|
||||
const NAME: &'static str = SERVICE_NAME;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user