gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 13:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ecstore): add remote snapshot lease RPCs (#5389)

Browse Source 
* feat(ecstore): add local snapshot leases

* feat(ecstore): add remote snapshot lease RPCs

* fix(rpc): keep snapshot lease checks CI-compatible
This commit is contained in:
cxymds
2026-07-29 15:06:18 +08:00
committed by GitHub
parent 2801b2500d
commit d9efd6b853
14 changed files with 805 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/Cargo.toml
+1 -1
View File
@@ -125,7 +125,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "sig
tokio-rustls = { workspace = true, default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
tokio-stream.workspace = true
tokio-util = { workspace = true, features = ["io", "compat"] }
tokio-util = { workspace = true, features = ["io", "compat", "time"] }
tonic = { workspace = true, features = ["gzip", "deflate"] }
tower = { workspace = true, features = ["timeout"] }
tower-http = { workspace = true, features = ["trace", "compression-full", "cors", "catch-panic", "timeout", "limit", "request-id", "add-extension"] }
rustfs/src/storage/rpc/node_service.rs
+24 -1
View File
@@ -343,6 +343,7 @@ mod metrics;
pub struct NodeService {
local_peer: LocalPeerS3Client,
context: Option<Arc<runtime_sources::AppContext>>,
snapshot_lease_expiry: disk::SnapshotLeaseExpiryScheduler,
}
impl std::fmt::Debug for NodeService {
@@ -361,7 +362,11 @@ pub fn make_server() -> NodeService {
pub fn make_server_for_context(context: Option<Arc<runtime_sources::AppContext>>) -> NodeService {
let local_peer = LocalPeerS3Client::new(None, None);
NodeService { local_peer, context }
NodeService {
local_peer,
context,
snapshot_lease_expiry: disk::SnapshotLeaseExpiryScheduler::new(),
}
}
#[derive(Clone, Debug, Default)]
@@ -1124,6 +1129,24 @@ impl Node for NodeService {
async fn delete_paths(&self, request: Request<DeletePathsRequest>) -> Result<Response<DeletePathsResponse>, Status> {
self.handle_delete_paths(request).await
}
async fn acquire_snapshot_lease(
&self,
request: Request<SnapshotLeaseRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
self.handle_acquire_snapshot_lease(request).await
}
async fn renew_snapshot_lease(
&self,
request: Request<SnapshotLeaseRenewRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
self.handle_renew_snapshot_lease(request).await
}
async fn release_snapshot_lease(
&self,
request: Request<SnapshotLeaseReleaseRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
self.handle_release_snapshot_lease(request).await
}
async fn read_metadata(&self, request: Request<ReadMetadataRequest>) -> Result<Response<ReadMetadataResponse>, Status> {
self.handle_read_metadata(request).await
}
rustfs/src/storage/rpc/node_service/disk.rs
+239 -6
View File
@@ -14,11 +14,12 @@
use super::NodeService;
use crate::storage::storage_api::rpc_consumer::node_service::{
BatchReadVersionReq, BatchReadVersionResp, DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq,
ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count,
BatchReadVersionReq, BatchReadVersionResp, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions,
ReadMultipleReq, ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts,
validate_batch_read_version_item_count,
};
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use crate::storage::storage_api::{PartTransactionAction, verify_tonic_mutation_body_digest};
use crate::storage::storage_api::{PartTransactionAction, SnapshotLeaseToken, verify_tonic_mutation_body_digest};
use bytes::Bytes;
use rustfs_filemeta::FileInfo;
use rustfs_io_metrics::internode_metrics::{
@@ -28,13 +29,96 @@ use rustfs_io_metrics::internode_metrics::{
};
use rustfs_protos::proto_gen::node_service::*;
use serde::de::DeserializeOwned;
use std::io::Cursor;
use std::{collections::HashMap, io::Cursor, time::Duration};
use tokio::sync::mpsc;
use tokio_util::time::DelayQueue;
use tonic::{Request, Response, Status};
use tracing::debug;
/// Initial capacity hint (bytes) for msgpack encode buffers, sized to cover a typical single-
/// version `FileInfo` without repeated growth reallocations. Larger payloads still grow as needed.
const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512;
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
const SNAPSHOT_LEASE_MIN_TTL: Duration = Duration::from_secs(5);
const SNAPSHOT_LEASE_MAX_TTL: Duration = Duration::from_secs(5 * 60);
struct SnapshotLeaseExpiry {
disk: DiskStore,
volume: String,
path: String,
token: SnapshotLeaseToken,
}
pub(super) struct SnapshotLeaseExpiryScheduler {
tx: mpsc::UnboundedSender<SnapshotLeaseExpiryCommand>,
}
enum SnapshotLeaseExpiryCommand {
Schedule(SnapshotLeaseExpiry, Duration),
Cancel(SnapshotLeaseToken),
}
impl SnapshotLeaseExpiryScheduler {
pub(super) fn new() -> Self {
let (tx, mut rx) = mpsc::unbounded_channel();
tokio::spawn(async move {
let mut expirations: DelayQueue<SnapshotLeaseExpiry> = DelayQueue::new();
let mut keys = HashMap::new();
loop {
tokio::select! {
Some(command) = rx.recv() => {
match command {
SnapshotLeaseExpiryCommand::Schedule(expiry, ttl) => {
if let Some(key) = keys.remove(&expiry.token) {
expirations.remove(&key);
}
let token = expiry.token;
let key = expirations.insert(expiry, ttl);
keys.insert(token, key);
}
SnapshotLeaseExpiryCommand::Cancel(token) => {
if let Some(key) = keys.remove(&token) {
expirations.remove(&key);
}
}
}
}
Some(expired) = futures_util::StreamExt::next(&mut expirations), if !expirations.is_empty() => {
let expiry = expired.into_inner();
keys.remove(&expiry.token);
let _ = expiry
.disk
.release_snapshot_lease(&expiry.volume, &expiry.path, expiry.token)
.await;
}
else => break,
}
}
});
Self { tx }
}
fn schedule(&self, expiry: SnapshotLeaseExpiry, ttl: Duration) -> Result<(), SnapshotLeaseExpiry> {
self.tx
.send(SnapshotLeaseExpiryCommand::Schedule(expiry, ttl))
.map_err(|err| match err.0 {
SnapshotLeaseExpiryCommand::Schedule(expiry, _) => expiry,
SnapshotLeaseExpiryCommand::Cancel(_) => unreachable!(),
})
}
fn cancel(&self, token: SnapshotLeaseToken) {
let _ = self.tx.send(SnapshotLeaseExpiryCommand::Cancel(token));
}
}
fn snapshot_lease_ttl(ttl_ms: u64) -> Result<Duration, Status> {
let ttl = Duration::from_millis(ttl_ms);
if !(SNAPSHOT_LEASE_MIN_TTL..=SNAPSHOT_LEASE_MAX_TTL).contains(&ttl) {
return Err(Status::invalid_argument("snapshot lease TTL is outside the supported range"));
}
Ok(ttl)
}
fn decode_msgpack_or_json<T: DeserializeOwned>(
binary: &[u8],
@@ -165,6 +249,146 @@ fn encode_batch_read_version_response_payloads(
}
impl NodeService {
pub(super) async fn handle_acquire_snapshot_lease(
&self,
request: Request<SnapshotLeaseRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref()),
"acquire_snapshot_lease",
)?;
let request = request.into_inner();
let ttl = snapshot_lease_ttl(request.ttl_ms)?;
let Some(disk) = self.find_disk(&request.disk).await else {
return Ok(Response::new(SnapshotLeaseResponse {
success: false,
token: Bytes::new(),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: Some(DiskError::other("cannot find disk").into()),
}));
};
match disk.acquire_snapshot_lease(&request.volume, &request.path).await {
Ok(token) => {
if let Err(expiry) = self.snapshot_lease_expiry.schedule(
SnapshotLeaseExpiry {
disk,
volume: request.volume,
path: request.path,
token,
},
ttl,
) {
let _ = expiry
.disk
.release_snapshot_lease(&expiry.volume, &expiry.path, expiry.token)
.await;
return Err(Status::internal("snapshot lease expiry scheduler is unavailable"));
}
Ok(Response::new(SnapshotLeaseResponse {
success: true,
token: Bytes::copy_from_slice(token.as_bytes()),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: None,
}))
}
Err(err) => Ok(Response::new(SnapshotLeaseResponse {
success: false,
token: Bytes::new(),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: Some(err.into()),
})),
}
}
pub(super) async fn handle_renew_snapshot_lease(
&self,
request: Request<SnapshotLeaseRenewRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref()),
"renew_snapshot_lease",
)?;
let request = request.into_inner();
let ttl = snapshot_lease_ttl(request.ttl_ms)?;
let token =
SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?;
let Some(disk) = self.find_disk(&request.disk).await else {
return Ok(Response::new(SnapshotLeaseResponse {
success: false,
token: Bytes::new(),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: Some(DiskError::other("cannot find disk").into()),
}));
};
match disk.renew_snapshot_lease(&request.volume, &request.path, token).await {
Ok(renewed) => {
if let Err(expiry) = self.snapshot_lease_expiry.schedule(
SnapshotLeaseExpiry {
disk,
volume: request.volume,
path: request.path,
token: renewed,
},
ttl,
) {
let _ = expiry
.disk
.release_snapshot_lease(&expiry.volume, &expiry.path, expiry.token)
.await;
return Err(Status::internal("snapshot lease expiry scheduler is unavailable"));
}
self.snapshot_lease_expiry.cancel(token);
Ok(Response::new(SnapshotLeaseResponse {
success: true,
token: Bytes::copy_from_slice(renewed.as_bytes()),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: None,
}))
}
Err(err) => Ok(Response::new(SnapshotLeaseResponse {
success: false,
token: Bytes::new(),
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
error: Some(err.into()),
})),
}
}
pub(super) async fn handle_release_snapshot_lease(
&self,
request: Request<SnapshotLeaseReleaseRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
verify_disk_mutation_digest(
&request,
rustfs_protos::canonical_snapshot_lease_release_request_body(request.get_ref()),
"release_snapshot_lease",
)?;
let request = request.into_inner();
let token =
SnapshotLeaseToken::from_slice(&request.token).map_err(|_| Status::invalid_argument("invalid lease token"))?;
let Some(disk) = self.find_disk(&request.disk).await else {
return Ok(Response::new(SnapshotLeaseMutationResponse {
success: false,
error: Some(DiskError::other("cannot find disk").into()),
}));
};
match disk.release_snapshot_lease(&request.volume, &request.path, token).await {
Ok(()) => {
self.snapshot_lease_expiry.cancel(token);
Ok(Response::new(SnapshotLeaseMutationResponse {
success: true,
error: None,
}))
}
Err(err) => Ok(Response::new(SnapshotLeaseMutationResponse {
success: false,
error: Some(err.into()),
})),
}
}
pub(super) async fn handle_disk_info(&self, request: Request<DiskInfoRequest>) -> Result<Response<DiskInfoResponse>, Status> {
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
@@ -1368,8 +1592,9 @@ impl NodeService {
#[cfg(test)]
mod tests {
use super::{
compat_response_json, decode_msgpack_or_json, encode_batch_read_version_response_payloads, encode_msgpack,
encode_msgpack_named, encode_read_multiple_response_payloads,
SNAPSHOT_LEASE_MAX_TTL, SNAPSHOT_LEASE_MIN_TTL, compat_response_json, decode_msgpack_or_json,
encode_batch_read_version_response_payloads, encode_msgpack, encode_msgpack_named,
encode_read_multiple_response_payloads, snapshot_lease_ttl,
};
use crate::storage::storage_api::ReadMultipleResp;
use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp;
@@ -1382,6 +1607,14 @@ mod tests {
count: u32,
}
#[test]
fn snapshot_lease_ttl_rejects_values_outside_server_bounds() {
assert!(snapshot_lease_ttl(4_999).is_err());
assert_eq!(snapshot_lease_ttl(5_000).unwrap(), SNAPSHOT_LEASE_MIN_TTL);
assert_eq!(snapshot_lease_ttl(300_000).unwrap(), SNAPSHOT_LEASE_MAX_TTL);
assert!(snapshot_lease_ttl(300_001).is_err());
}
#[test]
fn decode_msgpack_or_json_prefers_binary_payload() {
let payload = SamplePayload {
rustfs/src/storage/storage_api.rs
+17 -1
View File
@@ -431,7 +431,7 @@ pub(crate) mod ecstore_disk {
pub(crate) use rustfs_ecstore::api::disk::{
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore,
FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq,
ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
get_object_disk_read_timeout, validate_batch_read_version_item_count,
};
pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce};
@@ -606,6 +606,7 @@ pub(crate) type ExpiryState = ecstore_bucket::lifecycle::bucket_lifecycle_ops::E
pub(crate) type FileInfoVersions = ecstore_disk::FileInfoVersions;
pub(crate) type FileReader = ecstore_disk::FileReader;
pub(crate) type FileWriter = ecstore_disk::FileWriter;
pub(crate) type SnapshotLeaseToken = ecstore_disk::SnapshotLeaseToken;
pub(crate) type FS = super::ecfs::FS;
pub(crate) type HashReader = ecstore_rio::HashReader;
pub(crate) type InstanceContext = ecstore_runtime::InstanceContext;
@@ -1075,6 +1076,9 @@ pub(crate) trait StorageDiskRpcExt {
) -> DiskResult<()>;
async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult<bytes::Bytes>;
async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>;
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult<SnapshotLeaseToken>;
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<SnapshotLeaseToken>;
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()>;
async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo>;
async fn list_volumes(&self) -> DiskResult<Vec<VolumeInfo>>;
async fn make_volume(&self, volume: &str) -> DiskResult<()>;
@@ -1201,6 +1205,18 @@ where
ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await
}
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> DiskResult<SnapshotLeaseToken> {
ecstore_disk::DiskAPI::acquire_snapshot_lease(self, volume, path).await
}
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<SnapshotLeaseToken> {
ecstore_disk::DiskAPI::renew_snapshot_lease(self, volume, path, token).await
}
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> DiskResult<()> {
ecstore_disk::DiskAPI::release_snapshot_lease(self, volume, path, token).await
}
async fn stat_volume(&self, volume: &str) -> DiskResult<VolumeInfo> {
ecstore_disk::DiskAPI::stat_volume(self, volume).await
}