gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-10 23:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(storage): optimize internode RPC transfer path (#2262)

Browse Source 
Co-authored-by: momoda693 <momoda693@gmail.com>
This commit is contained in:
weisd
2026-03-23 17:09:49 +08:00
committed by GitHub
parent 236142a682
commit 05dc131a49
43 changed files with 1846 additions and 566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/config/com.rs
+31 -4
View File
@@ -15,6 +15,7 @@
use crate::config::{Config, GLOBAL_STORAGE_CLASS, storageclass};
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::global::is_first_cluster_node_local;
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_config::{DEFAULT_DELIMITER, RUSTFS_REGION};
@@ -44,6 +45,19 @@ pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u
Ok(data)
}
pub async fn read_config_no_lock<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
let (data, _obj) = read_config_with_metadata(
api,
file,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await?;
Ok(data)
}
pub async fn read_config_with_metadata<S: StorageAPI>(
api: Arc<S>,
file: &str,
@@ -287,7 +301,14 @@ fn is_object_not_found(err: &Error) -> bool {
pub async fn try_migrate_server_config<S: StorageAPI>(api: Arc<S>) {
let config_file = get_config_file();
match api
.get_object_info(RUSTFS_META_BUCKET, &config_file, &ObjectOptions::default())
.get_object_info(
RUSTFS_META_BUCKET,
&config_file,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
{
Ok(_) => {
@@ -360,7 +381,13 @@ pub async fn try_migrate_server_config<S: StorageAPI>(api: Arc<S>) {
/// Handle the situation where the configuration file does not exist, create and save a new configuration
async fn handle_missing_config<S: StorageAPI>(api: Arc<S>, context: &str) -> Result<Config> {
warn!("Configuration not found ({}): Start initializing new configuration", context);
let cfg = new_and_save_server_config(api).await?;
let cfg = if is_first_cluster_node_local().await {
new_and_save_server_config(api.clone()).await?
} else {
let mut cfg = new_server_config();
lookup_configs(&mut cfg, api).await;
cfg
};
warn!("Configuration initialization complete ({})", context);
Ok(cfg)
}
@@ -375,7 +402,7 @@ pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<C
let config_file = get_config_file();
// Try to read the configuration file
match read_config(api.clone(), &config_file).await {
match read_config_no_lock(api.clone(), &config_file).await {
Ok(data) => read_server_config(api, &data).await,
Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await,
Err(err) => handle_config_read_error(err, &config_file),
@@ -389,7 +416,7 @@ async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<C
warn!("Received empty configuration data, try to reread from '{}'", config_file);
// Try to read the configuration again
match read_config(api.clone(), &config_file).await {
match read_config_no_lock(api.clone(), &config_file).await {
Ok(cfg_data) => {
// TODO: decrypt
let cfg = decode_server_config_blob(&cfg_data)?;
crates/ecstore/src/config/mod.rs
+3 -3
View File
@@ -79,7 +79,7 @@ pub async fn try_migrate_server_config(api: Arc<ECStore>) {
com::try_migrate_server_config(api).await
}
#[derive(Debug, Deserialize, Serialize, Clone)]
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct KV {
pub key: String,
pub value: String,
@@ -87,7 +87,7 @@ pub struct KV {
pub hidden_if_empty: bool,
}
#[derive(Debug, Deserialize, Serialize, Clone)]
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct KVS(pub Vec<KV>);
impl Default for KVS {
@@ -163,7 +163,7 @@ impl KVS {
}
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Config(pub HashMap<String, HashMap<String, KVS>>);
impl Default for Config {
crates/ecstore/src/disk/disk_store.rs
+13
View File
@@ -18,6 +18,7 @@ use crate::disk::{
WalkDirOptions,
local::{LocalDisk, ScanGuard},
};
use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use std::{
@@ -410,7 +411,19 @@ impl LocalDiskWrapper {
/// Set the disk ID
pub async fn set_disk_id_internal(&self, id: Option<Uuid>) -> Result<()> {
let mut disk_id = self.disk_id.write().await;
let previous = *disk_id;
*disk_id = id;
drop(disk_id);
if self.disk.is_local() {
let mut disk_id_map = GLOBAL_LOCAL_DISK_ID_MAP.write().await;
if let Some(previous_id) = previous {
disk_id_map.remove(&previous_id);
}
if let Some(current_id) = id {
disk_id_map.insert(current_id, self.disk.endpoint().to_string());
}
}
Ok(())
}
crates/ecstore/src/erasure_coding/bitrot.rs
+164 -6
View File
@@ -15,6 +15,7 @@
use bytes::Bytes;
use pin_project_lite::pin_project;
use rustfs_utils::HashAlgorithm;
use std::io::IoSlice;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tracing::error;
use uuid::Uuid;
@@ -155,23 +156,49 @@ where
error!("bitrot writer write hash error: hash is empty");
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "hash is empty"));
}
self.inner.write_all(hash.as_ref()).await?;
write_all_vectored(&mut self.inner, hash.as_ref(), buf).await?;
} else {
self.inner.write_all(buf).await?;
}
self.inner.write_all(buf).await?;
self.inner.flush().await?;
let n = buf.len();
Ok(n)
}
pub async fn shutdown(&mut self) -> std::io::Result<()> {
self.inner.flush().await?;
self.inner.shutdown().await
}
}
async fn write_all_vectored<W>(writer: &mut W, hash: &[u8], data: &[u8]) -> std::io::Result<()>
where
W: AsyncWrite + Unpin,
{
let mut hash_offset = 0;
let mut data_offset = 0;
while hash_offset < hash.len() || data_offset < data.len() {
let slices = [IoSlice::new(&hash[hash_offset..]), IoSlice::new(&data[data_offset..])];
let written = writer.write_vectored(&slices).await?;
if written == 0 {
return Err(std::io::Error::new(std::io::ErrorKind::WriteZero, "failed to write hash and data"));
}
let hash_remaining = hash.len() - hash_offset;
if written < hash_remaining {
hash_offset += written;
continue;
}
hash_offset = hash.len();
data_offset += written - hash_remaining;
}
Ok(())
}
pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize {
if algo != HashAlgorithm::HighwayHash256S && algo != HashAlgorithm::HighwayHash256SLegacy {
return size;
@@ -292,6 +319,33 @@ impl AsyncWrite for CustomWriter {
}
}
}
fn poll_write_vectored(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
bufs: &[IoSlice<'_>],
) -> std::task::Poll<std::io::Result<usize>> {
match self.get_mut() {
Self::InlineBuffer(data) => {
let total = bufs.iter().map(|buf| buf.len()).sum::<usize>();
for buf in bufs {
data.extend_from_slice(buf);
}
std::task::Poll::Ready(Ok(total))
}
Self::Other(writer) => {
let pinned_writer = std::pin::Pin::new(writer.as_mut());
pinned_writer.poll_write_vectored(cx, bufs)
}
}
}
fn is_write_vectored(&self) -> bool {
match self {
Self::InlineBuffer(_) => true,
Self::Other(writer) => writer.is_write_vectored(),
}
}
}
/// Wrapper around BitrotWriter that uses our custom writer
@@ -361,7 +415,74 @@ mod tests {
use super::BitrotReader;
use super::BitrotWriter;
use rustfs_utils::HashAlgorithm;
use std::io::Cursor;
use std::io::{Cursor, IoSlice};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use std::task::{Context, Poll};
use tokio::io::AsyncWrite;
#[derive(Default)]
struct VectoredCountingWriter {
vectored_writes: Arc<AtomicUsize>,
writes: Vec<u8>,
}
impl AsyncWrite for VectoredCountingWriter {
fn poll_write(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
Poll::Ready(Err(std::io::Error::other("poll_write should not be used")))
}
fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_write_vectored(
mut self: std::pin::Pin<&mut Self>,
_cx: &mut Context<'_>,
bufs: &[IoSlice<'_>],
) -> Poll<std::io::Result<usize>> {
self.vectored_writes.fetch_add(1, Ordering::SeqCst);
let total = bufs.iter().map(|buf| buf.len()).sum::<usize>();
for buf in bufs {
self.writes.extend_from_slice(buf);
}
Poll::Ready(Ok(total))
}
fn is_write_vectored(&self) -> bool {
true
}
}
#[derive(Default)]
struct CountingWriter {
flushes: Arc<AtomicUsize>,
shutdowns: Arc<AtomicUsize>,
writes: Vec<u8>,
}
impl AsyncWrite for CountingWriter {
fn poll_write(mut self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
self.writes.extend_from_slice(buf);
Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
self.flushes.fetch_add(1, Ordering::SeqCst);
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
self.shutdowns.fetch_add(1, Ordering::SeqCst);
Poll::Ready(Ok(()))
}
}
#[tokio::test]
async fn test_bitrot_read_write_ok() {
@@ -471,4 +592,41 @@ mod tests {
assert_eq!(n, data_size);
assert_eq!(data, &out[..]);
}
#[tokio::test]
async fn test_bitrot_writer_flushes_once_on_shutdown() {
let flushes = Arc::new(AtomicUsize::new(0));
let shutdowns = Arc::new(AtomicUsize::new(0));
let writer = CountingWriter {
flushes: flushes.clone(),
shutdowns: shutdowns.clone(),
writes: Vec::new(),
};
let mut bitrot_writer = BitrotWriter::new(writer, 8, HashAlgorithm::None);
bitrot_writer.write(b"12345678").await.unwrap();
bitrot_writer.write(b"abc").await.unwrap();
assert_eq!(flushes.load(Ordering::SeqCst), 0);
assert_eq!(shutdowns.load(Ordering::SeqCst), 0);
bitrot_writer.shutdown().await.unwrap();
assert_eq!(flushes.load(Ordering::SeqCst), 1);
assert_eq!(shutdowns.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_bitrot_writer_uses_vectored_write_for_hash_and_data() {
let vectored_writes = Arc::new(AtomicUsize::new(0));
let writer = VectoredCountingWriter {
vectored_writes: vectored_writes.clone(),
writes: Vec::new(),
};
let mut bitrot_writer = BitrotWriter::new(writer, 8, HashAlgorithm::HighwayHash256);
bitrot_writer.write(b"payload").await.unwrap();
assert!(vectored_writes.load(Ordering::SeqCst) > 0);
}
}
crates/ecstore/src/erasure_coding/encode.rs
+122 -5
View File
@@ -112,11 +112,66 @@ impl<'a> MultiWriter<'a> {
)))
}
pub async fn _shutdown(&mut self) -> std::io::Result<()> {
for writer in self.writers.iter_mut().flatten() {
writer.shutdown().await?;
async fn shutdown_writer(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>) {
match writer_opt {
Some(writer) => match writer.shutdown().await {
Ok(()) => {
*err = None;
}
Err(e) => {
*err = Some(Error::from(e));
*writer_opt = None;
}
},
None => {
*err = Some(Error::DiskNotFound);
}
}
Ok(())
}
pub async fn shutdown(&mut self) -> std::io::Result<()> {
{
let mut futures = FuturesUnordered::new();
for (writer_opt, err) in self.writers.iter_mut().zip(self.errs.iter_mut()) {
if err.is_some() {
continue;
}
futures.push(Self::shutdown_writer(writer_opt, err));
}
while let Some(()) = futures.next().await {}
}
let nil_count = self.errs.iter().filter(|&e| e.is_none()).count();
if nil_count >= self.write_quorum {
return Ok(());
}
if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
error!(
"reduce_write_quorum_errs during shutdown: {:?}, offline-disks={}/{}, errs={:?}",
write_err,
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len(),
self.errs
);
return Err(std::io::Error::other(format!(
"Failed to shutdown writers: {} (offline-disks={}/{})",
write_err,
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len()
)));
}
Err(std::io::Error::other(format!(
"Failed to shutdown writers: (offline-disks={}/{}): {}",
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len(),
self.errs
.iter()
.map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
.collect::<Vec<_>>()
.join(", ")
)))
}
}
@@ -176,7 +231,69 @@ impl Erasure {
}
let (reader, total) = task.await??;
// writers.shutdown().await?;
writers.shutdown().await?;
Ok((reader, total))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::erasure_coding::{BitrotWriterWrapper, CustomWriter};
use rustfs_utils::HashAlgorithm;
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use tokio::io::AsyncWrite;
#[derive(Clone, Default)]
struct DeferredCommitWriter {
buffered: Vec<u8>,
committed: Arc<Mutex<Vec<u8>>>,
}
impl DeferredCommitWriter {
fn new(committed: Arc<Mutex<Vec<u8>>>) -> Self {
Self {
buffered: Vec::new(),
committed,
}
}
}
impl AsyncWrite for DeferredCommitWriter {
fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
self.buffered.extend_from_slice(buf);
Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
let buffered = std::mem::take(&mut self.buffered);
let mut committed = self.committed.lock().unwrap();
committed.extend_from_slice(&buffered);
Poll::Ready(Ok(()))
}
}
#[tokio::test]
async fn encode_shutdowns_writers_after_small_shards() {
let committed = Arc::new(Mutex::new(Vec::new()));
let writer = DeferredCommitWriter::new(committed.clone());
let mut writers = vec![Some(BitrotWriterWrapper::new(
CustomWriter::new_tokio_writer(writer),
16,
HashAlgorithm::HighwayHash256S,
))];
let erasure = Arc::new(Erasure::new(1, 0, 16));
let reader = tokio::io::BufReader::new(std::io::Cursor::new(b"small payload".to_vec()));
let (_reader, written) = erasure.encode(reader, &mut writers, 1).await.unwrap();
assert_eq!(written, b"small payload".len());
assert!(!committed.lock().unwrap().is_empty());
}
}
crates/ecstore/src/global.rs
+5
View File
@@ -46,6 +46,7 @@ lazy_static! {
pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
pub static ref GLOBAL_LOCAL_DISK_ID_MAP: Arc<RwLock<HashMap<Uuid, String>>> = Arc::new(RwLock::new(HashMap::new()));
pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new();
pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
@@ -153,6 +154,10 @@ pub fn get_global_endpoints_opt() -> Option<EndpointServerPools> {
GLOBAL_Endpoints.get().cloned()
}
pub async fn is_first_cluster_node_local() -> bool {
get_global_endpoints().first_local()
}
pub fn get_global_tier_config_mgr() -> Arc<RwLock<TierConfigMgr>> {
GLOBAL_TierConfigMgr.clone()
}
crates/ecstore/src/metrics_realtime.rs
+73 -5
View File
@@ -14,8 +14,11 @@
use crate::{admin_server_info::get_local_server_property, new_object_layer_fn, store_api::StorageAPI};
use chrono::Utc;
use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, metrics::global_metrics};
use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, RealtimeMetrics};
use rustfs_common::{
GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, internode_metrics::global_internode_metrics,
metrics::global_metrics,
};
use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics};
use rustfs_utils::os::get_drive_stats;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
@@ -120,13 +123,50 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
// if types.contains(&MetricType::SITE_RESYNC) {}
// if types.contains(&MetricType::NET) {}
if types.contains(&MetricType::NET) {
let snapshot = global_internode_metrics().snapshot();
real_time_metrics.aggregated.net = Some(NetMetrics {
collected_at: Utc::now(),
interface_name: "internode".to_string(),
net_stats: NetDevLine {
name: "internode".to_string(),
rx_bytes: snapshot.recv_bytes_total,
tx_bytes: snapshot.sent_bytes_total,
..Default::default()
},
});
}
// if types.contains(&MetricType::MEM) {}
// if types.contains(&MetricType::CPU) {}
// if types.contains(&MetricType::RPC) {}
if types.contains(&MetricType::RPC) {
let collected_at = Utc::now();
let snapshot = global_internode_metrics().snapshot();
let last_connect_time =
chrono::DateTime::<Utc>::from_timestamp_millis(snapshot.last_dial_unix_millis as i64).unwrap_or(collected_at);
real_time_metrics.aggregated.rpc = Some(RPCMetrics {
collected_at,
connected: i32::from(snapshot.last_dial_unix_millis > 0),
reconnect_count: snapshot.dial_errors_total.min(i32::MAX as u64) as i32,
disconnected: 0,
outgoing_streams: 0,
incoming_streams: 0,
outgoing_bytes: snapshot.sent_bytes_total.min(i64::MAX as u64) as i64,
incoming_bytes: snapshot.recv_bytes_total.min(i64::MAX as u64) as i64,
outgoing_messages: snapshot.outgoing_requests_total.min(i64::MAX as u64) as i64,
incoming_messages: snapshot.incoming_requests_total.min(i64::MAX as u64) as i64,
out_queue: 0,
last_pong_time: collected_at,
last_ping_ms: snapshot.dial_avg_time_nanos as f64 / 1_000_000.0,
max_ping_dur_ms: snapshot.dial_avg_time_nanos as f64 / 1_000_000.0,
last_connect_time,
by_destination: None,
by_caller: None,
});
}
real_time_metrics
.by_host
@@ -211,7 +251,9 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
#[cfg(test)]
mod test {
use super::MetricType;
use super::*;
use rustfs_common::internode_metrics::global_internode_metrics;
use std::time::Duration;
#[test]
fn tes_types() {
@@ -229,4 +271,30 @@ mod test {
let disk = MetricType::new(1 << 1);
assert!(disk.contains(&MetricType::DISK));
}
#[tokio::test]
async fn collect_local_metrics_reports_internode_net_and_rpc() {
let metrics = global_internode_metrics();
metrics.reset_for_test();
metrics.record_sent_bytes(128);
metrics.record_recv_bytes(64);
metrics.record_outgoing_request();
metrics.record_incoming_request();
metrics.record_dial_result(Duration::from_millis(4), true);
let realtime = collect_local_metrics(MetricType::NET, &CollectMetricsOpts::default()).await;
let net = realtime.aggregated.net.expect("net metrics");
assert_eq!(net.net_stats.tx_bytes, 128);
assert_eq!(net.net_stats.rx_bytes, 64);
let realtime = collect_local_metrics(MetricType::RPC, &CollectMetricsOpts::default()).await;
let rpc = realtime.aggregated.rpc.expect("rpc metrics");
assert_eq!(rpc.outgoing_bytes, 128);
assert_eq!(rpc.incoming_bytes, 64);
assert_eq!(rpc.outgoing_messages, 1);
assert_eq!(rpc.incoming_messages, 1);
assert!(rpc.last_ping_ms > 0.0);
metrics.reset_for_test();
}
}
crates/ecstore/src/rpc/remote_disk.rs
+145 -46
View File
@@ -23,6 +23,7 @@ use crate::disk::{
};
use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use crate::set_disk::DEFAULT_READ_BUFFER_SIZE;
use crate::{
disk::error::{Error, Result},
rpc::build_auth_headers,
@@ -40,7 +41,9 @@ use rustfs_protos::proto_gen::node_service::{
node_service_client::NodeServiceClient,
};
use rustfs_rio::{HttpReader, HttpWriter};
use serde::{Serialize, de::DeserializeOwned};
use std::{
io::Cursor,
path::PathBuf,
sync::{
Arc,
@@ -49,12 +52,36 @@ use std::{
time::Duration,
};
use tokio::time;
use tokio::{io::AsyncWrite, net::TcpStream, time::timeout};
use tokio::{
io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt},
net::TcpStream,
time::timeout,
};
use tokio_util::sync::CancellationToken;
use tonic::{Request, service::interceptor::InterceptedService, transport::Channel};
use tracing::{debug, info, warn};
use uuid::Uuid;
async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
where
R: AsyncRead + Unpin,
W: AsyncWrite + Unpin,
{
let mut copied = 0_u64;
let mut buffer = vec![0_u8; buffer_size];
loop {
let bytes_read = reader.read(&mut buffer).await?;
if bytes_read == 0 {
writer.flush().await?;
return Ok(copied);
}
writer.write_all(&buffer[..bytes_read]).await?;
copied += bytes_read as u64;
}
}
#[derive(Debug)]
pub struct RemoteDisk {
pub id: Mutex<Option<Uuid>>,
@@ -259,6 +286,27 @@ impl RemoteDisk {
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))
}
async fn disk_ref(&self) -> String {
(*self.id.lock().await)
.map(|id| id.to_string())
.unwrap_or_else(|| self.endpoint.to_string())
}
}
fn encode_msgpack<T: Serialize>(value: &T) -> Result<Vec<u8>> {
let mut serializer = rmp_serde::Serializer::new(Vec::new());
value.serialize(&mut serializer)?;
Ok(serializer.into_inner())
}
fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str) -> Result<T> {
if !binary.is_empty() {
let mut deserializer = rmp_serde::Deserializer::new(Cursor::new(binary));
return T::deserialize(&mut deserializer).map_err(Error::from);
}
serde_json::from_str(json).map_err(Error::from)
}
// TODO: all api need to handle errors
@@ -707,18 +755,21 @@ impl DiskAPI for RemoteDisk {
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
info!("write_metadata {}/{}", volume, path);
let file_info = serde_json::to_string(&fi)?;
let file_info_bin = encode_msgpack(&fi)?;
self.execute_with_timeout(
|| async {
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(WriteMetadataRequest {
disk: self.endpoint.to_string(),
disk,
volume: volume.to_string(),
path: path.to_string(),
file_info: file_info.clone(),
file_info_bin: file_info_bin.clone(),
});
let response = client.write_metadata(request).await?.into_inner();
@@ -735,6 +786,7 @@ impl DiskAPI for RemoteDisk {
}
async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
@@ -742,7 +794,7 @@ impl DiskAPI for RemoteDisk {
let request = Request::new(ReadMetadataRequest {
volume: volume.to_string(),
path: path.to_string(),
disk: self.endpoint.to_string(),
disk,
});
let response = client.read_metadata(request).await?.into_inner();
@@ -759,19 +811,24 @@ impl DiskAPI for RemoteDisk {
info!("update_metadata");
let file_info = serde_json::to_string(&fi)?;
let opts_str = serde_json::to_string(&opts)?;
let file_info_bin = encode_msgpack(&fi)?;
let opts_bin = encode_msgpack(opts)?;
self.execute_with_timeout(
|| async {
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(UpdateMetadataRequest {
disk: self.endpoint.to_string(),
disk,
volume: volume.to_string(),
path: path.to_string(),
file_info: file_info.clone(),
opts: opts_str.clone(),
file_info_bin: file_info_bin.clone(),
opts_bin: opts_bin.clone(),
});
let response = client.update_metadata(request).await?.into_inner();
@@ -798,19 +855,22 @@ impl DiskAPI for RemoteDisk {
) -> Result<FileInfo> {
info!("read_version");
let opts_str = serde_json::to_string(opts)?;
let opts_bin = encode_msgpack(opts)?;
self.execute_with_timeout(
|| async {
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(ReadVersionRequest {
disk: self.endpoint.to_string(),
disk,
volume: volume.to_string(),
path: path.to_string(),
version_id: version_id.to_string(),
opts: opts_str.clone(),
opts_bin: opts_bin.clone(),
});
let response = client.read_version(request).await?.into_inner();
@@ -819,7 +879,7 @@ impl DiskAPI for RemoteDisk {
return Err(response.error.unwrap_or_default().into());
}
let file_info = serde_json::from_str::<FileInfo>(&response.file_info)?;
let file_info = decode_msgpack_or_json::<FileInfo>(&response.file_info_bin, &response.file_info)?;
Ok(file_info)
},
@@ -834,12 +894,13 @@ impl DiskAPI for RemoteDisk {
self.execute_with_timeout(
|| async {
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(ReadXlRequest {
disk: self.endpoint.to_string(),
disk,
volume: volume.to_string(),
path: path.to_string(),
read_data,
@@ -851,7 +912,7 @@ impl DiskAPI for RemoteDisk {
return Err(response.error.unwrap_or_default().into());
}
let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
let raw_file_info = decode_msgpack_or_json::<RawFileInfo>(&response.raw_file_info_bin, &response.raw_file_info)?;
Ok(raw_file_info)
},
@@ -909,13 +970,14 @@ impl DiskAPI for RemoteDisk {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(ListDirRequest {
disk: self.endpoint.to_string(),
disk,
volume: volume.to_string(),
dir_path: dir_path.to_string(),
count,
@@ -937,12 +999,9 @@ impl DiskAPI for RemoteDisk {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let disk = self.disk_ref().await;
let url = format!(
"{}/rustfs/rpc/walk_dir?disk={}",
self.endpoint.grid_host(),
urlencoding::encode(self.endpoint.to_string().as_str()),
);
let url = format!("{}/rustfs/rpc/walk_dir?disk={}", self.endpoint.grid_host(), urlencoding::encode(&disk),);
let opts = serde_json::to_vec(&opts)?;
@@ -952,33 +1011,14 @@ impl DiskAPI for RemoteDisk {
let mut reader = HttpReader::new(url, Method::GET, headers, Some(opts)).await?;
tokio::io::copy(&mut reader, wr).await?;
copy_stream_with_buffer(&mut reader, wr, DEFAULT_READ_BUFFER_SIZE).await?;
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
info!("read_file {}/{}", volume, path);
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let url = format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
self.endpoint.grid_host(),
urlencoding::encode(self.endpoint.to_string().as_str()),
urlencoding::encode(volume),
urlencoding::encode(path),
0,
0
);
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
build_auth_headers(&url, &Method::GET, &mut headers);
Ok(Box::new(HttpReader::new(url, Method::GET, headers, None).await?))
self.read_file_stream(volume, path, 0, 0).await
}
#[tracing::instrument(level = "debug", skip(self))]
@@ -995,11 +1035,12 @@ impl DiskAPI for RemoteDisk {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let disk = self.disk_ref().await;
let url = format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
self.endpoint.grid_host(),
urlencoding::encode(self.endpoint.to_string().as_str()),
urlencoding::encode(&disk),
urlencoding::encode(volume),
urlencoding::encode(path),
offset,
@@ -1019,11 +1060,12 @@ impl DiskAPI for RemoteDisk {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let disk = self.disk_ref().await;
let url = format!(
"{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
self.endpoint.grid_host(),
urlencoding::encode(self.endpoint.to_string().as_str()),
urlencoding::encode(&disk),
urlencoding::encode(volume),
urlencoding::encode(path),
true,
@@ -1049,11 +1091,12 @@ impl DiskAPI for RemoteDisk {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
let disk = self.disk_ref().await;
let url = format!(
"{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
self.endpoint.grid_host(),
urlencoding::encode(self.endpoint.to_string().as_str()),
urlencoding::encode(&disk),
urlencoding::encode(volume),
urlencoding::encode(path),
false,
@@ -1261,13 +1304,16 @@ impl DiskAPI for RemoteDisk {
self.execute_with_timeout(
|| async {
let read_multiple_req = serde_json::to_string(&req)?;
let read_multiple_req_bin = encode_msgpack(&req)?;
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(ReadMultipleRequest {
disk: self.endpoint.to_string(),
disk,
read_multiple_req,
read_multiple_req_bin,
});
let response = client.read_multiple(request).await?.into_inner();
@@ -1276,11 +1322,19 @@ impl DiskAPI for RemoteDisk {
return Err(response.error.unwrap_or_default().into());
}
let read_multiple_resps = response
.read_multiple_resps
.into_iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
.collect();
let read_multiple_resps = if !response.read_multiple_resps_bin.is_empty() {
response
.read_multiple_resps_bin
.into_iter()
.filter_map(|buf| decode_msgpack_or_json::<ReadMultipleResp>(&buf, "").ok())
.collect()
} else {
response
.read_multiple_resps
.into_iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
.collect()
};
Ok(read_multiple_resps)
},
@@ -1295,12 +1349,13 @@ impl DiskAPI for RemoteDisk {
self.execute_with_timeout(
|| async {
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(WriteAllRequest {
disk: self.endpoint.to_string(),
disk,
volume: volume.to_string(),
path: path.to_string(),
data,
@@ -1325,12 +1380,13 @@ impl DiskAPI for RemoteDisk {
self.execute_with_timeout(
|| async {
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(ReadAllRequest {
disk: self.endpoint.to_string(),
disk,
volume: volume.to_string(),
path: path.to_string(),
});
@@ -1386,6 +1442,7 @@ impl DiskAPI for RemoteDisk {
mod tests {
use super::*;
use std::sync::Once;
use tokio::io::duplex;
use tokio::net::TcpListener;
use tracing::Level;
use uuid::Uuid;
@@ -1543,6 +1600,24 @@ mod tests {
assert!(!remote_disk.is_online().await);
}
#[tokio::test]
async fn test_copy_stream_with_buffer_copies_full_payload() {
let payload = b"walk-dir-stream".repeat(1024);
let expected = payload.clone();
let (mut write_half, mut read_half) = duplex(128);
let copy_task = tokio::spawn(async move {
let mut cursor = Cursor::new(payload);
copy_stream_with_buffer(&mut cursor, &mut write_half, 4 * 1024).await.unwrap();
});
let mut copied = Vec::new();
read_half.read_to_end(&mut copied).await.unwrap();
copy_task.await.unwrap();
assert_eq!(copied, expected);
}
#[tokio::test]
async fn test_remote_disk_disk_id() {
let url = url::Url::parse("http://remote-server:9000").unwrap();
@@ -1579,6 +1654,30 @@ mod tests {
assert!(cleared_id.is_none());
}
#[tokio::test]
async fn test_remote_disk_ref_prefers_disk_id() {
let url = url::Url::parse("http://remote-server:9000").unwrap();
let endpoint = Endpoint {
url,
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: 0,
};
let disk_option = DiskOption {
cleanup: false,
health_check: false,
};
let remote_disk = RemoteDisk::new(&endpoint, &disk_option).await.unwrap();
assert_eq!(remote_disk.disk_ref().await, endpoint.to_string());
let disk_id = Uuid::new_v4();
remote_disk.set_disk_id(Some(disk_id)).await.unwrap();
assert_eq!(remote_disk.disk_ref().await, disk_id.to_string());
}
#[tokio::test]
async fn test_remote_disk_endpoints_with_different_schemes() {
let test_cases = vec![
crates/ecstore/src/rpc/remote_locker.rs
+1
View File
@@ -52,6 +52,7 @@ impl RemoteClient {
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
suppress_contention_logs: false,
}
}
crates/ecstore/src/store.rs
+2 -2
View File
@@ -142,8 +142,8 @@ mod rebalance;
use peer::init_local_peer;
pub use peer::{
all_local_disk, all_local_disk_path, find_local_disk, get_disk_infos, get_disk_via_endpoint, has_space_for, init_local_disks,
init_lock_clients,
all_local_disk, all_local_disk_path, find_local_disk, find_local_disk_by_ref, get_disk_infos, get_disk_via_endpoint,
has_space_for, init_local_disks, init_lock_clients,
};
#[derive(Debug)]
crates/ecstore/src/store/init.rs
+5 -1
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use super::*;
use crate::global::is_first_cluster_node_local;
impl ECStore {
#[allow(clippy::new_ret_no_self)]
@@ -203,6 +204,7 @@ impl ECStore {
let mut meta = PoolMeta::default();
meta.load(self.pools[0].clone(), self.pools.clone()).await?;
let update = meta.validate(self.pools.clone())?;
let should_persist_pool_meta = is_first_cluster_node_local().await;
if !update {
{
@@ -211,7 +213,9 @@ impl ECStore {
}
} else {
let new_meta = PoolMeta::new(&self.pools, &meta);
new_meta.save(self.pools.clone()).await?;
if should_persist_pool_meta {
new_meta.save(self.pools.clone()).await?;
}
{
let mut pool_meta = self.pool_meta.write().await;
*pool_meta = new_meta;
crates/ecstore/src/store/peer.rs
+14
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use super::*;
use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await;
@@ -24,6 +25,19 @@ pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
}
}
pub async fn find_local_disk_by_ref(disk_ref: &str) -> Option<DiskStore> {
if let Some(disk) = find_local_disk(&disk_ref.to_string()).await {
return Some(disk);
}
let Ok(disk_id) = Uuid::parse_str(disk_ref) else {
return None;
};
let disk_path = GLOBAL_LOCAL_DISK_ID_MAP.read().await.get(&disk_id).cloned()?;
find_local_disk(&disk_path).await
}
pub async fn get_disk_via_endpoint(endpoint: &Endpoint) -> Option<DiskStore> {
let global_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.read().await;
if global_set_drives.is_empty() {
crates/ecstore/src/tier/tier.rs
+18 -2
View File
@@ -49,6 +49,7 @@ use crate::{
StorageAPI,
config::com::{CONFIG_PREFIX, read_config},
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
global::{get_global_endpoints, is_first_cluster_node_local},
store::ECStore,
store_api::{ObjectIO as _, ObjectOptions, PutObjReader},
};
@@ -1118,7 +1119,15 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
}
Err(legacy_err) if is_err_config_not_found(&legacy_err) => {
warn!("config not found, start to init");
new_and_save_tiering_config(api).await.map_err(io::Error::other)
if is_first_cluster_node_local().await {
new_and_save_tiering_config(api).await.map_err(io::Error::other)
} else {
Ok(TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
})
}
}
Err(legacy_err) => Err(io::Error::other(legacy_err)),
}
@@ -1166,7 +1175,14 @@ async fn write_tier_config_to_rustfs<S: StorageAPI>(api: Arc<S>, path: &str, dat
pub async fn try_migrate_tiering_config<S: StorageAPI>(api: Arc<S>) {
let target_path = tier_config_path(TIER_CONFIG_FILE);
if api
.get_object_info(RUSTFS_META_BUCKET, &target_path, &ObjectOptions::default())
.get_object_info(
RUSTFS_META_BUCKET,
&target_path,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.is_ok()
{