mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-17 10:17:55 +00:00
feat(storage): harden internode data-path controls (#4224)
* fix(rio): propagate http writer shutdown errors * fix(ecstore): unify remote lock rpc deadlines * fix(storage): reject corrupt read multiple payloads * feat(rio): add internode http tuning profiles * feat(metrics): add internode baseline signals * feat(ecstore): observe shard locality topology * feat(ecstore): gate shard locality scheduling * feat(ecstore): gate batch read version rpc * feat(ecstore): observe batch processor adaptation * feat(ecstore): gate batch processor observation * docs: add get benchmark regression analysis * docs: add issue 797 execution plan status * fix(ecstore): require explicit batch rpc support * fix(ecstore): honor documented batch read gate * fix(ecstore): keep batch read gate stable per call * chore: update workspace dependencies * feat(ecstore): log batch read gate decisions * feat(ecstore): count batch read gate decisions * test(issue-797): add local internode A/B runner * test(rio): fix tuning profile spelling fixture * fix(protocols): adapt sftp channel open callbacks * fix(metrics): wrap batch processor observation args * chore(docs): keep issue notes local only * fix(storage): address internode review feedback * fix(storage): address internode data-path review findings - Run the BatchReadVersion auto-mode unary fallback outside the batch RPC deadline so each read_version keeps its own per-op timeout and health accounting instead of racing the whole batch against one drive timeout. - Cap adaptive batch-processor concurrency growth at a hard multiple of the configured baseline so sustained fast batches cannot ratchet past the configured limit. - Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE, and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading the environment on hot paths. - Skip shard read-cost collection in observe mode when stage metrics are disabled, and cache the local endpoint host list instead of rebuilding it on every read. - Allow --warp-extra-args values starting with -- and drop the unused warp_hosts_csv helper in the issue-797 A/B runner. * fix(storage): address internode data-path review findings - Run the BatchReadVersion auto-mode unary fallback outside the batch RPC deadline so each read_version keeps its own per-op timeout and health accounting instead of racing the whole batch against one drive timeout. - Cap adaptive batch-processor concurrency growth at a hard multiple of the configured baseline so sustained fast batches cannot ratchet past the configured limit. - Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE, and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading the environment on hot paths. - Skip shard read-cost collection in observe mode when stage metrics are disabled, and cache the local endpoint host list instead of rebuilding it on every read. - Allow --warp-extra-args values starting with -- and drop the unused warp_hosts_csv helper in the issue-797 A/B runner. Co-Authored-By: heihutu<heihutu@gmail.com> * fix(storage): align buffer clamp test with media cap * fix(ecstore): release optimized read locks before streaming --------- Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
houseme
@@ -36,8 +36,8 @@ use super::wedge_watchdog;
|
||||
use crate::common::client::s3::StorageBackend;
|
||||
use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
|
||||
use russh::keys::{self, PrivateKey, PublicKeyBase64};
|
||||
use russh::server::{Auth, Msg, Session};
|
||||
use russh::{Channel, ChannelId, MethodKind, MethodSet, Pty, Sig};
|
||||
use russh::server::{Auth, ChannelOpenHandle, Msg, Session};
|
||||
use russh::{Channel, ChannelId, ChannelOpenFailure, MethodKind, MethodSet, Pty, Sig};
|
||||
use rustfs_config::{
|
||||
DEFAULT_SFTP_HOST_KEY_RELOAD_ENABLE, DEFAULT_SFTP_HOST_KEY_RELOAD_INTERVAL, ENV_SFTP_HOST_KEY_RELOAD_ENABLE,
|
||||
ENV_SFTP_HOST_KEY_RELOAD_INTERVAL,
|
||||
@@ -1135,15 +1135,19 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, channel, _session), fields(peer = %self.peer_addr))]
|
||||
#[tracing::instrument(level = "debug", skip(self, channel, reply, _session), fields(peer = %self.peer_addr))]
|
||||
fn channel_open_session(
|
||||
&mut self,
|
||||
channel: Channel<Msg>,
|
||||
reply: ChannelOpenHandle,
|
||||
_session: &mut Session,
|
||||
) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
|
||||
) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
|
||||
let id = channel.id();
|
||||
self.channels.insert(id, channel);
|
||||
async { Ok(true) }
|
||||
async move {
|
||||
reply.accept().await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, _session), fields(peer = %self.peer_addr, channel = ?channel))]
|
||||
@@ -1404,13 +1408,13 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
|
||||
async { Ok(false) }
|
||||
}
|
||||
|
||||
// Channel-open rejections. russh 0.60 defaults all of these to
|
||||
// Ok(false), but we override them explicitly with a warn log so
|
||||
// Channel-open rejections. russh defaults to rejecting dropped channel-open
|
||||
// handles, but we override them explicitly with a warn log so
|
||||
// (a) probe attempts are visible in operator logs and
|
||||
// (b) a future russh default flip cannot silently allow these
|
||||
// channel types.
|
||||
|
||||
#[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
|
||||
#[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
|
||||
fn channel_open_direct_tcpip(
|
||||
&mut self,
|
||||
_channel: Channel<Msg>,
|
||||
@@ -1418,12 +1422,16 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
|
||||
port_to_connect: u32,
|
||||
_originator_address: &str,
|
||||
_originator_port: u32,
|
||||
reply: ChannelOpenHandle,
|
||||
_session: &mut Session,
|
||||
) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
|
||||
async { Ok(false) }
|
||||
) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
|
||||
async move {
|
||||
reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
|
||||
#[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, host = %host_to_connect, port = port_to_connect))]
|
||||
fn channel_open_forwarded_tcpip(
|
||||
&mut self,
|
||||
_channel: Channel<Msg>,
|
||||
@@ -1431,30 +1439,42 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
|
||||
port_to_connect: u32,
|
||||
_originator_address: &str,
|
||||
_originator_port: u32,
|
||||
reply: ChannelOpenHandle,
|
||||
_session: &mut Session,
|
||||
) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
|
||||
async { Ok(false) }
|
||||
) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
|
||||
async move {
|
||||
reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr))]
|
||||
#[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr))]
|
||||
fn channel_open_x11(
|
||||
&mut self,
|
||||
_channel: Channel<Msg>,
|
||||
_originator_address: &str,
|
||||
_originator_port: u32,
|
||||
reply: ChannelOpenHandle,
|
||||
_session: &mut Session,
|
||||
) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
|
||||
async { Ok(false) }
|
||||
) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
|
||||
async move {
|
||||
reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "warn", skip(self, _channel, _session), fields(peer = %self.peer_addr, socket = %socket_path))]
|
||||
#[tracing::instrument(level = "warn", skip(self, _channel, reply, _session), fields(peer = %self.peer_addr, socket = %socket_path))]
|
||||
fn channel_open_direct_streamlocal(
|
||||
&mut self,
|
||||
_channel: Channel<Msg>,
|
||||
socket_path: &str,
|
||||
reply: ChannelOpenHandle,
|
||||
_session: &mut Session,
|
||||
) -> impl std::future::Future<Output = Result<bool, Self::Error>> + Send {
|
||||
async { Ok(false) }
|
||||
) -> impl std::future::Future<Output = Result<(), Self::Error>> + Send {
|
||||
async move {
|
||||
reply.reject(ChannelOpenFailure::AdministrativelyProhibited).await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user