gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 11:56:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(storage): harden internode data-path controls (#4224)

Browse Source 
* fix(rio): propagate http writer shutdown errors

* fix(ecstore): unify remote lock rpc deadlines

* fix(storage): reject corrupt read multiple payloads

* feat(rio): add internode http tuning profiles

* feat(metrics): add internode baseline signals

* feat(ecstore): observe shard locality topology

* feat(ecstore): gate shard locality scheduling

* feat(ecstore): gate batch read version rpc

* feat(ecstore): observe batch processor adaptation

* feat(ecstore): gate batch processor observation

* docs: add get benchmark regression analysis

* docs: add issue 797 execution plan status

* fix(ecstore): require explicit batch rpc support

* fix(ecstore): honor documented batch read gate

* fix(ecstore): keep batch read gate stable per call

* chore: update workspace dependencies

* feat(ecstore): log batch read gate decisions

* feat(ecstore): count batch read gate decisions

* test(issue-797): add local internode A/B runner

* test(rio): fix tuning profile spelling fixture

* fix(protocols): adapt sftp channel open callbacks

* fix(metrics): wrap batch processor observation args

* chore(docs): keep issue notes local only

* fix(storage): address internode review feedback

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

Co-Authored-By: heihutu<heihutu@gmail.com>

* fix(storage): align buffer clamp test with media cap

* fix(ecstore): release optimized read locks before streaming

---------

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
houseme
2026-07-03 17:08:15 +08:00
committed by GitHub
parent a9ac5f578d
commit 25d80d7c60
26 changed files with 3627 additions and 483 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/api/mod.rs
+4 -2
View File
@@ -84,9 +84,11 @@ pub mod disk {
pub use crate::disk::error_reduce::is_all_buckets_not_found;
pub use crate::disk::local::ScanGuard;
pub use crate::disk::{
BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
DiskStore, FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
validate_batch_read_version_item_count,
};
pub use crate::disk::{endpoint, error, error_reduce};
pub use bytes::Bytes;
crates/ecstore/src/cluster/rpc/remote_disk.rs
+441 -21
View File
@@ -20,8 +20,9 @@ use crate::cluster::rpc::internode_data_transport::{
};
use crate::disk::error::{Error, Result};
use crate::disk::{
CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader,
FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
disk_store::{
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -30,6 +31,7 @@ use crate::disk::{
},
endpoint::Endpoint,
health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state},
validate_batch_read_version_item_count,
};
use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
use crate::set_disk::DEFAULT_READ_BUFFER_SIZE;
@@ -40,11 +42,11 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_protos::evict_failed_connection;
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_protos::proto_gen::node_service::{
CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest,
ReadMetadataRequest, ReadMultipleRequest, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
RenameFileRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
node_service_client::NodeServiceClient,
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, StatVolumeRequest,
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
};
use serde::{Serialize, de::DeserializeOwned};
use std::{
@@ -63,7 +65,7 @@ use tokio::{
time::timeout,
};
use tokio_util::sync::CancellationToken;
use tonic::{Request, service::interceptor::InterceptedService, transport::Channel};
use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
use tracing::{Instrument, debug, warn};
use uuid::Uuid;
@@ -75,11 +77,78 @@ enum FailureHealthAction {
const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2;
const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20);
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
const BATCH_METADATA_RPC_OFF: &str = "off";
const BATCH_METADATA_RPC_AUTO: &str = "auto";
const BATCH_METADATA_RPC_ON: &str = "on";
const BATCH_READ_VERSION_GATE_ATTEMPT: &str = "attempt";
const BATCH_READ_VERSION_GATE_OFF_UNARY: &str = "off_unary";
const BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED: &str = "fallback_unimplemented";
const BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK: &str = "unsupported_no_fallback";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum BatchMetadataRpcMode {
Off,
Auto,
On,
}
impl BatchMetadataRpcMode {
fn as_str(self) -> &'static str {
match self {
Self::Off => BATCH_METADATA_RPC_OFF,
Self::Auto => BATCH_METADATA_RPC_AUTO,
Self::On => BATCH_METADATA_RPC_ON,
}
}
fn should_attempt(self) -> bool {
matches!(self, Self::Auto | Self::On)
}
fn should_fallback_on_unimplemented(self) -> bool {
matches!(self, Self::Auto)
}
}
fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
match raw.trim() {
value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_AUTO) => BatchMetadataRpcMode::Auto,
value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_ON) => BatchMetadataRpcMode::On,
value if value.eq_ignore_ascii_case(BATCH_METADATA_RPC_OFF) => BatchMetadataRpcMode::Off,
_ => BatchMetadataRpcMode::Off,
}
}
fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
.or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
.as_deref()
.map(parse_batch_metadata_rpc_mode)
.unwrap_or(BatchMetadataRpcMode::Off)
}
fn batch_metadata_rpc_mode() -> BatchMetadataRpcMode {
// The gate cannot change at runtime; parse it once instead of re-reading
// the environment on every batch RPC.
static MODE: std::sync::LazyLock<BatchMetadataRpcMode> = std::sync::LazyLock::new(batch_metadata_rpc_mode_from_env);
*MODE
}
fn record_batch_read_version_gate_decision(mode: BatchMetadataRpcMode, decision: &'static str) {
counter!(
"rustfs_remote_disk_batch_read_version_gate_total",
"mode" => mode.as_str(),
"decision" => decision
)
.increment(1);
}
async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
where
R: AsyncRead + Unpin,
@@ -769,6 +838,86 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str) -> Res
serde_json::from_str(json).map_err(Error::from)
}
fn decode_read_multiple_response_items(response: ReadMultipleResponse, endpoint: &Endpoint) -> Result<Vec<ReadMultipleResp>> {
if !response.read_multiple_resps_bin.is_empty() {
if !response.read_multiple_resps.is_empty()
&& response.read_multiple_resps.len() != response.read_multiple_resps_bin.len()
{
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
json_count = response.read_multiple_resps.len(),
msgpack_count = response.read_multiple_resps_bin.len(),
op = "read_multiple",
state = "response_count_mismatch",
"Remote disk ReadMultiple compatibility payload counts differ"
);
}
let mut read_multiple_resps = Vec::with_capacity(response.read_multiple_resps_bin.len());
for (index, buf) in response.read_multiple_resps_bin.iter().enumerate() {
let resp = decode_msgpack_or_json::<ReadMultipleResp>(buf, "").map_err(|err| {
Error::other(format!("decode ReadMultipleResp msgpack item {index} from {endpoint} failed: {err}"))
})?;
read_multiple_resps.push(resp);
}
return Ok(read_multiple_resps);
}
let mut read_multiple_resps = Vec::with_capacity(response.read_multiple_resps.len());
for (index, json_str) in response.read_multiple_resps.iter().enumerate() {
let resp = serde_json::from_str::<ReadMultipleResp>(json_str)
.map_err(|err| Error::other(format!("decode ReadMultipleResp json item {index} from {endpoint} failed: {err}")))?;
read_multiple_resps.push(resp);
}
Ok(read_multiple_resps)
}
fn decode_batch_read_version_response_items(
response: BatchReadVersionResponse,
endpoint: &Endpoint,
) -> Result<Vec<BatchReadVersionResp>> {
if !response.batch_read_version_resps_bin.is_empty() {
if !response.batch_read_version_resps.is_empty()
&& response.batch_read_version_resps.len() != response.batch_read_version_resps_bin.len()
{
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %endpoint,
json_count = response.batch_read_version_resps.len(),
msgpack_count = response.batch_read_version_resps_bin.len(),
op = "batch_read_version",
state = "response_count_mismatch",
"Remote disk BatchReadVersion compatibility payload counts differ"
);
}
let mut batch_read_version_resps = Vec::with_capacity(response.batch_read_version_resps_bin.len());
for (index, buf) in response.batch_read_version_resps_bin.iter().enumerate() {
let resp = decode_msgpack_or_json::<BatchReadVersionResp>(buf, "").map_err(|err| {
Error::other(format!("decode BatchReadVersionResp msgpack item {index} from {endpoint} failed: {err}"))
})?;
batch_read_version_resps.push(resp);
}
return Ok(batch_read_version_resps);
}
let mut batch_read_version_resps = Vec::with_capacity(response.batch_read_version_resps.len());
for (index, json_str) in response.batch_read_version_resps.iter().enumerate() {
let resp = serde_json::from_str::<BatchReadVersionResp>(json_str).map_err(|err| {
Error::other(format!("decode BatchReadVersionResp json item {index} from {endpoint} failed: {err}"))
})?;
batch_read_version_resps.push(resp);
}
Ok(batch_read_version_resps)
}
// TODO: all api need to handle errors
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
@@ -1462,6 +1611,99 @@ impl DiskAPI for RemoteDisk {
.await
}
#[tracing::instrument(skip(self, req))]
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
validate_batch_read_version_item_count(req.items.len())?;
let mode = batch_metadata_rpc_mode();
if !mode.should_attempt() {
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_OFF_UNARY);
return batch_read_version_one_by_one(self, req).await;
}
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_ATTEMPT);
debug!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
item_count = req.items.len(),
batch_metadata_rpc_mode = mode.as_str(),
op = "batch_read_version",
state = "started",
"Remote disk RPC started"
);
let batch_read_version_req = serde_json::to_string(&req)?;
let batch_read_version_req_bin = encode_msgpack(&req)?;
let batch_result = self
.execute_with_timeout_for_op(
"batch_read_version",
move || async move {
let disk = self.disk_ref().await;
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(BatchReadVersionRequest {
disk,
batch_read_version_req,
batch_read_version_req_bin: batch_read_version_req_bin.into(),
});
let response = match client.batch_read_version(request).await {
Ok(response) => response.into_inner(),
Err(status) if status.code() == Code::Unimplemented => {
if mode.should_fallback_on_unimplemented() {
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
batch_metadata_rpc_mode = mode.as_str(),
op = "batch_read_version",
state = "fallback_unimplemented",
"Remote disk BatchReadVersion unsupported; falling back to unary read_version"
);
return Ok(None);
}
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
endpoint = %self.endpoint,
batch_metadata_rpc_mode = mode.as_str(),
op = "batch_read_version",
state = "unsupported_no_fallback",
"Remote disk BatchReadVersion unsupported and explicit batch RPC mode forbids fallback"
);
return Err(Error::from(status));
}
Err(status) => return Err(Error::from(status)),
};
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
},
get_max_timeout_duration(),
)
.await?;
match batch_result {
Some(batch_read_version_resps) => Ok(batch_read_version_resps),
// Run the unary fallback outside the batch RPC deadline so each
// read_version keeps its own per-op timeout and health accounting
// instead of racing the whole batch against one drive timeout.
None => batch_read_version_one_by_one(self, req).await,
}
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
debug!(
@@ -2069,19 +2311,7 @@ impl DiskAPI for RemoteDisk {
return Err(response.error.unwrap_or_default().into());
}
let read_multiple_resps = if !response.read_multiple_resps_bin.is_empty() {
response
.read_multiple_resps_bin
.into_iter()
.filter_map(|buf| decode_msgpack_or_json::<ReadMultipleResp>(&buf, "").ok())
.collect()
} else {
response
.read_multiple_resps
.into_iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
.collect()
};
let read_multiple_resps = decode_read_multiple_response_items(response, &self.endpoint)?;
Ok(read_multiple_resps)
},
@@ -2413,6 +2643,196 @@ mod tests {
}
}
fn sample_read_multiple_resp(file: &str, data: &[u8]) -> ReadMultipleResp {
ReadMultipleResp {
bucket: "bucket".to_string(),
prefix: "prefix".to_string(),
file: file.to_string(),
exists: true,
data: data.to_vec(),
..Default::default()
}
}
fn sample_remote_endpoint() -> Endpoint {
Endpoint {
url: url::Url::parse("http://server:9000/disk-a").expect("endpoint URL should parse"),
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: 0,
}
}
#[test]
fn read_multiple_response_decode_prefers_msgpack_payloads() {
let endpoint = sample_remote_endpoint();
let msgpack_resp = sample_read_multiple_resp("msgpack", b"binary");
let json_resp = sample_read_multiple_resp("json", b"fallback");
let response = ReadMultipleResponse {
success: true,
read_multiple_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")],
read_multiple_resps_bin: vec![encode_msgpack(&msgpack_resp).expect("msgpack response should encode").into()],
error: None,
};
let decoded = decode_read_multiple_response_items(response, &endpoint).expect("msgpack response should decode");
assert_eq!(decoded.len(), 1);
assert_eq!(decoded[0].file, "msgpack");
assert_eq!(decoded[0].data, b"binary");
}
#[test]
fn read_multiple_response_decode_falls_back_to_json_payloads() {
let endpoint = sample_remote_endpoint();
let json_resp = sample_read_multiple_resp("json", b"fallback");
let response = ReadMultipleResponse {
success: true,
read_multiple_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")],
read_multiple_resps_bin: Vec::new(),
error: None,
};
let decoded = decode_read_multiple_response_items(response, &endpoint).expect("json response should decode");
assert_eq!(decoded.len(), 1);
assert_eq!(decoded[0].file, "json");
assert_eq!(decoded[0].data, b"fallback");
}
#[test]
fn read_multiple_response_decode_reports_corrupt_msgpack_item() {
let endpoint = sample_remote_endpoint();
let response = ReadMultipleResponse {
success: true,
read_multiple_resps: Vec::new(),
read_multiple_resps_bin: vec![
encode_msgpack(&sample_read_multiple_resp("ok", b"data"))
.expect("msgpack response should encode")
.into(),
bytes::Bytes::from_static(b"not-msgpack"),
],
error: None,
};
let err = decode_read_multiple_response_items(response, &endpoint).expect_err("corrupt msgpack item should fail");
let err = err.to_string();
assert!(err.contains("ReadMultipleResp msgpack item 1"), "unexpected error: {err}");
assert!(err.contains("server:9000"), "unexpected error: {err}");
}
fn sample_batch_read_version_resp(index: usize, path: &str, success: bool) -> BatchReadVersionResp {
BatchReadVersionResp {
index,
path: path.to_string(),
version_id: "version-a".to_string(),
success,
error: if success {
String::new()
} else {
"file version not found".to_string()
},
..Default::default()
}
}
#[test]
fn batch_read_version_response_decode_prefers_msgpack_payloads() {
let endpoint = sample_remote_endpoint();
let msgpack_resp = sample_batch_read_version_resp(7, "msgpack-object", true);
let json_resp = sample_batch_read_version_resp(1, "json-object", false);
let response = BatchReadVersionResponse {
success: true,
batch_read_version_resps: vec![serde_json::to_string(&json_resp).expect("json fallback should encode")],
batch_read_version_resps_bin: vec![encode_msgpack(&msgpack_resp).expect("msgpack response should encode").into()],
error: None,
};
let decoded = decode_batch_read_version_response_items(response, &endpoint).expect("msgpack response should decode");
assert_eq!(decoded.len(), 1);
assert_eq!(decoded[0].index, 7);
assert_eq!(decoded[0].path, "msgpack-object");
assert!(decoded[0].success);
}
#[test]
fn batch_read_version_response_decode_reports_corrupt_msgpack_item() {
let endpoint = sample_remote_endpoint();
let response = BatchReadVersionResponse {
success: true,
batch_read_version_resps: Vec::new(),
batch_read_version_resps_bin: vec![
encode_msgpack(&sample_batch_read_version_resp(0, "ok", true))
.expect("msgpack response should encode")
.into(),
bytes::Bytes::from_static(b"not-msgpack"),
],
error: None,
};
let err = decode_batch_read_version_response_items(response, &endpoint)
.expect_err("corrupt msgpack item should fail")
.to_string();
assert!(err.contains("BatchReadVersionResp msgpack item 1"), "unexpected error: {err}");
assert!(err.contains("server:9000"), "unexpected error: {err}");
}
#[test]
fn batch_metadata_rpc_mode_defaults_to_off_and_parses_supported_values() {
assert_eq!(parse_batch_metadata_rpc_mode(""), BatchMetadataRpcMode::Off);
assert_eq!(parse_batch_metadata_rpc_mode("off"), BatchMetadataRpcMode::Off);
assert_eq!(parse_batch_metadata_rpc_mode("auto"), BatchMetadataRpcMode::Auto);
assert_eq!(parse_batch_metadata_rpc_mode("on"), BatchMetadataRpcMode::On);
assert_eq!(parse_batch_metadata_rpc_mode("unknown"), BatchMetadataRpcMode::Off);
assert_eq!(BatchMetadataRpcMode::Off.as_str(), "off");
assert_eq!(BatchMetadataRpcMode::Auto.as_str(), "auto");
assert_eq!(BatchMetadataRpcMode::On.as_str(), "on");
assert!(!BatchMetadataRpcMode::Off.should_attempt());
assert!(BatchMetadataRpcMode::Auto.should_attempt());
assert!(BatchMetadataRpcMode::On.should_attempt());
assert_eq!(BATCH_READ_VERSION_GATE_ATTEMPT, "attempt");
assert_eq!(BATCH_READ_VERSION_GATE_OFF_UNARY, "off_unary");
assert_eq!(BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED, "fallback_unimplemented");
assert_eq!(BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK, "unsupported_no_fallback");
}
#[test]
fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
|| {
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::Auto);
},
);
}
#[test]
fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
|| {
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
},
);
}
#[test]
fn batch_metadata_rpc_mode_only_auto_falls_back_on_unimplemented() {
assert!(!BatchMetadataRpcMode::Off.should_fallback_on_unimplemented());
assert!(BatchMetadataRpcMode::Auto.should_fallback_on_unimplemented());
assert!(!BatchMetadataRpcMode::On.should_fallback_on_unimplemented());
}
#[test]
fn rename_data_file_info_named_msgpack_is_smaller_than_json() {
let file_info = sample_rename_data_file_info();
crates/ecstore/src/cluster/rpc/remote_locker.rs
+178 -68
View File
@@ -249,6 +249,22 @@ impl RemoteClient {
}
}
}
fn unknown_lock_info(lock_id: &LockId) -> LockInfo {
LockInfo {
id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive,
status: LockStatus::Acquired,
owner: "unknown".to_string(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}
}
}
#[async_trait]
@@ -338,11 +354,11 @@ impl LockClient for RemoteClient {
let request_string = serde_json::to_string(&unlock_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
let mut client = self.get_client().await?;
let resource_summary = unlock_request.resource.to_string();
let req = Request::new(GenerallyLockRequest { args: request_string });
let resp = client
.un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
let resp = self
.execute_rpc("release", &resource_summary, client.un_lock(req))
.await?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
@@ -351,21 +367,25 @@ impl LockClient for RemoteClient {
}
async fn release_locks_batch(&self, lock_ids: &[LockId]) -> Result<Vec<bool>> {
if lock_ids.is_empty() {
return Ok(Vec::new());
}
let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
let mut client = self.get_client().await?;
let resource_summary = Self::summarize_resources(&unlock_requests);
let req = Request::new(BatchGenerallyLockRequest {
args: lock_ids
args: unlock_requests
.iter()
.map(|lock_id| {
serde_json::to_string(&Self::create_unlock_request(lock_id))
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
.map(|request| {
serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
})
.collect::<Result<Vec<_>>>()?,
});
let resp = client
.un_lock_batch(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
let resp = self
.execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req))
.await?
.into_inner();
Ok(lock_ids
@@ -379,14 +399,14 @@ impl LockClient for RemoteClient {
info!("remote refresh for {}", lock_id);
let refresh_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
let resource_summary = refresh_request.resource.to_string();
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&refresh_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.refresh(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
let resp = self
.execute_rpc("refresh", &resource_summary, client.refresh(req))
.await?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
@@ -398,14 +418,14 @@ impl LockClient for RemoteClient {
info!("remote force_release for {}", lock_id);
let force_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
let resource_summary = force_request.resource.to_string();
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&force_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.force_un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
let resp = self
.execute_rpc("force_release", &resource_summary, client.force_un_lock(req))
.await?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
@@ -419,6 +439,7 @@ impl LockClient for RemoteClient {
// Since there's no direct status query in the gRPC service,
// we attempt a non-blocking lock acquisition to check if the resource is available
let status_request = Self::create_unlock_request(lock_id);
let resource_summary = status_request.resource.to_string();
let mut client = self.get_client().await?;
// Try to acquire a very short-lived lock to test availability
@@ -428,56 +449,27 @@ impl LockClient for RemoteClient {
});
// Try exclusive lock first with very short timeout
let resp = client.lock(req).await;
let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await {
Ok(response) => response.into_inner(),
Err(_) => return Ok(Some(Self::unknown_lock_info(lock_id))),
};
match resp {
Ok(response) => {
let resp = response.into_inner();
if resp.success {
// If we successfully acquired the lock, the resource was free
// Immediately release it
let release_req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&status_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let _ = client.un_lock(release_req).await; // Best effort release
if resp.success {
// If we successfully acquired the lock, the resource was free.
// Immediately release it on a best-effort basis.
let release_req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&status_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let _ = self
.execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req))
.await;
// Return None since no one was holding the lock
Ok(None)
} else {
// Lock acquisition failed, meaning someone is holding it
// We can't determine the exact details remotely, so return a generic status
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive, // We can't know the exact type
status: LockStatus::Acquired,
owner: "unknown".to_string(), // Remote client can't determine owner
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}))
}
}
Err(_) => {
// Communication error or lock is held
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive,
status: LockStatus::Acquired,
owner: "unknown".to_string(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}))
}
Ok(None)
} else {
// Lock acquisition failed, meaning someone is holding it.
// We can't determine the exact details remotely, so return a generic status.
Ok(Some(Self::unknown_lock_info(lock_id)))
}
}
@@ -558,6 +550,17 @@ mod tests {
Some((addr, task))
}
async fn closed_listener_addr() -> Option<String> {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
Err(err) => panic!("test listener should bind: {err}"),
};
let addr = format!("http://{}", listener.local_addr().expect("listener local address should be available"));
drop(listener);
Some(addr)
}
async fn cache_lazy_channel(addr: &str) {
let channel = TonicEndpoint::from_shared(addr.to_string()).unwrap().connect_lazy();
runtime_sources::cache_test_node_channel(addr.to_string(), channel).await;
@@ -664,6 +667,113 @@ mod tests {
accept_task.abort();
}
#[tokio::test]
#[serial_test::serial]
async fn test_remote_client_release_uses_rpc_timeout_and_evicts_connection() {
ensure_test_rpc_secret();
let Some((addr, accept_task)) = spawn_hanging_listener().await else {
return;
};
cache_lazy_channel(&addr).await;
assert!(runtime_sources::test_node_channel_is_cached(&addr).await);
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("50"))], async {
let client = RemoteClient::new(addr.clone());
let request = test_lock_request(Duration::from_millis(5));
let started_at = tokio::time::Instant::now();
let err = client.release(&request.lock_id).await.expect_err("release should time out");
let elapsed = started_at.elapsed();
assert!(
elapsed >= Duration::from_millis(40),
"remote release RPC should use configured transport timeout, got {elapsed:?}"
);
assert!(
elapsed < Duration::from_secs(1),
"test RPC timeout should keep the test fast, got {elapsed:?}"
);
assert!(matches!(err, LockError::Timeout { .. }), "expected remote release timeout, got {err:?}");
assert!(
!runtime_sources::test_node_channel_is_cached(&addr).await,
"release timeout should evict cached connection"
);
})
.await;
accept_task.abort();
}
#[tokio::test]
#[serial_test::serial]
async fn test_remote_client_refresh_tonic_error_evicts_connection() {
ensure_test_rpc_secret();
let Some(addr) = closed_listener_addr().await else {
return;
};
cache_lazy_channel(&addr).await;
assert!(runtime_sources::test_node_channel_is_cached(&addr).await);
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("500"))], async {
let client = RemoteClient::new(addr.clone());
let request = test_lock_request(Duration::from_millis(5));
let err = client
.refresh(&request.lock_id)
.await
.expect_err("refresh should report tonic failure");
assert!(
err.to_string().contains("refresh RPC failed"),
"expected refresh RPC failure marker, got {err}"
);
assert!(
!runtime_sources::test_node_channel_is_cached(&addr).await,
"refresh tonic error should evict cached connection"
);
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_remote_client_check_status_timeout_evicts_connection_and_preserves_status_shape() {
ensure_test_rpc_secret();
let Some((addr, accept_task)) = spawn_hanging_listener().await else {
return;
};
cache_lazy_channel(&addr).await;
assert!(runtime_sources::test_node_channel_is_cached(&addr).await);
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_RPC_TIMEOUT_MS, Some("50"))], async {
let client = RemoteClient::new(addr.clone());
let request = test_lock_request(Duration::from_millis(5));
let started_at = tokio::time::Instant::now();
let status = client.check_status(&request.lock_id).await.unwrap();
let elapsed = started_at.elapsed();
assert!(
elapsed >= Duration::from_millis(40),
"remote check_status RPC should use configured transport timeout, got {elapsed:?}"
);
assert!(
elapsed < Duration::from_secs(1),
"test RPC timeout should keep the test fast, got {elapsed:?}"
);
let info = status.expect("communication failure should preserve unknown lock status shape");
assert_eq!(info.id, request.lock_id);
assert_eq!(info.owner, "unknown");
assert!(
!runtime_sources::test_node_channel_is_cached(&addr).await,
"check_status timeout should evict cached connection"
);
})
.await;
accept_task.abort();
}
#[test]
#[serial_test::serial]
fn test_remote_client_rpc_timeout_honors_configured_deadline() {
crates/ecstore/src/disk/mod.rs
+73
View File
@@ -577,6 +577,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo>;
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
batch_read_version_one_by_one(self, req).await
}
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo>;
async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes>;
async fn rename_data(
@@ -637,6 +640,41 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
fn start_scan(&self) -> ScanGuard;
}
pub async fn batch_read_version_one_by_one<D>(disk: &D, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>>
where
D: DiskAPI + ?Sized,
{
validate_batch_read_version_item_count(req.items.len())?;
let mut responses = Vec::with_capacity(req.items.len());
for (index, item) in req.items.iter().enumerate() {
let response = match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: true,
file_info,
error: String::new(),
},
Err(err) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
},
};
responses.push(response);
}
Ok(responses)
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CheckPartsResp {
pub results: Vec<usize>,
@@ -807,6 +845,41 @@ pub struct ReadMultipleResp {
pub mod_time: Option<OffsetDateTime>,
}
pub const BATCH_READ_VERSION_MAX_ITEMS: usize = 128;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BatchReadVersionItem {
pub org_volume: String,
pub volume: String,
pub path: String,
pub version_id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BatchReadVersionReq {
pub items: Vec<BatchReadVersionItem>,
pub opts: ReadOptions,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct BatchReadVersionResp {
pub index: usize,
pub path: String,
pub version_id: String,
pub success: bool,
pub file_info: FileInfo,
pub error: String,
}
pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
if item_count > BATCH_READ_VERSION_MAX_ITEMS {
return Err(DiskError::other(format!(
"batch read version item count {item_count} exceeds limit {BATCH_READ_VERSION_MAX_ITEMS}"
)));
}
Ok(())
}
#[derive(Debug, Deserialize, Serialize)]
pub struct VolumeInfo {
pub name: String,
crates/ecstore/src/erasure/coding/decode.rs
+317 -115
View File
@@ -40,18 +40,54 @@ use tracing::{error, warn};
type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, bool)> + Send + 'a>>;
const ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING: &str = "RUSTFS_SHARD_LOCALITY_SCHEDULING";
const ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: &str = "RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE";
const DEFAULT_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: bool = false;
const SHARD_LOCALITY_SCHEDULING_OFF: &str = "off";
const SHARD_LOCALITY_SCHEDULING_OBSERVE: &str = "observe";
const SHARD_LOCALITY_SCHEDULING_ON: &str = "on";
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ShardLocalitySchedulingMode {
Off,
Observe,
On,
}
impl ShardLocalitySchedulingMode {
fn is_on(self) -> bool {
matches!(self, ShardLocalitySchedulingMode::On)
}
}
fn parse_shard_locality_scheduling_mode(value: &str) -> ShardLocalitySchedulingMode {
match value.trim() {
value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_OFF) => ShardLocalitySchedulingMode::Off,
value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_OBSERVE) => ShardLocalitySchedulingMode::Observe,
value if value.eq_ignore_ascii_case(SHARD_LOCALITY_SCHEDULING_ON) => ShardLocalitySchedulingMode::On,
_ => ShardLocalitySchedulingMode::Off,
}
}
fn get_shard_locality_scheduling_mode() -> ShardLocalitySchedulingMode {
if let Some(value) = rustfs_utils::get_env_opt_str(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING) {
return parse_shard_locality_scheduling_mode(&value);
}
if rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, false) {
return ShardLocalitySchedulingMode::On;
}
ShardLocalitySchedulingMode::Off
}
fn get_shard_locality_preference_enabled() -> bool {
rustfs_utils::get_env_bool(
ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE,
DEFAULT_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE,
)
get_shard_locality_scheduling_mode().is_on()
}
pub(crate) fn should_collect_shard_read_costs() -> bool {
rustfs_io_metrics::get_stage_metrics_enabled() || get_shard_locality_preference_enabled()
// `observe` mode only feeds the stage-metrics histograms, so skip the
// cost-collection overhead when those metrics cannot be reported anyway.
rustfs_io_metrics::get_stage_metrics_enabled() || get_shard_locality_scheduling_mode().is_on()
}
/// Number of stripes to prefetch in the legacy decode path.
@@ -108,9 +144,10 @@ impl ShardReadCostCounts {
}
fn shard_read_launch_rank(cost: ShardReadCost) -> u8 {
match cost {
ShardReadCost::Local | ShardReadCost::SameNode => 0,
ShardReadCost::Unknown => 1,
ShardReadCost::Local => 0,
ShardReadCost::SameNode => 1,
ShardReadCost::Remote => 2,
ShardReadCost::Unknown => 3,
}
}
@@ -533,6 +570,11 @@ fn shard_read_hedge_delay(read_timeout: Duration) -> Option<Duration> {
}
}
fn shard_locality_remote_avoid_potential(remote_scheduled: usize, low_cost_available: usize, data_shards: usize) -> usize {
let theoretical_remote_needed = data_shards.saturating_sub(low_cost_available);
remote_scheduled.saturating_sub(theoretical_remote_needed)
}
#[allow(clippy::too_many_arguments)]
fn record_scheduled_read_cost(
read_cost: ShardReadCost,
@@ -544,15 +586,13 @@ fn record_scheduled_read_cost(
remote_scheduled: &mut usize,
fallback_to_remote: &mut usize,
) {
if !locality_preference_enabled {
return;
}
if read_cost.is_low_cost() {
*local_preferred += 1;
if locality_preference_enabled {
*local_preferred += 1;
}
} else if read_cost.is_remote() {
*remote_scheduled += 1;
if count_remote_as_fallback || low_cost_available < data_shards {
if locality_preference_enabled && (count_remote_as_fallback || low_cost_available < data_shards) {
*fallback_to_remote += 1;
}
}
@@ -889,6 +929,13 @@ where
fallback_to_remote,
);
} else {
let remote_avoid_potential =
shard_locality_remote_avoid_potential(remote_scheduled, low_cost_available, self.data_shards);
rustfs_io_metrics::record_get_object_shard_locality_observe_only(
path,
remote_scheduled,
remote_avoid_potential,
);
rustfs_io_metrics::record_get_object_shard_locality_policy_disabled(path);
}
}
@@ -1598,14 +1645,138 @@ mod tests {
}
#[test]
#[serial_test::serial]
fn test_shard_locality_preference_gate_defaults_disabled() {
temp_env::with_var(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>, || {
assert!(!get_shard_locality_preference_enabled());
});
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Off);
assert!(!get_shard_locality_preference_enabled());
},
);
}
#[test]
fn test_shard_read_launch_order_is_gated() {
#[serial_test::serial]
fn test_shard_locality_scheduling_mode_parses_supported_values() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("observe")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Observe);
assert!(!get_shard_locality_preference_enabled());
},
);
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::On);
assert!(get_shard_locality_preference_enabled());
assert!(should_collect_shard_read_costs());
},
);
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("unexpected")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::Off);
assert!(!get_shard_locality_preference_enabled());
},
);
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
],
|| {
assert_eq!(get_shard_locality_scheduling_mode(), ShardLocalitySchedulingMode::On);
assert!(get_shard_locality_preference_enabled());
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_scheduling_off_does_not_collect_without_metrics() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("off")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
assert!(!should_collect_shard_read_costs());
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_scheduling_observe_collects_only_with_stage_metrics() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("observe")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
// Without stage metrics there is no reporting channel, so
// observe mode must not pay the cost-collection overhead.
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
assert!(!should_collect_shard_read_costs());
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
assert!(should_collect_shard_read_costs());
assert!(!get_shard_locality_preference_enabled());
let read_costs = [ShardReadCost::Remote, ShardReadCost::Local, ShardReadCost::SameNode];
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), false), vec![0, 1, 2]);
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_scheduling_on_enables_reordering() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
|| {
assert!(get_shard_locality_preference_enabled());
},
);
}
#[test]
#[serial_test::serial]
fn test_shard_locality_legacy_preference_gate_still_enables_on() {
temp_env::with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, None::<&str>),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true")),
],
|| {
assert!(get_shard_locality_preference_enabled());
},
);
}
#[test]
fn test_shard_locality_read_launch_order_is_gated() {
let read_costs = [
ShardReadCost::Remote,
ShardReadCost::Local,
@@ -1615,7 +1786,14 @@ mod tests {
];
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), false), vec![0, 1, 2, 3, 4]);
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 2, 0, 4]);
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 0, 4, 2]);
}
#[test]
fn test_shard_locality_remote_avoid_potential_is_observe_only() {
assert_eq!(shard_locality_remote_avoid_potential(2, 4, 4), 2);
assert_eq!(shard_locality_remote_avoid_potential(2, 2, 4), 0);
assert_eq!(shard_locality_remote_avoid_potential(3, 3, 4), 2);
}
#[test]
@@ -1639,90 +1817,107 @@ mod tests {
#[tokio::test]
#[serial_test::serial]
async fn test_parallel_reader_local_first_avoids_remote_when_local_quorum_exists() {
temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[]).await;
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::Local,
ShardReadCost::SameNode,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[]).await;
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::Local,
ShardReadCost::SameNode,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let (bufs, errs) = parallel_reader.read().await;
let (bufs, errs) = parallel_reader.read().await;
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(bufs[0].is_none());
assert!(bufs[1].is_none());
for (index, buf) in bufs.iter().enumerate().take(DATA_SHARDS + PARITY_SHARDS).skip(2) {
assert_eq!(buf.as_deref(), Some(&[(index % 256) as u8; SHARD_SIZE][..]));
}
assert!(errs.iter().all(Option::is_none));
})
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(bufs[0].is_none());
assert!(bufs[1].is_none());
for (index, buf) in bufs.iter().enumerate().take(DATA_SHARDS + PARITY_SHARDS).skip(2) {
assert_eq!(buf.as_deref(), Some(&[(index % 256) as u8; SHARD_SIZE][..]));
}
assert!(errs.iter().all(Option::is_none));
},
)
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_parallel_reader_local_missing_falls_back_to_remote() {
temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[1], &[]).await;
let read_costs = vec![
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Remote,
ShardReadCost::Remote,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let hash_algo = HashAlgorithm::HighwayHash256;
let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[1], &[]).await;
let read_costs = vec![
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::Remote,
ShardReadCost::Remote,
];
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure,
0,
NUM_SHARDS * BLOCK_SIZE,
None,
read_costs,
);
let (bufs, errs) = parallel_reader.read().await;
let (bufs, errs) = parallel_reader.read().await;
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(matches!(errs[1], Some(Error::FileNotFound)));
assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
assert!(bufs[5].is_none());
})
assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
assert!(matches!(errs[1], Some(Error::FileNotFound)));
assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
assert!(bufs[5].is_none());
},
)
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_parallel_reader_local_corrupt_falls_back_to_remote() {
temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const NUM_SHARDS: usize = 1;
const BLOCK_SIZE: usize = 64;
const DATA_SHARDS: usize = 4;
@@ -1757,45 +1952,52 @@ mod tests {
);
assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
assert!(bufs[5].is_none());
})
},
)
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn test_erasure_decode_local_first_preserves_output_order() {
temp_env::async_with_vars([(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, Some("true"))], async {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
temp_env::async_with_vars(
[
(ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING, Some("on")),
(ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
],
async {
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 64;
let total_data: Vec<u8> = (0..BLOCK_SIZE as u32).map(|i| i as u8).collect();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let shard_bufs = encode_test_object(&erasure, &total_data, shard_size, &hash_algo).await;
let readers = shard_bufs
.iter()
.map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
.collect();
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::SameNode,
];
let total_data: Vec<u8> = (0..BLOCK_SIZE as u32).map(|i| i as u8).collect();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let shard_bufs = encode_test_object(&erasure, &total_data, shard_size, &hash_algo).await;
let readers = shard_bufs
.iter()
.map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
.collect();
let read_costs = vec![
ShardReadCost::Remote,
ShardReadCost::Remote,
ShardReadCost::Local,
ShardReadCost::Local,
ShardReadCost::SameNode,
ShardReadCost::SameNode,
];
let mut output = Vec::new();
let (written, err) = erasure
.decode_with_read_costs(&mut output, readers, 0, total_data.len(), total_data.len(), read_costs)
.await;
let mut output = Vec::new();
let (written, err) = erasure
.decode_with_read_costs(&mut output, readers, 0, total_data.len(), total_data.len(), read_costs)
.await;
assert!(err.is_none(), "unexpected decode error: {err:?}");
assert_eq!(written, total_data.len());
assert_eq!(output, total_data);
})
assert!(err.is_none(), "unexpected decode error: {err:?}");
assert_eq!(written, total_data.len());
assert_eq!(output, total_data);
},
)
.await;
}
crates/ecstore/src/services/batch_processor.rs
+545 -16
View File
@@ -19,17 +19,255 @@
use crate::disk::error::{Error, Result};
use std::future::Future;
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tokio::task::JoinSet;
const BATCH_PROCESSOR_OPERATION_CUSTOM: &str = "custom";
const BATCH_PROCESSOR_OPERATION_READ: &str = "read";
const BATCH_PROCESSOR_OPERATION_WRITE: &str = "write";
const BATCH_PROCESSOR_OPERATION_METADATA: &str = "metadata";
const ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE: &str = "RUSTFS_BATCH_PROCESSOR_ADAPTIVE";
const BATCH_PROCESSOR_ADAPTIVE_OFF: &str = "off";
const BATCH_PROCESSOR_ADAPTIVE_OBSERVE: &str = "observe";
const BATCH_PROCESSOR_ADAPTIVE_ON: &str = "on";
const BATCH_SUGGESTION_REASON_IMPROVING: &str = "improving";
const BATCH_SUGGESTION_REASON_DEGRADING: &str = "degrading";
const BATCH_SUGGESTION_REASON_STABLE: &str = "stable";
const BATCH_SUGGESTION_REASON_COOLDOWN: &str = "cooldown";
const BATCH_OBSERVATION_IMPROVING_LATENCY: Duration = Duration::from_millis(50);
const BATCH_OBSERVATION_DEGRADING_LATENCY: Duration = Duration::from_millis(250);
const BATCH_OBSERVATION_COOLDOWN: Duration = Duration::from_secs(30);
const BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR: usize = 4;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BatchProcessorAdaptiveMode {
Off,
Observe,
On,
}
impl BatchProcessorAdaptiveMode {
fn should_observe(self) -> bool {
matches!(self, Self::Observe | Self::On)
}
fn should_apply(self) -> bool {
matches!(self, Self::On)
}
}
fn parse_batch_processor_adaptive_mode(raw: &str) -> BatchProcessorAdaptiveMode {
match raw.trim() {
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OBSERVE) => BatchProcessorAdaptiveMode::Observe,
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_ON) => BatchProcessorAdaptiveMode::On,
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OFF) => BatchProcessorAdaptiveMode::Off,
_ => BatchProcessorAdaptiveMode::Off,
}
}
fn batch_processor_adaptive_mode_from_env() -> BatchProcessorAdaptiveMode {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE)
.as_deref()
.map(parse_batch_processor_adaptive_mode)
.unwrap_or(BatchProcessorAdaptiveMode::Off)
}
fn batch_processor_adaptive_mode() -> BatchProcessorAdaptiveMode {
// The gate cannot change at runtime; parse it once instead of re-reading
// the environment on every batch.
static MODE: std::sync::LazyLock<BatchProcessorAdaptiveMode> =
std::sync::LazyLock::new(batch_processor_adaptive_mode_from_env);
*MODE
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct BatchConcurrencySuggestion {
concurrency: usize,
reason: &'static str,
}
#[derive(Debug, Clone, Copy)]
struct BatchObservation {
batch_size: usize,
success_count: usize,
error_count: usize,
timeout_count: usize,
max_queue_wait: Duration,
execution_latency: Duration,
}
#[derive(Debug)]
struct BatchObservationState {
last_suggestion_at: Option<Instant>,
last_suggested_concurrency: usize,
}
impl BatchObservationState {
fn new(configured_concurrency: usize) -> Self {
Self {
last_suggestion_at: None,
last_suggested_concurrency: configured_concurrency,
}
}
fn suggest(
&mut self,
current_concurrency: usize,
configured_concurrency: usize,
observation: BatchObservation,
now: Instant,
) -> BatchConcurrencySuggestion {
let raw = calculate_batch_concurrency_suggestion(current_concurrency, configured_concurrency, observation);
if raw.concurrency == current_concurrency {
self.last_suggested_concurrency = current_concurrency;
return raw;
}
if self
.last_suggestion_at
.is_some_and(|last_suggestion_at| now.duration_since(last_suggestion_at) < BATCH_OBSERVATION_COOLDOWN)
{
return BatchConcurrencySuggestion {
concurrency: self.last_suggested_concurrency,
reason: BATCH_SUGGESTION_REASON_COOLDOWN,
};
}
self.last_suggestion_at = Some(now);
self.last_suggested_concurrency = raw.concurrency;
raw
}
}
fn calculate_batch_concurrency_suggestion(
current_concurrency: usize,
configured_concurrency: usize,
observation: BatchObservation,
) -> BatchConcurrencySuggestion {
let current_concurrency = current_concurrency.max(1);
let configured_concurrency = configured_concurrency.max(1);
if observation.batch_size == 0 {
return BatchConcurrencySuggestion {
concurrency: current_concurrency,
reason: BATCH_SUGGESTION_REASON_STABLE,
};
}
if observation.timeout_count > 0
|| observation.error_count > 0
|| (observation.batch_size >= current_concurrency && observation.execution_latency >= BATCH_OBSERVATION_DEGRADING_LATENCY)
{
return BatchConcurrencySuggestion {
concurrency: decrease_batch_concurrency(current_concurrency),
reason: BATCH_SUGGESTION_REASON_DEGRADING,
};
}
if observation.success_count == observation.batch_size
&& observation.batch_size >= current_concurrency
&& observation.execution_latency <= BATCH_OBSERVATION_IMPROVING_LATENCY
{
return BatchConcurrencySuggestion {
concurrency: increase_batch_concurrency(current_concurrency, configured_concurrency, observation.batch_size),
reason: BATCH_SUGGESTION_REASON_IMPROVING,
};
}
BatchConcurrencySuggestion {
concurrency: current_concurrency,
reason: BATCH_SUGGESTION_REASON_STABLE,
}
}
fn decrease_batch_concurrency(current_concurrency: usize) -> usize {
(current_concurrency.saturating_mul(3) / 4).max(1)
}
fn increase_batch_concurrency(current_concurrency: usize, configured_concurrency: usize, batch_size: usize) -> usize {
let step = (current_concurrency / 4).max(1);
// Suggestions build on the previous suggestion, so bound the ratchet at a
// hard multiple of the configured baseline instead of the current value.
let upper_bound = configured_concurrency
.saturating_mul(BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR)
.max(current_concurrency);
current_concurrency
.saturating_add(step)
.min(upper_bound)
.min(batch_size.max(current_concurrency))
}
fn is_timeout_error(err: &Error) -> bool {
matches!(err, Error::Timeout | Error::SourceStalled)
|| matches!(err, Error::Io(io_err) if io_err.kind() == std::io::ErrorKind::TimedOut)
}
/// Batch processor that executes tasks concurrently with a semaphore
pub struct AsyncBatchProcessor {
max_concurrent: usize,
operation: &'static str,
observation_state: Mutex<BatchObservationState>,
}
impl AsyncBatchProcessor {
pub fn new(max_concurrent: usize) -> Self {
Self { max_concurrent }
Self::new_with_operation(max_concurrent, BATCH_PROCESSOR_OPERATION_CUSTOM)
}
fn new_with_operation(max_concurrent: usize, operation: &'static str) -> Self {
let max_concurrent = max_concurrent.max(1);
Self {
max_concurrent,
operation,
observation_state: Mutex::new(BatchObservationState::new(max_concurrent)),
}
}
fn execution_concurrency(&self, mode: BatchProcessorAdaptiveMode) -> usize {
if !mode.should_apply() {
return self.max_concurrent;
}
self.observation_state
.lock()
.map(|state| state.last_suggested_concurrency.max(1))
.unwrap_or(self.max_concurrent)
}
fn observe_batch_with_mode(
&self,
mode: BatchProcessorAdaptiveMode,
execution_concurrency: usize,
observation: BatchObservation,
) -> BatchConcurrencySuggestion {
let execution_concurrency = execution_concurrency.max(1);
if !mode.should_observe() {
return BatchConcurrencySuggestion {
concurrency: execution_concurrency,
reason: BATCH_SUGGESTION_REASON_STABLE,
};
}
let suggestion = match self.observation_state.lock() {
Ok(mut state) => state.suggest(execution_concurrency, self.max_concurrent, observation, Instant::now()),
Err(_) => calculate_batch_concurrency_suggestion(execution_concurrency, self.max_concurrent, observation),
};
rustfs_io_metrics::record_batch_processor_observation(rustfs_io_metrics::BatchProcessorObservation {
operation: self.operation,
batch_size: observation.batch_size,
configured_concurrency: execution_concurrency,
max_queue_wait_secs: observation.max_queue_wait.as_secs_f64(),
execution_latency_secs: observation.execution_latency.as_secs_f64(),
successes: observation.success_count,
errors: observation.error_count,
timeouts: observation.timeout_count,
suggested_concurrency: suggestion.concurrency,
suggestion_reason: suggestion.reason,
});
suggestion
}
/// Execute a batch of tasks concurrently with concurrency control
@@ -42,27 +280,35 @@ impl AsyncBatchProcessor {
return Vec::new();
}
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
let batch_size = tasks.len();
let batch_started_at = Instant::now();
let adaptive_mode = batch_processor_adaptive_mode();
let execution_concurrency = self.execution_concurrency(adaptive_mode);
let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency));
let mut join_set = JoinSet::new();
let mut results = Vec::with_capacity(tasks.len());
for _ in 0..tasks.len() {
let mut results = Vec::with_capacity(batch_size);
for _ in 0..batch_size {
results.push(Err(Error::other("Not completed")));
}
// Spawn all tasks with semaphore control
for (i, task) in tasks.into_iter().enumerate() {
let sem = semaphore.clone();
let queued_at = Instant::now();
join_set.spawn(async move {
let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
let queue_wait = queued_at.elapsed();
let result = task.await;
Ok::<(usize, Result<T>), Error>((i, result))
Ok::<(usize, Result<T>, Duration), Error>((i, result, queue_wait))
});
}
// Collect results
let mut max_queue_wait = Duration::ZERO;
while let Some(join_result) = join_set.join_next().await {
match join_result {
Ok(Ok((index, task_result))) => {
Ok(Ok((index, task_result, queue_wait))) => {
max_queue_wait = max_queue_wait.max(queue_wait);
if index < results.len() {
results[index] = task_result;
}
@@ -78,6 +324,29 @@ impl AsyncBatchProcessor {
}
}
let mut success_count = 0;
let mut error_count = 0;
let mut timeout_count = 0;
for result in &results {
match result {
Ok(_) => success_count += 1,
Err(err) if is_timeout_error(err) => timeout_count += 1,
Err(_) => error_count += 1,
}
}
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count,
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
results
}
@@ -97,17 +366,29 @@ impl AsyncBatchProcessor {
)));
}
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
let batch_size = tasks.len();
let batch_started_at = Instant::now();
let adaptive_mode = batch_processor_adaptive_mode();
let execution_concurrency = self.execution_concurrency(adaptive_mode);
let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency));
let mut join_set = JoinSet::new();
let mut successes = Vec::new();
let mut pending_tasks = tasks.len();
let mut pending_tasks = batch_size;
let mut first_error = None;
let mut error_count = 0;
let mut timeout_count = 0;
let mut max_queue_wait = Duration::ZERO;
for task in tasks {
let sem = semaphore.clone();
let queued_at = Instant::now();
join_set.spawn(async move {
let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
task.await
let _permit = sem
.acquire()
.await
.map_err(|_| (Error::other("Semaphore error"), queued_at.elapsed()))?;
let queue_wait = queued_at.elapsed();
task.await.map(|value| (value, queue_wait)).map_err(|err| (err, queue_wait))
});
}
@@ -115,18 +396,38 @@ impl AsyncBatchProcessor {
pending_tasks = pending_tasks.saturating_sub(1);
match join_result {
Ok(Ok(value)) => {
Ok(Ok((value, queue_wait))) => {
max_queue_wait = max_queue_wait.max(queue_wait);
successes.push(value);
if successes.len() >= required_successes {
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count: successes.len(),
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
return Ok(successes);
}
}
Ok(Err(err)) => {
Ok(Err((err, queue_wait))) => {
max_queue_wait = max_queue_wait.max(queue_wait);
if is_timeout_error(&err) {
timeout_count += 1;
} else {
error_count += 1;
}
if first_error.is_none() {
first_error = Some(err);
}
}
Err(join_error) => {
error_count += 1;
if first_error.is_none() {
first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}")));
}
@@ -134,6 +435,18 @@ impl AsyncBatchProcessor {
}
if successes.len() + pending_tasks < required_successes {
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count: successes.len(),
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
return Err(first_error.unwrap_or_else(|| {
Error::other(format!(
"Insufficient successful results: got {}, needed {}",
@@ -144,6 +457,18 @@ impl AsyncBatchProcessor {
}
}
self.observe_batch_with_mode(
adaptive_mode,
execution_concurrency,
BatchObservation {
batch_size,
success_count: successes.len(),
error_count,
timeout_count,
max_queue_wait,
execution_latency: batch_started_at.elapsed(),
},
);
Err(first_error.unwrap_or_else(|| {
Error::other(format!(
"Insufficient successful results: got {}, needed {}",
@@ -164,9 +489,9 @@ pub struct GlobalBatchProcessors {
impl GlobalBatchProcessors {
pub fn new() -> Self {
Self {
read_processor: AsyncBatchProcessor::new(16), // Higher concurrency for reads
write_processor: AsyncBatchProcessor::new(8), // Lower concurrency for writes
metadata_processor: AsyncBatchProcessor::new(12), // Medium concurrency for metadata
read_processor: AsyncBatchProcessor::new_with_operation(16, BATCH_PROCESSOR_OPERATION_READ),
write_processor: AsyncBatchProcessor::new_with_operation(8, BATCH_PROCESSOR_OPERATION_WRITE),
metadata_processor: AsyncBatchProcessor::new_with_operation(12, BATCH_PROCESSOR_OPERATION_METADATA),
}
}
@@ -321,4 +646,208 @@ mod tests {
assert!(err.to_string().contains("first failure"));
assert!(started.elapsed() < Duration::from_millis(120));
}
#[tokio::test]
async fn test_batch_processor_observe_only_keeps_configured_concurrency() {
let processor = AsyncBatchProcessor::new(2);
let tasks: Vec<_> = (0..4)
.map(|i| async move {
tokio::time::sleep(Duration::from_millis(5)).await;
Ok::<i32, Error>(i)
})
.collect();
let results = processor.execute_batch(tasks).await;
assert_eq!(results.len(), 4);
assert_eq!(processor.max_concurrent, 2);
}
#[test]
fn batch_processor_adaptive_mode_defaults_to_off_and_parses_supported_values() {
assert_eq!(parse_batch_processor_adaptive_mode(""), BatchProcessorAdaptiveMode::Off);
assert_eq!(parse_batch_processor_adaptive_mode("off"), BatchProcessorAdaptiveMode::Off);
assert_eq!(parse_batch_processor_adaptive_mode("observe"), BatchProcessorAdaptiveMode::Observe);
assert_eq!(parse_batch_processor_adaptive_mode("on"), BatchProcessorAdaptiveMode::On);
assert_eq!(parse_batch_processor_adaptive_mode("unknown"), BatchProcessorAdaptiveMode::Off);
assert!(!BatchProcessorAdaptiveMode::Off.should_observe());
assert!(BatchProcessorAdaptiveMode::Observe.should_observe());
assert!(BatchProcessorAdaptiveMode::On.should_observe());
assert!(!BatchProcessorAdaptiveMode::Observe.should_apply());
assert!(BatchProcessorAdaptiveMode::On.should_apply());
}
#[test]
fn batch_processor_adaptive_mode_env_gate_is_parsed_from_env() {
temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, None::<&str>, || {
assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Off);
});
temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, Some("observe"), || {
assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Observe);
});
}
#[test]
fn batch_processor_observation_gate_keeps_default_off_stable() {
let processor = AsyncBatchProcessor::new(8);
let suggestion = processor.observe_batch_with_mode(
BatchProcessorAdaptiveMode::Off,
processor.execution_concurrency(BatchProcessorAdaptiveMode::Off),
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_STABLE);
assert_eq!(suggestion.concurrency, 8);
}
#[test]
fn batch_processor_observation_gate_observe_records_suggestion() {
let processor = AsyncBatchProcessor::new(8);
let suggestion = processor.observe_batch_with_mode(
BatchProcessorAdaptiveMode::Observe,
processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe),
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert!(suggestion.concurrency > 8);
assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe), 8);
}
#[test]
fn batch_processor_adaptive_on_applies_last_suggestion_to_next_batch() {
let processor = AsyncBatchProcessor::new(8);
let suggestion = processor.observe_batch_with_mode(
BatchProcessorAdaptiveMode::On,
processor.execution_concurrency(BatchProcessorAdaptiveMode::On),
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert!(suggestion.concurrency > 8);
assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::On), suggestion.concurrency);
}
#[test]
fn batch_processor_suggestion_increases_for_fast_successful_batches() {
let suggestion = calculate_batch_concurrency_suggestion(
8,
8,
BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert!(suggestion.concurrency > 8);
}
#[test]
fn batch_processor_suggestion_decreases_for_timeout_batches() {
let suggestion = calculate_batch_concurrency_suggestion(
8,
8,
BatchObservation {
batch_size: 16,
success_count: 14,
error_count: 0,
timeout_count: 2,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_DEGRADING);
assert!(suggestion.concurrency < 8);
assert!(suggestion.concurrency >= 1);
}
#[test]
fn batch_processor_suggestion_clamps_to_batch_size() {
let suggestion = calculate_batch_concurrency_suggestion(
64,
64,
BatchObservation {
batch_size: 65,
success_count: 65,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
},
);
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert_eq!(suggestion.concurrency, 65);
}
#[test]
fn batch_processor_suggestion_cooldown_prevents_rapid_oscillation() {
let mut state = BatchObservationState::new(8);
let now = Instant::now();
let observation = BatchObservation {
batch_size: 16,
success_count: 16,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
};
let first = state.suggest(8, 8, observation, now);
let second = state.suggest(8, 8, observation, now + Duration::from_secs(1));
assert_eq!(first.reason, BATCH_SUGGESTION_REASON_IMPROVING);
assert_eq!(second.reason, BATCH_SUGGESTION_REASON_COOLDOWN);
assert_eq!(second.concurrency, first.concurrency);
}
#[test]
fn batch_processor_suggestion_growth_is_capped_relative_to_configured_concurrency() {
const CONFIGURED: usize = 8;
let mut state = BatchObservationState::new(CONFIGURED);
let mut current = CONFIGURED;
let mut now = Instant::now();
let observation = BatchObservation {
batch_size: 1024,
success_count: 1024,
error_count: 0,
timeout_count: 0,
max_queue_wait: Duration::ZERO,
execution_latency: Duration::from_millis(20),
};
for _ in 0..64 {
current = state.suggest(current, CONFIGURED, observation, now).concurrency;
now += BATCH_OBSERVATION_COOLDOWN;
}
assert_eq!(current, CONFIGURED * BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR);
}
}
crates/ecstore/src/set_disk/mod.rs
+34 -8
View File
@@ -270,10 +270,19 @@ impl AsyncRead for SetDiskLockGuardedReader {
}
}
fn attach_set_disk_read_lock_guard(mut reader: GetObjectReader, read_lock_guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
if let Some(guard) = read_lock_guard
&& reader.buffered_body.is_none()
{
fn finish_set_disk_read_lock(
mut reader: GetObjectReader,
read_lock_guard: Option<ObjectLockDiagGuard>,
lock_optimization_enabled: bool,
bucket: &str,
object: &str,
) -> GetObjectReader {
if lock_optimization_enabled || reader.buffered_body.is_some() {
release_materialized_read_lock(bucket, object, read_lock_guard);
return reader;
}
if let Some(guard) = read_lock_guard {
reader.stream = Box::new(SetDiskLockGuardedReader {
inner: reader.stream,
guard: Some(guard),
@@ -2314,7 +2323,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
opts.part_number = Some(1);
}
let gr = get_transitioned_object_reader(bucket, object, &range, &h, &object_info, &opts).await?;
return Ok(attach_set_disk_read_lock_guard(gr, read_lock_guard.take()));
return Ok(finish_set_disk_read_lock(
gr,
read_lock_guard.take(),
lock_optimization_enabled,
bucket,
object,
));
}
if is_get_small_object_direct_memory_eligible(&range, &object_info, &fi, opts) {
@@ -2418,7 +2433,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
);
record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket);
let (reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?;
return Ok(attach_set_disk_read_lock_guard(reader, read_lock_guard.take()));
return Ok(finish_set_disk_read_lock(
reader,
read_lock_guard.take(),
lock_optimization_enabled,
bucket,
object,
));
}
read::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
record_get_codec_streaming_gate_decision(
@@ -2454,8 +2475,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
let set_index = self.set_index;
let pool_index = self.pool_index;
let skip_verify = opts.skip_verify_bitrot;
// Move the read-lock guard into the task so it lives for the duration of the read.
// Fully materialized paths release it before returning; streaming paths keep it.
if lock_optimization_enabled {
release_materialized_read_lock(&bucket, &object, read_lock_guard.take());
debug!(bucket, object, "Lock optimization: released read lock before streaming read");
}
// When lock optimization is disabled, keep the read-lock guard in the
// task so it lives for the duration of the streaming read.
tokio::spawn(async move {
let _guard = read_lock_guard;
let mut writer = wd;
crates/ecstore/src/set_disk/read.rs
+65 -14
View File
@@ -595,14 +595,64 @@ fn resolve_read_part_from_responses(
Err(DiskError::ErasureReadQuorum)
}
fn shard_read_cost_for_disk(disk: Option<&DiskStore>) -> ShardReadCost {
fn shard_read_costs_for_disks(disks: &[Option<DiskStore>]) -> Vec<ShardReadCost> {
let local_endpoint_hosts = local_endpoint_hosts_for_shard_costs();
disks
.iter()
.map(|disk| shard_read_cost_for_disk(disk.as_ref(), local_endpoint_hosts))
.collect()
}
fn shard_read_cost_for_disk(disk: Option<&DiskStore>, local_endpoint_hosts: &[String]) -> ShardReadCost {
match disk {
Some(disk) if disk.is_local() => ShardReadCost::Local,
Some(_) => ShardReadCost::Remote,
Some(disk) => shard_read_cost_for_endpoint(false, &disk.host_name(), local_endpoint_hosts),
None => ShardReadCost::Unknown,
}
}
fn shard_read_cost_for_endpoint(is_local: bool, host_name: &str, local_endpoint_hosts: &[String]) -> ShardReadCost {
if is_local {
return ShardReadCost::Local;
}
if !host_name.is_empty() && local_endpoint_hosts.iter().any(|host| host == host_name) {
return ShardReadCost::SameNode;
}
ShardReadCost::Remote
}
fn local_endpoint_hosts_for_shard_costs() -> &'static [String] {
// Endpoint pools are immutable after startup, so build the host list once
// instead of walking every pool on each read. Do not cache the empty
// pre-startup answer: only memoize once the pools are published.
static LOCAL_ENDPOINT_HOSTS: std::sync::OnceLock<Vec<String>> = std::sync::OnceLock::new();
if let Some(hosts) = LOCAL_ENDPOINT_HOSTS.get() {
return hosts;
}
let Some(endpoint_pools) = runtime_sources::endpoint_pools() else {
return &[];
};
let mut hosts = Vec::new();
for pool in endpoint_pools.as_ref() {
for endpoint in pool.endpoints.as_ref() {
if !endpoint.is_local {
continue;
}
let host = endpoint.host_port();
if !host.is_empty() && !hosts.contains(&host) {
hosts.push(host);
}
}
}
LOCAL_ENDPOINT_HOSTS.get_or_init(|| hosts)
}
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct ReadRepairHealCacheKey {
bucket: String,
@@ -2813,12 +2863,7 @@ impl SetDisks {
let use_mmap_read = object_mmap_read_enabled();
let reader_setup_stage_start = Instant::now();
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| {
disks
.iter()
.map(|disk| shard_read_cost_for_disk(disk.as_ref()))
.collect::<Vec<_>>()
});
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(&disks));
let reader_setup = create_bitrot_readers_until_quorum_with_preference(
&files,
&disks,
@@ -3244,12 +3289,7 @@ impl SetDisks {
bitrot_reader_init_stage: GET_STAGE_READER_TASK_BITROT_READER_INIT,
});
let reader_setup_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| {
disks
.iter()
.map(|disk| shard_read_cost_for_disk(disk.as_ref()))
.collect::<Vec<_>>()
});
let read_costs = coding::decode::should_collect_shard_read_costs().then(|| shard_read_costs_for_disks(disks));
let reader_setup = create_bitrot_readers_until_quorum_with_preference(
files,
disks,
@@ -3835,6 +3875,17 @@ mod tests {
const CODEC_STREAMING_TEST_BUCKET: &str = "bucket";
const CODEC_STREAMING_TEST_OBJECT: &str = "object";
#[test]
fn shard_read_cost_for_endpoint_maps_topology_classes() {
let local_hosts = vec!["node-a:9000".to_string()];
assert_eq!(shard_read_cost_for_endpoint(true, "node-a:9000", &local_hosts), ShardReadCost::Local);
assert_eq!(shard_read_cost_for_endpoint(false, "node-a:9000", &local_hosts), ShardReadCost::SameNode);
assert_eq!(shard_read_cost_for_endpoint(false, "node-b:9000", &local_hosts), ShardReadCost::Remote);
assert_eq!(shard_read_cost_for_endpoint(false, "", &local_hosts), ShardReadCost::Remote);
assert_eq!(shard_read_cost_for_disk(None, &local_hosts), ShardReadCost::Unknown);
}
fn metadata_fanout_test_fileinfo(object: &str) -> FileInfo {
let mut fi = FileInfo::new(object, 2, 2);
fi.volume = "bucket".to_string();
crates/ecstore/src/store/object.rs
+5 -8
View File
@@ -15,7 +15,7 @@
use super::*;
use crate::set_disk::{
get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
is_object_lock_diag_enabled,
is_lock_optimization_enabled, is_object_lock_diag_enabled,
};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use rustfs_io_metrics::{
@@ -442,7 +442,7 @@ impl ECStore {
}
fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
if reader.buffered_body.is_some() {
if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
return reader;
}
@@ -1978,7 +1978,7 @@ mod tests {
#[tokio::test]
#[serial_test::serial]
async fn reader_lock_is_held_for_stream_when_optimization_is_enabled() {
async fn reader_lock_is_not_held_for_stream_when_optimization_is_enabled() {
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
@@ -2004,13 +2004,10 @@ mod tests {
let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
lock.get_write_lock(key.clone(), "writer", Duration::from_millis(20))
.await
.expect_err("streaming reader should hold the read lock");
drop(reader);
lock.get_write_lock(key, "writer", Duration::from_secs(1))
.await
.expect("dropping the reader should release the read lock");
.expect("lock optimization should release the read lock before returning the stream");
drop(reader);
})
.await;
}