gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 11:56:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(get): observe shard locality cost (#3922)

Browse Source 
* feat(get): observe shard locality cost

* fix(storage): avoid request counter underflow panic

* fix(storage): import put guard in concurrency tests
This commit is contained in:
houseme
2026-06-26 23:50:37 +08:00
committed by GitHub
parent 741b4dab7f
commit c0ddc14bb8
8 changed files with 414 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/read.rs
+18 -2
View File
@@ -24,6 +24,7 @@ use crate::get_diagnostics::{
GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GetObjectFailureReason, classify_disk_error, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path,
};
use crate::set_disk::shard_source::ShardReadCost;
use metrics::counter;
use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
use std::{
@@ -361,6 +362,14 @@ fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
}
fn shard_read_cost_for_disk(disk: Option<&DiskStore>) -> ShardReadCost {
match disk {
Some(disk) if disk.is_local() => ShardReadCost::Local,
Some(_) => ShardReadCost::Remote,
None => ShardReadCost::Unknown,
}
}
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct ReadRepairHealCacheKey {
bucket: String,
@@ -1713,8 +1722,10 @@ impl SetDisks {
let reader_setup_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now);
let mut readers = Vec::with_capacity(disks.len());
let mut read_costs = Vec::with_capacity(disks.len());
let mut errors = Vec::with_capacity(disks.len());
for (idx, disk_op) in disks.iter().enumerate() {
read_costs.push(shard_read_cost_for_disk(disk_op.as_ref()));
match create_bitrot_reader(
files[idx].data.as_deref(),
disk_op.as_ref(),
@@ -1843,7 +1854,9 @@ impl SetDisks {
// part_number, part_offset, part_length, part_size
// );
let decode_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now);
let (written, err) = erasure.decode(writer, readers, part_offset, part_length, part_size).await;
let (written, err) = erasure
.decode_with_read_costs(writer, readers, part_offset, part_length, part_size, read_costs)
.await;
if let Some(decode_stage_start) = decode_stage_start {
rustfs_io_metrics::record_get_object_decode_duration(decode_stage_start.elapsed().as_secs_f64());
}
@@ -1961,8 +1974,10 @@ impl SetDisks {
let reader_setup_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now);
let mut readers = Vec::with_capacity(disks.len());
let mut read_costs = Vec::with_capacity(disks.len());
let mut errors = Vec::with_capacity(disks.len());
for (idx, disk_op) in disks.iter().enumerate() {
read_costs.push(shard_read_cost_for_disk(disk_op.as_ref()));
match create_bitrot_reader(
files[idx].data.as_deref(),
disk_op.as_ref(),
@@ -2030,12 +2045,13 @@ impl SetDisks {
.await;
}
let source = erasure_coding::decode::ParallelReader::new_with_metrics_path(
let source = erasure_coding::decode::ParallelReader::new_with_metrics_path_and_read_costs(
readers,
erasure.clone(),
0,
part_size,
Some(GET_OBJECT_PATH_CODEC_STREAMING),
read_costs,
);
let engine = crate::erasure_codec::bridge::LegacyEcDecodeEngine::new(erasure);
let reader = erasure_coding::decode_reader::ErasureDecodeReader::new(source, engine, part_length)?;
crates/ecstore/src/set_disk/shard_source.rs
+85 -5
View File
@@ -13,31 +13,79 @@
// limitations under the License.
use crate::disk::error::Error;
use crate::get_diagnostics::{
GET_SHARD_READ_COST_LOCAL, GET_SHARD_READ_COST_REMOTE, GET_SHARD_READ_COST_SAME_NODE, GET_SHARD_READ_COST_UNKNOWN,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ShardReadCost {
Local,
SameNode,
Remote,
Unknown,
}
impl ShardReadCost {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Local => GET_SHARD_READ_COST_LOCAL,
Self::SameNode => GET_SHARD_READ_COST_SAME_NODE,
Self::Remote => GET_SHARD_READ_COST_REMOTE,
Self::Unknown => GET_SHARD_READ_COST_UNKNOWN,
}
}
pub(crate) const fn is_low_cost(self) -> bool {
matches!(self, Self::Local | Self::SameNode)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ShardSlot {
index: usize,
read_cost: ShardReadCost,
data: Option<Vec<u8>>,
error: Option<Error>,
}
impl ShardSlot {
pub(crate) fn new(index: usize, data: Option<Vec<u8>>, error: Option<Error>) -> Self {
Self { index, data, error }
Self::with_read_cost(index, ShardReadCost::Unknown, data, error)
}
pub(crate) fn with_read_cost(index: usize, read_cost: ShardReadCost, data: Option<Vec<u8>>, error: Option<Error>) -> Self {
Self {
index,
read_cost,
data,
error,
}
}
pub(crate) fn data(index: usize, data: Vec<u8>) -> Self {
Self::new(index, Some(data), None)
}
pub(crate) fn data_with_read_cost(index: usize, read_cost: ShardReadCost, data: Vec<u8>) -> Self {
Self::with_read_cost(index, read_cost, Some(data), None)
}
pub(crate) fn missing(index: usize, error: Error) -> Self {
Self::new(index, None, Some(error))
}
pub(crate) fn missing_with_read_cost(index: usize, read_cost: ShardReadCost, error: Error) -> Self {
Self::with_read_cost(index, read_cost, None, Some(error))
}
pub(crate) fn index(&self) -> usize {
self.index
}
pub(crate) fn read_cost(&self) -> ShardReadCost {
self.read_cost
}
pub(crate) fn has_data(&self) -> bool {
self.data.is_some()
}
@@ -63,12 +111,27 @@ impl StripeReadState {
}
pub(crate) fn from_parts(shards: Vec<Option<Vec<u8>>>, errors: Vec<Option<Error>>, read_quorum: usize) -> Self {
Self::from_parts_with_read_costs(shards, errors, &[], read_quorum)
}
pub(crate) fn from_parts_with_read_costs(
shards: Vec<Option<Vec<u8>>>,
errors: Vec<Option<Error>>,
read_costs: &[ShardReadCost],
read_quorum: usize,
) -> Self {
let slot_count = shards.len().max(errors.len());
let mut slots = Vec::with_capacity(slot_count);
let mut shards = shards.into_iter();
let mut errors = errors.into_iter();
for index in 0..slot_count {
slots.push(ShardSlot::new(index, shards.next().flatten(), errors.next().flatten()));
let read_cost = read_costs.get(index).copied().unwrap_or(ShardReadCost::Unknown);
slots.push(ShardSlot::with_read_cost(
index,
read_cost,
shards.next().flatten(),
errors.next().flatten(),
));
}
Self::new(slots, read_quorum)
}
@@ -125,9 +188,9 @@ mod tests {
fn stripe_read_state_tracks_decode_quorum() {
let state = StripeReadState::new(
vec![
ShardSlot::data(0, vec![1]),
ShardSlot::missing(1, Error::FileNotFound),
ShardSlot::data(2, vec![2]),
ShardSlot::data_with_read_cost(0, ShardReadCost::Local, vec![1]),
ShardSlot::missing_with_read_cost(1, ShardReadCost::Remote, Error::FileNotFound),
ShardSlot::data_with_read_cost(2, ShardReadCost::SameNode, vec![2]),
],
2,
);
@@ -135,6 +198,8 @@ mod tests {
assert_eq!(state.available_shards(), 2);
assert!(state.can_decode());
assert_eq!(state.slots()[1].index(), 1);
assert_eq!(state.slots()[0].read_cost(), ShardReadCost::Local);
assert!(state.slots()[2].read_cost().is_low_cost());
}
#[test]
@@ -157,6 +222,21 @@ mod tests {
assert_eq!(state.slots()[1].error(), Some(&Error::FileNotFound));
}
#[test]
fn stripe_read_state_preserves_read_cost_hints() {
let state = StripeReadState::from_parts_with_read_costs(
vec![Some(vec![1]), None, Some(vec![3])],
vec![None, Some(Error::FileNotFound)],
&[ShardReadCost::Local, ShardReadCost::Remote, ShardReadCost::Unknown],
2,
);
assert_eq!(state.slots()[0].read_cost(), ShardReadCost::Local);
assert_eq!(state.slots()[1].read_cost(), ShardReadCost::Remote);
assert_eq!(state.slots()[2].read_cost(), ShardReadCost::Unknown);
assert_eq!(ShardReadCost::SameNode.as_str(), GET_SHARD_READ_COST_SAME_NODE);
}
#[test]
fn stripe_read_state_reports_complete_data_shards_without_parity() {
let state = StripeReadState::from_parts(vec![Some(vec![1]), Some(vec![2]), None], Vec::new(), 2);