gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(get): add codec engine ab matrix (#3940)

Browse Source 
* upgrade version

* feat(get): add codec engine ab matrix

* chore(get): drop unrelated dependency drift

* upgrade version

* upgrade version

* fix cargo deny
This commit is contained in:
houseme
2026-06-27 13:27:16 +08:00
committed by GitHub
parent 996e58fd9a
commit 58b76a3d45
8 changed files with 440 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/mod.rs
+11 -2
View File
@@ -24,8 +24,10 @@ use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
use crate::cluster::rpc::heal_bucket_local_on_disks;
use crate::diagnostics::get::{
GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION,
GET_STAGE_EMIT, GET_STAGE_METADATA, classify_storage_error, record_get_object_pipeline_failure,
GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE,
GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_LEGACY_DUPLEX,
GET_OBJECT_PATH_REMOTE_TRANSITION, GET_STAGE_EMIT, GET_STAGE_METADATA, classify_storage_error,
record_get_object_pipeline_failure,
};
use crate::disk::error_reduce::{
BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs,
@@ -410,6 +412,13 @@ fn build_get_codec_streaming_decode_engine(erasure: coding::Erasure) -> std::io:
}
}
fn get_codec_streaming_metrics_path() -> &'static str {
match get_codec_streaming_engine() {
GetCodecStreamingEngine::Legacy => GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE,
GetCodecStreamingEngine::Rustfs => GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE,
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum GetCodecStreamingDecision {
Use,
crates/ecstore/src/set_disk/read.rs
+28 -9
View File
@@ -2244,8 +2244,9 @@ impl SetDisks {
BitrotReaderSetupMode::VerifyReconstruction,
)
.await;
let metrics_path = get_codec_streaming_metrics_path();
rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_PATH_CODEC_STREAMING,
metrics_path,
GET_STAGE_READER_SETUP,
reader_setup_stage_start.elapsed().as_secs_f64(),
);
@@ -2254,14 +2255,10 @@ impl SetDisks {
if available_shards < erasure.data_shards {
if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) {
let reason = classify_disk_error(&read_err);
record_get_object_pipeline_failure_for_path(GET_OBJECT_PATH_CODEC_STREAMING, GET_STAGE_READER_SETUP, reason);
record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, reason);
return Err(to_object_err(read_err.into(), vec![bucket, object]));
}
record_get_object_pipeline_failure_for_path(
GET_OBJECT_PATH_CODEC_STREAMING,
GET_STAGE_READER_SETUP,
GetObjectFailureReason::ReadQuorum,
);
record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, GetObjectFailureReason::ReadQuorum);
return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
}
@@ -2287,11 +2284,16 @@ impl SetDisks {
erasure.clone(),
0,
part_size,
Some(GET_OBJECT_PATH_CODEC_STREAMING),
Some(metrics_path),
read_costs,
);
let engine = build_get_codec_streaming_decode_engine(erasure)?;
let reader = crate::erasure::coding::decode_reader::ErasureDecodeReader::new(source, engine, part_length)?;
let reader = crate::erasure::coding::decode_reader::ErasureDecodeReader::new_with_metrics_path(
source,
engine,
part_length,
metrics_path,
)?;
Ok(Box::new(crate::erasure::coding::decode_reader::SyncErasureDecodeReader::new(reader)))
}
}
@@ -3362,6 +3364,23 @@ mod tests {
});
}
#[test]
fn codec_streaming_metrics_path_matches_selected_engine() {
temp_env::with_var(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, None::<&str>, || {
assert_eq!(
get_codec_streaming_metrics_path(),
crate::diagnostics::get::GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE
);
});
temp_env::with_var(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS), || {
assert_eq!(
get_codec_streaming_metrics_path(),
crate::diagnostics::get::GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE
);
});
}
#[test]
fn codec_streaming_fallback_metric_labels_are_stable() {
assert_eq!(GetCodecStreamingFallbackReason::Disabled.as_str(), "disabled");