fix(ecstore): resolve erasure parity per pool (#4801) (#5015)

* fix(ecstore): add fallible erasure construction

(cherry picked from commit bd148b20f7)

* fix(ecstore): resolve storage parity per pool

(cherry picked from commit c05c2cb24b)

* fix(ecstore): keep carved per-pool parity core self-contained on main

Fixups so the cherry-picked fallible-erasure + per-pool-parity core builds standalone on current main without the excluded scope-creep commits:
- runtime/sources.rs: re-add backend_storage_class_parities (removed by the per-pool commit; its rebalance caller was updated in an unrelated reporting commit that was left out). Reimplemented over the snapshot API, behavior-identical.
- config/mod.rs: rename the storage-class publish test module (main independently added a mod tests, so the cherry-pick collided).
- rustfs storage_api.rs + startup_storage.rs: route the storage-class ENV consts through the startup storage facade and use a local const for the erasure-set-drive-count env name, satisfying the layer/facade guardrail (main's guardrail is stricter than when the core was authored).

* fix(ecstore): use struct-init in erasure test helper to satisfy clippy field_reassign_with_default

The cherry-picked fallible-erasure commit's `erasure_with_invalid_dimensions` test helper built `Erasure` via `default()` then reassigned fields, which trips `clippy::field_reassign_with_default` under `-D warnings` (only surfaced by `--all-targets`, which lints test code). #4977 fixed this in a later commit that was not part of the carved core. Use struct-init with `..Default::default()`, matching #4977's final form.

* fix(rustfs): gate the test-only storage-class ENV facade re-export behind cfg(test)

The ENV constants (INLINE_BLOCK_ENV/OPTIMIZE_ENV/RRS_ENV/STANDARD_ENV) re-exported through the startup storage facade are only consumed by a #[cfg(test)] test in startup_storage.rs, so in a non-test lib build the re-export is unused and trips -D unused-imports under clippy --all-targets. Gate it with #[cfg(test)], matching #4977's final form.

---------

Co-authored-by: cxymds <cxymds@gmail.com>
This commit is contained in:
Zhengchao An
2026-07-19 11:06:46 +08:00
committed by GitHub
parent 3ed682be42
commit 7f5873dac8
22 changed files with 1463 additions and 248 deletions
+2 -1
View File
@@ -163,12 +163,13 @@ impl SetDisks {
let erasure = if !latest_meta.deleted && !latest_meta.is_remote() {
// Initialize erasure coding; use legacy mode for old-version files
coding::Erasure::new_with_options(
coding::Erasure::try_new_with_options(
latest_meta.erasure.data_blocks,
latest_meta.erasure.parity_blocks,
latest_meta.erasure.block_size,
latest_meta.uses_legacy_checksum,
)
.map_err(DiskError::from)?
} else {
coding::Erasure::default()
};
+2 -1
View File
@@ -355,7 +355,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}"));
let result: Result<PartInfo> = async {
let erasure = coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let erasure = coding::Erasure::try_new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size)
.map_err(Error::from)?;
let writer_setup_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
let mut writers = Vec::with_capacity(shuffle_disks.len());
+31 -7
View File
@@ -25,6 +25,11 @@ use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idemp
use crate::disk::OldCurrentSize;
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
coding::Erasure::try_new_with_options(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size, uses_legacy)
.map_err(Error::from)
}
/// Length of the full plaintext body when — and only when — this read's output
/// is exactly the object's complete plaintext, so the app-layer body cache may
/// serve it in place of the erasure read.
@@ -292,12 +297,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
}
}
let erasure = coding::Erasure::new_with_options(
fi.erasure.data_blocks,
fi.erasure.parity_blocks,
fi.erasure.block_size,
fi.uses_legacy_checksum,
);
let erasure = erasure_from_file_info(&fi, fi.uses_legacy_checksum)?;
let read_length = erasure.shard_file_offset(0, object_size, object_size);
let total_shards = data_shards + fi.erasure.parity_blocks;
let (_disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(&disks, &files, &fi);
@@ -761,7 +761,7 @@ impl SetDisks {
let tmp_object = format!("{}/{}/part.1", tmp_dir, fi.data_dir.unwrap());
let result: Result<(ObjectInfo, Option<OldCurrentSize>)> = async {
let erasure = coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let erasure = erasure_from_file_info(&fi, false)?;
let put_object_size = known_put_object_storage_size(data.size());
let is_inline_buffer =
@@ -2886,6 +2886,30 @@ fn drop_failed_writer_disks<D, W>(disks: &mut [Option<D>], writers: &[Option<W>]
committed
}
#[cfg(test)]
mod erasure_construction_tests {
use super::*;
use crate::erasure::coding::ErasureConstructionError;
use std::error::Error as _;
#[test]
fn object_file_info_mapping_preserves_construction_error() {
let mut fi = FileInfo::new("object", 2, 2);
fi.erasure.block_size = 0;
let error = match erasure_from_file_info(&fi, false) {
Ok(_) => panic!("invalid object erasure metadata must be rejected"),
Err(error) => error,
};
assert!(error.to_string().contains("block_size must be greater than zero"));
let io_source = error.source().expect("StorageError::Io must expose its io::Error source");
let construction_source = io_source
.source()
.expect("io::Error must expose the erasure construction error");
assert!(construction_source.is::<ErasureConstructionError>());
}
}
#[cfg(test)]
mod b3_write_quorum_tests {
use super::drop_failed_writer_disks;