mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 20:06:37 +00:00
* fix(ecstore): add fallible erasure construction (cherry picked from commitbd148b20f7) * fix(ecstore): resolve storage parity per pool (cherry picked from commitc05c2cb24b) * fix(ecstore): keep carved per-pool parity core self-contained on main Fixups so the cherry-picked fallible-erasure + per-pool-parity core builds standalone on current main without the excluded scope-creep commits: - runtime/sources.rs: re-add backend_storage_class_parities (removed by the per-pool commit; its rebalance caller was updated in an unrelated reporting commit that was left out). Reimplemented over the snapshot API, behavior-identical. - config/mod.rs: rename the storage-class publish test module (main independently added a mod tests, so the cherry-pick collided). - rustfs storage_api.rs + startup_storage.rs: route the storage-class ENV consts through the startup storage facade and use a local const for the erasure-set-drive-count env name, satisfying the layer/facade guardrail (main's guardrail is stricter than when the core was authored). * fix(ecstore): use struct-init in erasure test helper to satisfy clippy field_reassign_with_default The cherry-picked fallible-erasure commit's `erasure_with_invalid_dimensions` test helper built `Erasure` via `default()` then reassigned fields, which trips `clippy::field_reassign_with_default` under `-D warnings` (only surfaced by `--all-targets`, which lints test code). #4977 fixed this in a later commit that was not part of the carved core. Use struct-init with `..Default::default()`, matching #4977's final form. * fix(rustfs): gate the test-only storage-class ENV facade re-export behind cfg(test) The ENV constants (INLINE_BLOCK_ENV/OPTIMIZE_ENV/RRS_ENV/STANDARD_ENV) re-exported through the startup storage facade are only consumed by a #[cfg(test)] test in startup_storage.rs, so in a non-test lib build the re-export is unused and trips -D unused-imports under clippy --all-targets. Gate it with #[cfg(test)], matching #4977's final form. --------- Co-authored-by: cxymds <cxymds@gmail.com>
This commit is contained in:
@@ -163,12 +163,13 @@ impl SetDisks {
|
||||
|
||||
let erasure = if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
// Initialize erasure coding; use legacy mode for old-version files
|
||||
coding::Erasure::new_with_options(
|
||||
coding::Erasure::try_new_with_options(
|
||||
latest_meta.erasure.data_blocks,
|
||||
latest_meta.erasure.parity_blocks,
|
||||
latest_meta.erasure.block_size,
|
||||
latest_meta.uses_legacy_checksum,
|
||||
)
|
||||
.map_err(DiskError::from)?
|
||||
} else {
|
||||
coding::Erasure::default()
|
||||
};
|
||||
|
||||
@@ -355,7 +355,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}"));
|
||||
|
||||
let result: Result<PartInfo> = async {
|
||||
let erasure = coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
|
||||
let erasure = coding::Erasure::try_new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size)
|
||||
.map_err(Error::from)?;
|
||||
let writer_setup_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
let mut writers = Vec::with_capacity(shuffle_disks.len());
|
||||
|
||||
@@ -25,6 +25,11 @@ use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idemp
|
||||
use crate::disk::OldCurrentSize;
|
||||
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
|
||||
|
||||
fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
|
||||
coding::Erasure::try_new_with_options(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size, uses_legacy)
|
||||
.map_err(Error::from)
|
||||
}
|
||||
|
||||
/// Length of the full plaintext body when — and only when — this read's output
|
||||
/// is exactly the object's complete plaintext, so the app-layer body cache may
|
||||
/// serve it in place of the erasure read.
|
||||
@@ -292,12 +297,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
let erasure = coding::Erasure::new_with_options(
|
||||
fi.erasure.data_blocks,
|
||||
fi.erasure.parity_blocks,
|
||||
fi.erasure.block_size,
|
||||
fi.uses_legacy_checksum,
|
||||
);
|
||||
let erasure = erasure_from_file_info(&fi, fi.uses_legacy_checksum)?;
|
||||
let read_length = erasure.shard_file_offset(0, object_size, object_size);
|
||||
let total_shards = data_shards + fi.erasure.parity_blocks;
|
||||
let (_disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(&disks, &files, &fi);
|
||||
@@ -761,7 +761,7 @@ impl SetDisks {
|
||||
let tmp_object = format!("{}/{}/part.1", tmp_dir, fi.data_dir.unwrap());
|
||||
|
||||
let result: Result<(ObjectInfo, Option<OldCurrentSize>)> = async {
|
||||
let erasure = coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
|
||||
let erasure = erasure_from_file_info(&fi, false)?;
|
||||
|
||||
let put_object_size = known_put_object_storage_size(data.size());
|
||||
let is_inline_buffer =
|
||||
@@ -2886,6 +2886,30 @@ fn drop_failed_writer_disks<D, W>(disks: &mut [Option<D>], writers: &[Option<W>]
|
||||
committed
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod erasure_construction_tests {
|
||||
use super::*;
|
||||
use crate::erasure::coding::ErasureConstructionError;
|
||||
use std::error::Error as _;
|
||||
|
||||
#[test]
|
||||
fn object_file_info_mapping_preserves_construction_error() {
|
||||
let mut fi = FileInfo::new("object", 2, 2);
|
||||
fi.erasure.block_size = 0;
|
||||
|
||||
let error = match erasure_from_file_info(&fi, false) {
|
||||
Ok(_) => panic!("invalid object erasure metadata must be rejected"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(error.to_string().contains("block_size must be greater than zero"));
|
||||
let io_source = error.source().expect("StorageError::Io must expose its io::Error source");
|
||||
let construction_source = io_source
|
||||
.source()
|
||||
.expect("io::Error must expose the erasure construction error");
|
||||
assert!(construction_source.is::<ErasureConstructionError>());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod b3_write_quorum_tests {
|
||||
use super::drop_failed_writer_disks;
|
||||
|
||||
@@ -449,15 +449,13 @@ impl SetDisks {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let erasure = coding::Erasure::new_with_options(
|
||||
let erasure = coding::Erasure::try_new_with_options(
|
||||
fi.erasure.data_blocks,
|
||||
fi.erasure.parity_blocks,
|
||||
fi.erasure.block_size,
|
||||
fi.uses_legacy_checksum,
|
||||
);
|
||||
if erasure.data_shards == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
|
||||
let checksum_info = fi.erasure.get_checksum_info(part.number);
|
||||
let checksum_algo = if fi.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
|
||||
@@ -679,21 +677,13 @@ impl SetDisks {
|
||||
object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds"
|
||||
);
|
||||
|
||||
let erasure = coding::Erasure::new_with_options(
|
||||
let erasure = coding::Erasure::try_new_with_options(
|
||||
fi.erasure.data_blocks,
|
||||
fi.erasure.parity_blocks,
|
||||
fi.erasure.block_size,
|
||||
fi.uses_legacy_checksum,
|
||||
);
|
||||
|
||||
// Erasure params come from on-disk metadata; zero values must fail the read
|
||||
// instead of panicking on the block/shard divisions below.
|
||||
if !erasure.has_valid_dimensions() {
|
||||
return Err(Error::other(format!(
|
||||
"invalid erasure metadata for {bucket}/{object}: block_size={}, data_blocks={}",
|
||||
erasure.block_size, erasure.data_shards
|
||||
)));
|
||||
}
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
|
||||
let part_indices: Vec<usize> = (part_index..=last_part_index).collect();
|
||||
debug!(bucket, object, ?part_indices, "Multipart part indices to stream");
|
||||
@@ -1122,22 +1112,13 @@ impl SetDisks {
|
||||
metrics_size_bucket: &'static str,
|
||||
prefer_data_blocks_first_reader_setup: bool,
|
||||
) -> Result<GetCodecStreamingReaderBuildOutcome> {
|
||||
let erasure = coding::Erasure::new_with_options(
|
||||
let erasure = coding::Erasure::try_new_with_options(
|
||||
fi.erasure.data_blocks,
|
||||
fi.erasure.parity_blocks,
|
||||
fi.erasure.block_size,
|
||||
fi.uses_legacy_checksum,
|
||||
);
|
||||
|
||||
// Erasure params come from on-disk metadata; zero values must fail the read
|
||||
// instead of panicking on the block/shard divisions inside the codec
|
||||
// streaming reader below. Mirrors the legacy multipart guard.
|
||||
if !erasure.has_valid_dimensions() {
|
||||
return Err(Error::other(format!(
|
||||
"invalid erasure metadata for {bucket}/{object}: block_size={}, data_blocks={}",
|
||||
erasure.block_size, erasure.data_shards
|
||||
)));
|
||||
}
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
|
||||
let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, files, fi);
|
||||
|
||||
@@ -2052,7 +2033,12 @@ mod metadata_cache_tests {
|
||||
)
|
||||
.await
|
||||
.expect_err("invalid erasure metadata must fail before reader setup");
|
||||
assert!(err.to_string().contains("invalid erasure metadata"));
|
||||
assert!(err.to_string().contains("block_size must be greater than zero"));
|
||||
let io_source = std::error::Error::source(&err).expect("StorageError::Io must expose its io::Error source");
|
||||
let construction_source = io_source
|
||||
.source()
|
||||
.expect("io::Error must expose the erasure construction error");
|
||||
assert!(construction_source.is::<crate::erasure::coding::ErasureConstructionError>());
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user