gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-09 22:59:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: rebuild wiped disks during admin heal (#3084)

Browse Source 
* fix: rebuild wiped disks during admin heal

* Preserve forceStart heal admission semantics

* Address heal regression test review comments

* fix(heal): address admin heal review comments

* fix(heal): fix force-start dedup and test polling

* fix(heal): address unresolved review comments

* fix(heal): simplify dedup key for clippy

---------

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
安正超
2026-05-26 21:54:29 +08:00
committed by GitHub
parent ea74fa7a95
commit 0875f09a39
9 changed files with 499 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk.rs
+65 -45
View File
@@ -30,7 +30,6 @@ use crate::disk::{
};
use crate::disk::{STORAGE_FORMAT_FILE, count_part_not_success};
use crate::erasure_coding;
use crate::erasure_coding::bitrot_verify;
use crate::error::{Error, Result, is_err_version_not_found};
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
@@ -4049,32 +4048,16 @@ async fn disks_with_all_parts(
continue;
}
// Always check data, if we got it.
// Inline data is stored inside xl.meta, so there is no separate part file to
// verify here. Treat the shard as present once metadata was read successfully;
// object reads/heal will validate the inline shard through the normal bitrot
// reader path. Running bitrot_verify directly here can falsely mark small
// inline shards corrupt when older metadata has no per-part checksum entries.
if (meta.data.is_some() || meta.size == 0) && !meta.parts.is_empty() {
if let Some(data) = &meta.data {
let checksum_info = meta.erasure.get_checksum_info(meta.parts[0].number);
let checksum_algo = if meta.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
let data_len = data.len();
let verify_err = bitrot_verify(
Box::new(Cursor::new(data.clone())),
data_len,
meta.erasure.shard_file_size(meta.size) as usize,
checksum_algo,
checksum_info.hash,
meta.erasure.shard_size(),
)
.await
.err();
if let Some(vec) = data_errs_by_part.get_mut(&0)
&& index < vec.len()
{
vec[index] = conv_part_err_to_int(&verify_err.map(|e| e.into()));
}
if let Some(vec) = data_errs_by_part.get_mut(&0)
&& index < vec.len()
{
vec[index] = CHECK_PART_SUCCESS;
}
continue;
}
@@ -5393,6 +5376,62 @@ mod tests {
assert!(has_part_err(&unknown_errors));
}
#[test]
fn test_populate_data_errs_by_disk_uses_disk_index_not_error_code() {
let mut data_errs_by_disk = HashMap::from([
(0, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
(1, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
(2, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
]);
let data_errs_by_part = HashMap::from([
(0, vec![CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]),
(1, vec![CHECK_PART_SUCCESS, CHECK_PART_FILE_CORRUPT, CHECK_PART_SUCCESS]),
]);
populate_data_errs_by_disk(&mut data_errs_by_disk, &data_errs_by_part);
assert_eq!(data_errs_by_disk.get(&0).unwrap(), &vec![CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS]);
assert_eq!(data_errs_by_disk.get(&1).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_FILE_CORRUPT]);
assert_eq!(data_errs_by_disk.get(&2).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]);
let mut data_errs_by_disk = HashMap::from([
(0, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
(1, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
(2, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
(3, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
]);
let data_errs_by_part = HashMap::from([
(
0,
vec![
CHECK_PART_FILE_NOT_FOUND,
CHECK_PART_SUCCESS,
CHECK_PART_SUCCESS,
CHECK_PART_SUCCESS,
],
),
(
1,
vec![
CHECK_PART_FILE_CORRUPT,
CHECK_PART_SUCCESS,
CHECK_PART_SUCCESS,
CHECK_PART_SUCCESS,
],
),
]);
populate_data_errs_by_disk(&mut data_errs_by_disk, &data_errs_by_part);
assert_eq!(
data_errs_by_disk.get(&0).unwrap(),
&vec![CHECK_PART_FILE_NOT_FOUND, CHECK_PART_FILE_CORRUPT]
);
assert_eq!(data_errs_by_disk.get(&1).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]);
assert_eq!(data_errs_by_disk.get(&2).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]);
assert_eq!(data_errs_by_disk.get(&3).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]);
}
#[test]
fn test_should_heal_object_on_disk() {
// Test healing decision logic
@@ -5413,25 +5452,6 @@ mod tests {
assert!(should_heal);
}
#[test]
fn test_populate_data_errs_by_disk_uses_disk_index_not_error_code() {
let mut data_errs_by_disk = HashMap::from([
(0, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
(1, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
(2, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
]);
let data_errs_by_part = HashMap::from([
(0, vec![CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]),
(1, vec![CHECK_PART_SUCCESS, CHECK_PART_FILE_CORRUPT, CHECK_PART_SUCCESS]),
]);
populate_data_errs_by_disk(&mut data_errs_by_disk, &data_errs_by_part);
assert_eq!(data_errs_by_disk.get(&0).unwrap(), &vec![CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS]);
assert_eq!(data_errs_by_disk.get(&1).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_FILE_CORRUPT]);
assert_eq!(data_errs_by_disk.get(&2).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]);
}
#[tokio::test]
async fn test_get_disks_info_preserves_runtime_state_for_suspect_and_offline_disks() {
let format = FormatV3::new(1, 3);
crates/ecstore/src/set_disk/heal.rs
+9 -9
View File
@@ -350,15 +350,6 @@ impl SetDisks {
for (part_index, part) in latest_meta.parts.iter().enumerate() {
let till_offset = erasure.shard_file_offset(0, part.size, part.size);
let checksum_info = erasure_info.get_checksum_info(part.number);
let checksum_algo = if latest_meta.uses_legacy_checksum
&& checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S
{
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
// Read zero-copy configuration from environment variable
// Default: enabled (true) for performance
let use_zero_copy =
@@ -382,6 +373,15 @@ impl SetDisks {
}
if let (Some(disk), Some(metadata)) = (disk, &copy_parts_metadata[index]) {
let checksum_info = metadata.erasure.get_checksum_info(part.number);
let checksum_algo = if metadata.uses_legacy_checksum
&& checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S
{
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
match create_bitrot_reader(
metadata.data.as_deref(),
Some(disk),