gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 21:46:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Admin Heal API and Add Pagination Support for Large Buckets (#933)

Browse Source 
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
weisd
2025-12-03 18:10:46 +08:00
committed by GitHub
parent e355d3db80
commit a8b7b28fd0
14 changed files with 494 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/disk/local.rs
+5 -4
View File
@@ -967,9 +967,7 @@ impl LocalDisk {
sum: &[u8],
shard_size: usize,
) -> Result<()> {
let file = super::fs::open_file(part_path, O_CREATE | O_WRONLY)
.await
.map_err(to_file_error)?;
let file = super::fs::open_file(part_path, O_RDONLY).await.map_err(to_file_error)?;
let meta = file.metadata().await.map_err(to_file_error)?;
let file_size = meta.len() as usize;
@@ -1465,6 +1463,7 @@ impl DiskAPI for LocalDisk {
resp.results[i] = conv_part_err_to_int(&err);
if resp.results[i] == CHECK_PART_UNKNOWN {
if let Some(err) = err {
error!("verify_file: failed to bitrot verify file: {:?}, error: {:?}", &part_path, &err);
if err == DiskError::FileAccessDenied {
continue;
}
@@ -1551,7 +1550,7 @@ impl DiskAPI for LocalDisk {
.join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string()))
.join(format!("part.{}", part.number));
match lstat(file_path).await {
match lstat(&file_path).await {
Ok(st) => {
if st.is_dir() {
resp.results[i] = CHECK_PART_FILE_NOT_FOUND;
@@ -1577,6 +1576,8 @@ impl DiskAPI for LocalDisk {
}
}
resp.results[i] = CHECK_PART_FILE_NOT_FOUND;
} else {
error!("check_parts: failed to stat file: {:?}, error: {:?}", &file_path, &e);
}
continue;
}
crates/ecstore/src/disk/mod.rs
+4 -1
View File
@@ -681,7 +681,10 @@ pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
Some(DiskError::VolumeNotFound) => CHECK_PART_VOLUME_NOT_FOUND,
Some(DiskError::DiskNotFound) => CHECK_PART_DISK_NOT_FOUND,
None => CHECK_PART_SUCCESS,
_ => CHECK_PART_UNKNOWN,
_ => {
tracing::warn!("conv_part_err_to_int: unknown error: {err:?}");
CHECK_PART_UNKNOWN
}
}
}
crates/ecstore/src/set_disk.rs
+131 -37
View File
@@ -25,7 +25,7 @@ use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
use crate::disk::STORAGE_FORMAT_FILE;
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
use crate::disk::{
self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS,
self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
conv_part_err_to_int, has_part_err,
};
use crate::erasure_coding;
@@ -6431,6 +6431,10 @@ fn join_errs(errs: &[Option<DiskError>]) -> String {
errs.join(", ")
}
/// disks_with_all_partsv2 is a corrected version based on Go implementation.
/// It sets partsMetadata and onlineDisks when xl.meta is inexistant/corrupted or outdated.
/// It also checks if the status of each part (corrupted, missing, ok) in each drive.
/// Returns (availableDisks, dataErrsByDisk, dataErrsByPart).
async fn disks_with_all_parts(
online_disks: &[Option<DiskStore>],
parts_metadata: &mut [FileInfo],
@@ -6440,39 +6444,66 @@ async fn disks_with_all_parts(
object: &str,
scan_mode: HealScanMode,
) -> disk::error::Result<(Vec<Option<DiskStore>>, HashMap<usize, Vec<usize>>, HashMap<usize, Vec<usize>>)> {
info!(
"disks_with_all_parts: starting with online_disks.len()={}, scan_mode={:?}",
let object_name = latest_meta.name.clone();
debug!(
"disks_with_all_partsv2: starting with object_name={}, online_disks.len()={}, scan_mode={:?}",
object_name,
online_disks.len(),
scan_mode
);
let mut available_disks = vec![None; online_disks.len()];
// Initialize dataErrsByDisk and dataErrsByPart with 0 (CHECK_PART_UNKNOWN) to match Go
let mut data_errs_by_disk: HashMap<usize, Vec<usize>> = HashMap::new();
for i in 0..online_disks.len() {
data_errs_by_disk.insert(i, vec![1; latest_meta.parts.len()]);
data_errs_by_disk.insert(i, vec![CHECK_PART_SUCCESS; latest_meta.parts.len()]);
}
let mut data_errs_by_part: HashMap<usize, Vec<usize>> = HashMap::new();
for i in 0..latest_meta.parts.len() {
data_errs_by_part.insert(i, vec![1; online_disks.len()]);
data_errs_by_part.insert(i, vec![CHECK_PART_SUCCESS; online_disks.len()]);
}
// Check for inconsistent erasure distribution
let mut inconsistent = 0;
parts_metadata.iter().enumerate().for_each(|(index, meta)| {
if meta.is_valid() && !meta.deleted && meta.erasure.distribution.len() != online_disks.len()
|| (!meta.erasure.distribution.is_empty() && meta.erasure.distribution[index] != meta.erasure.index)
{
warn!("file info inconsistent, meta: {:?}", meta);
inconsistent += 1;
for (index, meta) in parts_metadata.iter().enumerate() {
if !meta.is_valid() {
// Since for majority of the cases erasure.Index matches with erasure.Distribution we can
// consider the offline disks as consistent.
continue;
}
});
if !meta.deleted {
if meta.erasure.distribution.len() != online_disks.len() {
// Erasure distribution seems to have lesser
// number of items than number of online disks.
inconsistent += 1;
continue;
}
if !meta.erasure.distribution.is_empty()
&& index < meta.erasure.distribution.len()
&& meta.erasure.distribution[index] != meta.erasure.index
{
// Mismatch indexes with distribution order
inconsistent += 1;
}
}
}
let erasure_distribution_reliable = inconsistent <= parts_metadata.len() / 2;
// Initialize metaErrs
let mut meta_errs = Vec::with_capacity(errs.len());
for _ in 0..errs.len() {
meta_errs.push(None);
}
// Process meta errors
for (index, disk) in online_disks.iter().enumerate() {
if let Some(err) = &errs[index] {
meta_errs[index] = Some(err.clone());
continue;
}
let disk = if let Some(disk) = disk {
disk
} else {
@@ -6480,48 +6511,59 @@ async fn disks_with_all_parts(
continue;
};
if let Some(err) = &errs[index] {
meta_errs[index] = Some(err.clone());
continue;
}
if !disk.is_online().await {
meta_errs[index] = Some(DiskError::DiskNotFound);
continue;
}
let meta = &parts_metadata[index];
if !meta.mod_time.eq(&latest_meta.mod_time) || !meta.data_dir.eq(&latest_meta.data_dir) {
warn!("mod_time is not Eq, file corrupt, index: {index}");
// Check if metadata is corrupted (equivalent to filterByETag=false in Go)
let corrupted = !meta.mod_time.eq(&latest_meta.mod_time) || !meta.data_dir.eq(&latest_meta.data_dir);
if corrupted {
meta_errs[index] = Some(DiskError::FileCorrupt);
parts_metadata[index] = FileInfo::default();
continue;
}
if erasure_distribution_reliable {
if !meta.is_valid() {
warn!("file info is not valid, file corrupt, index: {index}");
parts_metadata[index] = FileInfo::default();
meta_errs[index] = Some(DiskError::FileCorrupt);
continue;
}
#[allow(clippy::collapsible_if)]
if !meta.deleted && meta.erasure.distribution.len() != online_disks.len() {
warn!("file info distribution len not Eq online_disks len, file corrupt, index: {index}");
// Erasure distribution is not the same as onlineDisks
// attempt a fix if possible, assuming other entries
// might have the right erasure distribution.
parts_metadata[index] = FileInfo::default();
meta_errs[index] = Some(DiskError::FileCorrupt);
continue;
}
}
}
// info!("meta_errs: {:?}, errs: {:?}", meta_errs, errs);
meta_errs.iter().enumerate().for_each(|(index, err)| {
// Copy meta errors to part errors
for (index, err) in meta_errs.iter().enumerate() {
if err.is_some() {
let part_err = conv_part_err_to_int(err);
for p in 0..latest_meta.parts.len() {
data_errs_by_part.entry(p).or_insert(vec![0; meta_errs.len()])[index] = part_err;
if let Some(vec) = data_errs_by_part.get_mut(&p) {
if index < vec.len() {
info!(
"data_errs_by_part: copy meta errors to part errors: object_name={}, index: {index}, part: {p}, part_err: {part_err}",
object_name
);
vec[index] = part_err;
}
}
}
}
});
}
// info!("data_errs_by_part: {:?}, data_errs_by_disk: {:?}", data_errs_by_part, data_errs_by_disk);
// Check data for each disk
for (index, disk) in online_disks.iter().enumerate() {
if meta_errs[index].is_some() {
continue;
@@ -6530,7 +6572,6 @@ async fn disks_with_all_parts(
let disk = if let Some(disk) = disk {
disk
} else {
meta_errs[index] = Some(DiskError::DiskNotFound);
continue;
};
@@ -6558,16 +6599,21 @@ async fn disks_with_all_parts(
if let Some(vec) = data_errs_by_part.get_mut(&0) {
if index < vec.len() {
vec[index] = conv_part_err_to_int(&verify_err.map(|e| e.into()));
info!("bitrot check result: {}", vec[index]);
info!(
"data_errs_by_part:bitrot check result: object_name={}, index: {index}, result: {}",
object_name, vec[index]
);
}
}
}
continue;
}
// Verify file or check parts
let mut verify_resp = CheckPartsResp::default();
let mut verify_err = None;
meta.data_dir = latest_meta.data_dir;
if scan_mode == HealScanMode::Deep {
// disk has a valid xl.meta but may not have all the
// parts. This is considered an outdated disk, since
@@ -6577,6 +6623,7 @@ async fn disks_with_all_parts(
verify_resp = v;
}
Err(err) => {
warn!("verify_file failed: {err:?}, object_name={}, index: {index}", object_name);
verify_err = Some(err);
}
}
@@ -6586,38 +6633,85 @@ async fn disks_with_all_parts(
verify_resp = v;
}
Err(err) => {
warn!("check_parts failed: {err:?}, object_name={}, index: {index}", object_name);
verify_err = Some(err);
}
}
}
// Update dataErrsByPart for all parts
for p in 0..latest_meta.parts.len() {
if let Some(vec) = data_errs_by_part.get_mut(&p) {
if index < vec.len() {
if verify_err.is_some() {
info!("verify_err");
info!(
"data_errs_by_part: verify_err: object_name={}, index: {index}, part: {p}, verify_err: {verify_err:?}",
object_name
);
vec[index] = conv_part_err_to_int(&verify_err.clone());
} else {
info!("verify_resp, verify_resp.results {}", verify_resp.results[p]);
vec[index] = verify_resp.results[p];
// Fix: verify_resp.results length is based on meta.parts, not latest_meta.parts
// We need to check bounds to avoid panic
if p < verify_resp.results.len() {
info!(
"data_errs_by_part: update data_errs_by_part: object_name={}, index: {}, part: {}, verify_resp.results: {:?}",
object_name, index, p, verify_resp.results[p]
);
vec[index] = verify_resp.results[p];
} else {
debug!(
"data_errs_by_part: verify_resp.results length mismatch: expected at least {}, got {}, object_name={}, index: {index}, part: {p}",
p + 1,
verify_resp.results.len(),
object_name
);
vec[index] = CHECK_PART_SUCCESS;
}
}
}
}
}
}
// info!("data_errs_by_part: {:?}, data_errs_by_disk: {:?}", data_errs_by_part, data_errs_by_disk);
// Build dataErrsByDisk from dataErrsByPart
for (part, disks) in data_errs_by_part.iter() {
for (idx, disk) in disks.iter().enumerate() {
if let Some(vec) = data_errs_by_disk.get_mut(&idx) {
vec[*part] = *disk;
for (disk_idx, disk_err) in disks.iter().enumerate() {
if let Some(vec) = data_errs_by_disk.get_mut(&disk_idx) {
if *part < vec.len() {
vec[*part] = *disk_err;
info!(
"data_errs_by_disk: update data_errs_by_disk: object_name={}, part: {part}, disk_idx: {disk_idx}, disk_err: {disk_err}",
object_name,
);
}
}
}
}
// info!("data_errs_by_part: {:?}, data_errs_by_disk: {:?}", data_errs_by_part, data_errs_by_disk);
// Calculate available_disks based on meta_errs and data_errs_by_disk
for (i, disk) in online_disks.iter().enumerate() {
if meta_errs[i].is_none() && disk.is_some() && !has_part_err(&data_errs_by_disk[&i]) {
available_disks[i] = Some(disk.clone().unwrap());
if let Some(disk_errs) = data_errs_by_disk.get(&i) {
if meta_errs[i].is_none() && disk.is_some() && !has_part_err(disk_errs) {
available_disks[i] = Some(disk.clone().unwrap());
} else {
warn!(
"disks_with_all_partsv2: disk is not available, object_name={}, index: {}, meta_errs={:?}, disk_errs={:?}, disk_is_some={:?}",
object_name,
i,
meta_errs[i],
disk_errs,
disk.is_some(),
);
parts_metadata[i] = FileInfo::default();
}
} else {
warn!(
"disks_with_all_partsv2: data_errs_by_disk missing entry for object_name={},index {}, meta_errs={:?}, disk_is_some={:?}",
object_name,
i,
meta_errs[i],
disk.is_some(),
);
parts_metadata[i] = FileInfo::default();
}
}