gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-03 20:07:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: move ecstore set heal helpers (#3648)

Browse Source
This commit is contained in:
安正超
2026-06-20 12:03:22 +08:00
committed by GitHub
parent 1ea7b36ab4
commit 3035cad082
4 changed files with 179 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/layout/mod.rs
+1
View File
@@ -8,4 +8,5 @@ pub(crate) mod disks_layout;
pub(crate) mod endpoint;
pub(crate) mod endpoints;
pub(crate) mod format;
pub(crate) mod set_heal;
pub(crate) mod set_layout;
crates/ecstore/src/layout/set_heal.rs
+139
View File
@@ -0,0 +1,139 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::{DiskInfo, error::DiskError};
use crate::layout::{endpoints::Endpoints, format::FormatV3};
use rustfs_common::heal_channel::DriveState;
use rustfs_madmin::heal_commands::HealDriveInfo;
pub(crate) fn formats_to_drives_info(
endpoints: &Endpoints,
formats: &[Option<FormatV3>],
errs: &[Option<DiskError>],
) -> Vec<HealDriveInfo> {
let mut before_drives = Vec::with_capacity(endpoints.as_ref().len());
for (index, format) in formats.iter().enumerate() {
let drive = endpoints.get_string(index);
let state = if format.is_some() {
DriveState::Ok.to_string()
} else if let Some(Some(err)) = errs.get(index) {
if *err == DiskError::UnformattedDisk {
DriveState::Missing.to_string()
} else if *err == DiskError::DiskNotFound {
DriveState::Offline.to_string()
} else {
DriveState::Corrupt.to_string()
}
} else {
DriveState::Corrupt.to_string()
};
let uuid = if let Some(format) = format {
format.erasure.this.to_string()
} else {
"".to_string()
};
before_drives.push(HealDriveInfo {
uuid,
endpoint: drive,
state: state.to_string(),
});
}
before_drives
}
pub(crate) fn new_heal_format_sets(
ref_format: &FormatV3,
set_count: usize,
set_drive_count: usize,
formats: &[Option<FormatV3>],
errs: &[Option<DiskError>],
) -> (Vec<Vec<Option<FormatV3>>>, Vec<Vec<DiskInfo>>) {
let mut new_formats = vec![vec![None; set_drive_count]; set_count];
let mut current_disks_info = vec![vec![DiskInfo::default(); set_drive_count]; set_count];
for (i, set) in ref_format.erasure.sets.iter().enumerate() {
for j in 0..set.len() {
if let Some(Some(err)) = errs.get(i * set_drive_count + j)
&& *err == DiskError::UnformattedDisk
{
let mut fm = FormatV3::new(set_count, set_drive_count);
fm.id = ref_format.id;
fm.format = ref_format.format.clone();
fm.version = ref_format.version.clone();
fm.erasure.this = ref_format.erasure.sets[i][j];
fm.erasure.sets = ref_format.erasure.sets.clone();
fm.erasure.version = ref_format.erasure.version.clone();
fm.erasure.distribution_algo = ref_format.erasure.distribution_algo.clone();
new_formats[i][j] = Some(fm);
}
if let (Some(format), None) = (&formats[i * set_drive_count + j], &errs[i * set_drive_count + j])
&& let Some(info) = &format.disk_info
&& !info.endpoint.is_empty()
{
current_disks_info[i][j] = info.clone();
}
}
}
(new_formats, current_disks_info)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::endpoint::Endpoint;
#[test]
fn formats_to_drives_info_maps_missing_offline_and_ok_states() {
let endpoints = Endpoints::from(vec![
Endpoint::try_from("/tmp/rustfs-set-heal-a").unwrap(),
Endpoint::try_from("/tmp/rustfs-set-heal-b").unwrap(),
Endpoint::try_from("/tmp/rustfs-set-heal-c").unwrap(),
]);
let format = FormatV3::new(1, 1);
let uuid = format.erasure.this.to_string();
let drives = formats_to_drives_info(
&endpoints,
&[Some(format), None, None],
&[None, Some(DiskError::UnformattedDisk), Some(DiskError::DiskNotFound)],
);
assert_eq!(drives.len(), 3);
assert_eq!(drives[0].uuid, uuid);
assert_eq!(drives[0].state, DriveState::Ok.to_string());
assert_eq!(drives[1].state, DriveState::Missing.to_string());
assert_eq!(drives[2].state, DriveState::Offline.to_string());
}
#[test]
fn new_heal_format_sets_only_recreates_unformatted_slots() {
let ref_format = FormatV3::new(1, 2);
let existing_format = ref_format.clone();
let (new_formats, current_disks_info) = new_heal_format_sets(
&ref_format,
1,
2,
&[Some(existing_format), None],
&[None, Some(DiskError::UnformattedDisk)],
);
assert!(new_formats[0][0].is_none());
let repaired = new_formats[0][1].as_ref().unwrap();
assert_eq!(repaired.id, ref_format.id);
assert_eq!(repaired.erasure.this, ref_format.erasure.sets[0][1]);
assert_eq!(repaired.erasure.sets, ref_format.erasure.sets);
assert_eq!(current_disks_info.len(), 1);
assert_eq!(current_disks_info[0].len(), 2);
}
}
crates/ecstore/src/sets.rs
+2 -69
View File
@@ -15,9 +15,10 @@
use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result};
use crate::layout::set_heal::{formats_to_drives_info, new_heal_format_sets};
use crate::{
disk::{
DiskAPI, DiskInfo, DiskOption, DiskStore,
DiskAPI, DiskOption, DiskStore,
error::DiskError,
format::{DistributionAlgoVersion, FormatV3},
new_disk,
@@ -1017,74 +1018,6 @@ async fn init_storage_disks_with_errors(
(disks, errs)
}
fn formats_to_drives_info(endpoints: &Endpoints, formats: &[Option<FormatV3>], errs: &[Option<DiskError>]) -> Vec<HealDriveInfo> {
let mut before_drives = Vec::with_capacity(endpoints.as_ref().len());
for (index, format) in formats.iter().enumerate() {
let drive = endpoints.get_string(index);
let state = if format.is_some() {
DriveState::Ok.to_string()
} else if let Some(Some(err)) = errs.get(index) {
if *err == DiskError::UnformattedDisk {
DriveState::Missing.to_string()
} else if *err == DiskError::DiskNotFound {
DriveState::Offline.to_string()
} else {
DriveState::Corrupt.to_string()
}
} else {
DriveState::Corrupt.to_string()
};
let uuid = if let Some(format) = format {
format.erasure.this.to_string()
} else {
"".to_string()
};
before_drives.push(HealDriveInfo {
uuid,
endpoint: drive,
state: state.to_string(),
});
}
before_drives
}
fn new_heal_format_sets(
ref_format: &FormatV3,
set_count: usize,
set_drive_count: usize,
formats: &[Option<FormatV3>],
errs: &[Option<DiskError>],
) -> (Vec<Vec<Option<FormatV3>>>, Vec<Vec<DiskInfo>>) {
let mut new_formats = vec![vec![None; set_drive_count]; set_count];
let mut current_disks_info = vec![vec![DiskInfo::default(); set_drive_count]; set_count];
for (i, set) in ref_format.erasure.sets.iter().enumerate() {
for j in 0..set.len() {
if let Some(Some(err)) = errs.get(i * set_drive_count + j)
&& *err == DiskError::UnformattedDisk
{
let mut fm = FormatV3::new(set_count, set_drive_count);
fm.id = ref_format.id;
fm.format = ref_format.format.clone();
fm.version = ref_format.version.clone();
fm.erasure.this = ref_format.erasure.sets[i][j];
fm.erasure.sets = ref_format.erasure.sets.clone();
fm.erasure.version = ref_format.erasure.version.clone();
fm.erasure.distribution_algo = ref_format.erasure.distribution_algo.clone();
new_formats[i][j] = Some(fm);
}
if let (Some(format), None) = (&formats[i * set_drive_count + j], &errs[i * set_drive_count + j])
&& let Some(info) = &format.disk_info
&& !info.endpoint.is_empty()
{
current_disks_info[i][j] = info.clone();
}
}
}
(new_formats, current_disks_info)
}
#[cfg(test)]
mod tests {
use super::*;
docs/architecture/migration-progress.md
+37 -11
View File
@@ -5,16 +5,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-ecstore-layout-endpoints-move`
- Baseline: merged `E-002/E-LAYOUT-001`.
- Stacked on: merged ECStore layout foundation and format layout ownership
slices.
- Branch: `overtrue/arch-ecstore-layout-set-heal-helpers`
- Baseline: merged `E-003/E-LAYOUT-002`.
- Stacked on: merged ECStore layout foundation, format layout ownership, and
endpoint layout move slices.
- PR type for this branch: `pure-move`
- Runtime behavior changes: none.
- Rust code changes: pure-move ECStore endpoint parsing and endpoint grouping
modules into the internal layout bucket while preserving old public paths.
- Rust code changes: move runtime-neutral ECStore set-format heal helpers into
the internal layout bucket while preserving `Sets` runtime orchestration.
- CI/script changes: none.
- Docs changes: record the ECStore endpoint pure move slice.
- Docs changes: record the ECStore set-format heal helper layout slice.
## Phase 0 Tasks
@@ -2253,10 +2253,24 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
checks, migration/layer guards, formatting, diff hygiene, Rust risk scan,
branch freshness check, pre-commit quality gate, and three-expert review.
- [x] `E-004/E-LAYOUT-003` Move ECStore set-format heal helpers.
- Do: move runtime-neutral set-format heal helper logic into the ECStore
layout bucket while keeping disk initialization and `Sets` orchestration in
`sets.rs`.
- Acceptance: `layout::set_heal` owns drive-info mapping and unformatted
format regeneration helpers, `Sets` keeps the same heal orchestration, and
focused tests cover the extracted helper behavior.
- Must preserve: disk format heal state mapping, unformatted disk format
regeneration, current disk-info preservation, dry-run behavior, save-format
behavior, and all `Sets` runtime control flow.
- Verification: focused ECStore set-heal tests, ECStore/RustFS/Heal compile
checks, migration/layer guards, formatting, diff hygiene, Rust risk scan,
branch freshness check, pre-commit quality gate, and three-expert review.
## Next PRs
1. `pure-move`: continue moving runtime-neutral pool/set layout helpers once
E-003/E-LAYOUT-002 lands.
E-004/E-LAYOUT-003 lands.
2. `pure-move`: continue pruning residual embedded startup-only orchestration
once the lifecycle helpers are merged.
@@ -2264,9 +2278,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | passed | E-003/E-LAYOUT-002 is a pure move into ECStore layout with compatibility stubs at old public paths; no new runtime owner or dependency boundary is introduced. |
| Migration preservation | passed | Endpoint parsing, local-host detection, pool/set/disk indexes, endpoint grouping, disk independence checks, setup type classification, and old public module paths remain preserved. |
| Testing/verification | passed | Focused endpoint/layout checks, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. |
| Quality/architecture | passed | E-004/E-LAYOUT-003 extracts runtime-neutral set-format heal helpers into ECStore layout without moving disk init or `Sets` orchestration. |
| Migration preservation | passed | Drive-info state mapping, unformatted format regeneration, current disk-info preservation, dry-run/save behavior, and `Sets` runtime control flow remain preserved. |
| Testing/verification | passed | Focused set-heal helper tests, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. |
## Verification Notes
@@ -2354,6 +2368,18 @@ Passed before push:
- `make pre-commit`: passed.
- Three-expert review: passed.
- Issue #660 E-004/E-LAYOUT-003 current slice:
- `cargo test -p rustfs-ecstore layout::set_heal -- --nocapture`: passed.
- `cargo check -p rustfs-ecstore -p rustfs -p rustfs-heal`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan on changed Rust files: passed; only test-only unwrap
expectations were added around deterministic helper construction.
- `make pre-commit`: passed.
- Three-expert review: passed.
- Issue #660 X-012 current slice:
- `cargo test -p rustfs-extension-schema`: passed.
- `cargo check -p rustfs-extension-schema`: passed.