gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-09 22:59:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): start with unresolved Kubernetes peers (#5460)

Browse Source 
* fix(ecstore): start with unresolved Kubernetes peers

* fix(ecstore): infer Kubernetes endpoint identity safely

* fix(ecstore): fail closed on unsafe format migration

* fix(ecstore): reject poisoned format heal candidates

* fix(ecstore): reject unsafe legacy migration outliers

* fix(ecstore): resume interrupted format migrations

* fix(ecstore): preserve Kubernetes startup compatibility
This commit is contained in:
cxymds
2026-07-30 11:14:38 +08:00
committed by GitHub
parent 2e5cef513f
commit 67904a6c18
30 changed files with 3618 additions and 461 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/core/sets.rs
+199 -20
View File
@@ -37,7 +37,9 @@ use crate::{
runtime::instance::{InstanceContext, bootstrap_ctx},
runtime::sources as runtime_sources,
set_disk::{PreparedGetObjectMetadata, SetDisks},
store::init_format::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
store::init_format::{
check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
},
};
use futures::{
future::join_all,
@@ -947,7 +949,7 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
#[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
let (disks, _) = init_storage_disks_with_errors(
let (disks, init_errs) = init_storage_disks_with_errors(
&self.endpoints.endpoints,
&DiskOption {
cleanup: false,
@@ -955,15 +957,36 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
},
)
.await;
let (formats, errs) = load_format_erasure_all(&disks, true).await;
let (formats, mut errs) = load_format_erasure_all(&disks, true).await;
for (err, init_err) in errs.iter_mut().zip(init_errs) {
if init_err.is_some() {
*err = init_err;
}
}
if errs.iter().any(|err| {
matches!(
err,
Some(DiskError::InconsistentDisk | DiskError::CorruptedFormat | DiskError::CorruptedBackend)
)
}) {
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
}
if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) {
info!("failed to check formats erasure values: {}", err);
return Ok((HealResultItem::default(), Some(err)));
}
let ref_format = match get_format_erasure_in_quorum(&formats) {
Ok(format) => format,
let (ref_format, quorum_members) = match select_format_erasure_in_quorum(&formats, 0) {
Ok((format, members)) if format.shared_identity() == self.format.shared_identity() => (format, members),
Ok(_) => return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))),
Err(err) => return Ok((HealResultItem::default(), Some(err))),
};
if formats
.iter()
.zip(quorum_members)
.any(|(format, member)| format.is_some() && !member)
{
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
}
let mut res = HealResultItem {
heal_item_type: HealItemType::Metadata.to_string(),
detail: "disk-format".to_string(),
@@ -985,11 +1008,6 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
return Ok((res, Some(StorageError::NoHealRequired)));
}
// if !self.format.eq(&ref_format) {
// info!("format ({:?}) not eq ref_format ({:?})", self.format, ref_format);
// return Ok((res, Some(Error::new(DiskError::CorruptedFormat))));
// }
let (new_format_sets, _) = new_heal_format_sets(&ref_format, self.set_count, self.set_drive_count, &formats, &errs);
if !dry_run {
let mut tmp_new_formats = vec![None; self.set_count * self.set_drive_count];
@@ -1298,7 +1316,7 @@ mod tests {
assert_eq!(result, (Some(3), Some(1), Some(0)));
}
async fn multipart_listing_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
async fn two_set_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
let format = FormatV3::new(2, 2);
let mut temp_dirs = Vec::new();
let mut all_endpoints = Vec::new();
@@ -1339,8 +1357,8 @@ mod tests {
Arc::new(RwLock::new(disks)),
2,
1,
0,
set_index,
0,
endpoints,
format.clone(),
vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())],
@@ -1373,11 +1391,114 @@ mod tests {
(temp_dirs, sets)
}
#[tokio::test]
async fn set_format_heal_accepts_quorum_from_a_nonzero_set() {
let (_temp_dirs, sets) = two_set_test_sets().await;
let (result, err) = sets.disk_set[1]
.heal_format(false)
.await
.expect("the second erasure set should load its own format quorum");
assert!(matches!(err, Some(StorageError::NoHealRequired)), "unexpected heal result: {err:?}");
assert_eq!(result.disk_count, 2);
assert_eq!(result.set_count, 1);
}
#[tokio::test]
async fn format_heal_rejects_foreign_majorities_at_set_and_pool_scopes() {
let (_temp_dirs, _canonical_format, sets) = setup_heal_format_sets(2, true).await;
let set_disks = set_level_heal_view(&sets).await;
let (_, set_err) = set_disks
.heal_format(false)
.await
.expect("set format heal should report a typed mismatch");
assert!(
matches!(set_err, Some(StorageError::CorruptedFormat)),
"foreign set majority must not replace the cached format: {set_err:?}"
);
let (_, pool_err) = sets
.heal_format(false)
.await
.expect("pool format heal should report a typed mismatch");
assert!(
matches!(pool_err, Some(StorageError::CorruptedFormat)),
"foreign pool majority must not replace the cached format: {pool_err:?}"
);
}
#[tokio::test]
async fn pool_format_heal_rejects_a_wrong_slot_minority() {
let (_temp_dirs, canonical_format, sets) = setup_heal_format_sets(3, false).await;
let mut poisoned_format = canonical_format.clone();
poisoned_format.erasure.this = canonical_format.erasure.sets[0][0];
replace_heal_test_format(&sets, 2, &poisoned_format).await;
let probe_err = new_disk(
&sets.endpoints.endpoints.as_ref()[2],
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect_err("a wrong-slot local format must fail disk initialization");
assert_eq!(probe_err, DiskError::InconsistentDisk);
let (_, pool_err) = sets
.heal_format(false)
.await
.expect("pool format heal should report a typed slot mismatch");
assert!(
matches!(pool_err, Some(StorageError::CorruptedFormat)),
"a wrong-slot minority must not be reported as no-heal-required: {pool_err:?}"
);
assert_eq!(
read_heal_test_format(&sets, 2).await,
poisoned_format,
"format heal must not overwrite a wrong-slot disk"
);
}
#[tokio::test]
async fn format_heal_rejects_a_foreign_minority_at_set_and_pool_scopes() {
let (_temp_dirs, canonical_format, sets) = setup_heal_format_sets(3, false).await;
let mut poisoned_format = canonical_format.clone();
poisoned_format.id = Uuid::new_v4();
poisoned_format.erasure.this = poisoned_format.erasure.sets[0][2];
replace_heal_test_format(&sets, 2, &poisoned_format).await;
let set_disks = set_level_heal_view(&sets).await;
let (_, set_err) = set_disks
.heal_format(false)
.await
.expect("set format heal should report a typed identity mismatch");
assert!(
matches!(set_err, Some(StorageError::CorruptedFormat)),
"a foreign minority must not be reported as no-heal-required: {set_err:?}"
);
let (_, pool_err) = sets
.heal_format(false)
.await
.expect("pool format heal should report a typed identity mismatch");
assert!(
matches!(pool_err, Some(StorageError::CorruptedFormat)),
"a foreign minority must not be reported as no-heal-required: {pool_err:?}"
);
assert_eq!(
read_heal_test_format(&sets, 2).await,
poisoned_format,
"format heal must not overwrite a foreign disk"
);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn list_multipart_uploads_merges_all_sets_without_pagination_loss() {
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await;
let (_temp_dirs, sets) = multipart_listing_test_sets().await;
let (_temp_dirs, sets) = two_set_test_sets().await;
let bucket = format!("multipart-list-{}", Uuid::new_v4().simple());
sets.make_bucket(&bucket, &MakeBucketOptions::default())
.await
@@ -1616,11 +1737,15 @@ mod tests {
// formatting the first `num_formatted` of them against a shared reference
// format and leaving the rest unformatted. Returns the live TempDir handles
// (must be kept alive), the reference format, and the assembled `Sets`.
// `disk_set` is intentionally empty: these tests only drive `heal_format`
// with `dry_run == true`, which never touches `disk_set`.
async fn setup_heal_format_sets(num_formatted: usize) -> (Vec<tempfile::TempDir>, FormatV3, Sets) {
// `disk_set` is intentionally empty: these tests only exercise paths that
// return before pool-level healing delegates into a set.
async fn setup_heal_format_sets(num_formatted: usize, foreign_identity: bool) -> (Vec<tempfile::TempDir>, FormatV3, Sets) {
const SET_DRIVE_COUNT: usize = 3;
let ref_format = FormatV3::new(1, SET_DRIVE_COUNT);
let mut stored_format = ref_format.clone();
if foreign_identity {
stored_format.id = Uuid::new_v4();
}
let mut dirs = Vec::with_capacity(SET_DRIVE_COUNT);
let mut endpoints = Vec::with_capacity(SET_DRIVE_COUNT);
@@ -1645,8 +1770,8 @@ mod tests {
)
.await
.expect("disk should be created");
let mut disk_format = ref_format.clone();
disk_format.erasure.this = ref_format.erasure.sets[0][i];
let mut disk_format = stored_format.clone();
disk_format.erasure.this = stored_format.erasure.sets[0][i];
save_format_file(&Some(disk), &Some(disk_format))
.await
.expect("format should be saved");
@@ -1677,6 +1802,60 @@ mod tests {
(dirs, ref_format, sets)
}
async fn set_level_heal_view(sets: &Sets) -> Arc<SetDisks> {
let endpoints = sets.endpoints.endpoints.as_ref().clone();
let mut disks = Vec::with_capacity(endpoints.len());
for endpoint in &endpoints {
disks.push(Some(
new_disk(
endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("fresh set-level disk handle should open"),
));
}
SetDisks::new(
"test-owner".to_string(),
Arc::new(RwLock::new(disks)),
endpoints.len(),
1,
0,
0,
endpoints,
sets.format.clone(),
Vec::new(),
)
.await
}
async fn replace_heal_test_format(sets: &Sets, disk_index: usize, format: &FormatV3) {
let disk = new_disk(
&sets.endpoints.endpoints.as_ref()[disk_index],
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("heal test disk should open");
save_format_file(&Some(disk.clone()), &Some(format.clone()))
.await
.expect("poisoned test format should be written");
}
async fn read_heal_test_format(sets: &Sets, disk_index: usize) -> FormatV3 {
let path = std::path::Path::new(&sets.endpoints.endpoints.as_ref()[disk_index].get_file_path())
.join(crate::disk::RUSTFS_META_BUCKET)
.join(crate::disk::FORMAT_CONFIG_FILE);
let data = tokio::fs::read(path).await.expect("test format should be readable");
FormatV3::try_from(data.as_slice()).expect("test format should parse")
}
// Regression for #956 (NoHealRequired path): with every disk already
// formatted, `heal_format` reports exactly one drive record per disk
// (N = set_count * set_drive_count), each carrying a real endpoint. Before
@@ -1685,7 +1864,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn heal_format_no_heal_required_reports_one_record_per_disk() {
let (_dirs, _ref_format, sets) = setup_heal_format_sets(3).await;
let (_dirs, _ref_format, sets) = setup_heal_format_sets(3, false).await;
let (res, err) = sets.heal_format(true).await.expect("heal_format should succeed");
// All disks formatted -> NoHealRequired early return, still returns `res`.
@@ -1715,7 +1894,7 @@ mod tests {
#[serial]
async fn heal_format_heal_path_reports_one_record_per_disk_aligned() {
// Disks 0 and 1 formatted (quorum), disk 2 unformatted.
let (_dirs, _ref_format, sets) = setup_heal_format_sets(2).await;
let (_dirs, _ref_format, sets) = setup_heal_format_sets(2, false).await;
let (res, err) = sets.heal_format(true).await.expect("heal_format should succeed");
// Unformatted disk present -> heal path, not NoHealRequired.