gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-09 22:59:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): start with unresolved Kubernetes peers (#5460)

Browse Source 
* fix(ecstore): start with unresolved Kubernetes peers

* fix(ecstore): infer Kubernetes endpoint identity safely

* fix(ecstore): fail closed on unsafe format migration

* fix(ecstore): reject poisoned format heal candidates

* fix(ecstore): reject unsafe legacy migration outliers

* fix(ecstore): resume interrupted format migrations

* fix(ecstore): preserve Kubernetes startup compatibility
This commit is contained in:
cxymds
2026-07-30 11:14:38 +08:00
committed by GitHub
parent 2e5cef513f
commit 67904a6c18
30 changed files with 3618 additions and 461 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/mod.rs
+4 -1
View File
@@ -110,7 +110,10 @@ use crate::{
object_api::{GetObjectReader, ObjectInfo, PutObjReader},
// event::name::EventName,
services::event_notification::{EventArgs, send_event},
store::init_format::{get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, save_format_file},
store::init_format::{
formats_match_reference_slots, get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all,
save_format_file,
},
};
use bytes::Bytes;
use bytesize::ByteSize;
crates/ecstore/src/set_disk/ops/heal.rs
+63 -4
View File
@@ -1373,11 +1373,24 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
let disks = self.disks.read().await.clone();
let (formats, errs) = load_format_erasure_all(&disks, true).await;
let ref_format = match get_format_erasure_in_quorum(&formats) {
Ok(format) => format,
if errs.iter().any(|err| {
matches!(
err,
Some(DiskError::InconsistentDisk | DiskError::CorruptedFormat | DiskError::CorruptedBackend)
)
}) {
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
}
let slot_offset = self
.set_index
.checked_mul(self.set_drive_count)
.ok_or_else(|| Error::other("erasure set slot offset overflow"))?;
let ref_format = match get_format_erasure_in_quorum(&formats, slot_offset) {
Ok(format) if format.shared_identity() == self.format.shared_identity() => format,
Ok(_) => return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))),
Err(err) => {
let can_use_cached_layout = count_errs(&errs, &DiskError::UnformattedDisk) > 0
&& formats.iter().flatten().all(|format| self.format.check_other(format).is_ok())
&& formats_match_reference_slots(&formats, &self.format, slot_offset)
&& errs
.iter()
.all(|err| err.is_none() || matches!(err, Some(DiskError::UnformattedDisk)));
@@ -1388,6 +1401,9 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
}
}
};
if !formats_match_reference_slots(&formats, &ref_format, slot_offset) {
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
}
let endpoints = crate::layout::endpoints::Endpoints::from(self.set_endpoints.clone());
let before_drives = crate::layout::set_heal::formats_to_drives_info(&endpoints, &formats, &errs);
@@ -1543,11 +1559,16 @@ mod heal_result_report_tests {
use crate::disk::error::DiskError;
use crate::disk::format::FormatV3;
use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk};
use crate::error::Error;
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage_api_contracts::heal::HealOperations as _;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use crate::{config::storageclass, store::init_format::save_format_file};
use crate::{
config::storageclass,
store::init_format::{load_format_erasure, save_format_file},
};
use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
use std::sync::Arc;
@@ -1863,6 +1884,44 @@ mod heal_result_report_tests {
}
}
#[tokio::test]
async fn format_heal_cached_layout_rejects_a_disk_from_another_slot() {
let mut _temp_dirs = Vec::new();
let mut endpoints = Vec::new();
let mut disks = Vec::new();
for disk_index in 0..3 {
let (temp_dir, mut endpoint, disk) = real_disk().await;
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
_temp_dirs.push(temp_dir);
endpoints.push(endpoint);
disks.push(Some(disk));
}
let set = set_disks_with(disks.clone(), endpoints, 1).await;
let mut wrong_slot = set.format.clone();
wrong_slot.erasure.this = set.format.erasure.sets[0][1];
save_format_file(&disks[0], &Some(wrong_slot))
.await
.expect("wrong-slot format fixture should be saved");
let mut correct_slot = set.format.clone();
correct_slot.erasure.this = set.format.erasure.sets[0][2];
save_format_file(&disks[2], &Some(correct_slot))
.await
.expect("correct format fixture should be saved");
let (_, heal_err) = set
.heal_format(false)
.await
.expect("format heal should report the quorum failure in its result");
assert!(matches!(heal_err, Some(Error::CorruptedFormat)));
let unformatted = load_format_erasure(disks[1].as_ref().expect("second disk should be online"), true)
.await
.expect_err("a rejected fallback must not format the missing slot");
assert_eq!(unformatted, DiskError::UnformattedDisk);
}
// Regression for #955: an offline disk must contribute exactly one drive
// record. Before the fix the offline branch fell through and pushed a second
// (Corrupt) record for the same disk, so `before/after.drives` grew to
crates/ecstore/src/set_disk/ops/locking.rs
+114 -9
View File
@@ -343,20 +343,23 @@ impl SetDisks {
}
};
// The drive's format may place it in a different erasure set than this
// one. Claiming a misplaced drive into `self.disks` would let two sets
// manage the same drive and degrade together, so reject it here
// (backlog#799 B19).
if set_idx != self.set_index {
// Claiming a misplaced drive into `self.disks` would let two slots or
// sets manage the same drive and degrade together (backlog#799 B19).
if set_idx != self.set_index || self.set_endpoints.get(disk_idx) != Some(ep) {
warn!(
"renew_disk: drive {:?} belongs to set {} but is being renewed on set {}; skipping",
ep, set_idx, self.set_index
endpoint = %ep,
format_set_index = set_idx,
format_disk_index = disk_idx,
endpoint_pool_index = ep.pool_idx,
endpoint_set_index = ep.set_idx,
endpoint_disk_index = ep.disk_idx,
expected_pool_index = self.pool_index,
expected_set_index = self.set_index,
"renew_disk rejected a drive whose endpoint and format do not identify the same topology slot"
);
return;
}
// Check that the endpoint matches
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
new_disk.enable_health_check();
@@ -715,6 +718,108 @@ mod tests {
drop(temp_dirs);
}
#[tokio::test]
async fn renew_disk_rejects_a_format_from_another_slot_or_cluster() {
let disk_count = 3;
let format = FormatV3::new(1, disk_count);
let mut temp_dirs = Vec::with_capacity(disk_count);
let mut endpoints = Vec::with_capacity(disk_count);
let mut fixture_disks = Vec::with_capacity(disk_count);
for disk_idx in 0..disk_count {
let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
temp_dirs.push(temp_dir);
endpoints.push(endpoint);
fixture_disks.push(disk);
}
let set_disks = SetDisks::new(
"test-owner".to_string(),
Arc::new(RwLock::new(vec![Some(fixture_disks[0].clone()), None, None])),
disk_count,
disk_count / 2,
0,
0,
endpoints.clone(),
format.clone(),
Vec::new(),
)
.await;
let mut other_cluster_format = format.clone();
other_cluster_format.id = Uuid::new_v4();
other_cluster_format.erasure.this = format.erasure.sets[0][2];
save_format_file(&Some(fixture_disks[2].clone()), &Some(other_cluster_format))
.await
.expect("other-cluster format should be written for the rejection test");
set_disks.renew_disk(&endpoints[2]).await;
let disks = set_disks.get_disks_internal().await;
assert_eq!(
disks[0]
.as_ref()
.expect("the canonical first slot must remain attached")
.endpoint(),
endpoints[0]
);
assert!(
disks[2].is_none(),
"a disk from another deployment must remain detached even when its slot UUID matches"
);
let mut correct_format = format.clone();
correct_format.erasure.this = format.erasure.sets[0][2];
let replacement_disk = new_disk(
&endpoints[2],
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("third endpoint should reopen after other-cluster rejection");
save_format_file(&Some(replacement_disk), &Some(correct_format))
.await
.expect("correct slot format should be restored");
set_disks.renew_disk(&endpoints[2]).await;
let disks = set_disks.get_disks_internal().await;
assert_eq!(
disks[0]
.as_ref()
.expect("the canonical first slot must remain attached")
.endpoint(),
endpoints[0]
);
assert_eq!(disks[2].as_ref().expect("the restored third slot should attach").endpoint(), endpoints[2]);
let third_disk = disks[2].clone();
let mut wrong_slot_format = format.clone();
wrong_slot_format.erasure.this = format.erasure.sets[0][0];
save_format_file(&third_disk, &Some(wrong_slot_format))
.await
.expect("wrong-slot format should be written for the rejection test");
set_disks.disks.write().await[2] = None;
let mut misplaced_endpoint = endpoints[2].clone();
misplaced_endpoint.set_disk_index(0);
set_disks.renew_disk(&misplaced_endpoint).await;
let disks = set_disks.get_disks_internal().await;
assert_eq!(
disks[0]
.as_ref()
.expect("the canonical first slot must remain attached")
.endpoint(),
endpoints[0]
);
assert!(disks[2].is_none(), "a disk claiming another endpoint's slot must remain detached");
drop(temp_dirs);
}
// SetDisks split P0 (#816): the borrow handle must mirror the core state and
// the List operation family must run identically through it.
#[tokio::test]