mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-20 19:42:17 +00:00
fix(heal): respect scoped object repair limits (#5855)
This commit is contained in:
@@ -21,7 +21,27 @@ const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||
const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
|
||||
const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
|
||||
|
||||
fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
|
||||
StorageError::InvalidArgument(
|
||||
"heal".to_string(),
|
||||
"pool".to_string(),
|
||||
format!("invalid heal pool index {pool_idx} for {pool_count} pools"),
|
||||
)
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
|
||||
match opts.pool {
|
||||
Some(pool_idx) => Ok(vec![
|
||||
self.pools
|
||||
.get(pool_idx)
|
||||
.cloned()
|
||||
.ok_or_else(|| invalid_heal_pool_index(pool_idx, self.pools.len()))?,
|
||||
]),
|
||||
None => Ok(self.pools.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
let mut r = HealResultItem {
|
||||
@@ -105,8 +125,10 @@ impl ECStore {
|
||||
);
|
||||
let object = encode_dir_object(object);
|
||||
|
||||
let mut futures = Vec::with_capacity(self.pools.len());
|
||||
for pool in self.pools.iter() {
|
||||
let pools = self.get_pools_for_heal_object(opts)?;
|
||||
|
||||
let mut futures = Vec::with_capacity(pools.len());
|
||||
for pool in pools.iter() {
|
||||
if self.is_suspended(pool.pool_idx).await {
|
||||
continue;
|
||||
}
|
||||
@@ -178,6 +200,82 @@ mod tests {
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
|
||||
use crate::store::init_format::{load_format_erasure, save_format_file};
|
||||
|
||||
async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx);
|
||||
let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(pool_idx);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
|
||||
Sets::new(
|
||||
vec![None],
|
||||
&PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: Endpoints::from(vec![endpoint]),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
},
|
||||
&format,
|
||||
pool_idx,
|
||||
0,
|
||||
)
|
||||
.await
|
||||
.expect("minimal pool should build")
|
||||
}
|
||||
|
||||
async fn minimal_heal_store() -> ECStore {
|
||||
ECStore {
|
||||
id: Uuid::new_v4(),
|
||||
disk_map: HashMap::new(),
|
||||
pools: vec![minimal_heal_pool(0).await, minimal_heal_pool(1).await],
|
||||
peer_sys: S3PeerSys {
|
||||
clients: Vec::new(),
|
||||
pools_count: 2,
|
||||
},
|
||||
pool_meta: RwLock::new(PoolMeta::default()),
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_pool_scope_selects_only_requested_pool() {
|
||||
let store = minimal_heal_store().await;
|
||||
let pools = store
|
||||
.get_pools_for_heal_object(&HealOpts {
|
||||
pool: Some(1),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("requested pool should be selected");
|
||||
|
||||
assert_eq!(pools.len(), 1);
|
||||
assert!(Arc::ptr_eq(&pools[0], &store.pools[1]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_pool_scope_rejects_invalid_pool() {
|
||||
let store = minimal_heal_store().await;
|
||||
let err = store
|
||||
.get_pools_for_heal_object(&HealOpts {
|
||||
pool: Some(2),
|
||||
..Default::default()
|
||||
})
|
||||
.expect_err("out-of-range pool scope must fail closed");
|
||||
|
||||
assert!(
|
||||
matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
|
||||
if field == "pool" && reason.contains("invalid heal pool index 2 for 2 pools")),
|
||||
"unexpected invalid pool error: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handle_heal_format_continues_after_a_pool_error() {
|
||||
let canonical_format = FormatV3::new(1, 3);
|
||||
|
||||
Reference in New Issue
Block a user