fix(heal): respect scoped object repair limits (#5855)

This commit is contained in:
cxymds
2026-08-09 07:23:16 +08:00
committed by GitHub
parent 134081b27b
commit b4133d69e6
3 changed files with 230 additions and 10 deletions
+100 -2
View File
@@ -21,7 +21,27 @@ const LOG_SUBSYSTEM_HEAL: &str = "heal";
const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
StorageError::InvalidArgument(
"heal".to_string(),
"pool".to_string(),
format!("invalid heal pool index {pool_idx} for {pool_count} pools"),
)
}
impl ECStore {
fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
match opts.pool {
Some(pool_idx) => Ok(vec![
self.pools
.get(pool_idx)
.cloned()
.ok_or_else(|| invalid_heal_pool_index(pool_idx, self.pools.len()))?,
]),
None => Ok(self.pools.clone()),
}
}
#[instrument(skip(self))]
pub(super) async fn handle_heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
let mut r = HealResultItem {
@@ -105,8 +125,10 @@ impl ECStore {
);
let object = encode_dir_object(object);
let mut futures = Vec::with_capacity(self.pools.len());
for pool in self.pools.iter() {
let pools = self.get_pools_for_heal_object(opts)?;
let mut futures = Vec::with_capacity(pools.len());
for pool in pools.iter() {
if self.is_suspended(pool.pool_idx).await {
continue;
}
@@ -178,6 +200,82 @@ mod tests {
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
use crate::store::init_format::{load_format_erasure, save_format_file};
async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
let format = FormatV3::new(1, 1);
let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx);
let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
endpoint.set_pool_index(pool_idx);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
Sets::new(
vec![None],
&PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 1,
endpoints: Endpoints::from(vec![endpoint]),
cmd_line: String::new(),
platform: String::new(),
},
&format,
pool_idx,
0,
)
.await
.expect("minimal pool should build")
}
async fn minimal_heal_store() -> ECStore {
ECStore {
id: Uuid::new_v4(),
disk_map: HashMap::new(),
pools: vec![minimal_heal_pool(0).await, minimal_heal_pool(1).await],
peer_sys: S3PeerSys {
clients: Vec::new(),
pools_count: 2,
},
pool_meta: RwLock::new(PoolMeta::default()),
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: Mutex::new(()),
pool_meta_save_gate: Mutex::new(()),
ctx: crate::runtime::instance::bootstrap_ctx(),
bucket_fence_registry: std::sync::Arc::default(),
}
}
#[tokio::test]
async fn heal_object_pool_scope_selects_only_requested_pool() {
let store = minimal_heal_store().await;
let pools = store
.get_pools_for_heal_object(&HealOpts {
pool: Some(1),
..Default::default()
})
.expect("requested pool should be selected");
assert_eq!(pools.len(), 1);
assert!(Arc::ptr_eq(&pools[0], &store.pools[1]));
}
#[tokio::test]
async fn heal_object_pool_scope_rejects_invalid_pool() {
let store = minimal_heal_store().await;
let err = store
.get_pools_for_heal_object(&HealOpts {
pool: Some(2),
..Default::default()
})
.expect_err("out-of-range pool scope must fail closed");
assert!(
matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
if field == "pool" && reason.contains("invalid heal pool index 2 for 2 pools")),
"unexpected invalid pool error: {err:?}"
);
}
#[tokio::test]
async fn handle_heal_format_continues_after_a_pool_error() {
let canonical_format = FormatV3::new(1, 3);