fix(heal): track root erasure-set rebuild status (#3625)

This commit is contained in:
Henry Guo
2026-06-19 21:33:58 +08:00
committed by GitHub
parent 68a061c548
commit 2052336a53
3 changed files with 227 additions and 8 deletions
+53 -5
View File
@@ -262,7 +262,13 @@ impl HealChannelProcessor {
"Heal query received"
);
let (summary, detail, items) = match self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await {
let report = if heal_path.trim_matches('/').is_empty() {
self.heal_manager.get_task_report(&client_token).await
} else {
self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await
};
let (summary, detail, items) = match report {
Ok(HealTaskReport {
status: HealTaskStatus::Pending | HealTaskStatus::Running,
result_items,
@@ -291,7 +297,9 @@ impl HealChannelProcessor {
status: HealTaskStatus::Failed { error },
result_items,
}) => ("stopped".to_string(), Some(error), result_items),
Err(crate::Error::TaskNotFound { .. }) => ("finished".to_string(), None, Vec::new()),
Err(crate::Error::TaskNotFound { .. }) => {
("notFound".to_string(), Some("heal task not found or expired".to_string()), Vec::new())
}
Err(crate::Error::InvalidClientToken) => {
let response = HealChannelResponse {
request_id: client_token,
@@ -908,7 +916,7 @@ mod tests {
}
#[tokio::test]
async fn test_process_query_request_reports_finished_when_task_is_not_active() {
async fn test_process_query_request_reports_not_found_when_task_is_unknown() {
let heal_manager = create_test_heal_manager();
let processor = HealChannelProcessor::new(heal_manager);
let (tx, rx) = oneshot::channel();
@@ -924,10 +932,11 @@ mod tests {
.expect("query response should be returned");
assert!(response.success);
assert_eq!(response.request_id, "completed-token");
assert_eq!(response.error.as_deref(), Some("heal task not found or expired"));
let payload: serde_json::Value =
serde_json::from_slice(response.data.as_deref().expect("status payload should be present"))
.expect("status payload should be json");
assert_eq!(payload["summary"], "finished");
assert_eq!(payload["summary"], "notFound");
assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
}
@@ -1018,7 +1027,46 @@ mod tests {
let payload: serde_json::Value =
serde_json::from_slice(response.data.as_deref().expect("status payload should be present"))
.expect("status payload should be json");
assert_eq!(payload["summary"], "finished");
assert_eq!(payload["summary"], "notFound");
assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
}
#[tokio::test]
async fn test_process_query_request_empty_path_uses_client_token_directly() {
let heal_manager = create_test_heal_manager();
let request = HealRequest::new(
HealType::ErasureSet {
buckets: vec![],
set_disk_id: "pool_0_set_1".to_string(),
},
HealOptions::default(),
HealPriority::High,
);
let task_id = request.id.clone();
heal_manager
.submit_heal_request(request)
.await
.expect("request should be accepted");
let processor = HealChannelProcessor::new(heal_manager);
let (tx, rx) = oneshot::channel();
processor
.process_query_request(String::new(), task_id.clone(), tx)
.await
.expect("query should process");
let response = rx
.await
.expect("oneshot should resolve")
.expect("query response should be returned");
assert!(response.success);
assert_eq!(response.request_id, task_id);
assert!(response.error.is_none());
let payload: serde_json::Value =
serde_json::from_slice(response.data.as_deref().expect("status payload should be present"))
.expect("status payload should be json");
assert_eq!(payload["summary"], "running");
assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
}
+88
View File
@@ -1200,6 +1200,65 @@ impl HealManager {
})
}
pub async fn get_task_report(&self, task_id: &str) -> Result<HealTaskReport> {
let canonical_task_id = self.canonical_task_id(task_id).await;
{
let active_heals = self.active_heals.lock().await;
if let Some(task) = active_heals.get(&canonical_task_id) {
return Ok(HealTaskReport {
status: task.get_status().await,
result_items: task.get_result_items().await,
});
}
}
{
let retrying_heals = self.retrying_heals.lock().await;
if let Some(retrying) = retrying_heals.get(&canonical_task_id) {
return Ok(HealTaskReport {
status: retrying.status(),
result_items: Vec::new(),
});
}
}
{
let mut completed_heals = self.completed_heals.lock().await;
prune_completed_heal_statuses(&mut completed_heals);
if let Some(completed) = completed_heals.get(&canonical_task_id)
&& completed_status_is_retrying(&completed.status)
{
return Ok(HealTaskReport {
status: completed.status.clone(),
result_items: completed.result_items.clone(),
});
}
}
{
let queue = self.heal_queue.lock().await;
if queue.contains_request_id(&canonical_task_id) {
return Ok(HealTaskReport {
status: HealTaskStatus::Pending,
result_items: Vec::new(),
});
}
}
let mut completed_heals = self.completed_heals.lock().await;
prune_completed_heal_statuses(&mut completed_heals);
if let Some(completed) = completed_heals.get(&canonical_task_id) {
return Ok(HealTaskReport {
status: completed.status.clone(),
result_items: completed.result_items.clone(),
});
}
Err(Error::TaskNotFound {
task_id: task_id.to_string(),
})
}
pub async fn get_task_report_for_path(&self, heal_path: &str, task_id: &str) -> Result<HealTaskReport> {
let canonical_task_id = self.canonical_task_id(task_id).await;
{
@@ -3245,6 +3304,35 @@ mod tests {
));
}
#[tokio::test]
async fn test_get_task_report_queries_queued_task_by_token_without_path() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let manager = HealManager::new(storage, None);
let request = HealRequest::new(
HealType::ErasureSet {
buckets: vec![],
set_disk_id: "pool_0_set_1".to_string(),
},
HealOptions::default(),
HealPriority::High,
);
let request_id = request.id.clone();
manager
.submit_heal_request(request)
.await
.expect("request should be accepted");
let report = manager
.get_task_report(&request_id)
.await
.expect("queued task should be queryable by token");
assert_eq!(report.status, HealTaskStatus::Pending);
assert!(report.result_items.is_empty());
}
#[tokio::test]
async fn test_get_task_status_reads_recent_completed_status() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);