mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-10 15:16:56 +00:00
fix(heal): track root erasure-set rebuild status (#3625)
This commit is contained in:
@@ -262,7 +262,13 @@ impl HealChannelProcessor {
|
||||
"Heal query received"
|
||||
);
|
||||
|
||||
let (summary, detail, items) = match self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await {
|
||||
let report = if heal_path.trim_matches('/').is_empty() {
|
||||
self.heal_manager.get_task_report(&client_token).await
|
||||
} else {
|
||||
self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await
|
||||
};
|
||||
|
||||
let (summary, detail, items) = match report {
|
||||
Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Pending | HealTaskStatus::Running,
|
||||
result_items,
|
||||
@@ -291,7 +297,9 @@ impl HealChannelProcessor {
|
||||
status: HealTaskStatus::Failed { error },
|
||||
result_items,
|
||||
}) => ("stopped".to_string(), Some(error), result_items),
|
||||
Err(crate::Error::TaskNotFound { .. }) => ("finished".to_string(), None, Vec::new()),
|
||||
Err(crate::Error::TaskNotFound { .. }) => {
|
||||
("notFound".to_string(), Some("heal task not found or expired".to_string()), Vec::new())
|
||||
}
|
||||
Err(crate::Error::InvalidClientToken) => {
|
||||
let response = HealChannelResponse {
|
||||
request_id: client_token,
|
||||
@@ -908,7 +916,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_process_query_request_reports_finished_when_task_is_not_active() {
|
||||
async fn test_process_query_request_reports_not_found_when_task_is_unknown() {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
let processor = HealChannelProcessor::new(heal_manager);
|
||||
let (tx, rx) = oneshot::channel();
|
||||
@@ -924,10 +932,11 @@ mod tests {
|
||||
.expect("query response should be returned");
|
||||
assert!(response.success);
|
||||
assert_eq!(response.request_id, "completed-token");
|
||||
assert_eq!(response.error.as_deref(), Some("heal task not found or expired"));
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(response.data.as_deref().expect("status payload should be present"))
|
||||
.expect("status payload should be json");
|
||||
assert_eq!(payload["summary"], "finished");
|
||||
assert_eq!(payload["summary"], "notFound");
|
||||
assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
|
||||
}
|
||||
|
||||
@@ -1018,7 +1027,46 @@ mod tests {
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(response.data.as_deref().expect("status payload should be present"))
|
||||
.expect("status payload should be json");
|
||||
assert_eq!(payload["summary"], "finished");
|
||||
assert_eq!(payload["summary"], "notFound");
|
||||
assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_process_query_request_empty_path_uses_client_token_directly() {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec![],
|
||||
set_disk_id: "pool_0_set_1".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
);
|
||||
let task_id = request.id.clone();
|
||||
heal_manager
|
||||
.submit_heal_request(request)
|
||||
.await
|
||||
.expect("request should be accepted");
|
||||
|
||||
let processor = HealChannelProcessor::new(heal_manager);
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
processor
|
||||
.process_query_request(String::new(), task_id.clone(), tx)
|
||||
.await
|
||||
.expect("query should process");
|
||||
|
||||
let response = rx
|
||||
.await
|
||||
.expect("oneshot should resolve")
|
||||
.expect("query response should be returned");
|
||||
assert!(response.success);
|
||||
assert_eq!(response.request_id, task_id);
|
||||
assert!(response.error.is_none());
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(response.data.as_deref().expect("status payload should be present"))
|
||||
.expect("status payload should be json");
|
||||
assert_eq!(payload["summary"], "running");
|
||||
assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
|
||||
}
|
||||
|
||||
|
||||
@@ -1200,6 +1200,65 @@ impl HealManager {
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn get_task_report(&self, task_id: &str) -> Result<HealTaskReport> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
{
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
if let Some(task) = active_heals.get(&canonical_task_id) {
|
||||
return Ok(HealTaskReport {
|
||||
status: task.get_status().await,
|
||||
result_items: task.get_result_items().await,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let retrying_heals = self.retrying_heals.lock().await;
|
||||
if let Some(retrying) = retrying_heals.get(&canonical_task_id) {
|
||||
return Ok(HealTaskReport {
|
||||
status: retrying.status(),
|
||||
result_items: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let mut completed_heals = self.completed_heals.lock().await;
|
||||
prune_completed_heal_statuses(&mut completed_heals);
|
||||
if let Some(completed) = completed_heals.get(&canonical_task_id)
|
||||
&& completed_status_is_retrying(&completed.status)
|
||||
{
|
||||
return Ok(HealTaskReport {
|
||||
status: completed.status.clone(),
|
||||
result_items: completed.result_items.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let queue = self.heal_queue.lock().await;
|
||||
if queue.contains_request_id(&canonical_task_id) {
|
||||
return Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Pending,
|
||||
result_items: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let mut completed_heals = self.completed_heals.lock().await;
|
||||
prune_completed_heal_statuses(&mut completed_heals);
|
||||
if let Some(completed) = completed_heals.get(&canonical_task_id) {
|
||||
return Ok(HealTaskReport {
|
||||
status: completed.status.clone(),
|
||||
result_items: completed.result_items.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
Err(Error::TaskNotFound {
|
||||
task_id: task_id.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn get_task_report_for_path(&self, heal_path: &str, task_id: &str) -> Result<HealTaskReport> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
{
|
||||
@@ -3245,6 +3304,35 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_task_report_queries_queued_task_by_token_without_path() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec![],
|
||||
set_disk_id: "pool_0_set_1".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
);
|
||||
let request_id = request.id.clone();
|
||||
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
.await
|
||||
.expect("request should be accepted");
|
||||
|
||||
let report = manager
|
||||
.get_task_report(&request_id)
|
||||
.await
|
||||
.expect("queued task should be queryable by token");
|
||||
|
||||
assert_eq!(report.status, HealTaskStatus::Pending);
|
||||
assert!(report.result_items.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_task_status_reads_recent_completed_status() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
|
||||
Reference in New Issue
Block a user