mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-17 18:27:49 +00:00
fix(heal): aggregate status across cluster nodes (#4990)
* fix(rpc): bind internode auth to exact targets * fix(heal): initialize the runtime atomically * fix(heal): aggregate status across cluster nodes --------- Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
cxymds
@@ -243,6 +243,19 @@ pub enum HealAdmissionResult {
|
||||
Dropped(HealAdmissionDropReason),
|
||||
}
|
||||
|
||||
/// Admission decision together with the canonical task identifier.
|
||||
///
|
||||
/// A merged request must return the identifier of the task that already owns
|
||||
/// the work instead of exposing the discarded request identifier as a new
|
||||
/// client token.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HealAdmissionReceipt {
|
||||
/// Admission decision for the submitted request.
|
||||
pub result: HealAdmissionResult,
|
||||
/// Canonical identifier of the accepted or merged task.
|
||||
pub task_id: String,
|
||||
}
|
||||
|
||||
impl HealAdmissionResult {
|
||||
pub fn result_label(self) -> &'static str {
|
||||
match self {
|
||||
@@ -382,8 +395,25 @@ pub type HealChannelSender = mpsc::UnboundedSender<HealChannelCommand>;
|
||||
/// Heal channel receiver
|
||||
pub type HealChannelReceiver = mpsc::UnboundedReceiver<HealChannelCommand>;
|
||||
|
||||
/// Canonical-receipt start command kept separate from the legacy public enum.
|
||||
#[derive(Debug)]
|
||||
pub struct HealReceiptCommand {
|
||||
/// Heal request to admit.
|
||||
pub request: HealChannelRequest,
|
||||
/// Completion channel for the admission receipt.
|
||||
pub response_tx: oneshot::Sender<Result<HealAdmissionReceipt, String>>,
|
||||
}
|
||||
|
||||
/// Canonical-receipt command receiver.
|
||||
pub type HealReceiptReceiver = mpsc::UnboundedReceiver<HealReceiptCommand>;
|
||||
|
||||
struct HealChannelSenders {
|
||||
command: HealChannelSender,
|
||||
receipt: mpsc::UnboundedSender<HealReceiptCommand>,
|
||||
}
|
||||
|
||||
/// Global heal channel sender
|
||||
static GLOBAL_HEAL_CHANNEL_SENDER: OnceLock<HealChannelSender> = OnceLock::new();
|
||||
static GLOBAL_HEAL_CHANNEL_SENDERS: OnceLock<HealChannelSenders> = OnceLock::new();
|
||||
|
||||
type HealResponseSender = broadcast::Sender<HealChannelResponse>;
|
||||
|
||||
@@ -392,17 +422,24 @@ static GLOBAL_HEAL_RESPONSE_SENDER: OnceLock<HealResponseSender> = OnceLock::new
|
||||
|
||||
/// Initialize global heal channel
|
||||
pub fn init_heal_channel() -> Result<HealChannelReceiver, &'static str> {
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
if GLOBAL_HEAL_CHANNEL_SENDER.set(tx).is_ok() {
|
||||
Ok(rx)
|
||||
} else {
|
||||
Err("Heal channel sender already initialized")
|
||||
}
|
||||
let (receiver, receipt_receiver) = init_heal_channels()?;
|
||||
drop(receipt_receiver);
|
||||
Ok(receiver)
|
||||
}
|
||||
|
||||
/// Initialize the legacy command and canonical-receipt channels atomically.
|
||||
pub fn init_heal_channels() -> Result<(HealChannelReceiver, HealReceiptReceiver), &'static str> {
|
||||
let (command, command_receiver) = mpsc::unbounded_channel();
|
||||
let (receipt, receipt_receiver) = mpsc::unbounded_channel();
|
||||
GLOBAL_HEAL_CHANNEL_SENDERS
|
||||
.set(HealChannelSenders { command, receipt })
|
||||
.map_err(|_| "Heal channel sender already initialized")?;
|
||||
Ok((command_receiver, receipt_receiver))
|
||||
}
|
||||
|
||||
/// Get global heal channel sender
|
||||
pub fn get_heal_channel_sender() -> Option<&'static HealChannelSender> {
|
||||
GLOBAL_HEAL_CHANNEL_SENDER.get()
|
||||
GLOBAL_HEAL_CHANNEL_SENDERS.get().map(|senders| &senders.command)
|
||||
}
|
||||
|
||||
/// Send heal command through global channel
|
||||
@@ -436,6 +473,21 @@ pub fn subscribe_heal_responses() -> broadcast::Receiver<HealChannelResponse> {
|
||||
heal_response_sender().subscribe()
|
||||
}
|
||||
|
||||
/// Send heal start request and wait for structured admission feedback.
|
||||
pub async fn send_heal_request_with_receipt(request: HealChannelRequest) -> Result<HealAdmissionReceipt, String> {
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
let senders = GLOBAL_HEAL_CHANNEL_SENDERS
|
||||
.get()
|
||||
.ok_or_else(|| "Heal channel not initialized".to_string())?;
|
||||
senders
|
||||
.receipt
|
||||
.send(HealReceiptCommand { request, response_tx })
|
||||
.map_err(|err| format!("Failed to send heal receipt command: {err}"))?;
|
||||
response_rx
|
||||
.await
|
||||
.map_err(|e| format!("Failed to receive heal admission response: {e}"))?
|
||||
}
|
||||
|
||||
/// Send heal start request and wait for structured admission feedback.
|
||||
pub async fn send_heal_request_with_admission(request: HealChannelRequest) -> Result<HealAdmissionResult, String> {
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
|
||||
@@ -29,11 +29,11 @@ use rustfs_madmin::{
|
||||
};
|
||||
use rustfs_protos::evict_failed_connection;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
CancelDecommissionRequest, ClearDecommissionRequest, DeleteBucketMetadataRequest, DeletePolicyRequest,
|
||||
DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest, GetLiveEventsRequest, GetMemInfoRequest, GetMetricsRequest,
|
||||
GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest,
|
||||
GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
BackgroundHealStatusRequest, CancelDecommissionRequest, ClearDecommissionRequest, DeleteBucketMetadataRequest,
|
||||
DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest, GetLiveEventsRequest, GetMemInfoRequest,
|
||||
GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest,
|
||||
GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest,
|
||||
LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
|
||||
ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
|
||||
StopRebalanceRequest, node_service_client::NodeServiceClient,
|
||||
@@ -60,6 +60,7 @@ pub const PEER_RESTSUB_SYS: &str = "sub-sys";
|
||||
pub const PEER_RESTDRY_RUN: &str = "dry-run";
|
||||
pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
|
||||
pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
|
||||
const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
|
||||
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
@@ -664,6 +665,38 @@ impl PeerRestClient {
|
||||
Err(Error::NotImplemented)
|
||||
}
|
||||
|
||||
pub async fn background_heal_status(&self) -> Result<Option<Vec<u8>>> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE);
|
||||
let response = match client
|
||||
.background_heal_status(Request::new(BackgroundHealStatusRequest::default()))
|
||||
.await
|
||||
{
|
||||
Ok(response) => response.into_inner(),
|
||||
Err(status) if status.code() == tonic::Code::Unimplemented => {
|
||||
// RUSTFS_COMPAT_TODO(heal-status-rpc-v1): accept old peers without node heal snapshots. Remove after the minimum supported RustFS peer version implements BackgroundHealStatus.
|
||||
return Ok(None);
|
||||
}
|
||||
Err(status) => return Err(status.into()),
|
||||
};
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer background heal status failed without an error".to_string()),
|
||||
));
|
||||
}
|
||||
Ok(Some(response.bg_heal_state.to_vec()))
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
|
||||
+242
-23
@@ -20,8 +20,8 @@ use crate::heal::{
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use rustfs_common::heal_channel::{
|
||||
HealAdmissionResult, HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealChannelResponse,
|
||||
HealRequestSource, HealScanMode, publish_heal_response,
|
||||
HealAdmissionReceipt, HealAdmissionResult, HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest,
|
||||
HealChannelResponse, HealReceiptCommand, HealReceiptReceiver, HealRequestSource, HealScanMode, publish_heal_response,
|
||||
};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use serde::Serialize;
|
||||
@@ -79,8 +79,19 @@ impl HealChannelProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
/// Start processing heal channel requests
|
||||
pub async fn start(&mut self, mut receiver: HealChannelReceiver) -> Result<()> {
|
||||
/// Start processing legacy heal channel requests.
|
||||
pub async fn start(&mut self, receiver: HealChannelReceiver) -> Result<()> {
|
||||
let (receipt_sender, receipt_receiver) = mpsc::unbounded_channel();
|
||||
drop(receipt_sender);
|
||||
self.start_with_receipts(receiver, receipt_receiver).await
|
||||
}
|
||||
|
||||
/// Start processing legacy and canonical-receipt heal channel requests.
|
||||
pub async fn start_with_receipts(
|
||||
&mut self,
|
||||
mut receiver: HealChannelReceiver,
|
||||
mut receipt_receiver: HealReceiptReceiver,
|
||||
) -> Result<()> {
|
||||
info!(
|
||||
target: "rustfs::heal::channel",
|
||||
event = EVENT_HEAL_CHANNEL_STATE,
|
||||
@@ -90,6 +101,7 @@ impl HealChannelProcessor {
|
||||
"Heal channel started"
|
||||
);
|
||||
|
||||
let mut receipt_channel_open = true;
|
||||
loop {
|
||||
tokio::select! {
|
||||
command = receiver.recv() => {
|
||||
@@ -120,6 +132,23 @@ impl HealChannelProcessor {
|
||||
}
|
||||
}
|
||||
}
|
||||
command = receipt_receiver.recv(), if receipt_channel_open => {
|
||||
let Some(HealReceiptCommand { request, response_tx }) = command else {
|
||||
receipt_channel_open = false;
|
||||
continue;
|
||||
};
|
||||
if let Err(e) = self.process_start_request(request, false, true, response_tx).await {
|
||||
error!(
|
||||
target: "rustfs::heal::channel",
|
||||
event = EVENT_HEAL_CHANNEL_REQUEST,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_CHANNEL,
|
||||
state = "receipt_process_failed",
|
||||
error = %e,
|
||||
"Heal receipt request processing failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
response = self.response_receiver.recv() => {
|
||||
if let Some(response) = response {
|
||||
// Handle response if needed
|
||||
@@ -152,7 +181,7 @@ impl HealChannelProcessor {
|
||||
/// Process heal command
|
||||
async fn process_command(&self, command: HealChannelCommand) -> Result<()> {
|
||||
match command {
|
||||
HealChannelCommand::Start { request, response_tx } => self.process_start_request(request, response_tx).await,
|
||||
HealChannelCommand::Start { request, response_tx } => self.process_legacy_start_request(request, response_tx).await,
|
||||
HealChannelCommand::Query {
|
||||
heal_path,
|
||||
client_token,
|
||||
@@ -166,11 +195,28 @@ impl HealChannelProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_legacy_start_request(
|
||||
&self,
|
||||
request: HealChannelRequest,
|
||||
response_tx: oneshot::Sender<std::result::Result<HealAdmissionResult, String>>,
|
||||
) -> Result<()> {
|
||||
let (receipt_tx, receipt_rx) = oneshot::channel();
|
||||
self.process_start_request(request, true, false, receipt_tx).await?;
|
||||
let result = receipt_rx
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("heal receipt channel closed: {err}")))?
|
||||
.map(|receipt| receipt.result);
|
||||
let _ = response_tx.send(result);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Process start request
|
||||
async fn process_start_request(
|
||||
&self,
|
||||
request: HealChannelRequest,
|
||||
response_tx: oneshot::Sender<std::result::Result<HealAdmissionResult, String>>,
|
||||
preserve_alias: bool,
|
||||
publish_canonical_id: bool,
|
||||
response_tx: oneshot::Sender<std::result::Result<HealAdmissionReceipt, String>>,
|
||||
) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::channel",
|
||||
@@ -201,8 +247,13 @@ impl HealChannelProcessor {
|
||||
};
|
||||
|
||||
// Submit to heal manager
|
||||
match self.heal_manager.submit_heal_request(heal_request).await {
|
||||
Ok(admission) => {
|
||||
match self
|
||||
.heal_manager
|
||||
.submit_heal_request_with_receipt_and_alias(heal_request, preserve_alias)
|
||||
.await
|
||||
{
|
||||
Ok(receipt) => {
|
||||
let admission = receipt.result;
|
||||
debug!(
|
||||
target: "rustfs::heal::channel",
|
||||
event = EVENT_HEAL_CHANNEL_REQUEST,
|
||||
@@ -214,9 +265,13 @@ impl HealChannelProcessor {
|
||||
"Heal admission decided"
|
||||
);
|
||||
|
||||
let _ = response_tx.send(Ok(admission));
|
||||
|
||||
self.publish_response(admission_response(request.id, admission));
|
||||
let response_id = if publish_canonical_id {
|
||||
receipt.task_id.clone()
|
||||
} else {
|
||||
request.id.clone()
|
||||
};
|
||||
self.publish_response(admission_response(response_id, receipt.result));
|
||||
let _ = response_tx.send(Ok(receipt));
|
||||
}
|
||||
Err(e) => {
|
||||
let error_text = e.to_string();
|
||||
@@ -537,6 +592,7 @@ mod tests {
|
||||
HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealRequestSource, HealScanMode,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
// Mock storage for testing
|
||||
struct MockStorage;
|
||||
@@ -1097,27 +1153,190 @@ mod tests {
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
processor
|
||||
.process_start_request(request.clone(), tx)
|
||||
.process_start_request(request.clone(), false, true, tx)
|
||||
.await
|
||||
.expect("first admission should succeed");
|
||||
assert_eq!(
|
||||
rx.await
|
||||
.expect("oneshot should resolve")
|
||||
.expect("admission should be returned"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let first = rx
|
||||
.await
|
||||
.expect("oneshot should resolve")
|
||||
.expect("admission should be returned");
|
||||
assert_eq!(first.result, HealAdmissionResult::Accepted);
|
||||
assert_eq!(first.task_id, "admission-id");
|
||||
|
||||
let mut duplicate = request;
|
||||
duplicate.id = "duplicate-id".to_string();
|
||||
let (tx, rx) = oneshot::channel();
|
||||
processor
|
||||
.process_start_request(duplicate, false, true, tx)
|
||||
.await
|
||||
.expect("duplicate admission should succeed");
|
||||
let merged = rx
|
||||
.await
|
||||
.expect("oneshot should resolve")
|
||||
.expect("admission should be returned");
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, "admission-id");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_legacy_start_preserves_duplicate_alias_and_submitted_response_id() {
|
||||
let manager = create_test_heal_manager();
|
||||
let processor = HealChannelProcessor::new(manager.clone());
|
||||
let request = HealChannelRequest {
|
||||
id: "legacy-original-id".to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
object_prefix: Some("object".to_string()),
|
||||
priority: HealChannelPriority::Low,
|
||||
source: HealRequestSource::Admin,
|
||||
..Default::default()
|
||||
};
|
||||
let mut responses = rustfs_common::heal_channel::subscribe_heal_responses();
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
processor
|
||||
.process_start_request(request, tx)
|
||||
.process_command(HealChannelCommand::Start {
|
||||
request: request.clone(),
|
||||
response_tx: tx,
|
||||
})
|
||||
.await
|
||||
.expect("duplicate admission should succeed");
|
||||
.expect("legacy start should be processed");
|
||||
assert_eq!(
|
||||
rx.await
|
||||
.expect("oneshot should resolve")
|
||||
.expect("admission should be returned"),
|
||||
.expect("legacy response should arrive")
|
||||
.expect("legacy start should succeed"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
|
||||
let mut duplicate = request;
|
||||
duplicate.id = "legacy-duplicate-id".to_string();
|
||||
let (tx, rx) = oneshot::channel();
|
||||
processor
|
||||
.process_command(HealChannelCommand::Start {
|
||||
request: duplicate,
|
||||
response_tx: tx,
|
||||
})
|
||||
.await
|
||||
.expect("legacy duplicate should be processed");
|
||||
assert_eq!(
|
||||
rx.await
|
||||
.expect("legacy duplicate response should arrive")
|
||||
.expect("legacy duplicate should succeed"),
|
||||
HealAdmissionResult::Merged
|
||||
);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let response = responses.recv().await.expect("legacy broadcast should stay open");
|
||||
if response.request_id == "legacy-duplicate-id" {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("legacy broadcast must retain the submitted request id");
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_task_status_for_path("bucket/object", "legacy-duplicate-id")
|
||||
.await
|
||||
.expect("legacy alias should resolve"),
|
||||
HealTaskStatus::Pending
|
||||
);
|
||||
manager
|
||||
.cancel_task("legacy-duplicate-id")
|
||||
.await
|
||||
.expect("legacy alias should cancel the canonical task");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_public_legacy_start_processes_commands_with_receipt_channel_closed() {
|
||||
let manager = create_test_heal_manager();
|
||||
let mut processor = HealChannelProcessor::new(manager);
|
||||
let (command_tx, command_rx) = mpsc::unbounded_channel();
|
||||
let processor_task = tokio::spawn(async move { processor.start(command_rx).await });
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
command_tx
|
||||
.send(HealChannelCommand::Start {
|
||||
request: HealChannelRequest {
|
||||
id: "legacy-start-loop-id".to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
object_prefix: Some("object".to_string()),
|
||||
source: HealRequestSource::Admin,
|
||||
..Default::default()
|
||||
},
|
||||
response_tx,
|
||||
})
|
||||
.expect("legacy command should send");
|
||||
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(1), response_rx)
|
||||
.await
|
||||
.expect("legacy processor should not starve")
|
||||
.expect("legacy response should arrive")
|
||||
.expect("legacy start should succeed"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
drop(command_tx);
|
||||
tokio::time::timeout(Duration::from_secs(1), processor_task)
|
||||
.await
|
||||
.expect("legacy processor should stop when command channel closes")
|
||||
.expect("legacy processor task should join")
|
||||
.expect("legacy processor should stop cleanly");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_receipt_channel_returns_canonical_id_end_to_end() {
|
||||
let manager = create_test_heal_manager();
|
||||
let mut processor = HealChannelProcessor::new(manager);
|
||||
let (command_tx, command_rx) = mpsc::unbounded_channel();
|
||||
let (receipt_tx, receipt_rx) = mpsc::unbounded_channel();
|
||||
let processor_task = tokio::spawn(async move { processor.start_with_receipts(command_rx, receipt_rx).await });
|
||||
let request = HealChannelRequest {
|
||||
id: "receipt-original-id".to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
object_prefix: Some("object".to_string()),
|
||||
source: HealRequestSource::Admin,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
receipt_tx
|
||||
.send(HealReceiptCommand {
|
||||
request: request.clone(),
|
||||
response_tx,
|
||||
})
|
||||
.expect("receipt command should send");
|
||||
let accepted = tokio::time::timeout(Duration::from_secs(1), response_rx)
|
||||
.await
|
||||
.expect("receipt processor should not starve")
|
||||
.expect("receipt response should arrive")
|
||||
.expect("receipt start should succeed");
|
||||
assert_eq!(accepted.result, HealAdmissionResult::Accepted);
|
||||
assert_eq!(accepted.task_id, "receipt-original-id");
|
||||
|
||||
let mut duplicate = request;
|
||||
duplicate.id = "receipt-duplicate-id".to_string();
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
receipt_tx
|
||||
.send(HealReceiptCommand {
|
||||
request: duplicate,
|
||||
response_tx,
|
||||
})
|
||||
.expect("duplicate receipt command should send");
|
||||
let merged = tokio::time::timeout(Duration::from_secs(1), response_rx)
|
||||
.await
|
||||
.expect("duplicate receipt should not starve")
|
||||
.expect("duplicate receipt response should arrive")
|
||||
.expect("duplicate receipt should succeed");
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, "receipt-original-id");
|
||||
|
||||
drop(receipt_tx);
|
||||
drop(command_tx);
|
||||
tokio::time::timeout(Duration::from_secs(1), processor_task)
|
||||
.await
|
||||
.expect("receipt processor should stop when channels close")
|
||||
.expect("receipt processor task should join")
|
||||
.expect("receipt processor should stop cleanly");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1147,7 +1366,7 @@ mod tests {
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
processor
|
||||
.process_start_request(request, tx)
|
||||
.process_start_request(request, false, true, tx)
|
||||
.await
|
||||
.expect("processor should surface invalid request through response channel");
|
||||
assert!(rx.await.expect("oneshot should resolve").is_err());
|
||||
|
||||
+397
-146
@@ -19,9 +19,11 @@ use crate::heal::{
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult, HealRequestSource};
|
||||
use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionReceipt, HealAdmissionResult, HealRequestSource};
|
||||
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
#[cfg(test)]
|
||||
use std::sync::LazyLock;
|
||||
use std::{
|
||||
collections::{BinaryHeap, HashMap, HashSet},
|
||||
sync::Arc,
|
||||
@@ -52,6 +54,54 @@ const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown";
|
||||
const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
|
||||
const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
|
||||
#[cfg(test)]
|
||||
struct RetryOwnershipTestHook {
|
||||
task_id: String,
|
||||
active_to_retrying_reached: Notify,
|
||||
active_to_retrying_release: Notify,
|
||||
retrying_to_queue_reached: Notify,
|
||||
retrying_to_queue_release: Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static RETRY_OWNERSHIP_TEST_HOOK: LazyLock<Mutex<Option<Arc<RetryOwnershipTestHook>>>> = LazyLock::new(|| Mutex::new(None));
|
||||
|
||||
#[cfg(test)]
|
||||
struct DuplicateAdmissionTestHook {
|
||||
request_id: String,
|
||||
active_lock_reached: Notify,
|
||||
active_lock_release: Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DUPLICATE_ADMISSION_TEST_HOOK: LazyLock<Mutex<Option<Arc<DuplicateAdmissionTestHook>>>> =
|
||||
LazyLock::new(|| Mutex::new(None));
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_retry_ownership_transition(task_id: &str, to_queue: bool) {
|
||||
let hook = RETRY_OWNERSHIP_TEST_HOOK.lock().await.clone();
|
||||
let Some(hook) = hook.filter(|hook| hook.task_id == task_id) else {
|
||||
return;
|
||||
};
|
||||
if to_queue {
|
||||
hook.retrying_to_queue_reached.notify_one();
|
||||
hook.retrying_to_queue_release.notified().await;
|
||||
} else {
|
||||
hook.active_to_retrying_reached.notify_one();
|
||||
hook.active_to_retrying_release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_duplicate_admission_after_active_lock(request_id: &str) {
|
||||
let hook = DUPLICATE_ADMISSION_TEST_HOOK.lock().await.clone();
|
||||
let Some(hook) = hook.filter(|hook| hook.request_id == request_id) else {
|
||||
return;
|
||||
};
|
||||
hook.active_lock_reached.notify_one();
|
||||
hook.active_lock_release.notified().await;
|
||||
}
|
||||
|
||||
type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
|
||||
|
||||
/// Priority queue wrapper for heal requests
|
||||
@@ -151,8 +201,8 @@ pub struct HealTaskReport {
|
||||
pub progress: Option<HealProgress>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct HealPriorityCounts {
|
||||
pub low: u64,
|
||||
pub normal: u64,
|
||||
@@ -171,8 +221,8 @@ impl HealPriorityCounts {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct HealSourceCounts {
|
||||
pub scanner: u64,
|
||||
pub admin: u64,
|
||||
@@ -193,15 +243,18 @@ impl HealSourceCounts {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct HealOperationsSnapshot {
|
||||
pub queue_length: u64,
|
||||
pub active_tasks: u64,
|
||||
pub retrying_tasks: u64,
|
||||
pub queued_by_priority: HealPriorityCounts,
|
||||
pub active_by_priority: HealPriorityCounts,
|
||||
pub retrying_by_priority: HealPriorityCounts,
|
||||
pub queued_by_source: HealSourceCounts,
|
||||
pub active_by_source: HealSourceCounts,
|
||||
pub retrying_by_source: HealSourceCounts,
|
||||
}
|
||||
|
||||
fn usize_to_u64_saturated(value: usize) -> u64 {
|
||||
@@ -1391,110 +1444,47 @@ impl HealManager {
|
||||
}
|
||||
|
||||
/// Submit heal request
|
||||
pub async fn submit_heal_request(&self, request: HealRequest) -> Result<HealAdmissionResult> {
|
||||
pub async fn submit_heal_request_with_receipt(&self, request: HealRequest) -> Result<HealAdmissionReceipt> {
|
||||
self.submit_heal_request_with_receipt_and_alias(request, false).await
|
||||
}
|
||||
|
||||
pub(crate) async fn submit_heal_request_with_receipt_and_alias(
|
||||
&self,
|
||||
request: HealRequest,
|
||||
preserve_alias: bool,
|
||||
) -> Result<HealAdmissionReceipt> {
|
||||
let config = self.config.read().await;
|
||||
let dedup_key = PriorityHealQueue::make_dedup_key(&request);
|
||||
|
||||
// Keep this lock order aligned with the scheduler so a request cannot
|
||||
// slip between queued and active states while duplicate admission runs.
|
||||
let active_duplicate = {
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
(!request.force_start)
|
||||
.then(|| active_heal_for_dedup_key(&active_heals, &dedup_key))
|
||||
.flatten()
|
||||
};
|
||||
if let Some((merged_task_id, _)) = active_duplicate {
|
||||
let admission = Self::duplicate_admission_for_request(&request, &config);
|
||||
|
||||
match admission {
|
||||
HealAdmissionResult::Merged => {
|
||||
self.insert_task_alias(&request.id, &merged_task_id).await;
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
merged_task_id = %merged_task_id,
|
||||
priority = ?request.priority,
|
||||
result = "merged_active_duplicate",
|
||||
"Heal queue admission decided"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Dropped(reason) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
reason = reason.as_str(),
|
||||
result = "dropped_active_duplicate",
|
||||
"Heal queue admission decided"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
|
||||
}
|
||||
|
||||
return Ok(admission);
|
||||
}
|
||||
|
||||
let retrying_duplicate = {
|
||||
let retrying_heals = self.retrying_heals.lock().await;
|
||||
(!request.force_start)
|
||||
.then(|| retrying_heal_for_dedup_key(&retrying_heals, &dedup_key))
|
||||
.flatten()
|
||||
};
|
||||
if let Some((merged_task_id, _)) = retrying_duplicate {
|
||||
let admission = Self::duplicate_admission_for_request(&request, &config);
|
||||
|
||||
match admission {
|
||||
HealAdmissionResult::Merged => {
|
||||
self.insert_task_alias(&request.id, &merged_task_id).await;
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
merged_task_id = %merged_task_id,
|
||||
priority = ?request.priority,
|
||||
result = "merged_retrying_duplicate",
|
||||
"Heal queue admission decided"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Dropped(reason) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
reason = reason.as_str(),
|
||||
result = "dropped_retrying_duplicate",
|
||||
"Heal queue admission decided"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
|
||||
}
|
||||
|
||||
return Ok(admission);
|
||||
}
|
||||
|
||||
// Match the scheduler's active -> queue order and keep retry ownership
|
||||
// in the same atomic view. Otherwise queue -> active and
|
||||
// active -> retrying transitions can slip between duplicate checks.
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
#[cfg(test)]
|
||||
pause_duplicate_admission_after_active_lock(&request.id).await;
|
||||
let mut queue = self.heal_queue.lock().await;
|
||||
|
||||
let queued_duplicate = (!request.force_start)
|
||||
.then(|| queue.request_for_dedup_key(&dedup_key).map(|queued| queued.id.clone()))
|
||||
.flatten();
|
||||
if let Some(merged_task_id) = queued_duplicate {
|
||||
let retrying_heals = self.retrying_heals.lock().await;
|
||||
let duplicate = (!request.force_start).then(|| {
|
||||
active_heal_for_dedup_key(&active_heals, &dedup_key)
|
||||
.map(|(task_id, _)| (task_id, "active"))
|
||||
.or_else(|| {
|
||||
queue
|
||||
.request_for_dedup_key(&dedup_key)
|
||||
.map(|queued| (queued.id.clone(), "queued"))
|
||||
})
|
||||
.or_else(|| retrying_heal_for_dedup_key(&retrying_heals, &dedup_key).map(|(task_id, _)| (task_id, "retrying")))
|
||||
});
|
||||
if let Some((merged_task_id, duplicate_state)) = duplicate.flatten() {
|
||||
let admission = Self::duplicate_admission_for_request(&request, &config);
|
||||
drop(retrying_heals);
|
||||
drop(queue);
|
||||
drop(active_heals);
|
||||
|
||||
match admission {
|
||||
HealAdmissionResult::Merged => {
|
||||
self.insert_task_alias(&request.id, &merged_task_id).await;
|
||||
if preserve_alias {
|
||||
self.insert_task_alias(&request.id, &merged_task_id).await;
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
@@ -1503,8 +1493,9 @@ impl HealManager {
|
||||
request_id = %request.id,
|
||||
merged_task_id = %merged_task_id,
|
||||
priority = ?request.priority,
|
||||
duplicate_state,
|
||||
result = "merged_duplicate",
|
||||
"Heal queue request merged"
|
||||
"Heal queue admission decided"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Dropped(reason) => {
|
||||
@@ -1516,25 +1507,45 @@ impl HealManager {
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
reason = reason.as_str(),
|
||||
duplicate_state,
|
||||
result = "dropped_duplicate",
|
||||
"Heal queue request dropped"
|
||||
"Heal queue admission decided"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
|
||||
}
|
||||
|
||||
return Ok(admission);
|
||||
return Ok(HealAdmissionReceipt {
|
||||
result: admission,
|
||||
task_id: merged_task_id,
|
||||
});
|
||||
}
|
||||
|
||||
let mut task_id = request.id.clone();
|
||||
let admission = Self::admit_request_to_queue(&mut queue, request, &config, "submit");
|
||||
if admission == HealAdmissionResult::Merged
|
||||
&& let Some(queued) = queue.request_for_dedup_key(&dedup_key)
|
||||
{
|
||||
task_id.clone_from(&queued.id);
|
||||
}
|
||||
let should_notify = matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
|
||||
drop(retrying_heals);
|
||||
drop(queue);
|
||||
drop(active_heals);
|
||||
|
||||
if should_notify {
|
||||
self.notify.notify_one();
|
||||
}
|
||||
|
||||
Ok(admission)
|
||||
Ok(HealAdmissionReceipt {
|
||||
result: admission,
|
||||
task_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Submit heal request.
|
||||
pub async fn submit_heal_request(&self, request: HealRequest) -> Result<HealAdmissionResult> {
|
||||
Ok(self.submit_heal_request_with_receipt_and_alias(request, true).await?.result)
|
||||
}
|
||||
|
||||
/// Get task status
|
||||
@@ -2012,43 +2023,51 @@ impl HealManager {
|
||||
}
|
||||
|
||||
pub async fn operations_snapshot(&self) -> HealOperationsSnapshot {
|
||||
let (queue_length, queued_by_priority, queued_by_source) = {
|
||||
let queue = self.heal_queue.lock().await;
|
||||
let (priority, source) = queue.operation_counts();
|
||||
publish_heal_queue_length(&queue);
|
||||
(usize_to_u64_saturated(queue.len()), priority, source)
|
||||
};
|
||||
|
||||
let (active_tasks, active_by_priority, active_by_source) = {
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
let mut priority = HealPriorityCounts::default();
|
||||
let mut source = HealSourceCounts::default();
|
||||
for task in active_heals.values() {
|
||||
priority.increment(task.priority);
|
||||
source.increment(task.source);
|
||||
}
|
||||
publish_active_heal_count(&active_heals);
|
||||
(usize_to_u64_saturated(active_heals.len()), priority, source)
|
||||
};
|
||||
// Match the scheduler's active -> queue order. Retry transitions also
|
||||
// acquire active before retrying, so the three sets form one snapshot.
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
let queue = self.heal_queue.lock().await;
|
||||
let retrying_heals = self.retrying_heals.lock().await;
|
||||
let (queued_by_priority, queued_by_source) = queue.operation_counts();
|
||||
let mut active_by_priority = HealPriorityCounts::default();
|
||||
let mut active_by_source = HealSourceCounts::default();
|
||||
for task in active_heals.values() {
|
||||
active_by_priority.increment(task.priority);
|
||||
active_by_source.increment(task.source);
|
||||
}
|
||||
let mut retrying_by_priority = HealPriorityCounts::default();
|
||||
let mut retrying_by_source = HealSourceCounts::default();
|
||||
for retrying in retrying_heals.values() {
|
||||
retrying_by_priority.increment(retrying.request.priority);
|
||||
retrying_by_source.increment(retrying.request.source);
|
||||
}
|
||||
publish_active_heal_count(&active_heals);
|
||||
publish_heal_queue_length(&queue);
|
||||
|
||||
HealOperationsSnapshot {
|
||||
queue_length,
|
||||
active_tasks,
|
||||
queue_length: usize_to_u64_saturated(queue.len()),
|
||||
active_tasks: usize_to_u64_saturated(active_heals.len()),
|
||||
retrying_tasks: usize_to_u64_saturated(retrying_heals.len()),
|
||||
queued_by_priority,
|
||||
active_by_priority,
|
||||
retrying_by_priority,
|
||||
queued_by_source,
|
||||
active_by_source,
|
||||
retrying_by_source,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn active_progress_snapshot(&self) -> Option<HealProgress> {
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
if active_heals.is_empty() {
|
||||
let active_tasks = {
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
active_heals.values().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
if active_tasks.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut snapshot = HealProgress::default();
|
||||
for task in active_heals.values() {
|
||||
for task in active_tasks {
|
||||
let progress = task.get_progress().await;
|
||||
snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned);
|
||||
snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed);
|
||||
@@ -2556,10 +2575,40 @@ impl HealManager {
|
||||
retry_attempt: request.retry_attempts,
|
||||
});
|
||||
let retry_request_for_queue = retry_request;
|
||||
let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new());
|
||||
let mut active_heals_guard = active_heals_clone.lock().await;
|
||||
if let Some(completed_task) = active_heals_guard.remove(&task_id) {
|
||||
// Keep retry ownership continuous: status snapshots acquire
|
||||
// these locks in the same active -> retrying order.
|
||||
let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
|
||||
(retry_request_for_queue.as_ref(), retry_cancel_token.as_ref())
|
||||
{
|
||||
let mut retrying = retrying_heals_clone.lock().await;
|
||||
if active_heals_guard.contains_key(&task_id) {
|
||||
retrying.insert(
|
||||
request.id.clone(),
|
||||
RetryingHeal {
|
||||
request: request.clone(),
|
||||
error: error.clone(),
|
||||
cancel_token: cancel_token.clone(),
|
||||
},
|
||||
);
|
||||
#[cfg(test)]
|
||||
pause_retry_ownership_transition(&task_id, false).await;
|
||||
}
|
||||
Some(retrying)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let completed_task = active_heals_guard.remove(&task_id);
|
||||
if let Some(completed_task) = completed_task.as_ref() {
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref());
|
||||
}
|
||||
let active_count = active_heals_guard.len();
|
||||
drop(retrying_heals_guard.take());
|
||||
drop(active_heals_guard);
|
||||
|
||||
if let Some(completed_task) = completed_task {
|
||||
let completed_status = if let Some(status) = retry_request_for_status {
|
||||
status
|
||||
} else {
|
||||
@@ -2585,10 +2634,12 @@ impl HealManager {
|
||||
stats.update_task_completion(false);
|
||||
}
|
||||
}
|
||||
stats.update_running_tasks(active_heals_guard.len() as u64);
|
||||
stats.update_running_tasks(usize_to_u64_saturated(active_count));
|
||||
}
|
||||
|
||||
if let Some((retry_request, retry_delay, retry_error)) = retry_request_for_queue {
|
||||
if let (Some((retry_request, retry_delay, retry_error)), Some(retry_cancel_token)) =
|
||||
(retry_request_for_queue, retry_cancel_token)
|
||||
{
|
||||
let retry_request_id = retry_request.id.clone();
|
||||
let retry_attempt = retry_request.retry_attempts;
|
||||
let retry_key = PriorityHealQueue::make_dedup_key(&retry_request);
|
||||
@@ -2598,17 +2649,8 @@ impl HealManager {
|
||||
let retrying_heals_for_spawn = retrying_heals_clone.clone();
|
||||
let retry_completed_heals = completed_heals_clone.clone();
|
||||
let retry_notify = notify_clone.clone();
|
||||
let retry_cancel_token = CancellationToken::new();
|
||||
let retry_manager_cancel_token = manager_cancel_token.clone();
|
||||
let retry_config = config_for_spawn.clone();
|
||||
retrying_heals_clone.lock().await.insert(
|
||||
retry_request_id.clone(),
|
||||
RetryingHeal {
|
||||
request: retry_request.clone(),
|
||||
error: retry_error.clone(),
|
||||
cancel_token: retry_cancel_token.clone(),
|
||||
},
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -2667,8 +2709,12 @@ impl HealManager {
|
||||
&& retry_config.event_driven_scheduler_enable;
|
||||
match admission {
|
||||
HealAdmissionResult::Accepted => {
|
||||
drop(queue);
|
||||
// Transfer ownership while holding queue -> retrying,
|
||||
// matching operations_snapshot's lock order.
|
||||
#[cfg(test)]
|
||||
pause_retry_ownership_transition(&retry_request_id, true).await;
|
||||
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
|
||||
drop(queue);
|
||||
retry_completed_heals.lock().await.remove(&retry_request_id);
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
@@ -2689,8 +2735,8 @@ impl HealManager {
|
||||
return;
|
||||
}
|
||||
HealAdmissionResult::Merged => {
|
||||
drop(queue);
|
||||
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
|
||||
drop(queue);
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
@@ -2963,8 +3009,8 @@ mod tests {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn object_exists(&self, _bucket: &str, _object: &str) -> Result<bool> {
|
||||
Ok(false)
|
||||
async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> {
|
||||
Ok(bucket == "retry-transition")
|
||||
}
|
||||
|
||||
async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result<Option<u64>> {
|
||||
@@ -2977,11 +3023,20 @@ mod tests {
|
||||
|
||||
async fn heal_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
bucket: &str,
|
||||
_object: &str,
|
||||
_version_id: Option<&str>,
|
||||
_opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
if bucket == "retry-transition" {
|
||||
return Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||
bucket.to_string(),
|
||||
"object".to_string(),
|
||||
))),
|
||||
));
|
||||
}
|
||||
Ok((HealResultItem::default(), None))
|
||||
}
|
||||
|
||||
@@ -3478,6 +3533,103 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_admin_duplicate_receipt_returns_canonical_task_without_alias() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new(storage, None);
|
||||
let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
original.source = HealRequestSource::Admin;
|
||||
let original_id = original.id.clone();
|
||||
let mut duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
duplicate.source = HealRequestSource::Admin;
|
||||
let duplicate_id = duplicate.id.clone();
|
||||
|
||||
let accepted = manager
|
||||
.submit_heal_request_with_receipt(original)
|
||||
.await
|
||||
.expect("first request should be accepted");
|
||||
let merged = manager
|
||||
.submit_heal_request_with_receipt(duplicate)
|
||||
.await
|
||||
.expect("duplicate request should merge");
|
||||
|
||||
assert_eq!(accepted.result, HealAdmissionResult::Accepted);
|
||||
assert_eq!(accepted.task_id, original_id);
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, original_id);
|
||||
assert_eq!(manager.canonical_task_id(&duplicate_id).await, duplicate_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = Arc::new(HealManager::new(storage.clone(), None));
|
||||
let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
original.source = HealRequestSource::Admin;
|
||||
let original_id = original.id.clone();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(original)
|
||||
.await
|
||||
.expect("original request should be accepted"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
|
||||
let mut duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
duplicate.source = HealRequestSource::Admin;
|
||||
let hook = Arc::new(DuplicateAdmissionTestHook {
|
||||
request_id: duplicate.id.clone(),
|
||||
active_lock_reached: Notify::new(),
|
||||
active_lock_release: Notify::new(),
|
||||
});
|
||||
*DUPLICATE_ADMISSION_TEST_HOOK.lock().await = Some(hook.clone());
|
||||
|
||||
let duplicate_manager = manager.clone();
|
||||
let duplicate_task = tokio::spawn(async move { duplicate_manager.submit_heal_request_with_receipt(duplicate).await });
|
||||
tokio::time::timeout(Duration::from_secs(1), hook.active_lock_reached.notified())
|
||||
.await
|
||||
.expect("duplicate admission should reach the active lock hook");
|
||||
|
||||
let transition_manager = manager.clone();
|
||||
let transition_storage = storage.clone();
|
||||
let (attempting_active_tx, attempting_active_rx) = tokio::sync::oneshot::channel();
|
||||
let (active_acquired_tx, mut active_acquired_rx) = tokio::sync::oneshot::channel();
|
||||
let transition = tokio::spawn(async move {
|
||||
let _ = attempting_active_tx.send(());
|
||||
let mut active = transition_manager.active_heals.lock().await;
|
||||
let _ = active_acquired_tx.send(());
|
||||
let mut queue = transition_manager.heal_queue.lock().await;
|
||||
let request = queue.pop_next().expect("original request should remain queued");
|
||||
let task = Arc::new(HealTask::from_request(request, transition_storage));
|
||||
active.insert(task.id.clone(), task);
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), attempting_active_rx)
|
||||
.await
|
||||
.expect("transition should attempt the active lock")
|
||||
.expect("transition attempt signal should be delivered");
|
||||
assert!(matches!(
|
||||
active_acquired_rx.try_recv(),
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Empty)
|
||||
));
|
||||
|
||||
hook.active_lock_release.notify_one();
|
||||
let receipt = tokio::time::timeout(Duration::from_secs(1), duplicate_task)
|
||||
.await
|
||||
.expect("duplicate admission should not hang")
|
||||
.expect("duplicate task should join")
|
||||
.expect("duplicate admission should succeed");
|
||||
tokio::time::timeout(Duration::from_secs(1), transition)
|
||||
.await
|
||||
.expect("queue to active transition should not hang")
|
||||
.expect("queue to active transition should join");
|
||||
*DUPLICATE_ADMISSION_TEST_HOOK.lock().await = None;
|
||||
|
||||
assert_eq!(receipt.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(receipt.task_id, original_id);
|
||||
assert_eq!(manager.get_queue_length().await, 0);
|
||||
assert_eq!(manager.get_active_task_count().await, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_returns_merged_for_active_duplicate() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
@@ -3988,6 +4140,105 @@ mod tests {
|
||||
assert_eq!(snapshot.active_by_source.scanner, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_operations_snapshot_counts_retry_backoff_as_owned_work() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new(storage, None);
|
||||
let mut request = HealRequest::bucket("bucket-retry".to_string());
|
||||
request.priority = HealPriority::Urgent;
|
||||
request.source = HealRequestSource::Admin;
|
||||
let task_id = request.id.clone();
|
||||
manager.retrying_heals.lock().await.insert(
|
||||
task_id,
|
||||
RetryingHeal {
|
||||
request,
|
||||
error: "transient".to_string(),
|
||||
cancel_token: CancellationToken::new(),
|
||||
},
|
||||
);
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
|
||||
assert_eq!(snapshot.queue_length, 0);
|
||||
assert_eq!(snapshot.active_tasks, 0);
|
||||
assert_eq!(snapshot.retrying_tasks, 1);
|
||||
assert_eq!(snapshot.retrying_by_priority.urgent, 1);
|
||||
assert_eq!(snapshot.retrying_by_source.admin, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scheduler_retry_transitions_keep_continuous_single_ownership() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = Arc::new(HealManager::new(storage, None));
|
||||
{
|
||||
let mut config = manager.config.write().await;
|
||||
config.enable_auto_heal = false;
|
||||
config.heal_interval = Duration::from_millis(10);
|
||||
config.event_driven_scheduler_enable = true;
|
||||
}
|
||||
manager.start().await.expect("manager should start");
|
||||
|
||||
let request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
|
||||
let task_id = request.id.clone();
|
||||
let hook = Arc::new(RetryOwnershipTestHook {
|
||||
task_id: task_id.clone(),
|
||||
active_to_retrying_reached: Notify::new(),
|
||||
active_to_retrying_release: Notify::new(),
|
||||
retrying_to_queue_reached: Notify::new(),
|
||||
retrying_to_queue_release: Notify::new(),
|
||||
});
|
||||
*RETRY_OWNERSHIP_TEST_HOOK.lock().await = Some(hook.clone());
|
||||
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
.await
|
||||
.expect("retry test request should be accepted");
|
||||
tokio::time::timeout(Duration::from_secs(2), hook.active_to_retrying_reached.notified())
|
||||
.await
|
||||
.expect("active to retrying transition should be reached");
|
||||
|
||||
let snapshot_manager = manager.clone();
|
||||
let mut snapshot = tokio::spawn(async move { snapshot_manager.operations_snapshot().await });
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut snapshot).await.is_err(),
|
||||
"snapshot must wait while active and retrying ownership locks are held"
|
||||
);
|
||||
hook.active_to_retrying_release.notify_one();
|
||||
let snapshot = tokio::time::timeout(Duration::from_secs(1), snapshot)
|
||||
.await
|
||||
.expect("snapshot should resume after active to retrying handoff")
|
||||
.expect("snapshot task should complete");
|
||||
assert_eq!(snapshot.active_tasks + snapshot.queue_length + snapshot.retrying_tasks, 1);
|
||||
assert_eq!(snapshot.retrying_tasks, 1);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.retrying_to_queue_reached.notified())
|
||||
.await
|
||||
.expect("retrying to queue transition should be reached");
|
||||
let snapshot_manager = manager.clone();
|
||||
let mut snapshot = tokio::spawn(async move { snapshot_manager.operations_snapshot().await });
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut snapshot).await.is_err(),
|
||||
"snapshot must wait while queue ownership is transferred"
|
||||
);
|
||||
hook.retrying_to_queue_release.notify_one();
|
||||
// The scheduler may immediately execute the retried request again;
|
||||
// leave a permit so a second test-only active handoff cannot stall it.
|
||||
hook.active_to_retrying_release.notify_one();
|
||||
let snapshot = tokio::time::timeout(Duration::from_secs(1), snapshot)
|
||||
.await
|
||||
.expect("snapshot should resume after retrying to queue handoff")
|
||||
.expect("snapshot task should complete");
|
||||
assert_eq!(snapshot.active_tasks + snapshot.queue_length + snapshot.retrying_tasks, 1);
|
||||
|
||||
*RETRY_OWNERSHIP_TEST_HOOK.lock().await = None;
|
||||
hook.active_to_retrying_release.notify_one();
|
||||
hook.retrying_to_queue_release.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(1), manager.stop())
|
||||
.await
|
||||
.expect("manager stop should not stall")
|
||||
.expect("manager should stop");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_active_progress_snapshot_sums_active_task_progress() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
|
||||
@@ -156,10 +156,10 @@ pub async fn init_heal_manager_with_workload_provider(
|
||||
let channel_receiver = if force_channel_failure {
|
||||
Err("forced heal channel initialization failure")
|
||||
} else {
|
||||
rustfs_common::heal_channel::init_heal_channel()
|
||||
rustfs_common::heal_channel::init_heal_channels()
|
||||
};
|
||||
let receiver = match channel_receiver {
|
||||
Ok(receiver) => receiver,
|
||||
let (receiver, receipt_receiver) = match channel_receiver {
|
||||
Ok(receivers) => receivers,
|
||||
Err(err) => {
|
||||
stop_initializing_manager(&heal_manager).await?;
|
||||
return Err(Error::Config(err.to_string()));
|
||||
@@ -175,7 +175,7 @@ pub async fn init_heal_manager_with_workload_provider(
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut processor = channel_processor.lock().await;
|
||||
if let Err(e) = processor.start(receiver).await {
|
||||
if let Err(e) = processor.start_with_receipts(receiver, receipt_receiver).await {
|
||||
error!(
|
||||
target: "rustfs::heal",
|
||||
event = EVENT_HEAL_RUNTIME_STATE,
|
||||
|
||||
Reference in New Issue
Block a user