// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus}; use serde::{Deserialize, Serialize}; use std::{ fmt::{self, Display}, sync::OnceLock, }; use tokio::sync::{broadcast, mpsc, oneshot}; use uuid::Uuid; pub const HEAL_DELETE_DANGLING: bool = true; pub const RUSTFS_RESERVED_BUCKET: &str = "rustfs"; pub const RUSTFS_RESERVED_BUCKET_PATH: &str = "/rustfs"; #[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub enum HealItemType { Metadata, Bucket, BucketMetadata, Object, } impl HealItemType { pub fn to_str(&self) -> &str { match self { HealItemType::Metadata => "metadata", HealItemType::Bucket => "bucket", HealItemType::BucketMetadata => "bucket-metadata", HealItemType::Object => "object", } } } impl Display for HealItemType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.to_str()) } } #[derive(Debug, Serialize, Deserialize)] pub enum DriveState { Ok, Offline, Corrupt, Missing, PermissionDenied, Faulty, RootMount, Unknown(String), Unformatted, // only returned by disk } impl DriveState { pub fn to_str(&self) -> &str { match self { DriveState::Ok => "ok", DriveState::Offline => "offline", DriveState::Corrupt => "corrupt", DriveState::Missing => "missing", DriveState::PermissionDenied => "permission-denied", DriveState::Faulty => "faulty", DriveState::RootMount => "root-mount", DriveState::Unknown(reason) => reason, DriveState::Unformatted => "unformatted", } } } impl Clone for DriveState { fn clone(&self) -> Self { match self { DriveState::Unknown(reason) => DriveState::Unknown(reason.clone()), DriveState::Ok => DriveState::Ok, DriveState::Offline => DriveState::Offline, DriveState::Corrupt => DriveState::Corrupt, DriveState::Missing => DriveState::Missing, DriveState::PermissionDenied => DriveState::PermissionDenied, DriveState::Faulty => DriveState::Faulty, DriveState::RootMount => DriveState::RootMount, DriveState::Unformatted => DriveState::Unformatted, } } } impl Display for DriveState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.to_str()) } } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[repr(u8)] pub enum HealScanMode { Unknown = 0, #[default] Normal = 1, Deep = 2, } impl HealScanMode { pub const fn as_str(self) -> &'static str { match self { Self::Unknown => "unknown", Self::Normal => "normal", Self::Deep => "deep", } } pub const fn from_u8(value: u8) -> Option { match value { 0 => Some(Self::Unknown), 1 => Some(Self::Normal), 2 => Some(Self::Deep), _ => None, } } } impl Serialize for HealScanMode { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_u8(*self as u8) } } impl<'de> Deserialize<'de> for HealScanMode { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { struct HealScanModeVisitor; impl<'de> serde::de::Visitor<'de> for HealScanModeVisitor { type Value = HealScanMode; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("an integer between 0 and 2") } fn visit_u8(self, value: u8) -> Result where E: serde::de::Error, { HealScanMode::from_u8(value).ok_or_else(|| E::custom(format!("invalid HealScanMode value: {value}"))) } fn visit_u64(self, value: u64) -> Result where E: serde::de::Error, { if value > u8::MAX as u64 { return Err(E::custom(format!("HealScanMode value too large: {value}"))); } self.visit_u8(value as u8) } fn visit_i64(self, value: i64) -> Result where E: serde::de::Error, { if value < 0 || value > u8::MAX as i64 { return Err(E::custom(format!("invalid HealScanMode value: {value}"))); } self.visit_u8(value as u8) } fn visit_str(self, value: &str) -> Result where E: serde::de::Error, { // Try parsing as number string first (for URL-encoded values) if let Ok(num) = value.parse::() { return self.visit_u8(num); } // Try parsing as named string match value { "Unknown" | "unknown" => Ok(HealScanMode::Unknown), "Normal" | "normal" => Ok(HealScanMode::Normal), "Deep" | "deep" => Ok(HealScanMode::Deep), _ => Err(E::custom(format!("invalid HealScanMode string: {value}"))), } } } deserializer.deserialize_any(HealScanModeVisitor) } } #[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] pub struct HealOpts { pub recursive: bool, #[serde(rename = "dryRun")] pub dry_run: bool, pub remove: bool, pub recreate: bool, #[serde(rename = "scanMode")] pub scan_mode: HealScanMode, #[serde(rename = "updateParity")] pub update_parity: bool, #[serde(rename = "nolock")] pub no_lock: bool, #[serde(rename = "pool", default)] pub pool: Option, #[serde(rename = "set", default)] pub set: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HealAdmissionDropReason { QueueFull, PolicyDropped, } impl HealAdmissionDropReason { pub fn as_str(self) -> &'static str { match self { Self::QueueFull => "queue_full", Self::PolicyDropped => "policy_dropped", } } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HealAdmissionResult { Accepted, Merged, Full, Dropped(HealAdmissionDropReason), } /// Admission decision together with the canonical task identifier. /// /// A merged request must return the identifier of the task that already owns /// the work instead of exposing the discarded request identifier as a new /// client token. #[derive(Debug, Clone, PartialEq, Eq)] pub struct HealAdmissionReceipt { /// Admission decision for the submitted request. pub result: HealAdmissionResult, /// Canonical identifier of the accepted or merged task. pub task_id: String, } impl HealAdmissionResult { pub fn result_label(self) -> &'static str { match self { Self::Accepted => "accepted", Self::Merged => "merged", Self::Full => "full", Self::Dropped(_) => "dropped", } } pub fn reason_label(self) -> &'static str { match self { Self::Dropped(reason) => reason.as_str(), _ => "none", } } pub fn is_admitted(self) -> bool { matches!(self, Self::Accepted | Self::Merged) } } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum HealRequestSource { #[default] Internal, Admin, Scanner, AutoHeal, ReadRepair, } impl HealRequestSource { pub const fn as_str(self) -> &'static str { match self { Self::Internal => "internal", Self::Admin => "admin", Self::Scanner => "scanner", Self::AutoHeal => "auto_heal", Self::ReadRepair => "read_repair", } } } /// Heal channel command type #[derive(Debug)] pub enum HealChannelCommand { /// Start a new heal task Start { request: HealChannelRequest, response_tx: oneshot::Sender>, }, /// Query heal task status Query { heal_path: String, client_token: String, response_tx: oneshot::Sender>, }, /// Cancel heal task Cancel { heal_path: String, client_token: String, response_tx: oneshot::Sender>, }, } /// Heal request from admin to ahm #[derive(Debug, Clone, Default)] pub struct HealChannelRequest { /// Unique request ID pub id: String, /// Disk ID for heal disk/erasure set task pub disk: Option, /// Bucket name pub bucket: String, /// Object prefix (optional) pub object_prefix: Option, /// Object version ID (optional) pub object_version_id: Option, /// Force start heal pub force_start: bool, /// Priority pub priority: HealChannelPriority, /// Pool index (optional) pub pool_index: Option, /// Set index (optional) pub set_index: Option, /// Scan mode (optional) pub scan_mode: Option, /// Whether to remove corrupted data pub remove_corrupted: Option, /// Whether to recreate missing data pub recreate_missing: Option, /// Whether to update parity pub update_parity: Option, /// Whether to recursively process pub recursive: Option, /// Whether to dry run pub dry_run: Option, /// Whether to skip namespace locking pub no_lock: Option, /// Timeout in seconds (optional) pub timeout_seconds: Option, /// Origin of the request for operational status and queue accounting pub source: HealRequestSource, } /// Heal response from ahm to admin #[derive(Debug, Clone)] pub struct HealChannelResponse { /// Request ID pub request_id: String, /// Success status pub success: bool, /// Response data (if successful) pub data: Option>, /// Error message (if failed) pub error: Option, } /// Heal priority #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub enum HealChannelPriority { /// Low priority Low, /// Normal priority #[default] Normal, /// High priority High, /// Critical priority Critical, } /// Heal channel sender pub type HealChannelSender = mpsc::UnboundedSender; /// Heal channel receiver pub type HealChannelReceiver = mpsc::UnboundedReceiver; /// Canonical-receipt start command kept separate from the legacy public enum. #[derive(Debug)] pub struct HealReceiptCommand { /// Heal request to admit. pub request: HealChannelRequest, /// Completion channel for the admission receipt. pub response_tx: oneshot::Sender>, } /// Canonical-receipt command receiver. pub type HealReceiptReceiver = mpsc::UnboundedReceiver; struct HealChannelSenders { command: HealChannelSender, receipt: mpsc::UnboundedSender, } /// Global heal channel sender static GLOBAL_HEAL_CHANNEL_SENDERS: OnceLock = OnceLock::new(); type HealResponseSender = broadcast::Sender; /// Global heal response broadcaster static GLOBAL_HEAL_RESPONSE_SENDER: OnceLock = OnceLock::new(); /// Initialize global heal channel pub fn init_heal_channel() -> Result { let (receiver, receipt_receiver) = init_heal_channels()?; drop(receipt_receiver); Ok(receiver) } /// Initialize the legacy command and canonical-receipt channels atomically. pub fn init_heal_channels() -> Result<(HealChannelReceiver, HealReceiptReceiver), &'static str> { let (command, command_receiver) = mpsc::unbounded_channel(); let (receipt, receipt_receiver) = mpsc::unbounded_channel(); GLOBAL_HEAL_CHANNEL_SENDERS .set(HealChannelSenders { command, receipt }) .map_err(|_| "Heal channel sender already initialized")?; Ok((command_receiver, receipt_receiver)) } /// Get global heal channel sender pub fn get_heal_channel_sender() -> Option<&'static HealChannelSender> { GLOBAL_HEAL_CHANNEL_SENDERS.get().map(|senders| &senders.command) } /// Send heal command through global channel pub async fn send_heal_command(command: HealChannelCommand) -> Result<(), String> { if let Some(sender) = get_heal_channel_sender() { sender .send(command) .map_err(|e| format!("Failed to send heal command: {e}"))?; Ok(()) } else { Err("Heal channel not initialized".to_string()) } } fn heal_response_sender() -> &'static HealResponseSender { GLOBAL_HEAL_RESPONSE_SENDER.get_or_init(|| { let (tx, _rx) = broadcast::channel(1024); tx }) } /// Publish a heal response to subscribers. pub fn publish_heal_response(response: HealChannelResponse) -> Result<(), broadcast::error::SendError> { let sender = heal_response_sender(); let _ = sender.send(response); Ok(()) } /// Subscribe to heal responses. pub fn subscribe_heal_responses() -> broadcast::Receiver { heal_response_sender().subscribe() } /// Send heal start request and wait for structured admission feedback. pub async fn send_heal_request_with_receipt(request: HealChannelRequest) -> Result { let (response_tx, response_rx) = oneshot::channel(); let senders = GLOBAL_HEAL_CHANNEL_SENDERS .get() .ok_or_else(|| "Heal channel not initialized".to_string())?; senders .receipt .send(HealReceiptCommand { request, response_tx }) .map_err(|err| format!("Failed to send heal receipt command: {err}"))?; response_rx .await .map_err(|e| format!("Failed to receive heal admission response: {e}"))? } /// Send heal start request and wait for structured admission feedback. pub async fn send_heal_request_with_admission(request: HealChannelRequest) -> Result { let (response_tx, response_rx) = oneshot::channel(); send_heal_command(HealChannelCommand::Start { request, response_tx }).await?; response_rx .await .map_err(|e| format!("Failed to receive heal admission response: {e}"))? } /// Send heal start request pub async fn send_heal_request(request: HealChannelRequest) -> Result<(), String> { match send_heal_request_with_admission(request).await? { HealAdmissionResult::Accepted | HealAdmissionResult::Merged => Ok(()), HealAdmissionResult::Full => Err("Heal request queue is full".to_string()), HealAdmissionResult::Dropped(reason) => Err(format!("Heal request dropped: {}", reason.as_str())), } } async fn receive_heal_channel_response( response_rx: oneshot::Receiver>, ) -> Result { response_rx .await .map_err(|e| format!("Failed to receive heal channel response: {e}"))? } /// Send heal query request pub async fn query_heal_status(heal_path: String, client_token: String) -> Result { let (response_tx, response_rx) = oneshot::channel(); send_heal_command(HealChannelCommand::Query { heal_path, client_token, response_tx, }) .await?; receive_heal_channel_response(response_rx).await } /// Send heal cancel request pub async fn cancel_heal_task(heal_path: String, client_token: String) -> Result { let (response_tx, response_rx) = oneshot::channel(); send_heal_command(HealChannelCommand::Cancel { heal_path, client_token, response_tx, }) .await?; receive_heal_channel_response(response_rx).await } /// Create a new heal request pub fn create_heal_request( bucket: String, object_prefix: Option, force_start: bool, priority: Option, ) -> HealChannelRequest { HealChannelRequest { id: Uuid::new_v4().to_string(), bucket, object_prefix, object_version_id: None, force_start, priority: priority.unwrap_or_default(), pool_index: None, set_index: None, scan_mode: None, remove_corrupted: None, recreate_missing: None, update_parity: None, recursive: None, dry_run: None, no_lock: None, timeout_seconds: None, source: HealRequestSource::Internal, disk: None, } } /// Create a new heal request with advanced options pub fn create_heal_request_with_options( bucket: String, object_prefix: Option, force_start: bool, priority: Option, pool_index: Option, set_index: Option, ) -> HealChannelRequest { HealChannelRequest { id: Uuid::new_v4().to_string(), bucket, object_prefix, object_version_id: None, force_start, priority: priority.unwrap_or_default(), pool_index, set_index, ..Default::default() } } /// Create a heal response pub fn create_heal_response( request_id: String, success: bool, data: Option>, error: Option, ) -> HealChannelResponse { HealChannelResponse { request_id, success, data, error, } } fn lc_get_prefix(rule: &LifecycleRule) -> String { if let Some(p) = &rule.prefix { return p.to_string(); } else if let Some(filter) = &rule.filter { if let Some(p) = &filter.prefix { return p.to_string(); } else if let Some(and) = &filter.and && let Some(p) = &and.prefix { return p.to_string(); } } "".into() } pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) -> bool { if config.rules.is_empty() { return false; } for rule in config.rules.iter() { if rule.status == ExpirationStatus::from_static(ExpirationStatus::DISABLED) { continue; } let rule_prefix = lc_get_prefix(rule); if !prefix.is_empty() && !rule_prefix.is_empty() && !prefix.starts_with(&rule_prefix) && !rule_prefix.starts_with(prefix) { continue; } if let Some(e) = &rule.noncurrent_version_expiration { if e.noncurrent_days.is_some() { return true; } if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) { return true; } } if rule.noncurrent_version_transitions.is_some() { return true; } if let Some(true) = rule.expiration.as_ref().map(|e| e.date.is_some()) { return true; } if let Some(true) = rule.expiration.as_ref().map(|e| e.days.is_some()) { return true; } if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker) { return true; } if let Some(true) = rule.transitions.as_ref().map(|t| !t.is_empty()) { return true; } if rule.transitions.is_some() { return true; } } false } pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool { if config.rules.is_empty() { return false; } for rule in config.rules.iter() { if rule .status .eq(&ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED)) { continue; } if !prefix.is_empty() && let Some(filter) = &rule.filter && let Some(r_prefix) = &filter.prefix && !r_prefix.is_empty() { // incoming prefix must be in rule prefix if !recursive && !prefix.starts_with(r_prefix) { continue; } // If recursive, we can skip this rule if it doesn't match the tested prefix or level below prefix // does not match if recursive && !r_prefix.starts_with(prefix) && !prefix.starts_with(r_prefix) { continue; } } return true; } false } pub async fn send_heal_disk(set_disk_id: String, priority: Option) -> Result<(), String> { let req = HealChannelRequest { id: Uuid::new_v4().to_string(), bucket: "".to_string(), object_prefix: None, disk: Some(set_disk_id), object_version_id: None, force_start: false, priority: priority.unwrap_or(HealChannelPriority::Low), pool_index: None, set_index: None, scan_mode: None, remove_corrupted: None, recreate_missing: None, update_parity: None, recursive: None, dry_run: None, no_lock: None, timeout_seconds: None, source: HealRequestSource::AutoHeal, }; send_heal_request(req).await } #[cfg(test)] mod tests { use super::*; #[test] fn heal_request_source_labels_are_stable() { assert_eq!(HealRequestSource::Scanner.as_str(), "scanner"); assert_eq!(HealRequestSource::Admin.as_str(), "admin"); assert_eq!(HealRequestSource::AutoHeal.as_str(), "auto_heal"); assert_eq!(HealRequestSource::Internal.as_str(), "internal"); assert_eq!(HealRequestSource::ReadRepair.as_str(), "read_repair"); let request = HealChannelRequest::default(); assert_eq!(request.source, HealRequestSource::Internal); } #[test] fn heal_admission_result_labels_are_stable() { assert_eq!(HealAdmissionResult::Accepted.result_label(), "accepted"); assert_eq!(HealAdmissionResult::Merged.result_label(), "merged"); assert_eq!(HealAdmissionResult::Full.result_label(), "full"); assert_eq!( HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull).reason_label(), "queue_full" ); assert!(HealAdmissionResult::Merged.is_admitted()); assert!(!HealAdmissionResult::Full.is_admitted()); } #[tokio::test] async fn heal_response_broadcast_reaches_subscriber() { let mut receiver = subscribe_heal_responses(); let response = create_heal_response("req-1".to_string(), true, None, None); publish_heal_response(response.clone()).expect("publish should succeed"); let received = receiver.recv().await.expect("should receive heal response"); assert_eq!(received.request_id, response.request_id); assert!(received.success); drop(receiver); let response = create_heal_response("req-no-subscriber".to_string(), true, None, None); publish_heal_response(response).expect("publish without subscribers should be ignored"); } }