feat: implement heal subsystem for automatic data repair

- Add heal module with core types (HealType, HealRequest, HealTask) - Implement HealManager for task scheduling and execution - Add HealStorageAPI trait and ECStoreHealStorage implementation - Integrate heal capabilities into scanner for automatic repair - Support multiple heal types: object, bucket, disk, metadata, MRF, EC decode - Add progress tracking and event system for heal operations - Merge heal and scanner error types for unified error handling - Include comprehensive logging and metrics for heal operations Signed-off-by: junxiang Mu <1948535941@qq.com>
2026-09-01 17:58:22 +00:00 · 2025-07-10 18:11:42 +08:00
parent 0aff736efd
commit 4fb3d187d0
10 changed files with 1774 additions and 12 deletions
@@ -0,0 +1,325 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
+use rustfs_ecstore::disk::endpoint::Endpoint;
+use serde::{Deserialize, Serialize};
+use std::time::SystemTime;
+
+/// Corruption type
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum CorruptionType {
+    /// Data corruption
+    DataCorruption,
+    /// Metadata corruption
+    MetadataCorruption,
+    /// Partial corruption
+    PartialCorruption,
+    /// Complete corruption
+    CompleteCorruption,
+}
+
+/// Severity level
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+pub enum Severity {
+    /// Low severity
+    Low = 0,
+    /// Medium severity
+    Medium = 1,
+    /// High severity
+    High = 2,
+    /// Critical severity
+    Critical = 3,
+}
+
+/// Heal event
+#[derive(Debug, Clone)]
+pub enum HealEvent {
+    /// Object corruption event
+    ObjectCorruption {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+        corruption_type: CorruptionType,
+        severity: Severity,
+    },
+    /// Object missing event
+    ObjectMissing {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+        expected_locations: Vec<usize>,
+        available_locations: Vec<usize>,
+    },
+    /// Metadata corruption event
+    MetadataCorruption {
+        bucket: String,
+        object: String,
+        corruption_type: CorruptionType,
+    },
+    /// Disk status change event
+    DiskStatusChange {
+        endpoint: Endpoint,
+        old_status: String,
+        new_status: String,
+    },
+    /// EC decode failure event
+    ECDecodeFailure {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+        missing_shards: Vec<usize>,
+        available_shards: Vec<usize>,
+    },
+    /// Checksum mismatch event
+    ChecksumMismatch {
+        bucket: String,
+        object: String,
+        version_id: Option<String>,
+        expected_checksum: String,
+        actual_checksum: String,
+    },
+    /// Bucket metadata corruption event
+    BucketMetadataCorruption {
+        bucket: String,
+        corruption_type: CorruptionType,
+    },
+    /// MRF metadata corruption event
+    MRFMetadataCorruption {
+        meta_path: String,
+        corruption_type: CorruptionType,
+    },
+}
+
+impl HealEvent {
+    /// Convert HealEvent to HealRequest
+    pub fn to_heal_request(&self) -> HealRequest {
+        match self {
+            HealEvent::ObjectCorruption { bucket, object, version_id, severity, .. } => {
+                HealRequest::new(
+                    HealType::Object {
+                        bucket: bucket.clone(),
+                        object: object.clone(),
+                        version_id: version_id.clone(),
+                    },
+                    HealOptions::default(),
+                    Self::severity_to_priority(severity),
+                )
+            }
+            HealEvent::ObjectMissing { bucket, object, version_id, .. } => {
+                HealRequest::new(
+                    HealType::Object {
+                        bucket: bucket.clone(),
+                        object: object.clone(),
+                        version_id: version_id.clone(),
+                    },
+                    HealOptions::default(),
+                    HealPriority::High,
+                )
+            }
+            HealEvent::MetadataCorruption { bucket, object, .. } => {
+                HealRequest::new(
+                    HealType::Metadata {
+                        bucket: bucket.clone(),
+                        object: object.clone(),
+                    },
+                    HealOptions::default(),
+                    HealPriority::High,
+                )
+            }
+            HealEvent::DiskStatusChange { endpoint, .. } => {
+                HealRequest::new(
+                    HealType::Disk {
+                        endpoint: endpoint.clone(),
+                    },
+                    HealOptions::default(),
+                    HealPriority::High,
+                )
+            }
+            HealEvent::ECDecodeFailure { bucket, object, version_id, .. } => {
+                HealRequest::new(
+                    HealType::ECDecode {
+                        bucket: bucket.clone(),
+                        object: object.clone(),
+                        version_id: version_id.clone(),
+                    },
+                    HealOptions::default(),
+                    HealPriority::Urgent,
+                )
+            }
+            HealEvent::ChecksumMismatch { bucket, object, version_id, .. } => {
+                HealRequest::new(
+                    HealType::Object {
+                        bucket: bucket.clone(),
+                        object: object.clone(),
+                        version_id: version_id.clone(),
+                    },
+                    HealOptions::default(),
+                    HealPriority::High,
+                )
+            }
+            HealEvent::BucketMetadataCorruption { bucket, .. } => {
+                HealRequest::new(
+                    HealType::Bucket {
+                        bucket: bucket.clone(),
+                    },
+                    HealOptions::default(),
+                    HealPriority::High,
+                )
+            }
+            HealEvent::MRFMetadataCorruption { meta_path, .. } => {
+                HealRequest::new(
+                    HealType::MRF {
+                        meta_path: meta_path.clone(),
+                    },
+                    HealOptions::default(),
+                    HealPriority::High,
+                )
+            }
+        }
+    }
+
+    /// Convert severity to priority
+    fn severity_to_priority(severity: &Severity) -> HealPriority {
+        match severity {
+            Severity::Low => HealPriority::Low,
+            Severity::Medium => HealPriority::Normal,
+            Severity::High => HealPriority::High,
+            Severity::Critical => HealPriority::Urgent,
+        }
+    }
+
+    /// Get event description
+    pub fn description(&self) -> String {
+        match self {
+            HealEvent::ObjectCorruption { bucket, object, corruption_type, .. } => {
+                format!("Object corruption detected: {}/{} - {:?}", bucket, object, corruption_type)
+            }
+            HealEvent::ObjectMissing { bucket, object, .. } => {
+                format!("Object missing: {}/{}", bucket, object)
+            }
+            HealEvent::MetadataCorruption { bucket, object, corruption_type, .. } => {
+                format!("Metadata corruption: {}/{} - {:?}", bucket, object, corruption_type)
+            }
+            HealEvent::DiskStatusChange { endpoint, old_status, new_status, .. } => {
+                format!("Disk status changed: {:?} {} -> {}", endpoint, old_status, new_status)
+            }
+            HealEvent::ECDecodeFailure { bucket, object, missing_shards, .. } => {
+                format!("EC decode failure: {}/{} - missing shards: {:?}", bucket, object, missing_shards)
+            }
+            HealEvent::ChecksumMismatch { bucket, object, expected_checksum, actual_checksum, .. } => {
+                format!("Checksum mismatch: {}/{} - expected: {}, actual: {}", bucket, object, expected_checksum, actual_checksum)
+            }
+            HealEvent::BucketMetadataCorruption { bucket, corruption_type, .. } => {
+                format!("Bucket metadata corruption: {} - {:?}", bucket, corruption_type)
+            }
+            HealEvent::MRFMetadataCorruption { meta_path, corruption_type, .. } => {
+                format!("MRF metadata corruption: {} - {:?}", meta_path, corruption_type)
+            }
+        }
+    }
+
+    /// Get event severity
+    pub fn severity(&self) -> Severity {
+        match self {
+            HealEvent::ObjectCorruption { severity, .. } => severity.clone(),
+            HealEvent::ObjectMissing { .. } => Severity::High,
+            HealEvent::MetadataCorruption { .. } => Severity::High,
+            HealEvent::DiskStatusChange { .. } => Severity::High,
+            HealEvent::ECDecodeFailure { .. } => Severity::Critical,
+            HealEvent::ChecksumMismatch { .. } => Severity::High,
+            HealEvent::BucketMetadataCorruption { .. } => Severity::High,
+            HealEvent::MRFMetadataCorruption { .. } => Severity::High,
+        }
+    }
+
+    /// Get event timestamp
+    pub fn timestamp(&self) -> SystemTime {
+        SystemTime::now()
+    }
+}
+
+/// Heal event handler
+pub struct HealEventHandler {
+    /// Event queue
+    events: Vec<HealEvent>,
+    /// Maximum number of events
+    max_events: usize,
+}
+
+impl HealEventHandler {
+    pub fn new(max_events: usize) -> Self {
+        Self {
+            events: Vec::new(),
+            max_events,
+        }
+    }
+
+    /// Add event
+    pub fn add_event(&mut self, event: HealEvent) {
+        if self.events.len() >= self.max_events {
+            // Remove oldest event
+            self.events.remove(0);
+        }
+        self.events.push(event);
+    }
+
+    /// Get all events
+    pub fn get_events(&self) -> &[HealEvent] {
+        &self.events
+    }
+
+    /// Clear events
+    pub fn clear_events(&mut self) {
+        self.events.clear();
+    }
+
+    /// Get event count
+    pub fn event_count(&self) -> usize {
+        self.events.len()
+    }
+
+    /// Filter events by severity
+    pub fn filter_by_severity(&self, min_severity: Severity) -> Vec<&HealEvent> {
+        self.events
+            .iter()
+            .filter(|event| event.severity() >= min_severity)
+            .collect()
+    }
+
+    /// Filter events by type
+    pub fn filter_by_type(&self, event_type: &str) -> Vec<&HealEvent> {
+        self.events
+            .iter()
+            .filter(|event| {
+                match event {
+                    HealEvent::ObjectCorruption { .. } => event_type == "ObjectCorruption",
+                    HealEvent::ObjectMissing { .. } => event_type == "ObjectMissing",
+                    HealEvent::MetadataCorruption { .. } => event_type == "MetadataCorruption",
+                    HealEvent::DiskStatusChange { .. } => event_type == "DiskStatusChange",
+                    HealEvent::ECDecodeFailure { .. } => event_type == "ECDecodeFailure",
+                    HealEvent::ChecksumMismatch { .. } => event_type == "ChecksumMismatch",
+                    HealEvent::BucketMetadataCorruption { .. } => event_type == "BucketMetadataCorruption",
+                    HealEvent::MRFMetadataCorruption { .. } => event_type == "MRFMetadataCorruption",
+                }
+            })
+            .collect()
+    }
+}
+
+impl Default for HealEventHandler {
+    fn default() -> Self {
+        Self::new(1000)
+    }
+}