gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 13:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement heal subsystem for automatic data repair

Browse Source 
- Add heal module with core types (HealType, HealRequest, HealTask)
- Implement HealManager for task scheduling and execution
- Add HealStorageAPI trait and ECStoreHealStorage implementation
- Integrate heal capabilities into scanner for automatic repair
- Support multiple heal types: object, bucket, disk, metadata, MRF, EC decode
- Add progress tracking and event system for heal operations
- Merge heal and scanner error types for unified error handling
- Include comprehensive logging and metrics for heal operations

Signed-off-by: junxiang Mu <1948535941@qq.com>
This commit is contained in:
junxiang Mu
2025-07-10 18:11:42 +08:00
parent 0aff736efd
commit 4fb3d187d0
10 changed files with 1774 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ahm/src/heal/task.rs
+415
View File
@@ -0,0 +1,415 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::error::Result;
use crate::heal::{progress::HealProgress, storage::HealStorageAPI};
use rustfs_ecstore::disk::endpoint::Endpoint;
use serde::{Deserialize, Serialize};
use std::{
sync::Arc,
time::{Duration, SystemTime},
};
use tokio::sync::RwLock;
use tracing::{debug, error, info};
use uuid::Uuid;
/// Heal 扫描模式
pub type HealScanMode = usize;
pub const HEAL_UNKNOWN_SCAN: HealScanMode = 0;
pub const HEAL_NORMAL_SCAN: HealScanMode = 1;
pub const HEAL_DEEP_SCAN: HealScanMode = 2;
/// Heal 类型
#[derive(Debug, Clone)]
pub enum HealType {
/// 对象 heal
Object {
bucket: String,
object: String,
version_id: Option<String>,
},
/// 桶 heal
Bucket {
bucket: String,
},
/// 磁盘 heal
Disk {
endpoint: Endpoint,
},
/// 元数据 heal
Metadata {
bucket: String,
object: String,
},
/// MRF heal
MRF {
meta_path: String,
},
/// EC 解码 heal
ECDecode {
bucket: String,
object: String,
version_id: Option<String>,
},
}
/// Heal 优先级
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub enum HealPriority {
/// 低优先级
Low = 0,
/// 普通优先级
Normal = 1,
/// 高优先级
High = 2,
/// 紧急优先级
Urgent = 3,
}
impl Default for HealPriority {
fn default() -> Self {
Self::Normal
}
}
/// Heal 选项
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealOptions {
/// 扫描模式
pub scan_mode: HealScanMode,
/// 是否删除损坏数据
pub remove_corrupted: bool,
/// 是否重新创建
pub recreate_missing: bool,
/// 是否更新奇偶校验
pub update_parity: bool,
/// 是否递归处理
pub recursive: bool,
/// 是否试运行
pub dry_run: bool,
/// 超时时间
pub timeout: Option<Duration>,
}
impl Default for HealOptions {
fn default() -> Self {
Self {
scan_mode: HEAL_NORMAL_SCAN,
remove_corrupted: false,
recreate_missing: true,
update_parity: true,
recursive: false,
dry_run: false,
timeout: Some(Duration::from_secs(300)), // 5分钟默认超时
}
}
}
/// Heal 任务状态
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum HealTaskStatus {
/// 等待中
Pending,
/// 运行中
Running,
/// 完成
Completed,
/// 失败
Failed { error: String },
/// 取消
Cancelled,
/// 超时
Timeout,
}
/// Heal 请求
#[derive(Debug, Clone)]
pub struct HealRequest {
/// 请求 ID
pub id: String,
/// Heal 类型
pub heal_type: HealType,
/// Heal 选项
pub options: HealOptions,
/// 优先级
pub priority: HealPriority,
/// 创建时间
pub created_at: SystemTime,
}
impl HealRequest {
pub fn new(heal_type: HealType, options: HealOptions, priority: HealPriority) -> Self {
Self {
id: Uuid::new_v4().to_string(),
heal_type,
options,
priority,
created_at: SystemTime::now(),
}
}
pub fn object(bucket: String, object: String, version_id: Option<String>) -> Self {
Self::new(
HealType::Object {
bucket,
object,
version_id,
},
HealOptions::default(),
HealPriority::Normal,
)
}
pub fn bucket(bucket: String) -> Self {
Self::new(
HealType::Bucket { bucket },
HealOptions::default(),
HealPriority::Normal,
)
}
pub fn disk(endpoint: Endpoint) -> Self {
Self::new(
HealType::Disk { endpoint },
HealOptions::default(),
HealPriority::High,
)
}
pub fn metadata(bucket: String, object: String) -> Self {
Self::new(
HealType::Metadata { bucket, object },
HealOptions::default(),
HealPriority::High,
)
}
pub fn ec_decode(bucket: String, object: String, version_id: Option<String>) -> Self {
Self::new(
HealType::ECDecode {
bucket,
object,
version_id,
},
HealOptions::default(),
HealPriority::Urgent,
)
}
}
/// Heal 任务
pub struct HealTask {
/// 任务 ID
pub id: String,
/// Heal 类型
pub heal_type: HealType,
/// Heal 选项
pub options: HealOptions,
/// 任务状态
pub status: Arc<RwLock<HealTaskStatus>>,
/// 进度跟踪
pub progress: Arc<RwLock<HealProgress>>,
/// 创建时间
pub created_at: SystemTime,
/// 开始时间
pub started_at: Arc<RwLock<Option<SystemTime>>>,
/// 完成时间
pub completed_at: Arc<RwLock<Option<SystemTime>>>,
/// 取消令牌
pub cancel_token: tokio_util::sync::CancellationToken,
/// 存储层接口
pub storage: Arc<dyn HealStorageAPI>,
}
impl HealTask {
pub fn from_request(request: HealRequest, storage: Arc<dyn HealStorageAPI>) -> Self {
Self {
id: request.id,
heal_type: request.heal_type,
options: request.options,
status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
progress: Arc::new(RwLock::new(HealProgress::new())),
created_at: request.created_at,
started_at: Arc::new(RwLock::new(None)),
completed_at: Arc::new(RwLock::new(None)),
cancel_token: tokio_util::sync::CancellationToken::new(),
storage,
}
}
pub async fn execute(&self) -> Result<()> {
// 更新状态为运行中
{
let mut status = self.status.write().await;
*status = HealTaskStatus::Running;
}
{
let mut started_at = self.started_at.write().await;
*started_at = Some(SystemTime::now());
}
info!("Starting heal task: {} with type: {:?}", self.id, self.heal_type);
let result = match &self.heal_type {
HealType::Object { bucket, object, version_id } => {
self.heal_object(bucket, object, version_id.as_deref()).await
}
HealType::Bucket { bucket } => {
self.heal_bucket(bucket).await
}
HealType::Disk { endpoint } => {
self.heal_disk(endpoint).await
}
HealType::Metadata { bucket, object } => {
self.heal_metadata(bucket, object).await
}
HealType::MRF { meta_path } => {
self.heal_mrf(meta_path).await
}
HealType::ECDecode { bucket, object, version_id } => {
self.heal_ec_decode(bucket, object, version_id.as_deref()).await
}
};
// 更新完成时间和状态
{
let mut completed_at = self.completed_at.write().await;
*completed_at = Some(SystemTime::now());
}
match &result {
Ok(_) => {
let mut status = self.status.write().await;
*status = HealTaskStatus::Completed;
info!("Heal task completed successfully: {}", self.id);
}
Err(e) => {
let mut status = self.status.write().await;
*status = HealTaskStatus::Failed {
error: e.to_string(),
};
error!("Heal task failed: {} with error: {}", self.id, e);
}
}
result
}
pub async fn cancel(&self) -> Result<()> {
self.cancel_token.cancel();
let mut status = self.status.write().await;
*status = HealTaskStatus::Cancelled;
info!("Heal task cancelled: {}", self.id);
Ok(())
}
pub async fn get_status(&self) -> HealTaskStatus {
self.status.read().await.clone()
}
pub async fn get_progress(&self) -> HealProgress {
self.progress.read().await.clone()
}
// 具体的 heal 实现方法
async fn heal_object(&self, bucket: &str, object: &str, _version_id: Option<&str>) -> Result<()> {
debug!("Healing object: {}/{}", bucket, object);
// 更新进度
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("{}/{}", bucket, object)));
}
// TODO: 实现具体的对象 heal 逻辑
// 1. 检查对象完整性
// 2. 如果损坏，尝试 EC 重建
// 3. 更新对象数据
// 4. 更新进度
{
let mut progress = self.progress.write().await;
progress.update_progress(1, 1, 0, 1024); // 示例数据
}
Ok(())
}
async fn heal_bucket(&self, bucket: &str) -> Result<()> {
debug!("Healing bucket: {}", bucket);
// TODO: 实现桶 heal 逻辑
// 1. 检查桶元数据
// 2. 修复桶配置
// 3. 更新进度
Ok(())
}
async fn heal_disk(&self, endpoint: &Endpoint) -> Result<()> {
debug!("Healing disk: {:?}", endpoint);
// TODO: 实现磁盘 heal 逻辑
// 1. 检查磁盘状态
// 2. 格式化磁盘（如果需要）
// 3. 更新进度
Ok(())
}
async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
debug!("Healing metadata: {}/{}", bucket, object);
// TODO: 实现元数据 heal 逻辑
// 1. 检查元数据完整性
// 2. 重建元数据
// 3. 更新进度
Ok(())
}
async fn heal_mrf(&self, meta_path: &str) -> Result<()> {
debug!("Healing MRF: {}", meta_path);
// TODO: 实现 MRF heal 逻辑
// 1. 检查元数据复制因子
// 2. 修复元数据
// 3. 更新进度
Ok(())
}
async fn heal_ec_decode(&self, bucket: &str, object: &str, _version_id: Option<&str>) -> Result<()> {
debug!("Healing EC decode: {}/{}", bucket, object);
// TODO: 实现 EC 解码 heal 逻辑
// 1. 检查 EC 分片
// 2. 使用 EC 算法重建数据
// 3. 更新进度
Ok(())
}
}
impl std::fmt::Debug for HealTask {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HealTask")
.field("id", &self.id)
.field("heal_type", &self.heal_type)
.field("options", &self.options)
.field("created_at", &self.created_at)
.finish()
}
}