mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-04 11:15:39 +00:00
Feature/scanner performance optimization (#498)
* Refactor: reimplement scanner Signed-off-by: RustFS Developer <dandan@rustfs.com> * comment lock Signed-off-by: junxiang Mu <1948535941@qq.com> * remove dirty file Signed-off-by: junxiang Mu <1948535941@qq.com> * Fix: fix rebase * fix(scanner): Improve error handling and logging Signed-off-by: junxiang Mu <1948535941@qq.com> --------- Signed-off-by: RustFS Developer <dandan@rustfs.com> Signed-off-by: junxiang Mu <1948535941@qq.com> Co-authored-by: RustFS Developer <dandan@rustfs.com>
This commit is contained in:
guojidan
@@ -0,0 +1,328 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use super::node_scanner::ScanProgress;
|
||||
use crate::{Error, error::Result};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct CheckpointData {
|
||||
pub version: u32,
|
||||
pub timestamp: SystemTime,
|
||||
pub progress: ScanProgress,
|
||||
pub node_id: String,
|
||||
pub checksum: u64,
|
||||
}
|
||||
|
||||
impl CheckpointData {
|
||||
pub fn new(progress: ScanProgress, node_id: String) -> Self {
|
||||
let mut checkpoint = Self {
|
||||
version: 1,
|
||||
timestamp: SystemTime::now(),
|
||||
progress,
|
||||
node_id,
|
||||
checksum: 0,
|
||||
};
|
||||
|
||||
checkpoint.checksum = checkpoint.calculate_checksum();
|
||||
checkpoint
|
||||
}
|
||||
|
||||
fn calculate_checksum(&self) -> u64 {
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
self.version.hash(&mut hasher);
|
||||
self.node_id.hash(&mut hasher);
|
||||
self.progress.current_cycle.hash(&mut hasher);
|
||||
self.progress.current_disk_index.hash(&mut hasher);
|
||||
|
||||
if let Some(ref bucket) = self.progress.current_bucket {
|
||||
bucket.hash(&mut hasher);
|
||||
}
|
||||
|
||||
if let Some(ref key) = self.progress.last_scan_key {
|
||||
key.hash(&mut hasher);
|
||||
}
|
||||
|
||||
hasher.finish()
|
||||
}
|
||||
|
||||
pub fn verify_integrity(&self) -> bool {
|
||||
let calculated_checksum = self.calculate_checksum();
|
||||
self.checksum == calculated_checksum
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CheckpointManager {
|
||||
checkpoint_file: PathBuf,
|
||||
backup_file: PathBuf,
|
||||
temp_file: PathBuf,
|
||||
save_interval: Duration,
|
||||
last_save: RwLock<SystemTime>,
|
||||
node_id: String,
|
||||
}
|
||||
|
||||
impl CheckpointManager {
|
||||
pub fn new(node_id: &str, data_dir: &Path) -> Self {
|
||||
if !data_dir.exists() {
|
||||
if let Err(e) = std::fs::create_dir_all(data_dir) {
|
||||
error!("create data dir failed {:?}: {}", data_dir, e);
|
||||
}
|
||||
}
|
||||
|
||||
let checkpoint_file = data_dir.join(format!("scanner_checkpoint_{}.json", node_id));
|
||||
let backup_file = data_dir.join(format!("scanner_checkpoint_{}.backup", node_id));
|
||||
let temp_file = data_dir.join(format!("scanner_checkpoint_{}.tmp", node_id));
|
||||
|
||||
Self {
|
||||
checkpoint_file,
|
||||
backup_file,
|
||||
temp_file,
|
||||
save_interval: Duration::from_secs(30), // 30s
|
||||
last_save: RwLock::new(SystemTime::UNIX_EPOCH),
|
||||
node_id: node_id.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn save_checkpoint(&self, progress: &ScanProgress) -> Result<()> {
|
||||
let now = SystemTime::now();
|
||||
let last_save = *self.last_save.read().await;
|
||||
|
||||
if now.duration_since(last_save).unwrap_or(Duration::ZERO) < self.save_interval {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let checkpoint_data = CheckpointData::new(progress.clone(), self.node_id.clone());
|
||||
|
||||
let json_data = serde_json::to_string_pretty(&checkpoint_data)
|
||||
.map_err(|e| Error::Serialization(format!("serialize checkpoint failed: {}", e)))?;
|
||||
|
||||
tokio::fs::write(&self.temp_file, json_data)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("write temp checkpoint file failed: {}", e)))?;
|
||||
|
||||
if self.checkpoint_file.exists() {
|
||||
tokio::fs::copy(&self.checkpoint_file, &self.backup_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("backup checkpoint file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
tokio::fs::rename(&self.temp_file, &self.checkpoint_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("replace checkpoint file failed: {}", e)))?;
|
||||
|
||||
*self.last_save.write().await = now;
|
||||
|
||||
debug!(
|
||||
"save checkpoint to {:?}, cycle: {}, disk index: {}",
|
||||
self.checkpoint_file, checkpoint_data.progress.current_cycle, checkpoint_data.progress.current_disk_index
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn load_checkpoint(&self) -> Result<Option<ScanProgress>> {
|
||||
// first try main checkpoint file
|
||||
match self.load_checkpoint_from_file(&self.checkpoint_file).await {
|
||||
Ok(checkpoint) => {
|
||||
info!(
|
||||
"restore scan progress from main checkpoint file: cycle={}, disk index={}, last scan key={:?}",
|
||||
checkpoint.current_cycle, checkpoint.current_disk_index, checkpoint.last_scan_key
|
||||
);
|
||||
Ok(Some(checkpoint))
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("main checkpoint file is corrupted or not exists: {}", e);
|
||||
|
||||
// try backup file
|
||||
match self.load_checkpoint_from_file(&self.backup_file).await {
|
||||
Ok(checkpoint) => {
|
||||
warn!(
|
||||
"restore scan progress from backup file: cycle={}, disk index={}",
|
||||
checkpoint.current_cycle, checkpoint.current_disk_index
|
||||
);
|
||||
|
||||
// copy backup file to main checkpoint file
|
||||
if let Err(copy_err) = tokio::fs::copy(&self.backup_file, &self.checkpoint_file).await {
|
||||
warn!("restore main checkpoint file failed: {}", copy_err);
|
||||
}
|
||||
|
||||
Ok(Some(checkpoint))
|
||||
}
|
||||
Err(backup_e) => {
|
||||
warn!("backup file is corrupted or not exists: {}", backup_e);
|
||||
info!("cannot restore scan progress, will start fresh scan");
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// load checkpoint from file
|
||||
async fn load_checkpoint_from_file(&self, file_path: &Path) -> Result<ScanProgress> {
|
||||
if !file_path.exists() {
|
||||
return Err(Error::NotFound(format!("checkpoint file not exists: {:?}", file_path)));
|
||||
}
|
||||
|
||||
// read file content
|
||||
let content = tokio::fs::read_to_string(file_path)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("read checkpoint file failed: {}", e)))?;
|
||||
|
||||
// deserialize
|
||||
let checkpoint_data: CheckpointData =
|
||||
serde_json::from_str(&content).map_err(|e| Error::Serialization(format!("deserialize checkpoint failed: {}", e)))?;
|
||||
|
||||
// validate checkpoint data
|
||||
self.validate_checkpoint(&checkpoint_data)?;
|
||||
|
||||
Ok(checkpoint_data.progress)
|
||||
}
|
||||
|
||||
/// validate checkpoint data
|
||||
fn validate_checkpoint(&self, checkpoint: &CheckpointData) -> Result<()> {
|
||||
// validate data integrity
|
||||
if !checkpoint.verify_integrity() {
|
||||
return Err(Error::InvalidCheckpoint(
|
||||
"checkpoint data verification failed, may be corrupted".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// validate node id match
|
||||
if checkpoint.node_id != self.node_id {
|
||||
return Err(Error::InvalidCheckpoint(format!(
|
||||
"checkpoint node id not match: expected {}, actual {}",
|
||||
self.node_id, checkpoint.node_id
|
||||
)));
|
||||
}
|
||||
|
||||
let now = SystemTime::now();
|
||||
let checkpoint_age = now.duration_since(checkpoint.timestamp).unwrap_or(Duration::MAX);
|
||||
|
||||
// checkpoint is too old (more than 24 hours), may be data expired
|
||||
if checkpoint_age > Duration::from_secs(24 * 3600) {
|
||||
return Err(Error::InvalidCheckpoint(format!("checkpoint data is too old: {:?}", checkpoint_age)));
|
||||
}
|
||||
|
||||
// validate version compatibility
|
||||
if checkpoint.version > 1 {
|
||||
return Err(Error::InvalidCheckpoint(format!(
|
||||
"unsupported checkpoint version: {}",
|
||||
checkpoint.version
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// clean checkpoint file
|
||||
///
|
||||
/// called when scanner stops or resets
|
||||
pub async fn cleanup_checkpoint(&self) -> Result<()> {
|
||||
// delete main file
|
||||
if self.checkpoint_file.exists() {
|
||||
tokio::fs::remove_file(&self.checkpoint_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("delete main checkpoint file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
// delete backup file
|
||||
if self.backup_file.exists() {
|
||||
tokio::fs::remove_file(&self.backup_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("delete backup checkpoint file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
// delete temp file
|
||||
if self.temp_file.exists() {
|
||||
tokio::fs::remove_file(&self.temp_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("delete temp checkpoint file failed: {}", e)))?;
|
||||
}
|
||||
|
||||
info!("cleaned up all checkpoint files");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// get checkpoint file info
|
||||
pub async fn get_checkpoint_info(&self) -> Result<Option<CheckpointInfo>> {
|
||||
if !self.checkpoint_file.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let metadata = tokio::fs::metadata(&self.checkpoint_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("get checkpoint file metadata failed: {}", e)))?;
|
||||
|
||||
let content = tokio::fs::read_to_string(&self.checkpoint_file)
|
||||
.await
|
||||
.map_err(|e| Error::IO(format!("read checkpoint file failed: {}", e)))?;
|
||||
|
||||
let checkpoint_data: CheckpointData =
|
||||
serde_json::from_str(&content).map_err(|e| Error::Serialization(format!("deserialize checkpoint failed: {}", e)))?;
|
||||
|
||||
Ok(Some(CheckpointInfo {
|
||||
file_size: metadata.len(),
|
||||
last_modified: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
|
||||
checkpoint_timestamp: checkpoint_data.timestamp,
|
||||
current_cycle: checkpoint_data.progress.current_cycle,
|
||||
current_disk_index: checkpoint_data.progress.current_disk_index,
|
||||
completed_disks_count: checkpoint_data.progress.completed_disks.len(),
|
||||
is_valid: checkpoint_data.verify_integrity(),
|
||||
}))
|
||||
}
|
||||
|
||||
/// force save checkpoint (ignore time interval limit)
|
||||
pub async fn force_save_checkpoint(&self, progress: &ScanProgress) -> Result<()> {
|
||||
// temporarily reset last save time, force save
|
||||
*self.last_save.write().await = SystemTime::UNIX_EPOCH;
|
||||
self.save_checkpoint(progress).await
|
||||
}
|
||||
|
||||
/// set save interval
|
||||
pub async fn set_save_interval(&mut self, interval: Duration) {
|
||||
self.save_interval = interval;
|
||||
info!("checkpoint save interval set to: {:?}", interval);
|
||||
}
|
||||
}
|
||||
|
||||
/// checkpoint info
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CheckpointInfo {
|
||||
/// file size
|
||||
pub file_size: u64,
|
||||
/// file last modified time
|
||||
pub last_modified: SystemTime,
|
||||
/// checkpoint creation time
|
||||
pub checkpoint_timestamp: SystemTime,
|
||||
/// current scan cycle
|
||||
pub current_cycle: u64,
|
||||
/// current disk index
|
||||
pub current_disk_index: usize,
|
||||
/// completed disks count
|
||||
pub completed_disks_count: usize,
|
||||
/// checkpoint is valid
|
||||
pub is_valid: bool,
|
||||
}
|
||||
Reference in New Issue
Block a user