mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 11:56:38 +00:00
refactor: clean scanner heal runtime boundaries (#3571)
This commit is contained in:
@@ -14,6 +14,8 @@
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::heal::storage_compat::{DiskError, EcstoreError};
|
||||
|
||||
/// Custom error type for heal operations
|
||||
/// This enum defines various error variants that can occur during
|
||||
/// the execution of heal-related tasks, such as I/O errors, storage errors,
|
||||
@@ -24,10 +26,10 @@ pub enum Error {
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("Storage error: {0}")]
|
||||
Storage(#[from] rustfs_ecstore::error::Error),
|
||||
Storage(#[from] EcstoreError),
|
||||
|
||||
#[error("Disk error: {0}")]
|
||||
Disk(#[from] rustfs_ecstore::disk::error::DiskError),
|
||||
Disk(#[from] DiskError),
|
||||
|
||||
#[error("Configuration error: {0}")]
|
||||
Config(String),
|
||||
|
||||
@@ -489,6 +489,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::heal::manager::HealConfig;
|
||||
use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage_compat::{DiskStore, Endpoint};
|
||||
use rustfs_common::heal_channel::{
|
||||
HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealRequestSource, HealScanMode,
|
||||
};
|
||||
@@ -516,13 +517,10 @@ mod tests {
|
||||
async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> crate::Result<Vec<u8>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn get_disk_status(
|
||||
&self,
|
||||
_endpoint: &rustfs_ecstore::disk::endpoint::Endpoint,
|
||||
) -> crate::Result<crate::heal::storage::DiskStatus> {
|
||||
async fn get_disk_status(&self, _endpoint: &Endpoint) -> crate::Result<crate::heal::storage::DiskStatus> {
|
||||
Ok(crate::heal::storage::DiskStatus::Ok)
|
||||
}
|
||||
async fn format_disk(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> crate::Result<()> {
|
||||
async fn format_disk(&self, _endpoint: &Endpoint) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> crate::Result<Option<rustfs_storage_api::BucketInfo>> {
|
||||
@@ -576,7 +574,7 @@ mod tests {
|
||||
) -> crate::Result<(Vec<String>, Option<String>, bool)> {
|
||||
Ok((vec![], None, false))
|
||||
}
|
||||
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> crate::Result<rustfs_ecstore::disk::DiskStore> {
|
||||
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> crate::Result<DiskStore> {
|
||||
Err(crate::Error::other("Not implemented in mock"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,6 @@ use crate::{Error, Result};
|
||||
use futures::{StreamExt, future::join_all, stream::FuturesUnordered};
|
||||
use metrics::gauge;
|
||||
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_ecstore::disk::DiskStore;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
@@ -29,6 +28,8 @@ use std::sync::{
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use super::storage_compat::DiskStore;
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_ERASURE_HEALER: &str = "erasure_healer";
|
||||
const EVENT_HEAL_ERASURE_RESUME_STATE: &str = "heal_erasure_resume_state";
|
||||
|
||||
@@ -14,10 +14,11 @@
|
||||
|
||||
use crate::heal::{HealOptions, HealPriority, HealRequest, HealType};
|
||||
use crate::{Error, Result};
|
||||
use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use super::storage_compat::Endpoint;
|
||||
|
||||
/// Corruption type
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum CorruptionType {
|
||||
|
||||
@@ -20,9 +20,6 @@ use crate::heal::{
|
||||
use crate::{Error, Result};
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult, HealRequestSource};
|
||||
use rustfs_ecstore::disk::DiskAPI;
|
||||
use rustfs_ecstore::disk::error::DiskError;
|
||||
use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP;
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::{
|
||||
collections::{BinaryHeap, HashMap},
|
||||
@@ -36,6 +33,8 @@ use tokio::{
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use super::storage_compat::{DiskAPI, DiskError, GLOBAL_LOCAL_DISK_MAP};
|
||||
|
||||
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
|
||||
@@ -2286,10 +2285,11 @@ mod tests {
|
||||
use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
|
||||
use rustfs_ecstore::disk::{DiskStore, endpoint::Endpoint};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_storage_api::BucketInfo;
|
||||
|
||||
use super::super::storage_compat::{DiskStore, Endpoint};
|
||||
|
||||
struct MockStorage;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
|
||||
@@ -19,7 +19,7 @@ pub mod manager;
|
||||
pub mod progress;
|
||||
pub mod resume;
|
||||
pub mod storage;
|
||||
mod storage_compat;
|
||||
pub(crate) mod storage_compat;
|
||||
pub mod task;
|
||||
pub mod utils;
|
||||
|
||||
|
||||
@@ -13,8 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{Error, Result};
|
||||
use rustfs_ecstore::disk::error::DiskError;
|
||||
use rustfs_ecstore::disk::{BUCKET_META_PREFIX, DiskAPI, DiskStore, RUSTFS_META_BUCKET};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
@@ -23,6 +21,8 @@ use tokio::sync::RwLock;
|
||||
use tracing::{debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::storage_compat::{BUCKET_META_PREFIX, DiskAPI, DiskError, DiskStore, RUSTFS_META_BUCKET};
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_RESUME: &str = "resume";
|
||||
const EVENT_HEAL_RESUME_STATE: &str = "heal_resume_state";
|
||||
@@ -744,7 +744,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_resumable_tasks_integration() {
|
||||
use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk};
|
||||
use super::super::storage_compat::{DiskOption, Endpoint, new_disk};
|
||||
use tempfile::TempDir;
|
||||
|
||||
// Create a temporary directory for testing
|
||||
|
||||
@@ -15,11 +15,6 @@
|
||||
use crate::{Error, Result};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_ecstore::{
|
||||
disk::{DiskStore, endpoint::Endpoint},
|
||||
error::StorageError,
|
||||
store::ECStore,
|
||||
};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
|
||||
@@ -28,6 +23,7 @@ use rustfs_storage_api::{
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::storage_compat::{DiskStore, ECStore, Endpoint, StorageError};
|
||||
pub use super::storage_compat::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
@@ -200,7 +196,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
Ok(info) => Ok(Some(info)),
|
||||
Err(e) => {
|
||||
// Map ObjectNotFound to None to align with Option return type
|
||||
if matches!(e, rustfs_ecstore::error::StorageError::ObjectNotFound(_, _)) {
|
||||
if matches!(e, StorageError::ObjectNotFound(_, _)) {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
@@ -810,7 +806,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
match self.ecstore.get_object_info(bucket, object, &opts).await {
|
||||
Ok(_) => Ok(true), // Object exists
|
||||
Err(e) => {
|
||||
if matches!(e, rustfs_ecstore::error::StorageError::ObjectNotFound(_, _)) {
|
||||
if matches!(e, StorageError::ObjectNotFound(_, _)) {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
@@ -1236,8 +1232,8 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::storage_compat::StorageError;
|
||||
use super::{is_transient_object_exists_error, is_transient_object_exists_message};
|
||||
use rustfs_ecstore::error::StorageError;
|
||||
|
||||
#[test]
|
||||
fn transient_object_exists_message_matches_lock_quorum_failures() {
|
||||
|
||||
@@ -12,10 +12,18 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_ecstore::store_api::{
|
||||
ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, PutObjReader as EcstorePutObjReader,
|
||||
pub(crate) use rustfs_ecstore::{
|
||||
data_usage::DATA_USAGE_CACHE_NAME,
|
||||
disk::{BUCKET_META_PREFIX, DiskAPI, DiskStore, RUSTFS_META_BUCKET, endpoint::Endpoint, error::DiskError},
|
||||
error::{Error as EcstoreError, StorageError},
|
||||
global::GLOBAL_LOCAL_DISK_MAP,
|
||||
store::ECStore,
|
||||
store_api::{ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, PutObjReader as EcstorePutObjReader},
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::disk::{DiskOption, new_disk};
|
||||
|
||||
pub type HealObjectInfo = EcstoreObjectInfo;
|
||||
pub type HealObjectOptions = EcstoreObjectOptions;
|
||||
pub type HealPutObjReader = EcstorePutObjReader;
|
||||
|
||||
@@ -16,10 +16,6 @@ use crate::heal::{ErasureSetHealer, progress::HealProgress, storage::HealStorage
|
||||
use crate::{Error, Result};
|
||||
use metrics::{counter, histogram};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_ecstore::{
|
||||
data_usage::DATA_USAGE_CACHE_NAME,
|
||||
disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
|
||||
};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
@@ -31,6 +27,8 @@ use tokio::sync::RwLock;
|
||||
use tracing::{debug, error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::storage_compat::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, RUSTFS_META_BUCKET};
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_TASK: &str = "task";
|
||||
const LOG_SUBSYSTEM_OBJECT: &str = "object";
|
||||
@@ -2049,12 +2047,9 @@ impl std::fmt::Debug for HealTask {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::storage_compat::{DiskStore, Endpoint};
|
||||
use super::*;
|
||||
use crate::heal::storage::{DiskStatus, HealObjectInfo};
|
||||
use rustfs_ecstore::{
|
||||
data_usage::DATA_USAGE_CACHE_NAME,
|
||||
disk::{BUCKET_META_PREFIX, DiskStore, RUSTFS_META_BUCKET, endpoint::Endpoint},
|
||||
};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_storage_api::BucketInfo;
|
||||
use std::sync::Mutex;
|
||||
|
||||
Reference in New Issue
Block a user