// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use thiserror::Error; use super::heal::{DiskError, EcstoreError}; /// Custom error type for heal operations /// This enum defines various error variants that can occur during /// the execution of heal-related tasks, such as I/O errors, storage errors, /// configuration errors, and specific errors related to healing operations. #[derive(Debug, Error)] pub enum Error { #[error("I/O error: {0}")] Io(#[from] std::io::Error), #[error("Storage error: {0}")] Storage(#[from] EcstoreError), #[error("Disk error: {0}")] Disk(#[from] DiskError), #[error("Configuration error: {0}")] Config(String), #[error("Other error: {0}")] Other(String), #[error("Serialization error: {0}")] Serialization(String), #[error("Invalid checkpoint: {0}")] InvalidCheckpoint(String), #[error("Heal task not found: {task_id}")] TaskNotFound { task_id: String }, #[error("Invalid heal client token")] InvalidClientToken, #[error("Heal task execution failed: {message}")] TaskExecutionFailed { message: String }, #[error("Invalid heal type: {heal_type}")] InvalidHealType { heal_type: String }, #[error("Transient heal skip: {message}")] TransientSkip { message: String }, #[error("Heal task cancelled")] TaskCancelled, #[error("Heal task timeout")] TaskTimeout, } /// A specialized Result type for heal operations pub type Result = std::result::Result; impl Error { /// Create an Other error from any error type pub fn other(error: impl std::fmt::Display) -> Self { Error::Other(error.to_string()) } /// Create a transient skip error for retryable background heal checks. pub fn transient_skip(message: impl Into) -> Self { Error::TransientSkip { message: message.into() } } /// Whether a heal operation can be retried without changing its inputs. /// /// Typed-first (backlog#1845): the primary classification reads error /// variants and typed helpers (`is_quorum_error`, `LockError::is_fatal`); /// the substring fallback in [`is_recoverable_heal_error_message`] only /// catches errors whose typed identity was destroyed upstream. pub(crate) fn is_recoverable_heal(&self) -> bool { match self { Error::TaskCancelled | Error::TaskTimeout => false, Error::TransientSkip { .. } => true, // Lock failures classify by LockError's own taxonomy: only the // fatal variants (ResourceNotFound / PermissionDenied / // Configuration) are terminal - retrying cannot fix them - while // contention and transport variants (Timeout, Network, Internal, // AlreadyLocked, QuorumNotReached, InsufficientNodes, ...) stay // recoverable, as the previous blanket `Lock(_) => true` treated // them. Error::Storage(EcstoreError::Lock(lock_err)) => !lock_err.is_fatal(), Error::Storage(err) => { err.is_quorum_error() || matches!( err, EcstoreError::DiskNotFound | EcstoreError::VolumeNotFound | EcstoreError::SlowDown | EcstoreError::OperationCanceled | EcstoreError::RemoteClientUnavailable(_) ) || is_recoverable_heal_error_message(&err.to_string()) } Error::Disk(err) => { matches!( err, DiskError::DiskNotFound | DiskError::ErasureReadQuorum | DiskError::ErasureWriteQuorum | DiskError::Timeout | DiskError::SourceStalled | DiskError::FaultyRemoteDisk | DiskError::FaultyDisk | DiskError::RemoteClientUnavailable(_) ) || is_recoverable_heal_error_message(&err.to_string()) } Error::TaskExecutionFailed { message } | Error::Other(message) => is_recoverable_heal_error_message(message), Error::Io(err) => is_recoverable_heal_error_message(&err.to_string()), _ => false, } } } /// Documented substring fallback for errors that reach heal with their typed /// identity destroyed (stringified through `TaskExecutionFailed`/`Other`, or /// boxed into `Io`). Every needle is annotated with the producer that emits /// it; when a producer becomes typed end-to-end, delete its needle here /// (backlog#1845 - this list only shrinks). fn is_recoverable_heal_error_message(error: &str) -> bool { let error = error.to_ascii_lowercase(); [ // set_disk/ops/locking.rs ns_loc lock failures rendered into messages. "failed to acquire read lock", // ecstore cluster/rpc/remote_locker.rs "Lock acquisition failed on // remote server" and lock/src local lock responses. "lock acquisition failed", // lock/src/distributed_lock.rs + client/local.rs LockResponse::failure. "lock acquisition timeout", // remote_locker.rs RPC deadline wrapper. "remote lock rpc timed out", // tokio/tonic deadline rendering, reaches heal via stringified RPC errors. "deadline has elapsed", // generic io/tonic timeout rendering. "timed out", // tonic transport failure rendering. "transport error", // LockError::Network display prefix. "network error", // io::Error ConnectionRefused rendering. "connection refused", // StorageError::OperationCanceled rendered through task messages. "operation canceled", // LockError::QuorumNotReached display, when stringified before typing. "quorum not reached", // set_disk/ops/heal.rs HEAL_RENAME_INCOMPLETE - the one needle with no // typed variant yet; the producer formats it into a plain message. "heal rename incomplete", ] .iter() .any(|pattern| error.contains(pattern)) } impl From for std::io::Error { fn from(err: Error) -> Self { std::io::Error::other(err) } } #[cfg(test)] mod tests { use super::Error; use crate::heal::{DiskError, EcstoreError}; #[test] fn incomplete_target_rename_is_recoverable() { let task_error = Error::TaskExecutionFailed { message: "heal rename incomplete: 1 of 2 targets committed".to_string(), }; let storage_error = Error::Storage(EcstoreError::Io(std::io::Error::other( "heal rename incomplete: 1 of 2 targets committed", ))); assert!(task_error.is_recoverable_heal()); assert!(storage_error.is_recoverable_heal()); } #[test] fn offline_disk_errors_are_recoverable() { assert!(Error::Disk(DiskError::DiskNotFound).is_recoverable_heal()); assert!(Error::Storage(EcstoreError::DiskNotFound).is_recoverable_heal()); assert!(Error::Storage(EcstoreError::VolumeNotFound).is_recoverable_heal()); } #[test] fn task_timeout_is_terminal() { assert!(!Error::TaskTimeout.is_recoverable_heal()); } #[test] fn lock_contention_and_transport_variants_stay_recoverable() { use rustfs_lock::LockError; for lock_err in [ LockError::Timeout { resource: "bucket/object".to_string(), timeout: std::time::Duration::from_secs(5), }, LockError::Network { message: "peer unreachable".to_string(), source: Box::new(std::io::Error::other("reset")), }, LockError::Internal { message: "channel busy".to_string(), }, LockError::AlreadyLocked { resource: "bucket/object".to_string(), owner: "node-2".to_string(), }, LockError::QuorumNotReached { required: 3, achieved: 1, }, LockError::InsufficientNodes { required: 3, available: 1, }, ] { assert!( Error::Storage(EcstoreError::Lock(lock_err)).is_recoverable_heal(), "contention/transport lock failures must stay retryable" ); } } #[test] fn fatal_lock_variants_are_terminal() { use rustfs_lock::LockError; for lock_err in [ LockError::ResourceNotFound { resource: "bucket/object".to_string(), }, LockError::PermissionDenied { reason: "acl".to_string(), }, LockError::Configuration { message: "bad quorum config".to_string(), }, ] { assert!( !Error::Storage(EcstoreError::Lock(lock_err)).is_recoverable_heal(), "fatal lock failures cannot be fixed by retrying" ); } } #[test] fn remote_client_unavailable_is_recoverable_via_typed_variant() { // The detail deliberately avoids every substring needle: the typed // variant alone must classify these as retryable. let detail = "auth interceptor rebuild".to_string(); assert!(Error::Disk(DiskError::RemoteClientUnavailable(detail.clone())).is_recoverable_heal()); assert!(Error::Storage(EcstoreError::RemoteClientUnavailable(detail)).is_recoverable_heal()); } }