mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
277d80de13
* Initial plan * Implement priority-based heal queue with deduplication Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Apply cargo fmt formatting fixes Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Add queue monitoring, better error handling, and adaptive processing - Add priority-based queue statistics tracking - Implement queue capacity warnings (>80% full) - Process multiple tasks per cycle when capacity allows - Add proper error logging for failed heal request submissions - Add Hash trait to HealPriority for HashMap support - Improve observability with detailed queue status logs Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Add comprehensive diagnostic logging to heal flow - Add detailed logging for shard availability checks during reads - Log successful heal request submissions - Add detailed logging in heal_object for disk checking - Log each disk's heal decision with reason and endpoint - Add logging for heal writing and renaming phases - Improve variable naming (nil_count -> available_shards) - Add success confirmation messages for renamed healed data These logs will help diagnose where the heal flow is failing. Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * fix * feat(heal): enhance robustness, safety, and observability of heal process - **Logging & Observability**: - Upgrade critical heal logs from [info](/crates/e2e_test/src/reliant/node_interact_test.rs:196:0-213:1) to `warn` for better visibility. - Implement structured logging with `tracing` fields for machine readability. - Add `#[tracing::instrument]` to [HealTask](c/crates/ahm/src/heal/task.rs:182:0-205:1) and [SetDisks](/crates/ecstore/src/set_disk.rs:120:0-131:1) methods for automatic context propagation. - **Robustness**: - Add exponential backoff retry (3 attempts) for acquiring write locks in [heal_object](/crates/ahm/src/heal/storage.rs:438:4-460:5) to handle contention. - Handle [rename_data](/crates/ecstore/src/set_disk.rs:392:4-516:5) failures gracefully by preserving temporary files instead of forcing deletion, preventing potential data loss. - **Data Safety**: - Fix [object_exists](/crates/ahm/src/heal/storage.rs:395:4-412:5) to propagate IO errors instead of treating them as "object not found". - Update [ErasureSetHealer](/crates/ahm/src/heal/erasure_healer.rs:28:0-33:1) to mark objects as failed rather than skipped when existence checks error, ensuring they are tracked for retry. * fix * fmt * improve code for heal_object * fix * fix * fix --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
897 lines
30 KiB
Rust
897 lines
30 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
// use crate::quorum::CheckErrorFn;
|
|
use std::hash::{Hash, Hasher};
|
|
use std::io::{self};
|
|
use std::path::PathBuf;
|
|
use tracing::error;
|
|
|
|
pub type Error = DiskError;
|
|
pub type Result<T> = core::result::Result<T, Error>;
|
|
|
|
// DiskError == StorageErr
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum DiskError {
|
|
#[error("maximum versions exceeded, please delete few versions to proceed")]
|
|
MaxVersionsExceeded,
|
|
|
|
#[error("unexpected error")]
|
|
Unexpected,
|
|
|
|
#[error("corrupted format")]
|
|
CorruptedFormat,
|
|
|
|
#[error("corrupted backend")]
|
|
CorruptedBackend,
|
|
|
|
#[error("unformatted disk error")]
|
|
UnformattedDisk,
|
|
|
|
#[error("inconsistent drive found")]
|
|
InconsistentDisk,
|
|
|
|
#[error("drive does not support O_DIRECT")]
|
|
UnsupportedDisk,
|
|
|
|
#[error("drive path full")]
|
|
DiskFull,
|
|
|
|
#[error("disk not a dir")]
|
|
DiskNotDir,
|
|
|
|
#[error("disk not found")]
|
|
DiskNotFound,
|
|
|
|
#[error("drive still did not complete the request")]
|
|
DiskOngoingReq,
|
|
|
|
#[error("drive is part of root drive, will not be used")]
|
|
DriveIsRoot,
|
|
|
|
#[error("remote drive is faulty")]
|
|
FaultyRemoteDisk,
|
|
|
|
#[error("drive is faulty")]
|
|
FaultyDisk,
|
|
|
|
#[error("drive access denied")]
|
|
DiskAccessDenied,
|
|
|
|
#[error("file not found")]
|
|
FileNotFound,
|
|
|
|
#[error("file version not found")]
|
|
FileVersionNotFound,
|
|
|
|
#[error("too many open files, please increase 'ulimit -n'")]
|
|
TooManyOpenFiles,
|
|
|
|
#[error("file name too long")]
|
|
FileNameTooLong,
|
|
|
|
#[error("volume already exists")]
|
|
VolumeExists,
|
|
|
|
#[error("not of regular file type")]
|
|
IsNotRegular,
|
|
|
|
#[error("path not found")]
|
|
PathNotFound,
|
|
|
|
#[error("volume not found")]
|
|
VolumeNotFound,
|
|
|
|
#[error("volume is not empty")]
|
|
VolumeNotEmpty,
|
|
|
|
#[error("volume access denied")]
|
|
VolumeAccessDenied,
|
|
|
|
#[error("disk access denied")]
|
|
FileAccessDenied,
|
|
|
|
#[error("file is corrupted")]
|
|
FileCorrupt,
|
|
|
|
#[error("short write")]
|
|
ShortWrite,
|
|
|
|
#[error("bit-rot hash algorithm is invalid")]
|
|
BitrotHashAlgoInvalid,
|
|
|
|
#[error("Rename across devices not allowed, please fix your backend configuration")]
|
|
CrossDeviceLink,
|
|
|
|
#[error("less data available than what was requested")]
|
|
LessData,
|
|
|
|
#[error("more data was sent than what was advertised")]
|
|
MoreData,
|
|
|
|
#[error("outdated XL meta")]
|
|
OutdatedXLMeta,
|
|
|
|
#[error("part missing or corrupt")]
|
|
PartMissingOrCorrupt,
|
|
|
|
#[error("No healing is required")]
|
|
NoHealRequired,
|
|
|
|
#[error("method not allowed")]
|
|
MethodNotAllowed,
|
|
|
|
#[error("erasure write quorum")]
|
|
ErasureWriteQuorum,
|
|
|
|
#[error("erasure read quorum")]
|
|
ErasureReadQuorum,
|
|
|
|
#[error("io error {0}")]
|
|
Io(io::Error),
|
|
|
|
#[error("source stalled")]
|
|
SourceStalled,
|
|
|
|
#[error("timeout")]
|
|
Timeout,
|
|
}
|
|
|
|
impl DiskError {
|
|
pub fn other<E>(error: E) -> Self
|
|
where
|
|
E: Into<Box<dyn std::error::Error + Send + Sync>>,
|
|
{
|
|
DiskError::Io(std::io::Error::other(error))
|
|
}
|
|
|
|
pub fn is_all_not_found(errs: &[Option<DiskError>]) -> bool {
|
|
for err in errs.iter() {
|
|
if let Some(err) = err {
|
|
if err == &DiskError::FileNotFound || err == &DiskError::FileVersionNotFound {
|
|
continue;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
!errs.is_empty()
|
|
}
|
|
|
|
pub fn is_err_object_not_found(err: &DiskError) -> bool {
|
|
matches!(err, &DiskError::FileNotFound) || matches!(err, &DiskError::VolumeNotFound)
|
|
}
|
|
|
|
pub fn is_err_version_not_found(err: &DiskError) -> bool {
|
|
matches!(err, &DiskError::FileVersionNotFound)
|
|
}
|
|
|
|
// /// If all errors are of the same fatal disk error type, returns the corresponding error.
|
|
// /// Otherwise, returns Ok.
|
|
// pub fn check_disk_fatal_errs(errs: &[Option<Error>]) -> Result<()> {
|
|
// if DiskError::UnsupportedDisk.count_errs(errs) == errs.len() {
|
|
// return Err(DiskError::UnsupportedDisk.into());
|
|
// }
|
|
|
|
// if DiskError::FileAccessDenied.count_errs(errs) == errs.len() {
|
|
// return Err(DiskError::FileAccessDenied.into());
|
|
// }
|
|
|
|
// if DiskError::DiskNotDir.count_errs(errs) == errs.len() {
|
|
// return Err(DiskError::DiskNotDir.into());
|
|
// }
|
|
|
|
// Ok(())
|
|
// }
|
|
|
|
// pub fn count_errs(&self, errs: &[Option<Error>]) -> usize {
|
|
// errs.iter()
|
|
// .filter(|&err| match err {
|
|
// None => false,
|
|
// Some(e) => self.is(e),
|
|
// })
|
|
// .count()
|
|
// }
|
|
|
|
// pub fn quorum_unformatted_disks(errs: &[Option<Error>]) -> bool {
|
|
// DiskError::UnformattedDisk.count_errs(errs) > (errs.len() / 2)
|
|
// }
|
|
|
|
// pub fn should_init_erasure_disks(errs: &[Option<Error>]) -> bool {
|
|
// DiskError::UnformattedDisk.count_errs(errs) == errs.len()
|
|
// }
|
|
|
|
// // Check if the error is a disk error
|
|
// pub fn is(&self, err: &DiskError) -> bool {
|
|
// if let Some(e) = err.downcast_ref::<DiskError>() {
|
|
// e == self
|
|
// } else {
|
|
// false
|
|
// }
|
|
// }
|
|
}
|
|
|
|
impl From<rustfs_filemeta::Error> for DiskError {
|
|
fn from(e: rustfs_filemeta::Error) -> Self {
|
|
match e {
|
|
rustfs_filemeta::Error::Io(e) => DiskError::other(e),
|
|
rustfs_filemeta::Error::FileNotFound => DiskError::FileNotFound,
|
|
rustfs_filemeta::Error::FileVersionNotFound => DiskError::FileVersionNotFound,
|
|
rustfs_filemeta::Error::FileCorrupt => DiskError::FileCorrupt,
|
|
rustfs_filemeta::Error::MethodNotAllowed => DiskError::MethodNotAllowed,
|
|
e => DiskError::other(e),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<std::io::Error> for DiskError {
|
|
fn from(e: std::io::Error) -> Self {
|
|
e.downcast::<DiskError>().unwrap_or_else(DiskError::Io)
|
|
}
|
|
}
|
|
|
|
impl From<DiskError> for std::io::Error {
|
|
fn from(e: DiskError) -> Self {
|
|
match e {
|
|
DiskError::Io(io_error) => io_error,
|
|
e => std::io::Error::other(e),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<tonic::Status> for DiskError {
|
|
fn from(e: tonic::Status) -> Self {
|
|
DiskError::other(e.message().to_string())
|
|
}
|
|
}
|
|
|
|
impl From<rustfs_protos::proto_gen::node_service::Error> for DiskError {
|
|
fn from(e: rustfs_protos::proto_gen::node_service::Error) -> Self {
|
|
if let Some(err) = DiskError::from_u32(e.code) {
|
|
if matches!(err, DiskError::Io(_)) {
|
|
DiskError::other(e.error_info)
|
|
} else {
|
|
err
|
|
}
|
|
} else {
|
|
DiskError::other(e.error_info)
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<DiskError> for rustfs_protos::proto_gen::node_service::Error {
|
|
fn from(e: DiskError) -> Self {
|
|
rustfs_protos::proto_gen::node_service::Error {
|
|
code: e.to_u32(),
|
|
error_info: e.to_string(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<serde_json::Error> for DiskError {
|
|
fn from(e: serde_json::Error) -> Self {
|
|
DiskError::other(e)
|
|
}
|
|
}
|
|
|
|
impl From<rmp_serde::encode::Error> for DiskError {
|
|
fn from(e: rmp_serde::encode::Error) -> Self {
|
|
DiskError::other(e)
|
|
}
|
|
}
|
|
|
|
impl From<rmp_serde::decode::Error> for DiskError {
|
|
fn from(e: rmp_serde::decode::Error) -> Self {
|
|
DiskError::other(e)
|
|
}
|
|
}
|
|
|
|
impl From<rmp::encode::ValueWriteError> for DiskError {
|
|
fn from(e: rmp::encode::ValueWriteError) -> Self {
|
|
DiskError::other(e)
|
|
}
|
|
}
|
|
|
|
impl From<rmp::decode::ValueReadError> for DiskError {
|
|
fn from(e: rmp::decode::ValueReadError) -> Self {
|
|
DiskError::other(e)
|
|
}
|
|
}
|
|
|
|
impl From<std::string::FromUtf8Error> for DiskError {
|
|
fn from(e: std::string::FromUtf8Error) -> Self {
|
|
DiskError::other(e)
|
|
}
|
|
}
|
|
|
|
impl From<rmp::decode::NumValueReadError> for DiskError {
|
|
fn from(e: rmp::decode::NumValueReadError) -> Self {
|
|
DiskError::other(e)
|
|
}
|
|
}
|
|
|
|
impl From<tokio::task::JoinError> for DiskError {
|
|
fn from(e: tokio::task::JoinError) -> Self {
|
|
DiskError::other(e)
|
|
}
|
|
}
|
|
|
|
impl Clone for DiskError {
|
|
fn clone(&self) -> Self {
|
|
match self {
|
|
DiskError::Io(io_error) => DiskError::Io(std::io::Error::new(io_error.kind(), io_error.to_string())),
|
|
DiskError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
|
|
DiskError::Unexpected => DiskError::Unexpected,
|
|
DiskError::CorruptedFormat => DiskError::CorruptedFormat,
|
|
DiskError::CorruptedBackend => DiskError::CorruptedBackend,
|
|
DiskError::UnformattedDisk => DiskError::UnformattedDisk,
|
|
DiskError::InconsistentDisk => DiskError::InconsistentDisk,
|
|
DiskError::UnsupportedDisk => DiskError::UnsupportedDisk,
|
|
DiskError::DiskFull => DiskError::DiskFull,
|
|
DiskError::DiskNotDir => DiskError::DiskNotDir,
|
|
DiskError::DiskNotFound => DiskError::DiskNotFound,
|
|
DiskError::DiskOngoingReq => DiskError::DiskOngoingReq,
|
|
DiskError::DriveIsRoot => DiskError::DriveIsRoot,
|
|
DiskError::FaultyRemoteDisk => DiskError::FaultyRemoteDisk,
|
|
DiskError::FaultyDisk => DiskError::FaultyDisk,
|
|
DiskError::DiskAccessDenied => DiskError::DiskAccessDenied,
|
|
DiskError::FileNotFound => DiskError::FileNotFound,
|
|
DiskError::FileVersionNotFound => DiskError::FileVersionNotFound,
|
|
DiskError::TooManyOpenFiles => DiskError::TooManyOpenFiles,
|
|
DiskError::FileNameTooLong => DiskError::FileNameTooLong,
|
|
DiskError::VolumeExists => DiskError::VolumeExists,
|
|
DiskError::IsNotRegular => DiskError::IsNotRegular,
|
|
DiskError::PathNotFound => DiskError::PathNotFound,
|
|
DiskError::VolumeNotFound => DiskError::VolumeNotFound,
|
|
DiskError::VolumeNotEmpty => DiskError::VolumeNotEmpty,
|
|
DiskError::VolumeAccessDenied => DiskError::VolumeAccessDenied,
|
|
DiskError::FileAccessDenied => DiskError::FileAccessDenied,
|
|
DiskError::FileCorrupt => DiskError::FileCorrupt,
|
|
DiskError::BitrotHashAlgoInvalid => DiskError::BitrotHashAlgoInvalid,
|
|
DiskError::CrossDeviceLink => DiskError::CrossDeviceLink,
|
|
DiskError::LessData => DiskError::LessData,
|
|
DiskError::MoreData => DiskError::MoreData,
|
|
DiskError::OutdatedXLMeta => DiskError::OutdatedXLMeta,
|
|
DiskError::PartMissingOrCorrupt => DiskError::PartMissingOrCorrupt,
|
|
DiskError::NoHealRequired => DiskError::NoHealRequired,
|
|
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
|
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
|
|
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
|
|
DiskError::ShortWrite => DiskError::ShortWrite,
|
|
DiskError::SourceStalled => DiskError::SourceStalled,
|
|
DiskError::Timeout => DiskError::Timeout,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl DiskError {
|
|
pub fn to_u32(&self) -> u32 {
|
|
match self {
|
|
DiskError::MaxVersionsExceeded => 0x01,
|
|
DiskError::Unexpected => 0x02,
|
|
DiskError::CorruptedFormat => 0x03,
|
|
DiskError::CorruptedBackend => 0x04,
|
|
DiskError::UnformattedDisk => 0x05,
|
|
DiskError::InconsistentDisk => 0x06,
|
|
DiskError::UnsupportedDisk => 0x07,
|
|
DiskError::DiskFull => 0x08,
|
|
DiskError::DiskNotDir => 0x09,
|
|
DiskError::DiskNotFound => 0x0A,
|
|
DiskError::DiskOngoingReq => 0x0B,
|
|
DiskError::DriveIsRoot => 0x0C,
|
|
DiskError::FaultyRemoteDisk => 0x0D,
|
|
DiskError::FaultyDisk => 0x0E,
|
|
DiskError::DiskAccessDenied => 0x0F,
|
|
DiskError::FileNotFound => 0x10,
|
|
DiskError::FileVersionNotFound => 0x11,
|
|
DiskError::TooManyOpenFiles => 0x12,
|
|
DiskError::FileNameTooLong => 0x13,
|
|
DiskError::VolumeExists => 0x14,
|
|
DiskError::IsNotRegular => 0x15,
|
|
DiskError::PathNotFound => 0x16,
|
|
DiskError::VolumeNotFound => 0x17,
|
|
DiskError::VolumeNotEmpty => 0x18,
|
|
DiskError::VolumeAccessDenied => 0x19,
|
|
DiskError::FileAccessDenied => 0x1A,
|
|
DiskError::FileCorrupt => 0x1B,
|
|
DiskError::BitrotHashAlgoInvalid => 0x1C,
|
|
DiskError::CrossDeviceLink => 0x1D,
|
|
DiskError::LessData => 0x1E,
|
|
DiskError::MoreData => 0x1F,
|
|
DiskError::OutdatedXLMeta => 0x20,
|
|
DiskError::PartMissingOrCorrupt => 0x21,
|
|
DiskError::NoHealRequired => 0x22,
|
|
DiskError::MethodNotAllowed => 0x23,
|
|
DiskError::Io(_) => 0x24,
|
|
DiskError::ErasureWriteQuorum => 0x25,
|
|
DiskError::ErasureReadQuorum => 0x26,
|
|
DiskError::ShortWrite => 0x27,
|
|
DiskError::SourceStalled => 0x28,
|
|
DiskError::Timeout => 0x29,
|
|
}
|
|
}
|
|
|
|
pub fn from_u32(error: u32) -> Option<Self> {
|
|
match error {
|
|
0x01 => Some(DiskError::MaxVersionsExceeded),
|
|
0x02 => Some(DiskError::Unexpected),
|
|
0x03 => Some(DiskError::CorruptedFormat),
|
|
0x04 => Some(DiskError::CorruptedBackend),
|
|
0x05 => Some(DiskError::UnformattedDisk),
|
|
0x06 => Some(DiskError::InconsistentDisk),
|
|
0x07 => Some(DiskError::UnsupportedDisk),
|
|
0x08 => Some(DiskError::DiskFull),
|
|
0x09 => Some(DiskError::DiskNotDir),
|
|
0x0A => Some(DiskError::DiskNotFound),
|
|
0x0B => Some(DiskError::DiskOngoingReq),
|
|
0x0C => Some(DiskError::DriveIsRoot),
|
|
0x0D => Some(DiskError::FaultyRemoteDisk),
|
|
0x0E => Some(DiskError::FaultyDisk),
|
|
0x0F => Some(DiskError::DiskAccessDenied),
|
|
0x10 => Some(DiskError::FileNotFound),
|
|
0x11 => Some(DiskError::FileVersionNotFound),
|
|
0x12 => Some(DiskError::TooManyOpenFiles),
|
|
0x13 => Some(DiskError::FileNameTooLong),
|
|
0x14 => Some(DiskError::VolumeExists),
|
|
0x15 => Some(DiskError::IsNotRegular),
|
|
0x16 => Some(DiskError::PathNotFound),
|
|
0x17 => Some(DiskError::VolumeNotFound),
|
|
0x18 => Some(DiskError::VolumeNotEmpty),
|
|
0x19 => Some(DiskError::VolumeAccessDenied),
|
|
0x1A => Some(DiskError::FileAccessDenied),
|
|
0x1B => Some(DiskError::FileCorrupt),
|
|
0x1C => Some(DiskError::BitrotHashAlgoInvalid),
|
|
0x1D => Some(DiskError::CrossDeviceLink),
|
|
0x1E => Some(DiskError::LessData),
|
|
0x1F => Some(DiskError::MoreData),
|
|
0x20 => Some(DiskError::OutdatedXLMeta),
|
|
0x21 => Some(DiskError::PartMissingOrCorrupt),
|
|
0x22 => Some(DiskError::NoHealRequired),
|
|
0x23 => Some(DiskError::MethodNotAllowed),
|
|
0x24 => Some(DiskError::Io(std::io::Error::other(String::new()))),
|
|
0x25 => Some(DiskError::ErasureWriteQuorum),
|
|
0x26 => Some(DiskError::ErasureReadQuorum),
|
|
0x27 => Some(DiskError::ShortWrite),
|
|
0x28 => Some(DiskError::SourceStalled),
|
|
0x29 => Some(DiskError::Timeout),
|
|
_ => None,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl PartialEq for DiskError {
|
|
fn eq(&self, other: &Self) -> bool {
|
|
match (self, other) {
|
|
(DiskError::Io(e1), DiskError::Io(e2)) => e1.kind() == e2.kind() && e1.to_string() == e2.to_string(),
|
|
_ => self.to_u32() == other.to_u32(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Eq for DiskError {}
|
|
|
|
impl Hash for DiskError {
|
|
fn hash<H: Hasher>(&self, state: &mut H) {
|
|
self.to_u32().hash(state);
|
|
}
|
|
}
|
|
|
|
// NOTE: Remove commented out code later if not needed
|
|
// Some error-related helper functions and complex error handling logic
|
|
// is currently commented out to avoid complexity. These can be re-enabled
|
|
// when needed for specific disk quorum checking and error aggregation logic.
|
|
|
|
/// Bitrot errors
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum BitrotErrorType {
|
|
#[error("bitrot checksum verification failed")]
|
|
BitrotChecksumMismatch { expected: String, got: String },
|
|
}
|
|
|
|
impl From<BitrotErrorType> for DiskError {
|
|
fn from(e: BitrotErrorType) -> Self {
|
|
DiskError::other(e)
|
|
}
|
|
}
|
|
|
|
/// Context wrapper for file access errors
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub struct FileAccessDeniedWithContext {
|
|
pub path: PathBuf,
|
|
#[source]
|
|
pub source: io::Error,
|
|
}
|
|
|
|
impl std::fmt::Display for FileAccessDeniedWithContext {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
write!(f, "file access denied for path: {}", self.path.display())
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::collections::HashMap;
|
|
|
|
#[test]
|
|
fn test_disk_error_variants() {
|
|
let errors = vec![
|
|
DiskError::MaxVersionsExceeded,
|
|
DiskError::Unexpected,
|
|
DiskError::CorruptedFormat,
|
|
DiskError::CorruptedBackend,
|
|
DiskError::UnformattedDisk,
|
|
DiskError::InconsistentDisk,
|
|
DiskError::UnsupportedDisk,
|
|
DiskError::DiskFull,
|
|
DiskError::DiskNotDir,
|
|
DiskError::DiskNotFound,
|
|
DiskError::DiskOngoingReq,
|
|
DiskError::DriveIsRoot,
|
|
DiskError::FaultyRemoteDisk,
|
|
DiskError::FaultyDisk,
|
|
DiskError::DiskAccessDenied,
|
|
DiskError::FileNotFound,
|
|
DiskError::FileVersionNotFound,
|
|
DiskError::TooManyOpenFiles,
|
|
DiskError::FileNameTooLong,
|
|
DiskError::VolumeExists,
|
|
DiskError::IsNotRegular,
|
|
DiskError::PathNotFound,
|
|
DiskError::VolumeNotFound,
|
|
DiskError::VolumeNotEmpty,
|
|
DiskError::VolumeAccessDenied,
|
|
DiskError::FileAccessDenied,
|
|
DiskError::FileCorrupt,
|
|
DiskError::ShortWrite,
|
|
DiskError::BitrotHashAlgoInvalid,
|
|
DiskError::CrossDeviceLink,
|
|
DiskError::LessData,
|
|
DiskError::MoreData,
|
|
DiskError::OutdatedXLMeta,
|
|
DiskError::PartMissingOrCorrupt,
|
|
DiskError::NoHealRequired,
|
|
DiskError::MethodNotAllowed,
|
|
DiskError::ErasureWriteQuorum,
|
|
DiskError::ErasureReadQuorum,
|
|
];
|
|
|
|
for error in errors {
|
|
// Test error display
|
|
assert!(!error.to_string().is_empty());
|
|
|
|
// Test error conversion to u32 and back
|
|
let code = error.to_u32();
|
|
let converted_back = DiskError::from_u32(code);
|
|
assert!(converted_back.is_some());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_disk_error_other() {
|
|
let custom_error = DiskError::other("custom error message");
|
|
assert!(matches!(custom_error, DiskError::Io(_)));
|
|
// The error message format might vary, so just check it's not empty
|
|
assert!(!custom_error.to_string().is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn test_disk_error_from_io_error() {
|
|
let io_error = std::io::Error::new(std::io::ErrorKind::NotFound, "file not found");
|
|
let disk_error = DiskError::from(io_error);
|
|
assert!(matches!(disk_error, DiskError::Io(_)));
|
|
}
|
|
|
|
#[test]
|
|
fn test_is_all_not_found() {
|
|
// Empty slice
|
|
assert!(!DiskError::is_all_not_found(&[]));
|
|
|
|
// All file not found
|
|
let all_not_found = vec![
|
|
Some(DiskError::FileNotFound),
|
|
Some(DiskError::FileVersionNotFound),
|
|
Some(DiskError::FileNotFound),
|
|
];
|
|
assert!(DiskError::is_all_not_found(&all_not_found));
|
|
|
|
// Mixed errors
|
|
let mixed_errors = vec![
|
|
Some(DiskError::FileNotFound),
|
|
Some(DiskError::DiskNotFound),
|
|
Some(DiskError::FileNotFound),
|
|
];
|
|
assert!(!DiskError::is_all_not_found(&mixed_errors));
|
|
|
|
// Contains None
|
|
let with_none = vec![Some(DiskError::FileNotFound), None, Some(DiskError::FileNotFound)];
|
|
assert!(!DiskError::is_all_not_found(&with_none));
|
|
}
|
|
|
|
#[test]
|
|
fn test_is_err_object_not_found() {
|
|
assert!(DiskError::is_err_object_not_found(&DiskError::FileNotFound));
|
|
assert!(DiskError::is_err_object_not_found(&DiskError::VolumeNotFound));
|
|
assert!(!DiskError::is_err_object_not_found(&DiskError::DiskNotFound));
|
|
assert!(!DiskError::is_err_object_not_found(&DiskError::FileCorrupt));
|
|
}
|
|
|
|
#[test]
|
|
fn test_is_err_version_not_found() {
|
|
assert!(DiskError::is_err_version_not_found(&DiskError::FileVersionNotFound));
|
|
assert!(!DiskError::is_err_version_not_found(&DiskError::FileNotFound));
|
|
assert!(!DiskError::is_err_version_not_found(&DiskError::VolumeNotFound));
|
|
}
|
|
|
|
#[test]
|
|
fn test_disk_error_to_u32_from_u32() {
|
|
let test_cases = vec![
|
|
(DiskError::MaxVersionsExceeded, 1),
|
|
(DiskError::Unexpected, 2),
|
|
(DiskError::CorruptedFormat, 3),
|
|
(DiskError::UnformattedDisk, 5),
|
|
(DiskError::DiskNotFound, 10),
|
|
(DiskError::FileNotFound, 16),
|
|
(DiskError::VolumeNotFound, 23),
|
|
];
|
|
|
|
for (error, expected_code) in test_cases {
|
|
assert_eq!(error.to_u32(), expected_code);
|
|
assert_eq!(DiskError::from_u32(expected_code), Some(error));
|
|
}
|
|
|
|
// Test unknown error code
|
|
assert_eq!(DiskError::from_u32(999), None);
|
|
}
|
|
|
|
#[test]
|
|
fn test_disk_error_equality() {
|
|
assert_eq!(DiskError::FileNotFound, DiskError::FileNotFound);
|
|
assert_ne!(DiskError::FileNotFound, DiskError::VolumeNotFound);
|
|
|
|
let error1 = DiskError::other("test");
|
|
let error2 = DiskError::other("test");
|
|
// IO errors with the same message should be equal
|
|
assert_eq!(error1, error2);
|
|
}
|
|
|
|
#[test]
|
|
fn test_disk_error_clone() {
|
|
let original = DiskError::FileNotFound;
|
|
let cloned = original.clone();
|
|
assert_eq!(original, cloned);
|
|
|
|
let io_error = DiskError::other("test error");
|
|
let cloned_io = io_error.clone();
|
|
assert_eq!(io_error, cloned_io);
|
|
}
|
|
|
|
#[test]
|
|
fn test_disk_error_hash() {
|
|
let mut map = HashMap::new();
|
|
map.insert(DiskError::FileNotFound, "file not found");
|
|
map.insert(DiskError::VolumeNotFound, "volume not found");
|
|
|
|
assert_eq!(map.get(&DiskError::FileNotFound), Some(&"file not found"));
|
|
assert_eq!(map.get(&DiskError::VolumeNotFound), Some(&"volume not found"));
|
|
assert_eq!(map.get(&DiskError::DiskNotFound), None);
|
|
}
|
|
|
|
#[test]
|
|
fn test_error_conversions() {
|
|
// Test From implementations
|
|
let io_error = std::io::Error::new(std::io::ErrorKind::NotFound, "test");
|
|
let _disk_error: DiskError = io_error.into();
|
|
|
|
let json_str = r#"{"invalid": json}"#; // Invalid JSON
|
|
let json_error = serde_json::from_str::<serde_json::Value>(json_str).unwrap_err();
|
|
let _disk_error: DiskError = json_error.into();
|
|
}
|
|
|
|
#[test]
|
|
fn test_bitrot_error_type() {
|
|
let bitrot_error = BitrotErrorType::BitrotChecksumMismatch {
|
|
expected: "abc123".to_string(),
|
|
got: "def456".to_string(),
|
|
};
|
|
|
|
assert!(bitrot_error.to_string().contains("bitrot checksum verification failed"));
|
|
|
|
let disk_error: DiskError = bitrot_error.into();
|
|
assert!(matches!(disk_error, DiskError::Io(_)));
|
|
}
|
|
|
|
#[test]
|
|
fn test_file_access_denied_with_context() {
|
|
let path = PathBuf::from("/test/path");
|
|
let io_error = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "permission denied");
|
|
|
|
let context_error = FileAccessDeniedWithContext {
|
|
path: path.clone(),
|
|
source: io_error,
|
|
};
|
|
|
|
let display_str = format!("{context_error}");
|
|
assert!(display_str.contains("/test/path"));
|
|
assert!(display_str.contains("file access denied"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_error_debug_format() {
|
|
let error = DiskError::FileNotFound;
|
|
let debug_str = format!("{error:?}");
|
|
assert_eq!(debug_str, "FileNotFound");
|
|
|
|
let io_error = DiskError::other("test error");
|
|
let debug_str = format!("{io_error:?}");
|
|
assert!(debug_str.contains("Io"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_error_source() {
|
|
use std::error::Error;
|
|
|
|
let io_error = std::io::Error::new(std::io::ErrorKind::NotFound, "test");
|
|
let disk_error = DiskError::Io(io_error);
|
|
|
|
// DiskError should have a source
|
|
if let DiskError::Io(ref inner) = disk_error {
|
|
assert!(inner.source().is_none()); // std::io::Error typically doesn't have a source
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_io_error_roundtrip_conversion() {
|
|
// Test DiskError -> std::io::Error -> DiskError roundtrip
|
|
let original_disk_errors = vec![
|
|
DiskError::FileNotFound,
|
|
DiskError::VolumeNotFound,
|
|
DiskError::DiskFull,
|
|
DiskError::FileCorrupt,
|
|
DiskError::MethodNotAllowed,
|
|
];
|
|
|
|
for original_error in original_disk_errors {
|
|
// Convert to io::Error and back
|
|
let io_error: std::io::Error = original_error.clone().into();
|
|
let recovered_error: DiskError = io_error.into();
|
|
|
|
// For non-Io variants, they become Io(ErrorKind::Other) and then back to the original
|
|
match &original_error {
|
|
DiskError::Io(_) => {
|
|
// Io errors should maintain their kind
|
|
assert!(matches!(recovered_error, DiskError::Io(_)));
|
|
}
|
|
_ => {
|
|
// Other errors become Io(Other) and then are recovered via downcast
|
|
// The recovered error should be functionally equivalent
|
|
assert_eq!(original_error.to_u32(), recovered_error.to_u32());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_io_error_with_disk_error_inside() {
|
|
// Test that io::Error containing DiskError can be properly converted back
|
|
let original_disk_error = DiskError::FileNotFound;
|
|
let io_with_disk_error = std::io::Error::other(original_disk_error.clone());
|
|
|
|
// Convert io::Error back to DiskError
|
|
let recovered_disk_error: DiskError = io_with_disk_error.into();
|
|
assert_eq!(original_disk_error, recovered_disk_error);
|
|
}
|
|
|
|
#[test]
|
|
fn test_io_error_different_kinds() {
|
|
use std::io::ErrorKind;
|
|
|
|
let test_cases = vec![
|
|
(ErrorKind::NotFound, "file not found"),
|
|
(ErrorKind::PermissionDenied, "permission denied"),
|
|
(ErrorKind::ConnectionRefused, "connection refused"),
|
|
(ErrorKind::TimedOut, "timed out"),
|
|
(ErrorKind::InvalidInput, "invalid input"),
|
|
];
|
|
|
|
for (kind, message) in test_cases {
|
|
let io_error = std::io::Error::new(kind, message);
|
|
let disk_error: DiskError = io_error.into();
|
|
|
|
// Should become DiskError::Io with the same kind and message
|
|
match disk_error {
|
|
DiskError::Io(inner_io) => {
|
|
assert_eq!(inner_io.kind(), kind);
|
|
assert!(inner_io.to_string().contains(message));
|
|
}
|
|
_ => panic!("Expected DiskError::Io variant"),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_disk_error_to_io_error_preserves_information() {
|
|
let test_cases = vec![
|
|
DiskError::FileNotFound,
|
|
DiskError::VolumeNotFound,
|
|
DiskError::DiskFull,
|
|
DiskError::FileCorrupt,
|
|
DiskError::MethodNotAllowed,
|
|
DiskError::ErasureReadQuorum,
|
|
DiskError::ErasureWriteQuorum,
|
|
];
|
|
|
|
for disk_error in test_cases {
|
|
let io_error: std::io::Error = disk_error.clone().into();
|
|
|
|
// Error message should be preserved
|
|
assert!(io_error.to_string().contains(&disk_error.to_string()));
|
|
|
|
// Should be able to downcast back to DiskError
|
|
let recovered_error = io_error.downcast::<DiskError>();
|
|
assert!(recovered_error.is_ok());
|
|
assert_eq!(recovered_error.unwrap(), disk_error);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_io_error_downcast_chain() {
|
|
// Test nested error downcasting chain
|
|
let original_disk_error = DiskError::FileNotFound;
|
|
|
|
// Create a chain: DiskError -> io::Error -> DiskError -> io::Error
|
|
let io_error1: std::io::Error = original_disk_error.clone().into();
|
|
let disk_error2: DiskError = io_error1.into();
|
|
let io_error2: std::io::Error = disk_error2.into();
|
|
|
|
// Final io::Error should still contain the original DiskError
|
|
let final_disk_error = io_error2.downcast::<DiskError>();
|
|
assert!(final_disk_error.is_ok());
|
|
assert_eq!(final_disk_error.unwrap(), original_disk_error);
|
|
}
|
|
|
|
#[test]
|
|
fn test_io_error_with_original_io_content() {
|
|
// Test DiskError::Io variant preserves original io::Error
|
|
let original_io = std::io::Error::new(std::io::ErrorKind::BrokenPipe, "broken pipe");
|
|
let disk_error = DiskError::Io(original_io);
|
|
|
|
let converted_io: std::io::Error = disk_error.into();
|
|
assert_eq!(converted_io.kind(), std::io::ErrorKind::BrokenPipe);
|
|
assert!(converted_io.to_string().contains("broken pipe"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_error_display_preservation() {
|
|
let disk_errors = vec![
|
|
DiskError::MaxVersionsExceeded,
|
|
DiskError::CorruptedFormat,
|
|
DiskError::UnformattedDisk,
|
|
DiskError::DiskNotFound,
|
|
DiskError::FileAccessDenied,
|
|
];
|
|
|
|
for disk_error in disk_errors {
|
|
let original_message = disk_error.to_string();
|
|
let io_error: std::io::Error = disk_error.clone().into();
|
|
|
|
// The io::Error should contain the original error message
|
|
assert!(io_error.to_string().contains(&original_message));
|
|
}
|
|
}
|
|
}
|