add Error test, fix clippy

This commit is contained in:
weisd
2025-06-09 11:29:23 +08:00
parent 96de65ebab
commit 91c099e35f
108 changed files with 1594 additions and 282 deletions
+1 -1
View File
@@ -95,7 +95,7 @@ impl BucketMetadataError {
0x06 => Some(BucketMetadataError::BucketQuotaConfigNotFound),
0x07 => Some(BucketMetadataError::BucketReplicationConfigNotFound),
0x08 => Some(BucketMetadataError::BucketRemoteTargetNotFound),
0x09 => Some(BucketMetadataError::Io(std::io::Error::new(std::io::ErrorKind::Other, "Io error"))),
0x09 => Some(BucketMetadataError::Io(std::io::Error::other("Io error"))),
_ => None,
}
}
+6 -6
View File
@@ -3,15 +3,15 @@ use std::sync::OnceLock;
use std::time::Duration;
use std::{collections::HashMap, sync::Arc};
use crate::StorageAPI;
use crate::bucket::error::BucketMetadataError;
use crate::bucket::metadata::{load_bucket_metadata_parse, BUCKET_LIFECYCLE_CONFIG};
use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_parse};
use crate::bucket::utils::is_meta_bucketname;
use crate::error::{is_err_bucket_not_found, Error, Result};
use crate::global::{is_dist_erasure, is_erasure, new_object_layer_fn, GLOBAL_Endpoints};
use crate::error::{Error, Result, is_err_bucket_not_found};
use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_layer_fn};
use crate::heal::heal_commands::HealOpts;
use crate::store::ECStore;
use crate::utils::xml::deserialize;
use crate::StorageAPI;
use futures::future::join_all;
use policy::policy::BucketPolicy;
use s3s::dto::{
@@ -23,7 +23,7 @@ use tokio::sync::RwLock;
use tokio::time::sleep;
use tracing::{error, warn};
use super::metadata::{load_bucket_metadata, BucketMetadata};
use super::metadata::{BucketMetadata, load_bucket_metadata};
use super::quota::BucketQuota;
use super::target::BucketTargets;
@@ -363,7 +363,7 @@ impl BucketMetadataSys {
Err(Error::other("errBucketMetadataNotInitialized"))
} else {
Err(err)
}
};
}
};
+3 -3
View File
@@ -1,4 +1,4 @@
use super::{storageclass, Config, GLOBAL_StorageClass};
use super::{Config, GLOBAL_StorageClass, storageclass};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result};
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
@@ -123,7 +123,7 @@ pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<C
} else {
error!("read config err {:?}", &err);
Err(err)
}
};
}
};
@@ -145,7 +145,7 @@ async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<C
} else {
error!("read config err {:?}", &err);
Err(err)
}
};
}
};
// TODO: decrypt
+2 -6
View File
@@ -5,7 +5,7 @@ pub mod storageclass;
use crate::error::Result;
use crate::store::ECStore;
use com::{lookup_configs, read_config_without_migrate, STORAGE_CLASS_SUB_SYS};
use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -69,11 +69,7 @@ impl KVS {
KVS(Vec::new())
}
pub fn get(&self, key: &str) -> String {
if let Some(v) = self.lookup(key) {
v
} else {
"".to_owned()
}
if let Some(v) = self.lookup(key) { v } else { "".to_owned() }
}
pub fn lookup(&self, key: &str) -> Option<String> {
for kv in self.0.iter() {
+9 -9
View File
@@ -174,13 +174,7 @@ pub fn lookup_config(kvs: &KVS, set_drive_count: usize) -> Result<Config> {
parse_storage_class(&ssc_str)?
} else {
StorageClass {
parity: {
if set_drive_count == 1 {
0
} else {
DEFAULT_RRS_PARITY
}
},
parity: { if set_drive_count == 1 { 0 } else { DEFAULT_RRS_PARITY } },
}
}
};
@@ -193,7 +187,10 @@ pub fn lookup_config(kvs: &KVS, set_drive_count: usize) -> Result<Config> {
if let Ok(ev) = env::var(INLINE_BLOCK_ENV) {
if let Ok(block) = ev.parse::<bytesize::ByteSize>() {
if block.as_u64() as usize > DEFAULT_INLINE_BLOCK {
warn!("inline block value bigger than recommended max of 128KiB -> {}, performance may degrade for PUT please benchmark the changes",block);
warn!(
"inline block value bigger than recommended max of 128KiB -> {}, performance may degrade for PUT please benchmark the changes",
block
);
}
block.as_u64() as usize
} else {
@@ -295,7 +292,10 @@ pub fn validate_parity_inner(ss_parity: usize, rrs_parity: usize, set_drive_coun
}
if ss_parity > 0 && rrs_parity > 0 && ss_parity < rrs_parity {
return Err(Error::other(format!("Standard storage class parity drives {} should be greater than or equal to Reduced redundancy storage class parity drives {}", ss_parity, rrs_parity)));
return Err(Error::other(format!(
"Standard storage class parity drives {} should be greater than or equal to Reduced redundancy storage class parity drives {}",
ss_parity, rrs_parity
)));
}
Ok(())
}
+140
View File
@@ -721,4 +721,144 @@ mod tests {
assert!(inner.source().is_none()); // std::io::Error typically doesn't have a source
}
}
#[test]
fn test_io_error_roundtrip_conversion() {
// Test DiskError -> std::io::Error -> DiskError roundtrip
let original_disk_errors = vec![
DiskError::FileNotFound,
DiskError::VolumeNotFound,
DiskError::DiskFull,
DiskError::FileCorrupt,
DiskError::MethodNotAllowed,
];
for original_error in original_disk_errors {
// Convert to io::Error and back
let io_error: std::io::Error = original_error.clone().into();
let recovered_error: DiskError = io_error.into();
// For non-Io variants, they become Io(ErrorKind::Other) and then back to the original
match &original_error {
DiskError::Io(_) => {
// Io errors should maintain their kind
assert!(matches!(recovered_error, DiskError::Io(_)));
}
_ => {
// Other errors become Io(Other) and then are recovered via downcast
// The recovered error should be functionally equivalent
assert_eq!(original_error.to_u32(), recovered_error.to_u32());
}
}
}
}
#[test]
fn test_io_error_with_disk_error_inside() {
// Test that io::Error containing DiskError can be properly converted back
let original_disk_error = DiskError::FileNotFound;
let io_with_disk_error = std::io::Error::other(original_disk_error.clone());
// Convert io::Error back to DiskError
let recovered_disk_error: DiskError = io_with_disk_error.into();
assert_eq!(original_disk_error, recovered_disk_error);
}
#[test]
fn test_io_error_different_kinds() {
use std::io::ErrorKind;
let test_cases = vec![
(ErrorKind::NotFound, "file not found"),
(ErrorKind::PermissionDenied, "permission denied"),
(ErrorKind::ConnectionRefused, "connection refused"),
(ErrorKind::TimedOut, "timed out"),
(ErrorKind::InvalidInput, "invalid input"),
];
for (kind, message) in test_cases {
let io_error = std::io::Error::new(kind, message);
let disk_error: DiskError = io_error.into();
// Should become DiskError::Io with the same kind and message
match disk_error {
DiskError::Io(inner_io) => {
assert_eq!(inner_io.kind(), kind);
assert!(inner_io.to_string().contains(message));
}
_ => panic!("Expected DiskError::Io variant"),
}
}
}
#[test]
fn test_disk_error_to_io_error_preserves_information() {
let test_cases = vec![
DiskError::FileNotFound,
DiskError::VolumeNotFound,
DiskError::DiskFull,
DiskError::FileCorrupt,
DiskError::MethodNotAllowed,
DiskError::ErasureReadQuorum,
DiskError::ErasureWriteQuorum,
];
for disk_error in test_cases {
let io_error: std::io::Error = disk_error.clone().into();
// Error message should be preserved
assert!(io_error.to_string().contains(&disk_error.to_string()));
// Should be able to downcast back to DiskError
let recovered_error = io_error.downcast::<DiskError>();
assert!(recovered_error.is_ok());
assert_eq!(recovered_error.unwrap(), disk_error);
}
}
#[test]
fn test_io_error_downcast_chain() {
// Test nested error downcasting chain
let original_disk_error = DiskError::FileNotFound;
// Create a chain: DiskError -> io::Error -> DiskError -> io::Error
let io_error1: std::io::Error = original_disk_error.clone().into();
let disk_error2: DiskError = io_error1.into();
let io_error2: std::io::Error = disk_error2.into();
// Final io::Error should still contain the original DiskError
let final_disk_error = io_error2.downcast::<DiskError>();
assert!(final_disk_error.is_ok());
assert_eq!(final_disk_error.unwrap(), original_disk_error);
}
#[test]
fn test_io_error_with_original_io_content() {
// Test DiskError::Io variant preserves original io::Error
let original_io = std::io::Error::new(std::io::ErrorKind::BrokenPipe, "broken pipe");
let disk_error = DiskError::Io(original_io);
let converted_io: std::io::Error = disk_error.into();
assert_eq!(converted_io.kind(), std::io::ErrorKind::BrokenPipe);
assert!(converted_io.to_string().contains("broken pipe"));
}
#[test]
fn test_error_display_preservation() {
let disk_errors = vec![
DiskError::MaxVersionsExceeded,
DiskError::CorruptedFormat,
DiskError::UnformattedDisk,
DiskError::DiskNotFound,
DiskError::FileAccessDenied,
];
for disk_error in disk_errors {
let original_message = disk_error.to_string();
let io_error: std::io::Error = disk_error.clone().into();
// The io::Error should contain the original error message
assert!(io_error.to_string().contains(&original_message));
}
}
}
+8 -14
View File
@@ -6,7 +6,7 @@ use rustfs_rio::Reader;
use super::Erasure;
use crate::disk::error::Error;
use crate::disk::error_reduce::count_errs;
use crate::disk::error_reduce::{reduce_write_quorum_errs, OBJECT_OP_IGNORED_ERRS};
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs};
use std::sync::Arc;
use std::vec;
use tokio::sync::mpsc;
@@ -62,15 +62,12 @@ impl<'a> MultiWriter<'a> {
}
if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!(
"Failed to write data: {} (offline-disks={}/{})",
write_err,
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len()
),
));
return Err(std::io::Error::other(format!(
"Failed to write data: {} (offline-disks={}/{})",
write_err,
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len()
)));
}
Err(std::io::Error::other(format!(
@@ -103,10 +100,7 @@ impl Erasure {
total += n;
let res = self.encode_data(&buf[..n])?;
if let Err(err) = tx.send(res).await {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Failed to send encoded data : {}", err),
));
return Err(std::io::Error::other(format!("Failed to send encoded data : {}", err)));
}
}
Ok(_) => break,
+2 -3
View File
@@ -3,7 +3,6 @@ use reed_solomon_erasure::galois_8::ReedSolomon;
// use rustfs_rio::Reader;
use smallvec::SmallVec;
use std::io;
use std::io::ErrorKind;
use tracing::error;
use tracing::warn;
use uuid::Uuid;
@@ -98,7 +97,7 @@ impl Erasure {
if let Some(encoder) = self.encoder.as_ref() {
encoder.encode(data_slices).map_err(|e| {
error!("encode data error: {:?}", e);
io::Error::new(ErrorKind::Other, format!("encode data error {:?}", e))
io::Error::other(format!("encode data error {:?}", e))
})?;
} else {
warn!("parity_shards > 0, but encoder is None");
@@ -129,7 +128,7 @@ impl Erasure {
if let Some(encoder) = self.encoder.as_ref() {
encoder.reconstruct(shards).map_err(|e| {
error!("decode data error: {:?}", e);
io::Error::new(ErrorKind::Other, format!("decode data error {:?}", e))
io::Error::other(format!("decode data error {:?}", e))
})?;
} else {
warn!("parity_shards > 0, but encoder is None");
+193 -2
View File
@@ -732,7 +732,7 @@ mod tests {
#[test]
fn test_storage_error_to_u32() {
// Test Io error uses 0x01
let io_error = StorageError::Io(IoError::new(ErrorKind::Other, "test"));
let io_error = StorageError::Io(IoError::other("test"));
assert_eq!(io_error.to_u32(), 0x01);
// Test other errors have correct codes
@@ -781,7 +781,7 @@ mod tests {
#[test]
fn test_storage_error_from_disk_error() {
// Test conversion from DiskError
let disk_io = DiskError::Io(IoError::new(ErrorKind::Other, "disk io error"));
let disk_io = DiskError::Io(IoError::other("disk io error"));
let storage_error: StorageError = disk_io.into();
assert!(matches!(storage_error, StorageError::Io(_)));
@@ -877,4 +877,195 @@ mod tests {
}
}
}
#[test]
fn test_storage_error_io_roundtrip() {
// Test StorageError -> std::io::Error -> StorageError roundtrip conversion
let original_storage_errors = vec![
StorageError::FileNotFound,
StorageError::VolumeNotFound,
StorageError::DiskFull,
StorageError::FileCorrupt,
StorageError::MethodNotAllowed,
StorageError::BucketExists("test-bucket".to_string()),
StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
];
for original_error in original_storage_errors {
// Convert to io::Error and back
let io_error: std::io::Error = original_error.clone().into();
let recovered_error: StorageError = io_error.into();
// Check that conversion preserves the essential error information
match &original_error {
StorageError::Io(_) => {
// Io errors should maintain their inner structure
assert!(matches!(recovered_error, StorageError::Io(_)));
}
_ => {
// Other errors should be recoverable via downcast or match to equivalent type
assert_eq!(original_error.to_u32(), recovered_error.to_u32());
}
}
}
}
#[test]
fn test_io_error_with_storage_error_inside() {
// Test that io::Error containing StorageError can be properly converted back
let original_storage_error = StorageError::FileNotFound;
let io_with_storage_error = std::io::Error::other(original_storage_error.clone());
// Convert io::Error back to StorageError
let recovered_storage_error: StorageError = io_with_storage_error.into();
assert_eq!(original_storage_error, recovered_storage_error);
}
#[test]
fn test_io_error_with_disk_error_inside() {
// Test io::Error containing DiskError -> StorageError conversion
let original_disk_error = DiskError::FileNotFound;
let io_with_disk_error = std::io::Error::other(original_disk_error.clone());
// Convert io::Error to StorageError
let storage_error: StorageError = io_with_disk_error.into();
assert_eq!(storage_error, StorageError::FileNotFound);
}
#[test]
fn test_nested_error_conversion_chain() {
// Test complex conversion chain: DiskError -> StorageError -> io::Error -> StorageError
let original_disk_error = DiskError::DiskFull;
let storage_error1: StorageError = original_disk_error.into();
let io_error: std::io::Error = storage_error1.into();
let storage_error2: StorageError = io_error.into();
assert_eq!(storage_error2, StorageError::DiskFull);
}
#[test]
fn test_storage_error_different_io_kinds() {
use std::io::ErrorKind;
let test_cases = vec![
(ErrorKind::NotFound, "not found"),
(ErrorKind::PermissionDenied, "permission denied"),
(ErrorKind::ConnectionRefused, "connection refused"),
(ErrorKind::TimedOut, "timed out"),
(ErrorKind::InvalidInput, "invalid input"),
(ErrorKind::BrokenPipe, "broken pipe"),
];
for (kind, message) in test_cases {
let io_error = std::io::Error::new(kind, message);
let storage_error: StorageError = io_error.into();
// Should become StorageError::Io with the same kind and message
match storage_error {
StorageError::Io(inner_io) => {
assert_eq!(inner_io.kind(), kind);
assert!(inner_io.to_string().contains(message));
}
_ => panic!("Expected StorageError::Io variant for kind: {:?}", kind),
}
}
}
#[test]
fn test_storage_error_to_io_error_preserves_information() {
let test_cases = vec![
StorageError::FileNotFound,
StorageError::VolumeNotFound,
StorageError::DiskFull,
StorageError::FileCorrupt,
StorageError::MethodNotAllowed,
StorageError::StorageFull,
StorageError::SlowDown,
StorageError::BucketExists("test-bucket".to_string()),
];
for storage_error in test_cases {
let io_error: std::io::Error = storage_error.clone().into();
// Error message should be preserved
assert!(io_error.to_string().contains(&storage_error.to_string()));
// Should be able to downcast back to StorageError
let recovered_error = io_error.downcast::<StorageError>();
assert!(recovered_error.is_ok());
assert_eq!(recovered_error.unwrap(), storage_error);
}
}
#[test]
fn test_storage_error_io_variant_preservation() {
// Test StorageError::Io variant preserves original io::Error
let original_io = std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "unexpected eof");
let storage_error = StorageError::Io(original_io);
let converted_io: std::io::Error = storage_error.into();
assert_eq!(converted_io.kind(), std::io::ErrorKind::UnexpectedEof);
assert!(converted_io.to_string().contains("unexpected eof"));
}
#[test]
fn test_from_filemeta_error_conversions() {
// Test conversions from rustfs_filemeta::Error
use rustfs_filemeta::Error as FilemetaError;
let filemeta_errors = vec![
(FilemetaError::FileNotFound, StorageError::FileNotFound),
(FilemetaError::FileVersionNotFound, StorageError::FileVersionNotFound),
(FilemetaError::FileCorrupt, StorageError::FileCorrupt),
(FilemetaError::MethodNotAllowed, StorageError::MethodNotAllowed),
(FilemetaError::VolumeNotFound, StorageError::VolumeNotFound),
(FilemetaError::DoneForNow, StorageError::DoneForNow),
(FilemetaError::Unexpected, StorageError::Unexpected),
];
for (filemeta_error, expected_storage_error) in filemeta_errors {
let converted_storage_error: StorageError = filemeta_error.into();
assert_eq!(converted_storage_error, expected_storage_error);
// Test reverse conversion
let converted_back: rustfs_filemeta::Error = converted_storage_error.into();
assert_eq!(converted_back, expected_storage_error.into());
}
}
#[test]
fn test_error_message_consistency() {
let storage_errors = vec![
StorageError::BucketNotFound("test-bucket".to_string()),
StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
StorageError::VersionNotFound("bucket".to_string(), "object".to_string(), "v1".to_string()),
StorageError::InvalidUploadID("bucket".to_string(), "object".to_string(), "upload123".to_string()),
];
for storage_error in storage_errors {
let original_message = storage_error.to_string();
let io_error: std::io::Error = storage_error.clone().into();
// The io::Error should contain the original error message or info
assert!(io_error.to_string().contains(&original_message));
}
}
#[test]
fn test_error_equality_after_conversion() {
let storage_errors = vec![
StorageError::FileNotFound,
StorageError::VolumeNotFound,
StorageError::DiskFull,
StorageError::MethodNotAllowed,
];
for original_error in storage_errors {
// Test that equality is preserved through conversion
let io_error: std::io::Error = original_error.clone().into();
let recovered_error: StorageError = io_error.into();
assert_eq!(original_error, recovered_error);
}
}
}
+7 -7
View File
@@ -4,8 +4,8 @@ use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration};
use tokio::{
spawn,
sync::{
mpsc::{self, Receiver, Sender},
RwLock,
mpsc::{self, Receiver, Sender},
},
time::interval,
};
@@ -14,16 +14,16 @@ use uuid::Uuid;
use super::{
heal_commands::HealOpts,
heal_ops::{new_bg_heal_sequence, HealSequence},
heal_ops::{HealSequence, new_bg_heal_sequence},
};
use crate::error::{Error, Result};
use crate::global::GLOBAL_MRFState;
use crate::heal::error::ERR_RETRY_HEALING;
use crate::heal::heal_commands::{HealScanMode, HEAL_ITEM_BUCKET};
use crate::heal::heal_ops::{HealSource, BG_HEALING_UUID};
use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HealScanMode};
use crate::heal::heal_ops::{BG_HEALING_UUID, HealSource};
use crate::{
config::RUSTFS_CONFIG_PREFIX,
disk::{endpoint::Endpoint, error::DiskError, DiskAPI, DiskInfoOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
disk::{BUCKET_META_PREFIX, DiskAPI, DiskInfoOptions, RUSTFS_META_BUCKET, endpoint::Endpoint, error::DiskError},
global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP},
heal::{
data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT},
@@ -34,7 +34,7 @@ use crate::{
new_object_layer_fn,
store::get_disk_via_endpoint,
store_api::{BucketInfo, BucketOptions, StorageAPI},
utils::path::{path_join, SLASH_SEPARATOR},
utils::path::{SLASH_SEPARATOR, path_join},
};
pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10);
@@ -149,7 +149,7 @@ async fn heal_fresh_disk(endpoint: &Endpoint) -> Result<()> {
return Err(Error::other(format!(
"Unexpected error disk must be initialized by now after formatting: {}",
endpoint
)))
)));
}
};
+10 -11
View File
@@ -6,17 +6,17 @@ use std::{
path::{Path, PathBuf},
pin::Pin,
sync::{
atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
Arc,
atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
},
time::{Duration, SystemTime},
};
use super::{
data_scanner_metric::{globalScannerMetrics, ScannerMetric, ScannerMetrics},
data_usage::{store_data_usage_in_backend, DATA_USAGE_BLOOM_NAME_PATH},
data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics},
data_usage::{DATA_USAGE_BLOOM_NAME_PATH, store_data_usage_in_backend},
data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash},
heal_commands::{HealScanMode, HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN},
heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode},
};
use crate::{
bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys},
@@ -24,7 +24,7 @@ use crate::{
heal::data_usage::DATA_USAGE_ROOT,
};
use crate::{
cache_value::metacache_set::{list_path_raw, ListPathRawOptions},
cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
config::{
com::{read_config, save_config},
heal::Config,
@@ -33,22 +33,22 @@ use crate::{
global::{GLOBAL_BackgroundHealState, GLOBAL_IsErasure, GLOBAL_IsErasureSD},
heal::{
data_usage::BACKGROUND_HEAL_INFO_PATH,
data_usage_cache::{hash_path, DataUsageHashMap},
data_usage_cache::{DataUsageHashMap, hash_path},
error::ERR_IGNORE_FILE_CONTRIB,
heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT},
heal_ops::{HealSource, BG_HEALING_UUID},
heal_ops::{BG_HEALING_UUID, HealSource},
},
new_object_layer_fn,
peer::is_reserved_or_invalid_bucket,
store::ECStore,
utils::path::{path_join, path_to_bucket_object, path_to_bucket_object_with_base_path, SLASH_SEPARATOR},
utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path},
};
use crate::{disk::DiskAPI, store_api::ObjectInfo};
use crate::{
disk::error::DiskError,
error::{Error, Result},
};
use crate::{disk::local::LocalDisk, heal::data_scanner_metric::current_path_updater};
use crate::{disk::DiskAPI, store_api::ObjectInfo};
use chrono::{DateTime, Utc};
use lazy_static::lazy_static;
use rand::Rng;
@@ -58,9 +58,8 @@ use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, Re
use serde::{Deserialize, Serialize};
use tokio::{
sync::{
broadcast,
RwLock, broadcast,
mpsc::{self, Sender},
RwLock,
},
time::sleep,
};
+1 -1
View File
@@ -6,8 +6,8 @@ use std::{
collections::HashMap,
pin::Pin,
sync::{
atomic::{AtomicU64, Ordering},
Arc,
atomic::{AtomicU64, Ordering},
},
time::{Duration, SystemTime},
};
+1 -1
View File
@@ -18,7 +18,7 @@ use std::time::{Duration, SystemTime};
use tokio::sync::mpsc::Sender;
use tokio::time::sleep;
use super::data_scanner::{SizeSummary, DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS};
use super::data_scanner::{DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS, SizeSummary};
use super::data_usage::{BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo};
// DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals
+1 -1
View File
@@ -6,7 +6,7 @@ use std::{
use crate::{
config::storageclass::{RRS, STANDARD},
disk::{error::DiskError, DeleteOptions, DiskAPI, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
disk::{BUCKET_META_PREFIX, DeleteOptions, DiskAPI, DiskStore, RUSTFS_META_BUCKET, error::DiskError},
global::GLOBAL_BackgroundHealState,
heal::heal_ops::HEALING_TRACKER_FILENAME,
new_object_layer_fn,
+7 -5
View File
@@ -2,7 +2,7 @@ use super::{
background_heal_ops::HealTask,
data_scanner::HEAL_DELETE_DANGLING,
error::ERR_SKIP_FILE,
heal_commands::{HealOpts, HealScanMode, HealStopSuccess, HealingTracker, HEAL_ITEM_BUCKET_METADATA},
heal_commands::{HEAL_ITEM_BUCKET_METADATA, HealOpts, HealScanMode, HealStopSuccess, HealingTracker},
};
use crate::error::{Error, Result};
use crate::store_api::StorageAPI;
@@ -16,7 +16,7 @@ use crate::{
disk::endpoint::Endpoint,
endpoints::Endpoints,
global::GLOBAL_IsDistErasure,
heal::heal_commands::{HealStartSuccess, HEAL_UNKNOWN_SCAN},
heal::heal_commands::{HEAL_UNKNOWN_SCAN, HealStartSuccess},
new_object_layer_fn,
utils::path::has_prefix,
};
@@ -41,10 +41,9 @@ use std::{
use tokio::{
select, spawn,
sync::{
broadcast,
RwLock, broadcast,
mpsc::{self, Receiver as M_Receiver, Sender as M_Sender},
watch::{self, Receiver as W_Receiver, Sender as W_Sender},
RwLock,
},
time::{interval, sleep},
};
@@ -784,7 +783,10 @@ impl AllHealState {
self.stop_heal_sequence(path_s).await?;
} else if let Some(hs) = self.get_heal_sequence(path_s).await {
if !hs.has_ended().await {
return Err(Error::other(format!("Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {:?}, token is {}", heal_sequence.client_address, heal_sequence.start_time, heal_sequence.client_token)));
return Err(Error::other(format!(
"Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {:?}, token is {}",
heal_sequence.client_address, heal_sequence.start_time, heal_sequence.client_token
)));
}
}
+1 -1
View File
@@ -8,8 +8,8 @@ use regex::Regex;
use std::ops::Sub;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use tokio::sync::mpsc::{Receiver, Sender};
use tokio::sync::RwLock;
use tokio::sync::mpsc::{Receiver, Sender};
use tokio::time::sleep;
use tracing::error;
use uuid::Uuid;
+2 -2
View File
@@ -1,8 +1,8 @@
use crate::StorageAPI;
use crate::admin_server_info::get_commit_id;
use crate::error::{Error, Result};
use crate::global::{get_global_endpoints, GLOBAL_BOOT_TIME};
use crate::global::{GLOBAL_BOOT_TIME, get_global_endpoints};
use crate::peer_rest_client::PeerRestClient;
use crate::StorageAPI;
use crate::{endpoints::EndpointServerPools, new_object_layer_fn};
use futures::future::join_all;
use lazy_static::lazy_static;
+1 -1
View File
@@ -7,10 +7,10 @@ use crate::{
utils::net::XHost,
};
use madmin::{
ServerProperties,
health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysService},
metrics::RealtimeMetrics,
net::NetInfo,
ServerProperties,
};
use protos::{
node_service_time_out_client,
+4 -6
View File
@@ -3824,14 +3824,13 @@ impl ObjectIO for SetDisks {
let sc_parity_drives = {
if let Some(sc) = GLOBAL_StorageClass.get() {
let a = sc.get_parity_for_sc(
sc.get_parity_for_sc(
user_defined
.get(xhttp::AMZ_STORAGE_CLASS)
.cloned()
.unwrap_or_default()
.as_str(),
);
a
)
} else {
None
}
@@ -4806,14 +4805,13 @@ impl StorageAPI for SetDisks {
let sc_parity_drives = {
if let Some(sc) = GLOBAL_StorageClass.get() {
let a = sc.get_parity_for_sc(
sc.get_parity_for_sc(
user_defined
.get(xhttp::AMZ_STORAGE_CLASS)
.cloned()
.unwrap_or_default()
.as_str(),
);
a
)
} else {
None
}
+5 -4
View File
@@ -5,15 +5,16 @@ use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result};
use crate::{
disk::{
DiskAPI, DiskInfo, DiskOption, DiskStore,
error::DiskError,
format::{DistributionAlgoVersion, FormatV3},
new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore,
new_disk,
},
endpoints::{Endpoints, PoolEndpoints},
error::StorageError,
global::{is_dist_erasure, GLOBAL_LOCAL_DISK_SET_DRIVES},
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, is_dist_erasure},
heal::heal_commands::{
HealOpts, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_ITEM_METADATA,
DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_ITEM_METADATA, HealOpts,
},
set_disk::SetDisks,
store_api::{
@@ -27,7 +28,7 @@ use crate::{
use common::globals::GLOBAL_Local_Node_Name;
use futures::future::join_all;
use http::HeaderMap;
use lock::{namespace_lock::NsLockMap, new_lock_api, LockApi};
use lock::{LockApi, namespace_lock::NsLockMap, new_lock_api};
use madmin::heal_commands::{HealDriveInfo, HealResultItem};
use tokio::sync::RwLock;
use uuid::Uuid;
+2 -8
View File
@@ -160,13 +160,7 @@ impl HTTPRangeSpec {
Some(HTTPRangeSpec {
is_suffix_length: false,
start: start as usize,
end: {
if end < 0 {
None
} else {
Some(end as usize)
}
},
end: { if end < 0 { None } else { Some(end as usize) } },
})
}
@@ -827,7 +821,7 @@ pub trait StorageAPI: ObjectIO {
opts: &HealOpts,
) -> Result<(HealResultItem, Option<Error>)>;
async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, hs: Arc<HealSequence>, is_meta: bool)
-> Result<()>;
-> Result<()>;
async fn get_pool_and_set(&self, id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)>;
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()>;
}
+4 -3
View File
@@ -1,18 +1,19 @@
use crate::config::{storageclass, KVS};
use crate::config::{KVS, storageclass};
use crate::disk::error_reduce::{count_errs, reduce_write_quorum_errs};
use crate::disk::{self, DiskAPI};
use crate::error::{Error, Result};
use crate::{
disk::{
DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET,
error::DiskError,
format::{FormatErasureVersion, FormatMetaVersion, FormatV3},
new_disk, DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET,
new_disk,
},
endpoints::Endpoints,
heal::heal_commands::init_healing_tracker,
};
use futures::future::join_all;
use std::collections::{hash_map::Entry, HashMap};
use std::collections::{HashMap, hash_map::Entry};
use tracing::{debug, warn};
use uuid::Uuid;
+1 -1
View File
@@ -32,7 +32,7 @@ fn deep_match_rune(str_: &[u8], pattern: &[u8], simple: bool) -> bool {
} else {
deep_match_rune(str_, &pattern[1..], simple)
|| (!str_.is_empty() && deep_match_rune(&str_[1..], pattern, simple))
}
};
}
'?' => {
if str_.is_empty() {