refactor: clean external DTO consumers (#3566)

* refactor: clean external DTO consumers

* fix: handle empty erasure shard recovery
This commit is contained in:
安正超
2026-06-18 14:22:44 +08:00
committed by GitHub
parent 99941f7e7c
commit acdf439371
28 changed files with 292 additions and 151 deletions
+7 -1
View File
@@ -33,7 +33,7 @@ use rustfs_ecstore::{
config::{com::save_config, storageclass},
disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
error::{Error, Result as StorageResult, StorageError},
store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
store_api::{GetObjectReader, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader},
};
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
@@ -61,6 +61,12 @@ const EVENT_SCANNER_CACHE_LOAD_STATE: &str = "scanner_cache_load_state";
const EVENT_SCANNER_CACHE_SAVE_STATE: &str = "scanner_cache_save_state";
static CACHE_SAVE_METRICS_ONCE: Once = Once::new();
pub type ScannerGetObjectReader = GetObjectReader;
pub type ScannerObjectInfo = ObjectInfo;
pub type ScannerObjectOptions = ObjectOptions;
pub type ScannerObjectToDelete = ObjectToDelete;
pub type ScannerPutObjReader = PutObjReader;
pub trait ScannerObjectIO:
ObjectIO<
Error = Error,
+4 -1
View File
@@ -1070,7 +1070,10 @@ pub async fn store_data_usage_in_backend(
#[cfg(test)]
mod tests {
use super::*;
use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::{
ScannerGetObjectReader as GetObjectReader, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions,
ScannerPutObjReader as PutObjReader,
};
use serial_test::serial;
use std::collections::HashMap;
use std::io::Cursor;
+2 -1
View File
@@ -57,7 +57,6 @@ use rustfs_ecstore::disk::{Disk, DiskAPI as _, DiskInfoOptions};
use rustfs_ecstore::error::StorageError;
use rustfs_ecstore::global::is_erasure;
use rustfs_ecstore::pools::{path2_bucket_object, path2_bucket_object_with_base_path};
use rustfs_ecstore::store_api::{ObjectInfo, ObjectToDelete};
use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ReplicateObjectInfo, ReplicationStatusType, ReplicationType,
@@ -70,6 +69,8 @@ use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, warn};
use crate::{ScannerObjectInfo as ObjectInfo, ScannerObjectToDelete as ObjectToDelete};
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_FOLDER: &str = "folder";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
+2 -1
View File
@@ -41,7 +41,6 @@ use rustfs_ecstore::error::{Error, StorageError};
use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::set_disk::SetDisks;
use rustfs_ecstore::store_api::ObjectInfo;
use rustfs_ecstore::{error::Result, store::ECStore};
use rustfs_filemeta::FileMeta;
use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
@@ -59,6 +58,8 @@ use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, warn};
use crate::ScannerObjectInfo as ObjectInfo;
pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file";
const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_IO: &str = "io";
@@ -28,16 +28,18 @@ use rustfs_ecstore::{
global::GLOBAL_TierConfigMgr,
pools::path2_bucket_object_with_base_path,
store::ECStore,
store_api::{ObjectOptions, PutObjReader},
tier::{
tier_config::{TierConfig, TierMinIO, TierType},
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
},
};
use rustfs_filemeta::FileMeta;
use rustfs_scanner::scanner::init_data_scanner;
use rustfs_scanner::scanner_folder::ScannerItem;
use rustfs_scanner::scanner_io::ScannerIODisk;
use rustfs_scanner::{
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
scanner::init_data_scanner,
};
use rustfs_storage_api::{
BucketOperations, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, ObjectOperations as _,
};
@@ -784,12 +786,7 @@ async fn register_mock_tier(tier_name: &str) -> MockWarmBackend {
backend
}
async fn wait_for_transition(
ecstore: &Arc<ECStore>,
bucket: &str,
object: &str,
timeout: Duration,
) -> Option<rustfs_ecstore::store_api::ObjectInfo> {
async fn wait_for_transition(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> Option<ObjectInfo> {
let deadline = tokio::time::Instant::now() + timeout;
loop {
@@ -900,7 +897,7 @@ mod serial_tests {
println!("✅ Object was transitioned by lifecycle processing");
// Let's try to get object info to see its details
match ecstore
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
.get_object_info(bucket_name.as_str(), object_name, &ObjectOptions::default())
.await
{
Ok(obj_info) => {